From 1b6fd032e3def058133161a45a000dae4d0db729 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 26 Oct 2018 08:52:41 +0200 Subject: [PATCH 001/159] core/vm: check empty in extcodehash --- core/vm/instructions.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/vm/instructions.go b/core/vm/instructions.go index b7c3ca5323..6696c6e3d1 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -544,7 +544,12 @@ func opExtCodeCopy(pc *uint64, interpreter *EVMInterpreter, contract *Contract, // this account should be regarded as a non-existent account and zero should be returned. func opExtCodeHash(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { slot := stack.peek() - slot.SetBytes(interpreter.evm.StateDB.GetCodeHash(common.BigToAddress(slot)).Bytes()) + address := common.BigToAddress(slot) + if interpreter.evm.StateDB.Empty(address) { + slot.SetUint64(0) + } else { + slot.SetBytes(interpreter.evm.StateDB.GetCodeHash(address).Bytes()) + } return nil, nil } From 3e1cfbae934f72c94c00e515ca872997530eede8 Mon Sep 17 00:00:00 2001 From: Roc Yu Date: Mon, 29 Oct 2018 17:00:00 +0800 Subject: [PATCH 002/159] cmd/swarm/swarm-smoke: fix issue that loop variable capture in func (#17992) --- cmd/swarm/swarm-smoke/feed_upload_and_sync.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/cmd/swarm/swarm-smoke/feed_upload_and_sync.go b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go index c7a1475d6c..1371d66548 100644 --- a/cmd/swarm/swarm-smoke/feed_upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go @@ -13,16 +13,13 @@ import ( "sync" "time" - "github.com/pborman/uuid" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/multihash" "github.com/ethereum/go-ethereum/swarm/storage/feed" - colorable "github.com/mattn/go-colorable" - + "github.com/pborman/uuid" cli "gopkg.in/urfave/cli.v1" ) @@ -190,7 +187,7 @@ func cliFeedUploadAndSync(c *cli.Context) error { for _, hex := range []string{topicHex, subTopicOnlyHex, mergedSubTopicHex} { wg.Add(1) ruid := uuid.New()[:8] - go func(endpoint string, ruid string) { + go func(hex string, endpoint string, ruid string) { for { err := fetchFeed(hex, userHex, endpoint, dataHash, ruid) if err != nil { @@ -200,7 +197,7 @@ func cliFeedUploadAndSync(c *cli.Context) error { wg.Done() return } - }(endpoint, ruid) + }(hex, endpoint, ruid) } } @@ -268,7 +265,7 @@ func cliFeedUploadAndSync(c *cli.Context) error { for _, url := range []string{manifestWithTopic, manifestWithSubTopic, manifestWithMergedTopic} { wg.Add(1) ruid := uuid.New()[:8] - go func(endpoint string, ruid string) { + go func(url string, endpoint string, ruid string) { for { err := fetch(url, endpoint, fileHash, ruid) if err != nil { @@ -278,7 +275,7 @@ func cliFeedUploadAndSync(c *cli.Context) error { wg.Done() return } - }(endpoint, ruid) + }(url, endpoint, ruid) } } From f08f596a378e1bc0cfcb476868767b1f1d602c17 Mon Sep 17 00:00:00 2001 From: Elad Date: Wed, 31 Oct 2018 23:36:33 +0100 Subject: [PATCH 003/159] all: updated code owners file (#17987) --- .github/CODEOWNERS | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5a717da009..f7f6916c54 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -9,23 +9,26 @@ les/ @zsfelfoldi light/ @zsfelfoldi mobile/ @karalabe p2p/ @fjl @zsfelfoldi +p2p/simulations @lmars +p2p/protocols @zelig +swarm/api/http @justelad swarm/bmt @zelig swarm/dev @lmars swarm/fuse @jmozah @holisticode swarm/grafana_dashboards @nonsense swarm/metrics @nonsense @holisticode swarm/multihash @nolash -swarm/network/bitvector @zelig @janos @gbalint -swarm/network/priorityqueue @zelig @janos @gbalint -swarm/network/simulations @zelig -swarm/network/stream @janos @zelig @gbalint @holisticode @justelad +swarm/network/bitvector @zelig @janos +swarm/network/priorityqueue @zelig @janos +swarm/network/simulations @zelig @janos +swarm/network/stream @janos @zelig @holisticode @justelad swarm/network/stream/intervals @janos swarm/network/stream/testing @zelig swarm/pot @zelig swarm/pss @nolash @zelig @nonsense swarm/services @zelig swarm/state @justelad -swarm/storage/encryption @gbalint @zelig @nagydani +swarm/storage/encryption @zelig @nagydani swarm/storage/mock @janos swarm/storage/feed @nolash @jpeletier swarm/testutil @lmars From 126dfde6c9de1f8122bb3991453d2429b47542f8 Mon Sep 17 00:00:00 2001 From: Elad Date: Sun, 4 Nov 2018 07:59:58 +0100 Subject: [PATCH 004/159] cmd/swarm: auto resolve default path according to env flag (#17960) --- cmd/swarm/config.go | 1 + cmd/swarm/upload.go | 38 ++++++++++++++++++++++++++++---------- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index 16001010df..3eea3057b3 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -80,6 +80,7 @@ const ( SWARM_ENV_STORE_CAPACITY = "SWARM_STORE_CAPACITY" SWARM_ENV_STORE_CACHE_CAPACITY = "SWARM_STORE_CACHE_CAPACITY" SWARM_ACCESS_PASSWORD = "SWARM_ACCESS_PASSWORD" + SWARM_AUTO_DEFAULTPATH = "SWARM_AUTO_DEFAULTPATH" GETH_ENV_DATADIR = "GETH_DATADIR" ) diff --git a/cmd/swarm/upload.go b/cmd/swarm/upload.go index 0dbe896e23..992f2d6e97 100644 --- a/cmd/swarm/upload.go +++ b/cmd/swarm/upload.go @@ -26,8 +26,10 @@ import ( "os/user" "path" "path/filepath" + "strconv" "strings" + "github.com/ethereum/go-ethereum/log" swarm "github.com/ethereum/go-ethereum/swarm/api/client" "github.com/ethereum/go-ethereum/cmd/utils" @@ -47,17 +49,24 @@ var upCommand = cli.Command{ func upload(ctx *cli.Context) { args := ctx.Args() var ( - bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") - recursive = ctx.GlobalBool(SwarmRecursiveFlag.Name) - wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) - defaultPath = ctx.GlobalString(SwarmUploadDefaultPath.Name) - fromStdin = ctx.GlobalBool(SwarmUpFromStdinFlag.Name) - mimeType = ctx.GlobalString(SwarmUploadMimeType.Name) - client = swarm.NewClient(bzzapi) - toEncrypt = ctx.Bool(SwarmEncryptedFlag.Name) - file string + bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") + recursive = ctx.GlobalBool(SwarmRecursiveFlag.Name) + wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) + defaultPath = ctx.GlobalString(SwarmUploadDefaultPath.Name) + fromStdin = ctx.GlobalBool(SwarmUpFromStdinFlag.Name) + mimeType = ctx.GlobalString(SwarmUploadMimeType.Name) + client = swarm.NewClient(bzzapi) + toEncrypt = ctx.Bool(SwarmEncryptedFlag.Name) + autoDefaultPath = false + file string ) - + if autoDefaultPathString := os.Getenv(SWARM_AUTO_DEFAULTPATH); autoDefaultPathString != "" { + b, err := strconv.ParseBool(autoDefaultPathString) + if err != nil { + utils.Fatalf("invalid environment variable %s: %v", SWARM_AUTO_DEFAULTPATH, err) + } + autoDefaultPath = b + } if len(args) != 1 { if fromStdin { tmp, err := ioutil.TempFile("", "swarm-stdin") @@ -106,6 +115,15 @@ func upload(ctx *cli.Context) { if !recursive { return "", errors.New("Argument is a directory and recursive upload is disabled") } + if autoDefaultPath && defaultPath == "" { + defaultEntryCandidate := path.Join(file, "index.html") + log.Debug("trying to find default path", "path", defaultEntryCandidate) + defaultEntryStat, err := os.Stat(defaultEntryCandidate) + if err == nil && !defaultEntryStat.IsDir() { + log.Debug("setting auto detected default path", "path", defaultEntryCandidate) + defaultPath = defaultEntryCandidate + } + } if defaultPath != "" { // construct absolute default path absDefaultPath, _ := filepath.Abs(defaultPath) From baee850471603d7fbf37eed0074c9de01df92ee7 Mon Sep 17 00:00:00 2001 From: KimMachineGun Date: Tue, 6 Nov 2018 17:54:19 +0900 Subject: [PATCH 005/159] swarm: modify context key (#17925) * swarm: modify context key * gofmt sctx.go --- swarm/api/http/sctx.go | 14 +++++--------- swarm/sctx/sctx.go | 12 +++++------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/swarm/api/http/sctx.go b/swarm/api/http/sctx.go index 431e117354..b8dafab0b7 100644 --- a/swarm/api/http/sctx.go +++ b/swarm/api/http/sctx.go @@ -7,14 +7,10 @@ import ( "github.com/ethereum/go-ethereum/swarm/sctx" ) -type contextKey int - -const ( - uriKey contextKey = iota -) +type uriKey struct{} func GetRUID(ctx context.Context) string { - v, ok := ctx.Value(sctx.HTTPRequestIDKey).(string) + v, ok := ctx.Value(sctx.HTTPRequestIDKey{}).(string) if ok { return v } @@ -22,11 +18,11 @@ func GetRUID(ctx context.Context) string { } func SetRUID(ctx context.Context, ruid string) context.Context { - return context.WithValue(ctx, sctx.HTTPRequestIDKey, ruid) + return context.WithValue(ctx, sctx.HTTPRequestIDKey{}, ruid) } func GetURI(ctx context.Context) *api.URI { - v, ok := ctx.Value(uriKey).(*api.URI) + v, ok := ctx.Value(uriKey{}).(*api.URI) if ok { return v } @@ -34,5 +30,5 @@ func GetURI(ctx context.Context) *api.URI { } func SetURI(ctx context.Context, uri *api.URI) context.Context { - return context.WithValue(ctx, uriKey, uri) + return context.WithValue(ctx, uriKey{}, uri) } diff --git a/swarm/sctx/sctx.go b/swarm/sctx/sctx.go index bed2b11458..fb7d35b000 100644 --- a/swarm/sctx/sctx.go +++ b/swarm/sctx/sctx.go @@ -2,19 +2,17 @@ package sctx import "context" -type ContextKey int - -const ( - HTTPRequestIDKey ContextKey = iota - requestHostKey +type ( + HTTPRequestIDKey struct{} + requestHostKey struct{} ) func SetHost(ctx context.Context, domain string) context.Context { - return context.WithValue(ctx, requestHostKey, domain) + return context.WithValue(ctx, requestHostKey{}, domain) } func GetHost(ctx context.Context) string { - v, ok := ctx.Value(requestHostKey).(string) + v, ok := ctx.Value(requestHostKey{}).(string) if ok { return v } From 53eb4e0b0fffdc105fbe9f5eed671b96de6e2ba1 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 6 Nov 2018 12:34:34 +0100 Subject: [PATCH 006/159] swarm/api: unexport Respond methods (#18037) --- swarm/api/http/middleware.go | 2 +- swarm/api/http/response.go | 25 ++++++------ swarm/api/http/server.go | 76 ++++++++++++++++++------------------ 3 files changed, 51 insertions(+), 52 deletions(-) diff --git a/swarm/api/http/middleware.go b/swarm/api/http/middleware.go index 3b2dcc7d59..ccc040c54d 100644 --- a/swarm/api/http/middleware.go +++ b/swarm/api/http/middleware.go @@ -50,7 +50,7 @@ func ParseURI(h http.Handler) http.Handler { uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/")) if err != nil { w.WriteHeader(http.StatusBadRequest) - RespondError(w, r, fmt.Sprintf("invalid URI %q", r.URL.Path), http.StatusBadRequest) + respondError(w, r, fmt.Sprintf("invalid URI %q", r.URL.Path), http.StatusBadRequest) return } if uri.Addr != "" && strings.HasPrefix(uri.Addr, "0x") { diff --git a/swarm/api/http/response.go b/swarm/api/http/response.go index c9fb9d2855..d4e81d7f67 100644 --- a/swarm/api/http/response.go +++ b/swarm/api/http/response.go @@ -53,23 +53,23 @@ func ShowMultipleChoices(w http.ResponseWriter, r *http.Request, list api.Manife log.Debug("ShowMultipleChoices", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context())) msg := "" if list.Entries == nil { - RespondError(w, r, "Could not resolve", http.StatusInternalServerError) + respondError(w, r, "Could not resolve", http.StatusInternalServerError) return } requestUri := strings.TrimPrefix(r.RequestURI, "/") uri, err := api.Parse(requestUri) if err != nil { - RespondError(w, r, "Bad Request", http.StatusBadRequest) + respondError(w, r, "Bad Request", http.StatusBadRequest) } uri.Scheme = "bzz-list" msg += fmt.Sprintf("Disambiguation:
Your request may refer to multiple choices.
Click here if your browser does not redirect you within 5 seconds.
", "/"+uri.String()) - RespondTemplate(w, r, "error", msg, http.StatusMultipleChoices) + respondTemplate(w, r, "error", msg, http.StatusMultipleChoices) } -func RespondTemplate(w http.ResponseWriter, r *http.Request, templateName, msg string, code int) { - log.Debug("RespondTemplate", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context())) +func respondTemplate(w http.ResponseWriter, r *http.Request, templateName, msg string, code int) { + log.Debug("respondTemplate", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context())) respond(w, r, &ResponseParams{ Code: code, Msg: template.HTML(msg), @@ -78,13 +78,12 @@ func RespondTemplate(w http.ResponseWriter, r *http.Request, templateName, msg s }) } -func RespondError(w http.ResponseWriter, r *http.Request, msg string, code int) { - log.Debug("RespondError", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()), "code", code) - RespondTemplate(w, r, "error", msg, code) +func respondError(w http.ResponseWriter, r *http.Request, msg string, code int) { + log.Info("respondError", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()), "code", code) + respondTemplate(w, r, "error", msg, code) } func respond(w http.ResponseWriter, r *http.Request, params *ResponseParams) { - w.WriteHeader(params.Code) if params.Code >= 400 { @@ -96,7 +95,7 @@ func respond(w http.ResponseWriter, r *http.Request, params *ResponseParams) { // this cannot be in a switch since an Accept header can have multiple values: "Accept: */*, text/html, application/xhtml+xml, application/xml;q=0.9, */*;q=0.8" if strings.Contains(acceptHeader, "application/json") { if err := respondJSON(w, r, params); err != nil { - RespondError(w, r, "Internal server error", http.StatusInternalServerError) + respondError(w, r, "Internal server error", http.StatusInternalServerError) } } else if strings.Contains(acceptHeader, "text/html") { respondHTML(w, r, params) @@ -107,7 +106,7 @@ func respond(w http.ResponseWriter, r *http.Request, params *ResponseParams) { func respondHTML(w http.ResponseWriter, r *http.Request, params *ResponseParams) { htmlCounter.Inc(1) - log.Debug("respondHTML", "ruid", GetRUID(r.Context())) + log.Info("respondHTML", "ruid", GetRUID(r.Context()), "code", params.Code) err := params.template.Execute(w, params) if err != nil { log.Error(err.Error()) @@ -116,14 +115,14 @@ func respondHTML(w http.ResponseWriter, r *http.Request, params *ResponseParams) func respondJSON(w http.ResponseWriter, r *http.Request, params *ResponseParams) error { jsonCounter.Inc(1) - log.Debug("respondJSON", "ruid", GetRUID(r.Context())) + log.Info("respondJSON", "ruid", GetRUID(r.Context()), "code", params.Code) w.Header().Set("Content-Type", "application/json") return json.NewEncoder(w).Encode(params) } func respondPlaintext(w http.ResponseWriter, r *http.Request, params *ResponseParams) error { plaintextCounter.Inc(1) - log.Debug("respondPlaintext", "ruid", GetRUID(r.Context())) + log.Info("respondPlaintext", "ruid", GetRUID(r.Context()), "code", params.Code) w.Header().Set("Content-Type", "text/plain") strToWrite := "Code: " + fmt.Sprintf("%d", params.Code) + "\n" strToWrite += "Message: " + string(params.Msg) + "\n" diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index de1eb70dcf..e9005104ee 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -184,10 +184,10 @@ func (s *Server) HandleBzzGet(w http.ResponseWriter, r *http.Request) { if err != nil { if isDecryptError(err) { w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", uri.Address().String())) - RespondError(w, r, err.Error(), http.StatusUnauthorized) + respondError(w, r, err.Error(), http.StatusUnauthorized) return } - RespondError(w, r, fmt.Sprintf("Had an error building the tarball: %v", err), http.StatusInternalServerError) + respondError(w, r, fmt.Sprintf("Had an error building the tarball: %v", err), http.StatusInternalServerError) return } defer reader.Close() @@ -211,7 +211,7 @@ func (s *Server) HandleBzzGet(w http.ResponseWriter, r *http.Request) { func (s *Server) HandleRootPaths(w http.ResponseWriter, r *http.Request) { switch r.RequestURI { case "/": - RespondTemplate(w, r, "landing-page", "Swarm: Please request a valid ENS or swarm hash with the appropriate bzz scheme", 200) + respondTemplate(w, r, "landing-page", "Swarm: Please request a valid ENS or swarm hash with the appropriate bzz scheme", 200) return case "/robots.txt": w.Header().Set("Last-Modified", time.Now().Format(http.TimeFormat)) @@ -220,7 +220,7 @@ func (s *Server) HandleRootPaths(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) w.Write(faviconBytes) default: - RespondError(w, r, "Not Found", http.StatusNotFound) + respondError(w, r, "Not Found", http.StatusNotFound) } } @@ -240,26 +240,26 @@ func (s *Server) HandlePostRaw(w http.ResponseWriter, r *http.Request) { if uri.Path != "" { postRawFail.Inc(1) - RespondError(w, r, "raw POST request cannot contain a path", http.StatusBadRequest) + respondError(w, r, "raw POST request cannot contain a path", http.StatusBadRequest) return } if uri.Addr != "" && uri.Addr != "encrypt" { postRawFail.Inc(1) - RespondError(w, r, "raw POST request addr can only be empty or \"encrypt\"", http.StatusBadRequest) + respondError(w, r, "raw POST request addr can only be empty or \"encrypt\"", http.StatusBadRequest) return } if r.Header.Get("Content-Length") == "" { postRawFail.Inc(1) - RespondError(w, r, "missing Content-Length header in request", http.StatusBadRequest) + respondError(w, r, "missing Content-Length header in request", http.StatusBadRequest) return } addr, _, err := s.api.Store(r.Context(), r.Body, r.ContentLength, toEncrypt) if err != nil { postRawFail.Inc(1) - RespondError(w, r, err.Error(), http.StatusInternalServerError) + respondError(w, r, err.Error(), http.StatusInternalServerError) return } @@ -283,7 +283,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) { contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) if err != nil { postFilesFail.Inc(1) - RespondError(w, r, err.Error(), http.StatusBadRequest) + respondError(w, r, err.Error(), http.StatusBadRequest) return } @@ -298,7 +298,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) { addr, err = s.api.Resolve(r.Context(), uri.Addr) if err != nil { postFilesFail.Inc(1) - RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusInternalServerError) + respondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusInternalServerError) return } log.Debug("resolved key", "ruid", ruid, "key", addr) @@ -306,7 +306,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) { addr, err = s.api.NewManifest(r.Context(), toEncrypt) if err != nil { postFilesFail.Inc(1) - RespondError(w, r, err.Error(), http.StatusInternalServerError) + respondError(w, r, err.Error(), http.StatusInternalServerError) return } log.Debug("new manifest", "ruid", ruid, "key", addr) @@ -317,7 +317,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) { case "application/x-tar": _, err := s.handleTarUpload(r, mw) if err != nil { - RespondError(w, r, fmt.Sprintf("error uploading tarball: %v", err), http.StatusInternalServerError) + respondError(w, r, fmt.Sprintf("error uploading tarball: %v", err), http.StatusInternalServerError) return err } return nil @@ -330,7 +330,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) { }) if err != nil { postFilesFail.Inc(1) - RespondError(w, r, fmt.Sprintf("cannot create manifest: %s", err), http.StatusInternalServerError) + respondError(w, r, fmt.Sprintf("cannot create manifest: %s", err), http.StatusInternalServerError) return } @@ -439,7 +439,7 @@ func (s *Server) HandleDelete(w http.ResponseWriter, r *http.Request) { newKey, err := s.api.Delete(r.Context(), uri.Addr, uri.Path) if err != nil { deleteFail.Inc(1) - RespondError(w, r, fmt.Sprintf("could not delete from manifest: %v", err), http.StatusInternalServerError) + respondError(w, r, fmt.Sprintf("could not delete from manifest: %v", err), http.StatusInternalServerError) return } @@ -460,7 +460,7 @@ func (s *Server) HandlePostFeed(w http.ResponseWriter, r *http.Request) { // Creation and update must send feed.updateRequestJSON JSON structure body, err := ioutil.ReadAll(r.Body) if err != nil { - RespondError(w, r, err.Error(), http.StatusInternalServerError) + respondError(w, r, err.Error(), http.StatusInternalServerError) return } @@ -471,7 +471,7 @@ func (s *Server) HandlePostFeed(w http.ResponseWriter, r *http.Request) { if err == api.ErrCannotLoadFeedManifest || err == api.ErrCannotResolveFeedURI { httpStatus = http.StatusNotFound } - RespondError(w, r, fmt.Sprintf("cannot retrieve feed from manifest: %s", err), httpStatus) + respondError(w, r, fmt.Sprintf("cannot retrieve feed from manifest: %s", err), httpStatus) return } @@ -480,7 +480,7 @@ func (s *Server) HandlePostFeed(w http.ResponseWriter, r *http.Request) { query := r.URL.Query() if err := updateRequest.FromValues(query, body); err != nil { // decodes request from query parameters - RespondError(w, r, err.Error(), http.StatusBadRequest) + respondError(w, r, err.Error(), http.StatusBadRequest) return } @@ -489,12 +489,12 @@ func (s *Server) HandlePostFeed(w http.ResponseWriter, r *http.Request) { // to update this feed // Check this early, to avoid creating a feed and then not being able to set its first update. if err = updateRequest.Verify(); err != nil { - RespondError(w, r, err.Error(), http.StatusForbidden) + respondError(w, r, err.Error(), http.StatusForbidden) return } _, err = s.api.FeedsUpdate(r.Context(), &updateRequest) if err != nil { - RespondError(w, r, err.Error(), http.StatusInternalServerError) + respondError(w, r, err.Error(), http.StatusInternalServerError) return } } @@ -505,7 +505,7 @@ func (s *Server) HandlePostFeed(w http.ResponseWriter, r *http.Request) { // feed identification used to retrieve feed updates later m, err := s.api.NewFeedManifest(r.Context(), &updateRequest.Feed) if err != nil { - RespondError(w, r, fmt.Sprintf("failed to create feed manifest: %v", err), http.StatusInternalServerError) + respondError(w, r, fmt.Sprintf("failed to create feed manifest: %v", err), http.StatusInternalServerError) return } // the key to the manifest will be passed back to the client @@ -513,7 +513,7 @@ func (s *Server) HandlePostFeed(w http.ResponseWriter, r *http.Request) { // the manifest key can be set as content in the resolver of the ENS name outdata, err := json.Marshal(m) if err != nil { - RespondError(w, r, fmt.Sprintf("failed to create json response: %s", err), http.StatusInternalServerError) + respondError(w, r, fmt.Sprintf("failed to create json response: %s", err), http.StatusInternalServerError) return } fmt.Fprint(w, string(outdata)) @@ -550,7 +550,7 @@ func (s *Server) HandleGetFeed(w http.ResponseWriter, r *http.Request) { if err == api.ErrCannotLoadFeedManifest || err == api.ErrCannotResolveFeedURI { httpStatus = http.StatusNotFound } - RespondError(w, r, fmt.Sprintf("cannot retrieve feed information from manifest: %s", err), httpStatus) + respondError(w, r, fmt.Sprintf("cannot retrieve feed information from manifest: %s", err), httpStatus) return } @@ -559,12 +559,12 @@ func (s *Server) HandleGetFeed(w http.ResponseWriter, r *http.Request) { unsignedUpdateRequest, err := s.api.FeedsNewRequest(r.Context(), fd) if err != nil { getFail.Inc(1) - RespondError(w, r, fmt.Sprintf("cannot retrieve feed metadata for feed=%s: %s", fd.Hex(), err), http.StatusNotFound) + respondError(w, r, fmt.Sprintf("cannot retrieve feed metadata for feed=%s: %s", fd.Hex(), err), http.StatusNotFound) return } rawResponse, err := unsignedUpdateRequest.MarshalJSON() if err != nil { - RespondError(w, r, fmt.Sprintf("cannot encode unsigned feed update request: %v", err), http.StatusInternalServerError) + respondError(w, r, fmt.Sprintf("cannot encode unsigned feed update request: %v", err), http.StatusInternalServerError) return } w.Header().Add("Content-type", "application/json") @@ -575,7 +575,7 @@ func (s *Server) HandleGetFeed(w http.ResponseWriter, r *http.Request) { lookupParams := &feed.Query{Feed: *fd} if err = lookupParams.FromValues(r.URL.Query()); err != nil { // parse period, version - RespondError(w, r, fmt.Sprintf("invalid feed update request:%s", err), http.StatusBadRequest) + respondError(w, r, fmt.Sprintf("invalid feed update request:%s", err), http.StatusBadRequest) return } @@ -584,7 +584,7 @@ func (s *Server) HandleGetFeed(w http.ResponseWriter, r *http.Request) { // any error from the switch statement will end up here if err != nil { code, err2 := s.translateFeedError(w, r, "feed lookup fail", err) - RespondError(w, r, err2.Error(), code) + respondError(w, r, err2.Error(), code) return } @@ -630,7 +630,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *http.Request) { addr, err := s.api.ResolveURI(r.Context(), uri, pass) if err != nil { getFail.Inc(1) - RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) + respondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) return } w.Header().Set("Cache-Control", "max-age=2147483648, immutable") // url was of type bzz:///path, so we are sure it is immutable. @@ -654,7 +654,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *http.Request) { reader, isEncrypted := s.api.Retrieve(r.Context(), addr) if _, err := reader.Size(r.Context(), nil); err != nil { getFail.Inc(1) - RespondError(w, r, fmt.Sprintf("root chunk not found %s: %s", addr, err), http.StatusNotFound) + respondError(w, r, fmt.Sprintf("root chunk not found %s: %s", addr, err), http.StatusNotFound) return } @@ -694,7 +694,7 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *http.Request) { addr, err := s.api.Resolve(r.Context(), uri.Addr) if err != nil { getListFail.Inc(1) - RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) + respondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) return } log.Debug("handle.get.list: resolved", "ruid", ruid, "key", addr) @@ -704,10 +704,10 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *http.Request) { getListFail.Inc(1) if isDecryptError(err) { w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", addr.String())) - RespondError(w, r, err.Error(), http.StatusUnauthorized) + respondError(w, r, err.Error(), http.StatusUnauthorized) return } - RespondError(w, r, err.Error(), http.StatusInternalServerError) + respondError(w, r, err.Error(), http.StatusInternalServerError) return } @@ -755,7 +755,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) { manifestAddr, err = s.api.Resolve(r.Context(), uri.Addr) if err != nil { getFileFail.Inc(1) - RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) + respondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) return } } else { @@ -779,17 +779,17 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) { if err != nil { if isDecryptError(err) { w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", manifestAddr)) - RespondError(w, r, err.Error(), http.StatusUnauthorized) + respondError(w, r, err.Error(), http.StatusUnauthorized) return } switch status { case http.StatusNotFound: getFileNotFound.Inc(1) - RespondError(w, r, err.Error(), http.StatusNotFound) + respondError(w, r, err.Error(), http.StatusNotFound) default: getFileFail.Inc(1) - RespondError(w, r, err.Error(), http.StatusInternalServerError) + respondError(w, r, err.Error(), http.StatusInternalServerError) } return } @@ -802,10 +802,10 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) { getFileFail.Inc(1) if isDecryptError(err) { w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", manifestAddr)) - RespondError(w, r, err.Error(), http.StatusUnauthorized) + respondError(w, r, err.Error(), http.StatusUnauthorized) return } - RespondError(w, r, err.Error(), http.StatusInternalServerError) + respondError(w, r, err.Error(), http.StatusInternalServerError) return } @@ -818,7 +818,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) { // check the root chunk exists by retrieving the file's size if _, err := reader.Size(r.Context(), nil); err != nil { getFileNotFound.Inc(1) - RespondError(w, r, fmt.Sprintf("file not found %s: %s", uri, err), http.StatusNotFound) + respondError(w, r, fmt.Sprintf("file not found %s: %s", uri, err), http.StatusNotFound) return } From 79c7a69ac8066cc28ceee2ebaab3d0221a8adf57 Mon Sep 17 00:00:00 2001 From: holisticode Date: Tue, 6 Nov 2018 18:04:18 -0500 Subject: [PATCH 007/159] swarm: Better syncing and retrieval option definition (#17986) * swarm: Better syncing and retrieval option definition * swarm/network/stream: better comments * swarm/network/stream: addressed PR comments --- swarm/network/stream/delivery_test.go | 97 +++++++++++++------ swarm/network/stream/intervals_test.go | 2 + swarm/network/stream/lightnode_test.go | 12 ++- .../network/stream/snapshot_retrieval_test.go | 5 +- swarm/network/stream/snapshot_sync_test.go | 8 +- swarm/network/stream/stream.go | 57 ++++++++--- swarm/network/stream/streamer_test.go | 2 + swarm/network/stream/syncer_test.go | 2 + swarm/swarm.go | 20 ++-- 9 files changed, 147 insertions(+), 58 deletions(-) diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index 949645558b..29b4f2f69a 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -38,8 +38,13 @@ import ( "github.com/ethereum/go-ethereum/swarm/storage" ) +//Tests initializing a retrieve request func TestStreamerRetrieveRequest(t *testing.T) { - tester, streamer, _, teardown, err := newStreamerTester(t, nil) + regOpts := &RegistryOptions{ + Retrieval: RetrievalClientOnly, + Syncing: SyncingDisabled, + } + tester, streamer, _, teardown, err := newStreamerTester(t, regOpts) defer teardown() if err != nil { t.Fatal(err) @@ -55,10 +60,21 @@ func TestStreamerRetrieveRequest(t *testing.T) { ) streamer.delivery.RequestFromPeers(ctx, req) + stream := NewStream(swarmChunkServerStreamName, "", true) + err = tester.TestExchanges(p2ptest.Exchange{ Label: "RetrieveRequestMsg", Expects: []p2ptest.Expect{ - { + { //start expecting a subscription for RETRIEVE_REQUEST due to `RetrievalClientOnly` + Code: 4, + Msg: &SubscribeMsg{ + Stream: stream, + History: nil, + Priority: Top, + }, + Peer: node.ID(), + }, + { //expect a retrieve request message for the given hash Code: 5, Msg: &RetrieveRequestMsg{ Addr: hash0[:], @@ -74,9 +90,12 @@ func TestStreamerRetrieveRequest(t *testing.T) { } } +//Test requesting a chunk from a peer then issuing a "empty" OfferedHashesMsg (no hashes available yet) +//Should time out as the peer does not have the chunk (no syncing happened previously) func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { tester, streamer, _, teardown, err := newStreamerTester(t, &RegistryOptions{ - DoServeRetrieve: true, + Retrieval: RetrievalEnabled, + Syncing: SyncingDisabled, //do no syncing }) defer teardown() if err != nil { @@ -89,16 +108,31 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { peer := streamer.getPeer(node.ID()) + stream := NewStream(swarmChunkServerStreamName, "", true) + //simulate pre-subscription to RETRIEVE_REQUEST stream on peer peer.handleSubscribeMsg(context.TODO(), &SubscribeMsg{ - Stream: NewStream(swarmChunkServerStreamName, "", true), + Stream: stream, History: nil, Priority: Top, }) + //test the exchange err = tester.TestExchanges(p2ptest.Exchange{ + Expects: []p2ptest.Expect{ + { //first expect a subscription to the RETRIEVE_REQUEST stream + Code: 4, + Msg: &SubscribeMsg{ + Stream: stream, + History: nil, + Priority: Top, + }, + Peer: node.ID(), + }, + }, + }, p2ptest.Exchange{ Label: "RetrieveRequestMsg", Triggers: []p2ptest.Trigger{ - { + { //then the actual RETRIEVE_REQUEST.... Code: 5, Msg: &RetrieveRequestMsg{ Addr: chunk.Address()[:], @@ -107,7 +141,7 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { }, }, Expects: []p2ptest.Expect{ - { + { //to which the peer responds with offered hashes Code: 1, Msg: &OfferedHashesMsg{ HandoverProof: nil, @@ -120,7 +154,9 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { }, }) - expectedError := `exchange #0 "RetrieveRequestMsg": timed out` + //should fail with a timeout as the peer we are requesting + //the chunk from does not have the chunk + expectedError := `exchange #1 "RetrieveRequestMsg": timed out` if err == nil || err.Error() != expectedError { t.Fatalf("Expected error %v, got %v", expectedError, err) } @@ -130,7 +166,8 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { // offered hashes or delivery if skipHash is set to true func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { tester, streamer, localStore, teardown, err := newStreamerTester(t, &RegistryOptions{ - DoServeRetrieve: true, + Retrieval: RetrievalEnabled, + Syncing: SyncingDisabled, }) defer teardown() if err != nil { @@ -138,6 +175,7 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { } node := tester.Nodes[0] + peer := streamer.getPeer(node.ID()) stream := NewStream(swarmChunkServerStreamName, "", true) @@ -156,6 +194,18 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { } err = tester.TestExchanges(p2ptest.Exchange{ + Expects: []p2ptest.Expect{ + { + Code: 4, + Msg: &SubscribeMsg{ + Stream: stream, + History: nil, + Priority: Top, + }, + Peer: node.ID(), + }, + }, + }, p2ptest.Exchange{ Label: "RetrieveRequestMsg", Triggers: []p2ptest.Trigger{ { @@ -226,7 +276,8 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { tester, streamer, localStore, teardown, err := newStreamerTester(t, &RegistryOptions{ - DoServeRetrieve: true, + Retrieval: RetrievalDisabled, + Syncing: SyncingDisabled, }) defer teardown() if err != nil { @@ -241,6 +292,7 @@ func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { node := tester.Nodes[0] + //subscribe to custom stream stream := NewStream("foo", "", true) err = streamer.Subscribe(node.ID(), stream, NewRange(5, 8), Top) if err != nil { @@ -253,7 +305,7 @@ func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { err = tester.TestExchanges(p2ptest.Exchange{ Label: "Subscribe message", Expects: []p2ptest.Expect{ - { + { //first expect subscription to the custom stream... Code: 4, Msg: &SubscribeMsg{ Stream: stream, @@ -267,7 +319,8 @@ func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { p2ptest.Exchange{ Label: "ChunkDelivery message", Triggers: []p2ptest.Trigger{ - { + { //...then trigger a chunk delivery for the given chunk from peer in order for + //local node to get the chunk delivered Code: 6, Msg: &ChunkDeliveryMsg{ Addr: chunkKey, @@ -342,8 +395,9 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ - SkipCheck: skipCheck, - DoServeRetrieve: true, + SkipCheck: skipCheck, + Syncing: SyncingDisabled, + Retrieval: RetrievalEnabled, }) bucket.Store(bucketKeyRegistry, r) @@ -408,20 +462,6 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck return err } - //each of the nodes (except pivot node) subscribes to the stream of the next node - for j, node := range nodeIDs[0 : nodes-1] { - sid := nodeIDs[j+1] - item, ok := sim.NodeItem(node, bucketKeyRegistry) - if !ok { - return fmt.Errorf("No registry") - } - registry := item.(*Registry) - err = registry.Subscribe(sid, NewStream(swarmChunkServerStreamName, "", true), nil, Top) - if err != nil { - return err - } - } - //get the pivot node's filestore item, ok := sim.NodeItem(*sim.PivotNodeID(), bucketKeyFileStore) if !ok { @@ -530,7 +570,8 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skip r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ SkipCheck: skipCheck, - DoSync: true, + Syncing: SyncingDisabled, + Retrieval: RetrievalDisabled, SyncUpdateDelay: 0, }) diff --git a/swarm/network/stream/intervals_test.go b/swarm/network/stream/intervals_test.go index 3164193b34..0c95fabb70 100644 --- a/swarm/network/stream/intervals_test.go +++ b/swarm/network/stream/intervals_test.go @@ -83,6 +83,8 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ + Retrieval: RetrievalDisabled, + Syncing: SyncingRegisterOnly, SkipCheck: skipCheck, }) bucket.Store(bucketKeyRegistry, r) diff --git a/swarm/network/stream/lightnode_test.go b/swarm/network/stream/lightnode_test.go index 0d3bc7f544..65cde2411c 100644 --- a/swarm/network/stream/lightnode_test.go +++ b/swarm/network/stream/lightnode_test.go @@ -25,7 +25,8 @@ import ( // when it is serving Retrieve requests. func TestLigthnodeRetrieveRequestWithRetrieve(t *testing.T) { registryOptions := &RegistryOptions{ - DoServeRetrieve: true, + Retrieval: RetrievalClientOnly, + Syncing: SyncingDisabled, } tester, _, _, teardown, err := newStreamerTester(t, registryOptions) defer teardown() @@ -63,7 +64,8 @@ func TestLigthnodeRetrieveRequestWithRetrieve(t *testing.T) { // requests are disabled func TestLigthnodeRetrieveRequestWithoutRetrieve(t *testing.T) { registryOptions := &RegistryOptions{ - DoServeRetrieve: false, + Retrieval: RetrievalDisabled, + Syncing: SyncingDisabled, } tester, _, _, teardown, err := newStreamerTester(t, registryOptions) defer teardown() @@ -106,7 +108,8 @@ func TestLigthnodeRetrieveRequestWithoutRetrieve(t *testing.T) { // when syncing is enabled. func TestLigthnodeRequestSubscriptionWithSync(t *testing.T) { registryOptions := &RegistryOptions{ - DoSync: true, + Retrieval: RetrievalDisabled, + Syncing: SyncingRegisterOnly, } tester, _, _, teardown, err := newStreamerTester(t, registryOptions) defer teardown() @@ -150,7 +153,8 @@ func TestLigthnodeRequestSubscriptionWithSync(t *testing.T) { // when syncing is disabled. func TestLigthnodeRequestSubscriptionWithoutSync(t *testing.T) { registryOptions := &RegistryOptions{ - DoSync: false, + Retrieval: RetrievalDisabled, + Syncing: SyncingDisabled, } tester, _, _, teardown, err := newStreamerTester(t, registryOptions) defer teardown() diff --git a/swarm/network/stream/snapshot_retrieval_test.go b/swarm/network/stream/snapshot_retrieval_test.go index b81cfc0ca0..ad15193414 100644 --- a/swarm/network/stream/snapshot_retrieval_test.go +++ b/swarm/network/stream/snapshot_retrieval_test.go @@ -127,10 +127,9 @@ func retrievalStreamerFunc(ctx *adapters.ServiceContext, bucket *sync.Map) (s no netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ - DoSync: true, + Retrieval: RetrievalEnabled, + Syncing: SyncingAutoSubscribe, SyncUpdateDelay: 3 * time.Second, - DoRetrieve: true, - DoServeRetrieve: true, }) fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go index 8d89f28cba..2ddbed9368 100644 --- a/swarm/network/stream/snapshot_sync_test.go +++ b/swarm/network/stream/snapshot_sync_test.go @@ -165,8 +165,8 @@ func streamerFunc(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Servic netStore.NewNetFetcherFunc = network.NewFetcherFactory(dummyRequestFromPeers, true).New r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ - DoSync: true, - DoServeRetrieve: true, + Retrieval: RetrievalDisabled, + Syncing: SyncingAutoSubscribe, SyncUpdateDelay: 3 * time.Second, }) @@ -360,8 +360,8 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int) netStore.NewNetFetcherFunc = network.NewFetcherFactory(dummyRequestFromPeers, true).New r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ - DoServeRetrieve: true, - DoSync: true, + Retrieval: RetrievalDisabled, + Syncing: SyncingRegisterOnly, }) bucket.Store(bucketKeyRegistry, r) diff --git a/swarm/network/stream/stream.go b/swarm/network/stream/stream.go index 0ac374def1..695ff0c502 100644 --- a/swarm/network/stream/stream.go +++ b/swarm/network/stream/stream.go @@ -47,6 +47,31 @@ const ( HashSize = 32 ) +//Enumerate options for syncing and retrieval +type SyncingOption int +type RetrievalOption int + +//Syncing options +const ( + //Syncing disabled + SyncingDisabled SyncingOption = iota + //Register the client and the server but not subscribe + SyncingRegisterOnly + //Both client and server funcs are registered, subscribe sent automatically + SyncingAutoSubscribe +) + +const ( + //Retrieval disabled. Used mostly for tests to isolate syncing features (i.e. syncing only) + RetrievalDisabled RetrievalOption = iota + //Only the client side of the retrieve request is registered. + //(light nodes do not serve retrieve requests) + //once the client is registered, subscription to retrieve request stream is always sent + RetrievalClientOnly + //Both client and server funcs are registered, subscribe sent automatically + RetrievalEnabled +) + // Registry registry for outgoing and incoming streamer constructors type Registry struct { addr enode.ID @@ -60,16 +85,15 @@ type Registry struct { peers map[enode.ID]*Peer delivery *Delivery intervalsStore state.Store - doRetrieve bool + autoRetrieval bool //automatically subscribe to retrieve request stream maxPeerServers int } // RegistryOptions holds optional values for NewRegistry constructor. type RegistryOptions struct { SkipCheck bool - DoSync bool // Sets if the server syncs with peers. Default is true, set to false by lightnode or nosync flags. - DoRetrieve bool // Sets if the server issues Retrieve requests. Default is true. - DoServeRetrieve bool // Sets if the server serves Retrieve requests. Default is true, set to false by lightnode flag. + Syncing SyncingOption //Defines syncing behavior + Retrieval RetrievalOption //Defines retrieval behavior SyncUpdateDelay time.Duration MaxPeerServers int // The limit of servers for each peer in registry } @@ -82,6 +106,9 @@ func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.Sy if options.SyncUpdateDelay <= 0 { options.SyncUpdateDelay = 15 * time.Second } + //check if retriaval has been disabled + retrieval := options.Retrieval != RetrievalDisabled + streamer := &Registry{ addr: localID, skipCheck: options.SkipCheck, @@ -90,13 +117,14 @@ func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.Sy peers: make(map[enode.ID]*Peer), delivery: delivery, intervalsStore: intervalsStore, - doRetrieve: options.DoRetrieve, + autoRetrieval: retrieval, maxPeerServers: options.MaxPeerServers, } streamer.api = NewAPI(streamer) delivery.getPeer = streamer.getPeer - if options.DoServeRetrieve { + //if retrieval is enabled, register the server func, so that retrieve requests will be served (non-light nodes only) + if options.Retrieval == RetrievalEnabled { streamer.RegisterServerFunc(swarmChunkServerStreamName, func(_ *Peer, _ string, live bool) (Server, error) { if !live { return nil, errors.New("only live retrieval requests supported") @@ -105,16 +133,21 @@ func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.Sy }) } - streamer.RegisterClientFunc(swarmChunkServerStreamName, func(p *Peer, t string, live bool) (Client, error) { - return NewSwarmSyncerClient(p, syncChunkStore, NewStream(swarmChunkServerStreamName, t, live)) - }) + //if retrieval is not disabled, register the client func (both light nodes and normal nodes can issue retrieve requests) + if options.Retrieval != RetrievalDisabled { + streamer.RegisterClientFunc(swarmChunkServerStreamName, func(p *Peer, t string, live bool) (Client, error) { + return NewSwarmSyncerClient(p, syncChunkStore, NewStream(swarmChunkServerStreamName, t, live)) + }) + } - if options.DoSync { + //If syncing is not disabled, the syncing functions are registered (both client and server) + if options.Syncing != SyncingDisabled { RegisterSwarmSyncerServer(streamer, syncChunkStore) RegisterSwarmSyncerClient(streamer, syncChunkStore) } - if options.DoSync { + //if syncing is set to automatically subscribe to the syncing stream, start the subscription process + if options.Syncing == SyncingAutoSubscribe { // latestIntC function ensures that // - receiving from the in chan is not blocked by processing inside the for loop // - the latest int value is delivered to the loop after the processing is done @@ -385,7 +418,7 @@ func (r *Registry) Run(p *network.BzzPeer) error { defer close(sp.quit) defer sp.close() - if r.doRetrieve { + if r.autoRetrieval && !p.LightNode { err := r.Subscribe(p.ID(), NewStream(swarmChunkServerStreamName, "", true), nil, Top) if err != nil { return err diff --git a/swarm/network/stream/streamer_test.go b/swarm/network/stream/streamer_test.go index e7f79e7a12..16c74d3b34 100644 --- a/swarm/network/stream/streamer_test.go +++ b/swarm/network/stream/streamer_test.go @@ -765,6 +765,8 @@ func TestStreamerRequestSubscriptionQuitMsgExchange(t *testing.T) { func TestMaxPeerServersWithUnsubscribe(t *testing.T) { var maxPeerServers = 6 tester, streamer, _, teardown, err := newStreamerTester(t, &RegistryOptions{ + Retrieval: RetrievalDisabled, + Syncing: SyncingDisabled, MaxPeerServers: maxPeerServers, }) defer teardown() diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index f2be3bef99..113807b98a 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -114,6 +114,8 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck bucket.Store(bucketKeyDelivery, delivery) r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ + Retrieval: RetrievalDisabled, + Syncing: SyncingAutoSubscribe, SkipCheck: skipCheck, }) diff --git a/swarm/swarm.go b/swarm/swarm.go index 7214abbda4..1fb5443fd4 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -175,18 +175,24 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e if err := nodeID.UnmarshalText([]byte(config.NodeID)); err != nil { return nil, err } + + syncing := stream.SyncingAutoSubscribe + if !config.SyncEnabled || config.LightNodeEnabled { + syncing = stream.SyncingDisabled + } + + retrieval := stream.RetrievalEnabled + if config.LightNodeEnabled { + retrieval = stream.RetrievalClientOnly + } + registryOptions := &stream.RegistryOptions{ SkipCheck: config.DeliverySkipCheck, - DoSync: config.SyncEnabled, - DoRetrieve: true, - DoServeRetrieve: true, + Syncing: syncing, + Retrieval: retrieval, SyncUpdateDelay: config.SyncUpdateDelay, MaxPeerServers: config.MaxStreamPeerServers, } - if config.LightNodeEnabled { - registryOptions.DoSync = false - registryOptions.DoRetrieve = false - } self.streamer = stream.NewRegistry(nodeID, delivery, self.netStore, stateStore, registryOptions) // Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage From e2640a96d4311e3ac48de467a6eb3c88b95dfb3e Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 7 Nov 2018 09:55:56 +0100 Subject: [PATCH 008/159] miner: fix miner stress test (#18039) --- miner/stress_clique.go | 23 ++++++++--------------- miner/stress_ethash.go | 23 ++++++++--------------- 2 files changed, 16 insertions(+), 30 deletions(-) diff --git a/miner/stress_clique.go b/miner/stress_clique.go index 8961091d56..7e19975aec 100644 --- a/miner/stress_clique.go +++ b/miner/stress_clique.go @@ -22,7 +22,6 @@ package main import ( "bytes" "crypto/ecdsa" - "fmt" "io/ioutil" "math/big" "math/rand" @@ -40,7 +39,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/params" ) @@ -62,11 +61,11 @@ func main() { var ( nodes []*node.Node - enodes []string + enodes []*enode.Node ) for _, sealer := range sealers { // Start the node and wait until it's up - node, err := makeSealer(genesis, enodes) + node, err := makeSealer(genesis) if err != nil { panic(err) } @@ -76,18 +75,12 @@ func main() { time.Sleep(250 * time.Millisecond) } // Connect the node to al the previous ones - for _, enode := range enodes { - enode, err := discover.ParseNode(enode) - if err != nil { - panic(err) - } - node.Server().AddPeer(enode) + for _, n := range enodes { + node.Server().AddPeer(n) } - // Start tracking the node and it's enode url + // Start tracking the node and it's enode nodes = append(nodes, node) - - enode := fmt.Sprintf("enode://%s@127.0.0.1:%d", node.Server().NodeInfo().ID, node.Server().NodeInfo().Ports.Listener) - enodes = append(enodes, enode) + enodes = append(enodes, node.Server().Self()) // Inject the signer key and start sealing with it store := node.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore) @@ -177,7 +170,7 @@ func makeGenesis(faucets []*ecdsa.PrivateKey, sealers []*ecdsa.PrivateKey) *core return genesis } -func makeSealer(genesis *core.Genesis, nodes []string) (*node.Node, error) { +func makeSealer(genesis *core.Genesis) (*node.Node, error) { // Define the basic configurations for the Ethereum node datadir, _ := ioutil.TempDir("", "") diff --git a/miner/stress_ethash.go b/miner/stress_ethash.go index 5ed11d73a5..044ca9a218 100644 --- a/miner/stress_ethash.go +++ b/miner/stress_ethash.go @@ -21,7 +21,6 @@ package main import ( "crypto/ecdsa" - "fmt" "io/ioutil" "math/big" "math/rand" @@ -41,7 +40,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/params" ) @@ -62,11 +61,11 @@ func main() { var ( nodes []*node.Node - enodes []string + enodes []*enode.Node ) for i := 0; i < 4; i++ { // Start the node and wait until it's up - node, err := makeMiner(genesis, enodes) + node, err := makeMiner(genesis) if err != nil { panic(err) } @@ -76,18 +75,12 @@ func main() { time.Sleep(250 * time.Millisecond) } // Connect the node to al the previous ones - for _, enode := range enodes { - enode, err := discover.ParseNode(enode) - if err != nil { - panic(err) - } - node.Server().AddPeer(enode) + for _, n := range enodes { + node.Server().AddPeer(n) } - // Start tracking the node and it's enode url + // Start tracking the node and it's enode nodes = append(nodes, node) - - enode := fmt.Sprintf("enode://%s@127.0.0.1:%d", node.Server().NodeInfo().ID, node.Server().NodeInfo().Ports.Listener) - enodes = append(enodes, enode) + enodes = append(enodes, node.Server().Self()) // Inject the signer key and start sealing with it store := node.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore) @@ -155,7 +148,7 @@ func makeGenesis(faucets []*ecdsa.PrivateKey) *core.Genesis { return genesis } -func makeMiner(genesis *core.Genesis, nodes []string) (*node.Node, error) { +func makeMiner(genesis *core.Genesis) (*node.Node, error) { // Define the basic configurations for the Ethereum node datadir, _ := ioutil.TempDir("", "") From 80e2f3aca42583856669372cfc636cb7fd370fdb Mon Sep 17 00:00:00 2001 From: Samuel Marks Date: Wed, 7 Nov 2018 22:17:41 +1100 Subject: [PATCH 009/159] travis, appveyor: bump to Go 1.11.2 (#18031) --- .travis.yml | 2 +- appveyor.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 69535b7ef3..c1cc7c4aa7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -148,7 +148,7 @@ matrix: git: submodules: false # avoid cloning ethereum/tests before_install: - - curl https://storage.googleapis.com/golang/go1.11.1.linux-amd64.tar.gz | tar -xz + - curl https://storage.googleapis.com/golang/go1.11.2.linux-amd64.tar.gz | tar -xz - export PATH=`pwd`/go/bin:$PATH - export GOROOT=`pwd`/go - export GOPATH=$HOME/go diff --git a/appveyor.yml b/appveyor.yml index 11848ddb95..e5126b2529 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,8 +23,8 @@ environment: install: - git submodule update --init - rmdir C:\go /s /q - - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.1.windows-%GETH_ARCH%.zip - - 7z x go1.11.1.windows-%GETH_ARCH%.zip -y -oC:\ > NUL + - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.2.windows-%GETH_ARCH%.zip + - 7z x go1.11.2.windows-%GETH_ARCH%.zip -y -oC:\ > NUL - go version - gcc --version From 0fe0b8f7b9904767cdd73cf1aac4d7ad543d95a7 Mon Sep 17 00:00:00 2001 From: Corey Lin <514971757@qq.com> Date: Wed, 7 Nov 2018 19:22:31 +0800 Subject: [PATCH 010/159] p2p/protocols: use keyed fields for struct instantiation (#18017) --- p2p/protocols/protocol_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/protocols/protocol_test.go b/p2p/protocols/protocol_test.go index 2874af48d2..a26222cd8b 100644 --- a/p2p/protocols/protocol_test.go +++ b/p2p/protocols/protocol_test.go @@ -318,7 +318,7 @@ func TestProtocolHook(t *testing.T) { <-testHook.waitC time.Sleep(100 * time.Millisecond) - err = tester.TestDisconnected(&p2ptest.Disconnect{tester.Nodes[1].ID(), testHook.err}) + err = tester.TestDisconnected(&p2ptest.Disconnect{Peer: tester.Nodes[1].ID(), Error: testHook.err}) if err != nil { t.Fatalf("Expected a specific disconnect error, but got different one: %v", err) } From dc6648bb58fb37f7b5d9fd8e7ffbeb4cf45492fd Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 7 Nov 2018 13:18:07 +0100 Subject: [PATCH 011/159] downloader: measure successfull deliveries, not failed (#17983) * downloader: measure successfull deliveries, not failed * downloader: fix typos --- eth/downloader/statesync.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/eth/downloader/statesync.go b/eth/downloader/statesync.go index 8d33dfec74..29d5ee4ddb 100644 --- a/eth/downloader/statesync.go +++ b/eth/downloader/statesync.go @@ -313,11 +313,12 @@ func (s *stateSync) loop() (err error) { s.d.dropPeer(req.peer.id) } // Process all the received blobs and check for stale delivery - if err = s.process(req); err != nil { + delivered, err := s.process(req) + if err != nil { log.Warn("Node data write error", "err", err) return err } - req.peer.SetNodeDataIdle(len(req.response)) + req.peer.SetNodeDataIdle(delivered) } } return nil @@ -398,9 +399,11 @@ func (s *stateSync) fillTasks(n int, req *stateReq) { // process iterates over a batch of delivered state data, injecting each item // into a running state sync, re-queuing any items that were requested but not // delivered. -func (s *stateSync) process(req *stateReq) error { +// Returns whether the peer actually managed to deliver anything of value, +// and any error that occurred +func (s *stateSync) process(req *stateReq) (int, error) { // Collect processing stats and update progress if valid data was received - duplicate, unexpected := 0, 0 + duplicate, unexpected, successful := 0, 0, 0 defer func(start time.Time) { if duplicate > 0 || unexpected > 0 { @@ -410,7 +413,6 @@ func (s *stateSync) process(req *stateReq) error { // Iterate over all the delivered data and inject one-by-one into the trie progress := false - for _, blob := range req.response { prog, hash, err := s.processNodeData(blob) switch err { @@ -418,12 +420,13 @@ func (s *stateSync) process(req *stateReq) error { s.numUncommitted++ s.bytesUncommitted += len(blob) progress = progress || prog + successful++ case trie.ErrNotRequested: unexpected++ case trie.ErrAlreadyProcessed: duplicate++ default: - return fmt.Errorf("invalid state node %s: %v", hash.TerminalString(), err) + return successful, fmt.Errorf("invalid state node %s: %v", hash.TerminalString(), err) } if _, ok := req.tasks[hash]; ok { delete(req.tasks, hash) @@ -441,12 +444,12 @@ func (s *stateSync) process(req *stateReq) error { // If we've requested the node too many times already, it may be a malicious // sync where nobody has the right data. Abort. if len(task.attempts) >= npeers { - return fmt.Errorf("state node %s failed with all peers (%d tries, %d peers)", hash.TerminalString(), len(task.attempts), npeers) + return successful, fmt.Errorf("state node %s failed with all peers (%d tries, %d peers)", hash.TerminalString(), len(task.attempts), npeers) } // Missing item, place into the retry queue. s.tasks[hash] = task } - return nil + return successful, nil } // processNodeData tries to inject a trie node data blob delivered from a remote From eea3ae42a3d9bcbd33474c0e482754c5196a469f Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 7 Nov 2018 13:47:11 +0100 Subject: [PATCH 012/159] core, eth/downloader: fix validation flaw, fix downloader printout flaw (#17974) --- core/block_validator.go | 12 ++++++------ eth/downloader/downloader.go | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/core/block_validator.go b/core/block_validator.go index 1329f62427..3b9496fece 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -53,12 +53,6 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error { if v.bc.HasBlockAndState(block.Hash(), block.NumberU64()) { return ErrKnownBlock } - if !v.bc.HasBlockAndState(block.ParentHash(), block.NumberU64()-1) { - if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) { - return consensus.ErrUnknownAncestor - } - return consensus.ErrPrunedAncestor - } // Header validity is known at this point, check the uncles and transactions header := block.Header() if err := v.engine.VerifyUncles(v.bc, block); err != nil { @@ -70,6 +64,12 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error { if hash := types.DeriveSha(block.Transactions()); hash != header.TxHash { return fmt.Errorf("transaction root hash mismatch: have %x, want %x", hash, header.TxHash) } + if !v.bc.HasBlockAndState(block.ParentHash(), block.NumberU64()-1) { + if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) { + return consensus.ErrUnknownAncestor + } + return consensus.ErrPrunedAncestor + } return nil } diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index f01a8fdbd8..56c54c8ed5 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -740,6 +740,7 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err return 0, errBadPeer } start = check + hash = h case <-timeout: p.log.Debug("Waiting for search header timed out", "elapsed", ttl) From 5b74bb6445a7fb7220da74c6c6e891c20e9a26dc Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 7 Nov 2018 13:55:54 +0100 Subject: [PATCH 013/159] signer: remove ineffectual assignments (#18049) * signer: remove ineffectual assignments * signer: remove ineffectual assignments --- signer/core/abihelper_test.go | 6 +++--- signer/rules/rules_test.go | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/signer/core/abihelper_test.go b/signer/core/abihelper_test.go index 2afeec73e6..878210be1f 100644 --- a/signer/core/abihelper_test.go +++ b/signer/core/abihelper_test.go @@ -38,13 +38,13 @@ func verify(t *testing.T, jsondata, calldata string, exp []interface{}) { cd := common.Hex2Bytes(calldata) sigdata, argdata := cd[:4], cd[4:] method, err := abispec.MethodById(sigdata) - if err != nil { t.Fatal(err) } - data, err := method.Inputs.UnpackValues(argdata) - + if err != nil { + t.Fatal(err) + } if len(data) != len(exp) { t.Fatalf("Mismatched length, expected %d, got %d", len(exp), len(data)) } diff --git a/signer/rules/rules_test.go b/signer/rules/rules_test.go index 55614077ca..0b520a15bf 100644 --- a/signer/rules/rules_test.go +++ b/signer/rules/rules_test.go @@ -151,7 +151,7 @@ func TestListRequest(t *testing.T) { t.Errorf("Couldn't create evaluator %v", err) return } - resp, err := r.ApproveListing(&core.ListRequest{ + resp, _ := r.ApproveListing(&core.ListRequest{ Accounts: accs, Meta: core.Metadata{Remote: "remoteip", Local: "localip", Scheme: "inproc"}, }) @@ -515,7 +515,7 @@ func TestLimitWindow(t *testing.T) { r.OnApprovedTx(response) } // Fourth should fail - resp, err := r.ApproveTx(dummyTx(h)) + resp, _ := r.ApproveTx(dummyTx(h)) if resp.Approved { t.Errorf("Expected check to resolve to 'Reject'") } @@ -609,8 +609,8 @@ func TestContextIsCleared(t *testing.T) { t.Fatalf("Failed to load bootstrap js: %v", err) } tx := dummyTxWithV(0) - r1, err := r.ApproveTx(tx) - r2, err := r.ApproveTx(tx) + r1, _ := r.ApproveTx(tx) + r2, _ := r.ApproveTx(tx) if r1.Approved != r2.Approved { t.Errorf("Expected execution context to be cleared between executions") } From b35165555d737042d5f958413827c31a7a5f4805 Mon Sep 17 00:00:00 2001 From: Wenbiao Zheng Date: Wed, 7 Nov 2018 07:30:19 -0600 Subject: [PATCH 014/159] eth/downloader: remove the expired id directly (#17963) --- eth/downloader/queue.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index c6b635aff1..863cc8de14 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -664,12 +664,11 @@ func (q *queue) expire(timeout time.Duration, pendPool map[string]*fetchRequest, } // Add the peer to the expiry report along the number of failed requests expiries[id] = len(request.Headers) + + // Remove the expired requests from the pending pool directly + delete(pendPool, id) } } - // Remove the expired requests from the pending pool - for id := range expiries { - delete(pendPool, id) - } return expiries } From 36ca85fa1c2936087b2c3d976d3576f0f5d2157e Mon Sep 17 00:00:00 2001 From: Javier Peletier Date: Wed, 7 Nov 2018 14:49:42 +0100 Subject: [PATCH 015/159] swarm/api: Fix #18007, missing signature should return HTTP 400 (#18008) --- swarm/api/http/server.go | 10 ++++++---- swarm/api/http/server_test.go | 34 ++++++++++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index e9005104ee..803b789878 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -484,7 +484,8 @@ func (s *Server) HandlePostFeed(w http.ResponseWriter, r *http.Request) { return } - if updateRequest.IsUpdate() { + switch { + case updateRequest.IsUpdate(): // Verify that the signature is intact and that the signer is authorized // to update this feed // Check this early, to avoid creating a feed and then not being able to set its first update. @@ -497,9 +498,8 @@ func (s *Server) HandlePostFeed(w http.ResponseWriter, r *http.Request) { respondError(w, r, err.Error(), http.StatusInternalServerError) return } - } - - if query.Get("manifest") == "1" { + fallthrough + case query.Get("manifest") == "1": // we create a manifest so we can retrieve feed updates with bzz:// later // this manifest has a special "feed type" manifest, and saves the // feed identification used to retrieve feed updates later @@ -519,6 +519,8 @@ func (s *Server) HandlePostFeed(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, string(outdata)) w.Header().Add("Content-type", "application/json") + default: + respondError(w, r, "Missing signature in feed update request", http.StatusBadRequest) } } diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 1cf7ff5770..159c8a159f 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -333,15 +333,45 @@ func TestBzzFeed(t *testing.T) { } urlQuery = testUrl.Query() body = updateRequest.AppendValues(urlQuery) // this adds all query parameters + goodQueryParameters := urlQuery.Encode() // save the query parameters for a second attempt + + // create bad query parameters in which the signature is missing + urlQuery.Del("signature") testUrl.RawQuery = urlQuery.Encode() + // 1st attempt with bad query parameters in which the signature is missing resp, err = http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body)) if err != nil { t.Fatal(err) } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - t.Fatalf("Update returned %s", resp.Status) + expectedCode := http.StatusBadRequest + if resp.StatusCode != expectedCode { + t.Fatalf("Update returned %s. Expected %d", resp.Status, expectedCode) + } + + // 2nd attempt with bad query parameters in which the signature is of incorrect length + urlQuery.Set("signature", "0xabcd") // should be 130 hex chars + resp, err = http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + expectedCode = http.StatusBadRequest + if resp.StatusCode != expectedCode { + t.Fatalf("Update returned %s. Expected %d", resp.Status, expectedCode) + } + + // 3rd attempt, with good query parameters: + testUrl.RawQuery = goodQueryParameters + resp, err = http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + expectedCode = http.StatusOK + if resp.StatusCode != expectedCode { + t.Fatalf("Update returned %s. Expected %d", resp.Status, expectedCode) } // get latest update through bzz-feed directly From 0bcff8f52505fa1eece6c8fbfc5cab8aa9d9ed5d Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 7 Nov 2018 15:07:43 +0100 Subject: [PATCH 016/159] eth/downloader: speed up tests by generating chain only once (#17916) * core: speed up GenerateChain Use a mock implementation of ChainReader instead of creating and destroying a BlockChain object for each generated block. * eth/downloader: speed up tests by generating chain only once This change reworks the downloader tests so they share a common test blockchain instead of generating a chain in every test. The tests are roughly twice as fast now. --- core/chain_makers.go | 44 +- eth/downloader/downloader_test.go | 816 ++++++++++-------------------- eth/downloader/testchain_test.go | 221 ++++++++ 3 files changed, 520 insertions(+), 561 deletions(-) create mode 100644 eth/downloader/testchain_test.go diff --git a/core/chain_makers.go b/core/chain_makers.go index 0bc453fdf3..e3a5537a40 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -33,12 +33,11 @@ import ( // BlockGen creates blocks for testing. // See GenerateChain for a detailed explanation. type BlockGen struct { - i int - parent *types.Block - chain []*types.Block - chainReader consensus.ChainReader - header *types.Header - statedb *state.StateDB + i int + parent *types.Block + chain []*types.Block + header *types.Header + statedb *state.StateDB gasPool *GasPool txs []*types.Transaction @@ -138,7 +137,7 @@ func (b *BlockGen) AddUncle(h *types.Header) { // For index -1, PrevBlock returns the parent block given to GenerateChain. func (b *BlockGen) PrevBlock(index int) *types.Block { if index >= b.i { - panic("block index out of range") + panic(fmt.Errorf("block index %d out of range (%d,%d)", index, -1, b.i)) } if index == -1 { return b.parent @@ -154,7 +153,8 @@ func (b *BlockGen) OffsetTime(seconds int64) { if b.header.Time.Cmp(b.parent.Header().Time) <= 0 { panic("block time out of range") } - b.header.Difficulty = b.engine.CalcDifficulty(b.chainReader, b.header.Time.Uint64(), b.parent.Header()) + chainreader := &fakeChainReader{config: b.config} + b.header.Difficulty = b.engine.CalcDifficulty(chainreader, b.header.Time.Uint64(), b.parent.Header()) } // GenerateChain creates a chain of n blocks. The first block's @@ -174,14 +174,10 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse config = params.TestChainConfig } blocks, receipts := make(types.Blocks, n), make([]types.Receipts, n) + chainreader := &fakeChainReader{config: config} genblock := func(i int, parent *types.Block, statedb *state.StateDB) (*types.Block, types.Receipts) { - // TODO(karalabe): This is needed for clique, which depends on multiple blocks. - // It's nonetheless ugly to spin up a blockchain here. Get rid of this somehow. - blockchain, _ := NewBlockChain(db, nil, config, engine, vm.Config{}, nil) - defer blockchain.Stop() - - b := &BlockGen{i: i, parent: parent, chain: blocks, chainReader: blockchain, statedb: statedb, config: config, engine: engine} - b.header = makeHeader(b.chainReader, parent, statedb, b.engine) + b := &BlockGen{i: i, chain: blocks, parent: parent, statedb: statedb, config: config, engine: engine} + b.header = makeHeader(chainreader, parent, statedb, b.engine) // Mutate the state and block according to any hard-fork specs if daoBlock := config.DAOForkBlock; daoBlock != nil { @@ -201,7 +197,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse } if b.engine != nil { // Finalize and seal the block - block, _ := b.engine.Finalize(b.chainReader, b.header, statedb, b.txs, b.uncles, b.receipts) + block, _ := b.engine.Finalize(chainreader, b.header, statedb, b.txs, b.uncles, b.receipts) // Write state changes to db root, err := statedb.Commit(config.IsEIP158(b.header.Number)) @@ -269,3 +265,19 @@ func makeBlockChain(parent *types.Block, n int, engine consensus.Engine, db ethd }) return blocks } + +type fakeChainReader struct { + config *params.ChainConfig + genesis *types.Block +} + +// Config returns the chain configuration. +func (cr *fakeChainReader) Config() *params.ChainConfig { + return cr.config +} + +func (cr *fakeChainReader) CurrentHeader() *types.Header { return nil } +func (cr *fakeChainReader) GetHeaderByNumber(number uint64) *types.Header { return nil } +func (cr *fakeChainReader) GetHeaderByHash(hash common.Hash) *types.Header { return nil } +func (cr *fakeChainReader) GetHeader(hash common.Hash, number uint64) *types.Header { return nil } +func (cr *fakeChainReader) GetBlock(hash common.Hash, number uint64) *types.Block { return nil } diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index dad626e89a..1fe02d8849 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -25,22 +25,14 @@ import ( "testing" "time" + ethereum "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" ) -var ( - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - testAddress = crypto.PubkeyToAddress(testKey.PublicKey) -) - // Reduce some of the parameters to make the tester faster. func init() { MaxForkAncestry = uint64(10000) @@ -55,6 +47,7 @@ type downloadTester struct { genesis *types.Block // Genesis blocks used by the tester and peers stateDb ethdb.Database // Database used by the tester for syncing from peers peerDb ethdb.Database // Database of the peers containing all data + peers map[string]*downloadTesterPeer ownHashes []common.Hash // Hash chain belonging to the tester ownHeaders map[common.Hash]*types.Header // Headers belonging to the tester @@ -62,129 +55,27 @@ type downloadTester struct { ownReceipts map[common.Hash]types.Receipts // Receipts belonging to the tester ownChainTd map[common.Hash]*big.Int // Total difficulties of the blocks in the local chain - peerHashes map[string][]common.Hash // Hash chain belonging to different test peers - peerHeaders map[string]map[common.Hash]*types.Header // Headers belonging to different test peers - peerBlocks map[string]map[common.Hash]*types.Block // Blocks belonging to different test peers - peerReceipts map[string]map[common.Hash]types.Receipts // Receipts belonging to different test peers - peerChainTds map[string]map[common.Hash]*big.Int // Total difficulties of the blocks in the peer chains - - peerMissingStates map[string]map[common.Hash]bool // State entries that fast sync should not return - lock sync.RWMutex } // newTester creates a new downloader test mocker. func newTester() *downloadTester { - testdb := ethdb.NewMemDatabase() - genesis := core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000)) - tester := &downloadTester{ - genesis: genesis, - peerDb: testdb, - ownHashes: []common.Hash{genesis.Hash()}, - ownHeaders: map[common.Hash]*types.Header{genesis.Hash(): genesis.Header()}, - ownBlocks: map[common.Hash]*types.Block{genesis.Hash(): genesis}, - ownReceipts: map[common.Hash]types.Receipts{genesis.Hash(): nil}, - ownChainTd: map[common.Hash]*big.Int{genesis.Hash(): genesis.Difficulty()}, - peerHashes: make(map[string][]common.Hash), - peerHeaders: make(map[string]map[common.Hash]*types.Header), - peerBlocks: make(map[string]map[common.Hash]*types.Block), - peerReceipts: make(map[string]map[common.Hash]types.Receipts), - peerChainTds: make(map[string]map[common.Hash]*big.Int), - peerMissingStates: make(map[string]map[common.Hash]bool), + genesis: testGenesis, + peerDb: testDB, + peers: make(map[string]*downloadTesterPeer), + ownHashes: []common.Hash{testGenesis.Hash()}, + ownHeaders: map[common.Hash]*types.Header{testGenesis.Hash(): testGenesis.Header()}, + ownBlocks: map[common.Hash]*types.Block{testGenesis.Hash(): testGenesis}, + ownReceipts: map[common.Hash]types.Receipts{testGenesis.Hash(): nil}, + ownChainTd: map[common.Hash]*big.Int{testGenesis.Hash(): testGenesis.Difficulty()}, } tester.stateDb = ethdb.NewMemDatabase() - tester.stateDb.Put(genesis.Root().Bytes(), []byte{0x00}) - + tester.stateDb.Put(testGenesis.Root().Bytes(), []byte{0x00}) tester.downloader = New(FullSync, tester.stateDb, new(event.TypeMux), tester, nil, tester.dropPeer) - return tester } -// makeChain creates a chain of n blocks starting at and including parent. -// the returned hash chain is ordered head->parent. In addition, every 3rd block -// contains a transaction and every 5th an uncle to allow testing correct block -// reassembly. -func (dl *downloadTester) makeChain(n int, seed byte, parent *types.Block, parentReceipts types.Receipts, heavy bool) ([]common.Hash, map[common.Hash]*types.Header, map[common.Hash]*types.Block, map[common.Hash]types.Receipts) { - // Generate the block chain - blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), dl.peerDb, n, func(i int, block *core.BlockGen) { - block.SetCoinbase(common.Address{seed}) - - // If a heavy chain is requested, delay blocks to raise difficulty - if heavy { - block.OffsetTime(-1) - } - // If the block number is multiple of 3, send a bonus transaction to the miner - if parent == dl.genesis && i%3 == 0 { - signer := types.MakeSigner(params.TestChainConfig, block.Number()) - tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, nil, nil), signer, testKey) - if err != nil { - panic(err) - } - block.AddTx(tx) - } - // If the block number is a multiple of 5, add a bonus uncle to the block - if i > 0 && i%5 == 0 { - block.AddUncle(&types.Header{ - ParentHash: block.PrevBlock(i - 1).Hash(), - Number: big.NewInt(block.Number().Int64() - 1), - }) - } - }) - // Convert the block-chain into a hash-chain and header/block maps - hashes := make([]common.Hash, n+1) - hashes[len(hashes)-1] = parent.Hash() - - headerm := make(map[common.Hash]*types.Header, n+1) - headerm[parent.Hash()] = parent.Header() - - blockm := make(map[common.Hash]*types.Block, n+1) - blockm[parent.Hash()] = parent - - receiptm := make(map[common.Hash]types.Receipts, n+1) - receiptm[parent.Hash()] = parentReceipts - - for i, b := range blocks { - hashes[len(hashes)-i-2] = b.Hash() - headerm[b.Hash()] = b.Header() - blockm[b.Hash()] = b - receiptm[b.Hash()] = receipts[i] - } - return hashes, headerm, blockm, receiptm -} - -// makeChainFork creates two chains of length n, such that h1[:f] and -// h2[:f] are different but have a common suffix of length n-f. -func (dl *downloadTester) makeChainFork(n, f int, parent *types.Block, parentReceipts types.Receipts, balanced bool) ([]common.Hash, []common.Hash, map[common.Hash]*types.Header, map[common.Hash]*types.Header, map[common.Hash]*types.Block, map[common.Hash]*types.Block, map[common.Hash]types.Receipts, map[common.Hash]types.Receipts) { - // Create the common suffix - hashes, headers, blocks, receipts := dl.makeChain(n-f, 0, parent, parentReceipts, false) - - // Create the forks, making the second heavier if non balanced forks were requested - hashes1, headers1, blocks1, receipts1 := dl.makeChain(f, 1, blocks[hashes[0]], receipts[hashes[0]], false) - hashes1 = append(hashes1, hashes[1:]...) - - heavy := false - if !balanced { - heavy = true - } - hashes2, headers2, blocks2, receipts2 := dl.makeChain(f, 2, blocks[hashes[0]], receipts[hashes[0]], heavy) - hashes2 = append(hashes2, hashes[1:]...) - - for hash, header := range headers { - headers1[hash] = header - headers2[hash] = header - } - for hash, block := range blocks { - blocks1[hash] = block - blocks2[hash] = block - } - for hash, receipt := range receipts { - receipts1[hash] = receipt - receipts2[hash] = receipt - } - return hashes1, hashes2, headers1, headers2, blocks1, blocks2, receipts1, receipts2 -} - // terminate aborts any operations on the embedded downloader and releases all // held resources. func (dl *downloadTester) terminate() { @@ -194,13 +85,10 @@ func (dl *downloadTester) terminate() { // sync starts synchronizing with a remote peer, blocking until it completes. func (dl *downloadTester) sync(id string, td *big.Int, mode SyncMode) error { dl.lock.RLock() - hash := dl.peerHashes[id][0] + hash := dl.peers[id].chain.headBlock().Hash() // If no particular TD was requested, load from the peer's blockchain if td == nil { - td = big.NewInt(1) - if diff, ok := dl.peerChainTds[id][hash]; ok { - td = diff - } + td = dl.peers[id].chain.td(hash) } dl.lock.RUnlock() @@ -302,7 +190,7 @@ func (dl *downloadTester) GetTd(hash common.Hash, number uint64) *big.Int { } // InsertHeaderChain injects a new batch of headers into the simulated chain. -func (dl *downloadTester) InsertHeaderChain(headers []*types.Header, checkFreq int) (int, error) { +func (dl *downloadTester) InsertHeaderChain(headers []*types.Header, checkFreq int) (i int, err error) { dl.lock.Lock() defer dl.lock.Unlock() @@ -331,7 +219,7 @@ func (dl *downloadTester) InsertHeaderChain(headers []*types.Header, checkFreq i } // InsertChain injects a new batch of blocks into the simulated chain. -func (dl *downloadTester) InsertChain(blocks types.Blocks) (int, error) { +func (dl *downloadTester) InsertChain(blocks types.Blocks) (i int, err error) { dl.lock.Lock() defer dl.lock.Unlock() @@ -353,7 +241,7 @@ func (dl *downloadTester) InsertChain(blocks types.Blocks) (int, error) { } // InsertReceiptChain injects a new batch of receipts into the simulated chain. -func (dl *downloadTester) InsertReceiptChain(blocks types.Blocks, receipts []types.Receipts) (int, error) { +func (dl *downloadTester) InsertReceiptChain(blocks types.Blocks, receipts []types.Receipts) (i int, err error) { dl.lock.Lock() defer dl.lock.Unlock() @@ -387,60 +275,13 @@ func (dl *downloadTester) Rollback(hashes []common.Hash) { } // newPeer registers a new block download source into the downloader. -func (dl *downloadTester) newPeer(id string, version int, hashes []common.Hash, headers map[common.Hash]*types.Header, blocks map[common.Hash]*types.Block, receipts map[common.Hash]types.Receipts) error { - return dl.newSlowPeer(id, version, hashes, headers, blocks, receipts, 0) -} - -// newSlowPeer registers a new block download source into the downloader, with a -// specific delay time on processing the network packets sent to it, simulating -// potentially slow network IO. -func (dl *downloadTester) newSlowPeer(id string, version int, hashes []common.Hash, headers map[common.Hash]*types.Header, blocks map[common.Hash]*types.Block, receipts map[common.Hash]types.Receipts, delay time.Duration) error { +func (dl *downloadTester) newPeer(id string, version int, chain *testChain) error { dl.lock.Lock() defer dl.lock.Unlock() - var err = dl.downloader.RegisterPeer(id, version, &downloadTesterPeer{dl: dl, id: id, delay: delay}) - if err == nil { - // Assign the owned hashes, headers and blocks to the peer (deep copy) - dl.peerHashes[id] = make([]common.Hash, len(hashes)) - copy(dl.peerHashes[id], hashes) - - dl.peerHeaders[id] = make(map[common.Hash]*types.Header) - dl.peerBlocks[id] = make(map[common.Hash]*types.Block) - dl.peerReceipts[id] = make(map[common.Hash]types.Receipts) - dl.peerChainTds[id] = make(map[common.Hash]*big.Int) - dl.peerMissingStates[id] = make(map[common.Hash]bool) - - genesis := hashes[len(hashes)-1] - if header := headers[genesis]; header != nil { - dl.peerHeaders[id][genesis] = header - dl.peerChainTds[id][genesis] = header.Difficulty - } - if block := blocks[genesis]; block != nil { - dl.peerBlocks[id][genesis] = block - dl.peerChainTds[id][genesis] = block.Difficulty() - } - - for i := len(hashes) - 2; i >= 0; i-- { - hash := hashes[i] - - if header, ok := headers[hash]; ok { - dl.peerHeaders[id][hash] = header - if _, ok := dl.peerHeaders[id][header.ParentHash]; ok { - dl.peerChainTds[id][hash] = new(big.Int).Add(header.Difficulty, dl.peerChainTds[id][header.ParentHash]) - } - } - if block, ok := blocks[hash]; ok { - dl.peerBlocks[id][hash] = block - if _, ok := dl.peerBlocks[id][block.ParentHash()]; ok { - dl.peerChainTds[id][hash] = new(big.Int).Add(block.Difficulty(), dl.peerChainTds[id][block.ParentHash()]) - } - } - if receipt, ok := receipts[hash]; ok { - dl.peerReceipts[id][hash] = receipt - } - } - } - return err + peer := &downloadTesterPeer{dl: dl, id: id, chain: chain} + dl.peers[id] = peer + return dl.downloader.RegisterPeer(id, version, peer) } // dropPeer simulates a hard peer removal from the connection pool. @@ -448,89 +289,48 @@ func (dl *downloadTester) dropPeer(id string) { dl.lock.Lock() defer dl.lock.Unlock() - delete(dl.peerHashes, id) - delete(dl.peerHeaders, id) - delete(dl.peerBlocks, id) - delete(dl.peerChainTds, id) - + delete(dl.peers, id) dl.downloader.UnregisterPeer(id) } type downloadTesterPeer struct { - dl *downloadTester - id string - delay time.Duration - lock sync.RWMutex -} - -// setDelay is a thread safe setter for the network delay value. -func (dlp *downloadTesterPeer) setDelay(delay time.Duration) { - dlp.lock.Lock() - defer dlp.lock.Unlock() - - dlp.delay = delay -} - -// waitDelay is a thread safe way to sleep for the configured time. -func (dlp *downloadTesterPeer) waitDelay() { - dlp.lock.RLock() - delay := dlp.delay - dlp.lock.RUnlock() - - time.Sleep(delay) + dl *downloadTester + id string + lock sync.RWMutex + chain *testChain + missingStates map[common.Hash]bool // State entries that fast sync should not return } // Head constructs a function to retrieve a peer's current head hash // and total difficulty. func (dlp *downloadTesterPeer) Head() (common.Hash, *big.Int) { - dlp.dl.lock.RLock() - defer dlp.dl.lock.RUnlock() - - return dlp.dl.peerHashes[dlp.id][0], nil + b := dlp.chain.headBlock() + return b.Hash(), dlp.chain.td(b.Hash()) } // RequestHeadersByHash constructs a GetBlockHeaders function based on a hashed // origin; associated with a particular peer in the download tester. The returned // function can be used to retrieve batches of headers from the particular peer. func (dlp *downloadTesterPeer) RequestHeadersByHash(origin common.Hash, amount int, skip int, reverse bool) error { - // Find the canonical number of the hash - dlp.dl.lock.RLock() - number := uint64(0) - for num, hash := range dlp.dl.peerHashes[dlp.id] { - if hash == origin { - number = uint64(len(dlp.dl.peerHashes[dlp.id]) - num - 1) - break - } + if reverse { + panic("reverse header requests not supported") } - dlp.dl.lock.RUnlock() - // Use the absolute header fetcher to satisfy the query - return dlp.RequestHeadersByNumber(number, amount, skip, reverse) + result := dlp.chain.headersByHash(origin, amount, skip) + go dlp.dl.downloader.DeliverHeaders(dlp.id, result) + return nil } // RequestHeadersByNumber constructs a GetBlockHeaders function based on a numbered // origin; associated with a particular peer in the download tester. The returned // function can be used to retrieve batches of headers from the particular peer. func (dlp *downloadTesterPeer) RequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool) error { - dlp.waitDelay() - - dlp.dl.lock.RLock() - defer dlp.dl.lock.RUnlock() - - // Gather the next batch of headers - hashes := dlp.dl.peerHashes[dlp.id] - headers := dlp.dl.peerHeaders[dlp.id] - result := make([]*types.Header, 0, amount) - for i := 0; i < amount && len(hashes)-int(origin)-1-i*(skip+1) >= 0; i++ { - if header, ok := headers[hashes[len(hashes)-int(origin)-1-i*(skip+1)]]; ok { - result = append(result, header) - } + if reverse { + panic("reverse header requests not supported") } - // Delay delivery a bit to allow attacks to unfold - go func() { - time.Sleep(time.Millisecond) - dlp.dl.downloader.DeliverHeaders(dlp.id, result) - }() + + result := dlp.chain.headersByNumber(origin, amount, skip) + go dlp.dl.downloader.DeliverHeaders(dlp.id, result) return nil } @@ -538,24 +338,8 @@ func (dlp *downloadTesterPeer) RequestHeadersByNumber(origin uint64, amount int, // peer in the download tester. The returned function can be used to retrieve // batches of block bodies from the particularly requested peer. func (dlp *downloadTesterPeer) RequestBodies(hashes []common.Hash) error { - dlp.waitDelay() - - dlp.dl.lock.RLock() - defer dlp.dl.lock.RUnlock() - - blocks := dlp.dl.peerBlocks[dlp.id] - - transactions := make([][]*types.Transaction, 0, len(hashes)) - uncles := make([][]*types.Header, 0, len(hashes)) - - for _, hash := range hashes { - if block, ok := blocks[hash]; ok { - transactions = append(transactions, block.Transactions()) - uncles = append(uncles, block.Uncles()) - } - } - go dlp.dl.downloader.DeliverBodies(dlp.id, transactions, uncles) - + txs, uncles := dlp.chain.bodies(hashes) + go dlp.dl.downloader.DeliverBodies(dlp.id, txs, uncles) return nil } @@ -563,21 +347,8 @@ func (dlp *downloadTesterPeer) RequestBodies(hashes []common.Hash) error { // peer in the download tester. The returned function can be used to retrieve // batches of block receipts from the particularly requested peer. func (dlp *downloadTesterPeer) RequestReceipts(hashes []common.Hash) error { - dlp.waitDelay() - - dlp.dl.lock.RLock() - defer dlp.dl.lock.RUnlock() - - receipts := dlp.dl.peerReceipts[dlp.id] - - results := make([][]*types.Receipt, 0, len(hashes)) - for _, hash := range hashes { - if receipt, ok := receipts[hash]; ok { - results = append(results, receipt) - } - } - go dlp.dl.downloader.DeliverReceipts(dlp.id, results) - + receipts := dlp.chain.receipts(hashes) + go dlp.dl.downloader.DeliverReceipts(dlp.id, receipts) return nil } @@ -585,21 +356,18 @@ func (dlp *downloadTesterPeer) RequestReceipts(hashes []common.Hash) error { // peer in the download tester. The returned function can be used to retrieve // batches of node state data from the particularly requested peer. func (dlp *downloadTesterPeer) RequestNodeData(hashes []common.Hash) error { - dlp.waitDelay() - dlp.dl.lock.RLock() defer dlp.dl.lock.RUnlock() results := make([][]byte, 0, len(hashes)) for _, hash := range hashes { if data, err := dlp.dl.peerDb.Get(hash.Bytes()); err == nil { - if !dlp.dl.peerMissingStates[dlp.id][hash] { + if !dlp.missingStates[hash] { results = append(results, data) } } } go dlp.dl.downloader.DeliverNodeData(dlp.id, results) - return nil } @@ -639,21 +407,6 @@ func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, leng if rs := len(tester.ownReceipts); rs != receipts { t.Fatalf("synchronised receipts mismatch: have %v, want %v", rs, receipts) } - // Verify the state trie too for fast syncs - /*if tester.downloader.mode == FastSync { - pivot := uint64(0) - var index int - if pivot := int(tester.downloader.queue.fastSyncPivot); pivot < common { - index = pivot - } else { - index = len(tester.ownHashes) - lengths[len(lengths)-1] + int(tester.downloader.queue.fastSyncPivot) - } - if index > 0 { - if statedb, err := state.New(tester.ownHeaders[tester.ownHashes[index]].Root, state.NewDatabase(trie.NewDatabase(tester.stateDb))); statedb == nil || err != nil { - t.Fatalf("state reconstruction failed: %v", err) - } - } - }*/ } // Tests that simple synchronization against a canonical chain works correctly. @@ -673,16 +426,14 @@ func testCanonicalSynchronisation(t *testing.T, protocol int, mode SyncMode) { defer tester.terminate() // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) - - tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) + chain := testChainBase.shorten(blockCacheItems - 15) + tester.newPeer("peer", protocol, chain) // Synchronise with the peer and make sure all relevant data was retrieved if err := tester.sync("peer", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, chain.len()) } // Tests that if a large batch of blocks are being downloaded, it is throttled @@ -699,10 +450,8 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) { defer tester.terminate() // Create a long block chain to download and the tester - targetBlocks := 8 * blockCacheItems - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) - - tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) + targetBlocks := testChainBase.len() - 1 + tester.newPeer("peer", protocol, testChainBase) // Wrap the importer to allow stepping blocked, proceed := uint32(0), make(chan struct{}) @@ -734,9 +483,7 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) { cached = len(tester.downloader.queue.blockDonePool) if mode == FastSync { if receipts := len(tester.downloader.queue.receiptDonePool); receipts < cached { - //if tester.downloader.queue.resultCache[receipts].Header.Number.Uint64() < tester.downloader.queue.fastSyncPivot { cached = receipts - //} } } frozen = int(atomic.LoadUint32(&blocked)) @@ -786,24 +533,22 @@ func testForkedSync(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - // Create a long enough forked chain - common, fork := MaxHashFetch, 2*MaxHashFetch - hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, true) - - tester.newPeer("fork A", protocol, hashesA, headersA, blocksA, receiptsA) - tester.newPeer("fork B", protocol, hashesB, headersB, blocksB, receiptsB) + chainA := testChainForkLightA.shorten(testChainBase.len() + 80) + chainB := testChainForkLightB.shorten(testChainBase.len() + 80) + tester.newPeer("fork A", protocol, chainA) + tester.newPeer("fork B", protocol, chainB) // Synchronise with the peer and make sure all blocks were retrieved if err := tester.sync("fork A", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, common+fork+1) + assertOwnChain(t, tester, chainA.len()) // Synchronise with the second peer and make sure that fork is pulled too if err := tester.sync("fork B", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnForkedChain(t, tester, common+1, []int{common + fork + 1, common + fork + 1}) + assertOwnForkedChain(t, tester, testChainBase.len(), []int{chainA.len(), chainB.len()}) } // Tests that synchronising against a much shorter but much heavyer fork works @@ -821,24 +566,22 @@ func testHeavyForkedSync(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - // Create a long enough forked chain - common, fork := MaxHashFetch, 4*MaxHashFetch - hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, false) - - tester.newPeer("light", protocol, hashesA, headersA, blocksA, receiptsA) - tester.newPeer("heavy", protocol, hashesB[fork/2:], headersB, blocksB, receiptsB) + chainA := testChainForkLightA.shorten(testChainBase.len() + 80) + chainB := testChainForkHeavy.shorten(testChainBase.len() + 80) + tester.newPeer("light", protocol, chainA) + tester.newPeer("heavy", protocol, chainB) // Synchronise with the peer and make sure all blocks were retrieved if err := tester.sync("light", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, common+fork+1) + assertOwnChain(t, tester, chainA.len()) // Synchronise with the second peer and make sure that fork is pulled too if err := tester.sync("heavy", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnForkedChain(t, tester, common+1, []int{common + fork + 1, common + fork/2 + 1}) + assertOwnForkedChain(t, tester, testChainBase.len(), []int{chainA.len(), chainB.len()}) } // Tests that chain forks are contained within a certain interval of the current @@ -857,18 +600,16 @@ func testBoundedForkedSync(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - // Create a long enough forked chain - common, fork := 13, int(MaxForkAncestry+17) - hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, true) - - tester.newPeer("original", protocol, hashesA, headersA, blocksA, receiptsA) - tester.newPeer("rewriter", protocol, hashesB, headersB, blocksB, receiptsB) + chainA := testChainForkLightA + chainB := testChainForkLightB + tester.newPeer("original", protocol, chainA) + tester.newPeer("rewriter", protocol, chainB) // Synchronise with the peer and make sure all blocks were retrieved if err := tester.sync("original", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, common+fork+1) + assertOwnChain(t, tester, chainA.len()) // Synchronise with the second peer and ensure that the fork is rejected to being too old if err := tester.sync("rewriter", nil, mode); err != errInvalidAncestor { @@ -893,17 +634,16 @@ func testBoundedHeavyForkedSync(t *testing.T, protocol int, mode SyncMode) { defer tester.terminate() // Create a long enough forked chain - common, fork := 13, int(MaxForkAncestry+17) - hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, false) - - tester.newPeer("original", protocol, hashesA, headersA, blocksA, receiptsA) - tester.newPeer("heavy-rewriter", protocol, hashesB[MaxForkAncestry-17:], headersB, blocksB, receiptsB) // Root the fork below the ancestor limit + chainA := testChainForkLightA + chainB := testChainForkHeavy + tester.newPeer("original", protocol, chainA) + tester.newPeer("heavy-rewriter", protocol, chainB) // Synchronise with the peer and make sure all blocks were retrieved if err := tester.sync("original", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, common+fork+1) + assertOwnChain(t, tester, chainA.len()) // Synchronise with the second peer and ensure that the fork is rejected to being too old if err := tester.sync("heavy-rewriter", nil, mode); err != errInvalidAncestor { @@ -924,7 +664,7 @@ func TestInactiveDownloader62(t *testing.T) { t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive) } if err := tester.downloader.DeliverBodies("bad peer", [][]*types.Transaction{}, [][]*types.Header{}); err != errNoSyncActive { - t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive) + t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive) } } @@ -962,17 +702,8 @@ func testCancel(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - // Create a small enough block chain to download and the tester - targetBlocks := blockCacheItems - 15 - if targetBlocks >= MaxHashFetch { - targetBlocks = MaxHashFetch - 15 - } - if targetBlocks >= MaxHeaderFetch { - targetBlocks = MaxHeaderFetch - 15 - } - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) - - tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) + chain := testChainBase.shorten(MaxHeaderFetch) + tester.newPeer("peer", protocol, chain) // Make sure canceling works with a pristine downloader tester.downloader.Cancel() @@ -1005,17 +736,16 @@ func testMultiSynchronisation(t *testing.T, protocol int, mode SyncMode) { // Create various peers with various parts of the chain targetPeers := 8 - targetBlocks := targetPeers*blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(targetPeers * 100) for i := 0; i < targetPeers; i++ { id := fmt.Sprintf("peer #%d", i) - tester.newPeer(id, protocol, hashes[i*blockCacheItems:], headers, blocks, receipts) + tester.newPeer(id, protocol, chain.shorten(chain.len()/(i+1))) } if err := tester.sync("peer #0", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, chain.len()) } // Tests that synchronisations behave well in multi-version protocol environments @@ -1034,24 +764,23 @@ func testMultiProtoSync(t *testing.T, protocol int, mode SyncMode) { defer tester.terminate() // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(blockCacheItems - 15) // Create peers of every type - tester.newPeer("peer 62", 62, hashes, headers, blocks, nil) - tester.newPeer("peer 63", 63, hashes, headers, blocks, receipts) - tester.newPeer("peer 64", 64, hashes, headers, blocks, receipts) + tester.newPeer("peer 62", 62, chain) + tester.newPeer("peer 63", 63, chain) + tester.newPeer("peer 64", 64, chain) // Synchronise with the requested peer and make sure all blocks were retrieved if err := tester.sync(fmt.Sprintf("peer %d", protocol), nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, chain.len()) // Check that no peers have been dropped off for _, version := range []int{62, 63, 64} { peer := fmt.Sprintf("peer %d", version) - if _, ok := tester.peerHashes[peer]; !ok { + if _, ok := tester.peers[peer]; !ok { t.Errorf("%s dropped", peer) } } @@ -1073,10 +802,8 @@ func testEmptyShortCircuit(t *testing.T, protocol int, mode SyncMode) { defer tester.terminate() // Create a block chain to download - targetBlocks := 2*blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) - - tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) + chain := testChainBase + tester.newPeer("peer", protocol, chain) // Instrument the downloader to signal body requests bodiesHave, receiptsHave := int32(0), int32(0) @@ -1090,16 +817,16 @@ func testEmptyShortCircuit(t *testing.T, protocol int, mode SyncMode) { if err := tester.sync("peer", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, chain.len()) // Validate the number of block bodies that should have been requested bodiesNeeded, receiptsNeeded := 0, 0 - for _, block := range blocks { + for _, block := range chain.blockm { if mode != LightSync && block != tester.genesis && (len(block.Transactions()) > 0 || len(block.Uncles()) > 0) { bodiesNeeded++ } } - for _, receipt := range receipts { + for _, receipt := range chain.receiptm { if mode == FastSync && len(receipt) > 0 { receiptsNeeded++ } @@ -1127,24 +854,20 @@ func testMissingHeaderAttack(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) - - // Attempt a full sync with an attacker feeding gapped headers - tester.newPeer("attack", protocol, hashes, headers, blocks, receipts) - missing := targetBlocks / 2 - delete(tester.peerHeaders["attack"], hashes[missing]) + chain := testChainBase.shorten(blockCacheItems - 15) + brokenChain := chain.shorten(chain.len()) + delete(brokenChain.headerm, brokenChain.chain[brokenChain.len()/2]) + tester.newPeer("attack", protocol, brokenChain) if err := tester.sync("attack", nil, mode); err == nil { t.Fatalf("succeeded attacker synchronisation") } // Synchronise with the valid peer and make sure sync succeeds - tester.newPeer("valid", protocol, hashes, headers, blocks, receipts) + tester.newPeer("valid", protocol, chain) if err := tester.sync("valid", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, chain.len()) } // Tests that if requested headers are shifted (i.e. first is missing), the queue @@ -1162,25 +885,24 @@ func testShiftedHeaderAttack(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(blockCacheItems - 15) // Attempt a full sync with an attacker feeding shifted headers - tester.newPeer("attack", protocol, hashes, headers, blocks, receipts) - delete(tester.peerHeaders["attack"], hashes[len(hashes)-2]) - delete(tester.peerBlocks["attack"], hashes[len(hashes)-2]) - delete(tester.peerReceipts["attack"], hashes[len(hashes)-2]) - + brokenChain := chain.shorten(chain.len()) + delete(brokenChain.headerm, brokenChain.chain[1]) + delete(brokenChain.blockm, brokenChain.chain[1]) + delete(brokenChain.receiptm, brokenChain.chain[1]) + tester.newPeer("attack", protocol, brokenChain) if err := tester.sync("attack", nil, mode); err == nil { t.Fatalf("succeeded attacker synchronisation") } + // Synchronise with the valid peer and make sure sync succeeds - tester.newPeer("valid", protocol, hashes, headers, blocks, receipts) + tester.newPeer("valid", protocol, chain) if err := tester.sync("valid", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, chain.len()) } // Tests that upon detecting an invalid header, the recent ones are rolled back @@ -1198,13 +920,14 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) { // Create a small enough block chain to download targetBlocks := 3*fsHeaderSafetyNet + 256 + fsMinFullBlocks - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(targetBlocks) // Attempt to sync with an attacker that feeds junk during the fast sync phase. // This should result in the last fsHeaderSafetyNet headers being rolled back. - tester.newPeer("fast-attack", protocol, hashes, headers, blocks, receipts) missing := fsHeaderSafetyNet + MaxHeaderFetch + 1 - delete(tester.peerHeaders["fast-attack"], hashes[len(hashes)-missing]) + fastAttackChain := chain.shorten(chain.len()) + delete(fastAttackChain.headerm, fastAttackChain.chain[missing]) + tester.newPeer("fast-attack", protocol, fastAttackChain) if err := tester.sync("fast-attack", nil, mode); err == nil { t.Fatalf("succeeded fast attacker synchronisation") @@ -1212,13 +935,15 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) { if head := tester.CurrentHeader().Number.Int64(); int(head) > MaxHeaderFetch { t.Errorf("rollback head mismatch: have %v, want at most %v", head, MaxHeaderFetch) } + // Attempt to sync with an attacker that feeds junk during the block import phase. // This should result in both the last fsHeaderSafetyNet number of headers being // rolled back, and also the pivot point being reverted to a non-block status. - tester.newPeer("block-attack", protocol, hashes, headers, blocks, receipts) missing = 3*fsHeaderSafetyNet + MaxHeaderFetch + 1 - delete(tester.peerHeaders["fast-attack"], hashes[len(hashes)-missing]) // Make sure the fast-attacker doesn't fill in - delete(tester.peerHeaders["block-attack"], hashes[len(hashes)-missing]) + blockAttackChain := chain.shorten(chain.len()) + delete(fastAttackChain.headerm, fastAttackChain.chain[missing]) // Make sure the fast-attacker doesn't fill in + delete(blockAttackChain.headerm, blockAttackChain.chain[missing]) + tester.newPeer("block-attack", protocol, blockAttackChain) if err := tester.sync("block-attack", nil, mode); err == nil { t.Fatalf("succeeded block attacker synchronisation") @@ -1231,19 +956,18 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) { t.Errorf("fast sync pivot block #%d not rolled back", head) } } + // Attempt to sync with an attacker that withholds promised blocks after the // fast sync pivot point. This could be a trial to leave the node with a bad // but already imported pivot block. - tester.newPeer("withhold-attack", protocol, hashes, headers, blocks, receipts) - missing = 3*fsHeaderSafetyNet + MaxHeaderFetch + 1 - + withholdAttackChain := chain.shorten(chain.len()) + tester.newPeer("withhold-attack", protocol, withholdAttackChain) tester.downloader.syncInitHook = func(uint64, uint64) { - for i := missing; i <= len(hashes); i++ { - delete(tester.peerHeaders["withhold-attack"], hashes[len(hashes)-i]) + for i := missing; i < withholdAttackChain.len(); i++ { + delete(withholdAttackChain.headerm, withholdAttackChain.chain[i]) } tester.downloader.syncInitHook = nil } - if err := tester.sync("withhold-attack", nil, mode); err == nil { t.Fatalf("succeeded withholding attacker synchronisation") } @@ -1255,20 +979,21 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) { t.Errorf("fast sync pivot block #%d not rolled back", head) } } - // Synchronise with the valid peer and make sure sync succeeds. Since the last - // rollback should also disable fast syncing for this process, verify that we - // did a fresh full sync. Note, we can't assert anything about the receipts - // since we won't purge the database of them, hence we can't use assertOwnChain. - tester.newPeer("valid", protocol, hashes, headers, blocks, receipts) + + // synchronise with the valid peer and make sure sync succeeds. Since the last rollback + // should also disable fast syncing for this process, verify that we did a fresh full + // sync. Note, we can't assert anything about the receipts since we won't purge the + // database of them, hence we can't use assertOwnChain. + tester.newPeer("valid", protocol, chain) if err := tester.sync("valid", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - if hs := len(tester.ownHeaders); hs != len(headers) { - t.Fatalf("synchronised headers mismatch: have %v, want %v", hs, len(headers)) + if hs := len(tester.ownHeaders); hs != chain.len() { + t.Fatalf("synchronised headers mismatch: have %v, want %v", hs, chain.len()) } if mode != LightSync { - if bs := len(tester.ownBlocks); bs != len(blocks) { - t.Fatalf("synchronised blocks mismatch: have %v, want %v", bs, len(blocks)) + if bs := len(tester.ownBlocks); bs != chain.len() { + t.Fatalf("synchronised blocks mismatch: have %v, want %v", bs, chain.len()) } } } @@ -1288,9 +1013,8 @@ func testHighTDStarvationAttack(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - hashes, headers, blocks, receipts := tester.makeChain(0, 0, tester.genesis, nil, false) - tester.newPeer("attack", protocol, []common.Hash{hashes[0]}, headers, blocks, receipts) - + chain := testChainBase.shorten(1) + tester.newPeer("attack", protocol, chain) if err := tester.sync("attack", big.NewInt(1000000), mode); err != errStallingPeer { t.Fatalf("synchronisation error mismatch: have %v, want %v", err, errStallingPeer) } @@ -1333,21 +1057,22 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol int) { // Run the tests and check disconnection status tester := newTester() defer tester.terminate() + chain := testChainBase.shorten(1) for i, tt := range tests { // Register a new peer and ensure it's presence id := fmt.Sprintf("test %d", i) - if err := tester.newPeer(id, protocol, []common.Hash{tester.genesis.Hash()}, nil, nil, nil); err != nil { + if err := tester.newPeer(id, protocol, chain); err != nil { t.Fatalf("test %d: failed to register new peer: %v", i, err) } - if _, ok := tester.peerHashes[id]; !ok { + if _, ok := tester.peers[id]; !ok { t.Fatalf("test %d: registered peer not found", i) } // Simulate a synchronisation and check the required result tester.downloader.synchroniseMock = func(string, common.Hash) error { return tt.result } tester.downloader.Synchronise(id, tester.genesis.Hash(), big.NewInt(1000), FullSync) - if _, ok := tester.peerHashes[id]; !ok != tt.drop { + if _, ok := tester.peers[id]; !ok != tt.drop { t.Errorf("test %d: peer drop mismatch for %v: have %v, want %v", i, tt.result, !ok, tt.drop) } } @@ -1367,10 +1092,7 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - - // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(blockCacheItems - 15) // Set a sync init hook to catch progress changes starting := make(chan struct{}) @@ -1380,12 +1102,10 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { starting <- struct{}{} <-progress } - // Retrieve the sync progress and ensure they are zero (pristine sync) - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 { - t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0) - } + checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{}) + // Synchronise half the blocks and check initial progress - tester.newPeer("peer-half", protocol, hashes[targetBlocks/2:], headers, blocks, receipts) + tester.newPeer("peer-half", protocol, chain.shorten(chain.len()/2)) pending := new(sync.WaitGroup) pending.Add(1) @@ -1396,16 +1116,15 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(targetBlocks/2+1) { - t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, targetBlocks/2+1) - } + checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{ + HighestBlock: uint64(chain.len()/2 - 1), + }) progress <- struct{}{} pending.Wait() // Synchronise all the blocks and check continuation progress - tester.newPeer("peer-full", protocol, hashes, headers, blocks, receipts) + tester.newPeer("peer-full", protocol, chain) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("peer-full", nil, mode); err != nil { @@ -1413,15 +1132,29 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(targetBlocks/2+1) || progress.CurrentBlock != uint64(targetBlocks/2+1) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks/2+1, targetBlocks/2+1, targetBlocks) - } - progress <- struct{}{} - pending.Wait() + checkProgress(t, tester.downloader, "completing", ethereum.SyncProgress{ + StartingBlock: uint64(chain.len()/2 - 1), + CurrentBlock: uint64(chain.len()/2 - 1), + HighestBlock: uint64(chain.len() - 1), + }) // Check final progress after successful sync - if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(targetBlocks/2+1) || progress.CurrentBlock != uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Final progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks/2+1, targetBlocks, targetBlocks) + progress <- struct{}{} + pending.Wait() + checkProgress(t, tester.downloader, "final", ethereum.SyncProgress{ + StartingBlock: uint64(chain.len()/2 - 1), + CurrentBlock: uint64(chain.len() - 1), + HighestBlock: uint64(chain.len() - 1), + }) +} + +func checkProgress(t *testing.T, d *Downloader, stage string, want ethereum.SyncProgress) { + t.Helper() + p := d.Progress() + p.KnownStates, p.PulledStates = 0, 0 + want.KnownStates, want.PulledStates = 0, 0 + if p != want { + t.Fatalf("%s progress mismatch:\nhave %+v\nwant %+v", stage, p, want) } } @@ -1440,10 +1173,8 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - - // Create a forked chain to simulate origin revertal - common, fork := MaxHashFetch, 2*MaxHashFetch - hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, true) + chainA := testChainForkLightA.shorten(testChainBase.len() + MaxHashFetch) + chainB := testChainForkLightB.shorten(testChainBase.len() + MaxHashFetch) // Set a sync init hook to catch progress changes starting := make(chan struct{}) @@ -1453,15 +1184,12 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { starting <- struct{}{} <-progress } - // Retrieve the sync progress and ensure they are zero (pristine sync) - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 { - t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0) - } + checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{}) + // Synchronise with one of the forks and check progress - tester.newPeer("fork A", protocol, hashesA, headersA, blocksA, receiptsA) + tester.newPeer("fork A", protocol, chainA) pending := new(sync.WaitGroup) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("fork A", nil, mode); err != nil { @@ -1469,9 +1197,10 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(len(hashesA)-1) { - t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, len(hashesA)-1) - } + + checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{ + HighestBlock: uint64(chainA.len() - 1), + }) progress <- struct{}{} pending.Wait() @@ -1479,9 +1208,8 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { tester.downloader.syncStatsChainOrigin = tester.downloader.syncStatsChainHeight // Synchronise with the second fork and check progress resets - tester.newPeer("fork B", protocol, hashesB, headersB, blocksB, receiptsB) + tester.newPeer("fork B", protocol, chainB) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("fork B", nil, mode); err != nil { @@ -1489,16 +1217,20 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(common) || progress.CurrentBlock != uint64(len(hashesA)-1) || progress.HighestBlock != uint64(len(hashesB)-1) { - t.Fatalf("Forking progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, common, len(hashesA)-1, len(hashesB)-1) - } - progress <- struct{}{} - pending.Wait() + checkProgress(t, tester.downloader, "forking", ethereum.SyncProgress{ + StartingBlock: uint64(testChainBase.len()) - 1, + CurrentBlock: uint64(chainA.len() - 1), + HighestBlock: uint64(chainB.len() - 1), + }) // Check final progress after successful sync - if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(common) || progress.CurrentBlock != uint64(len(hashesB)-1) || progress.HighestBlock != uint64(len(hashesB)-1) { - t.Fatalf("Final progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, common, len(hashesB)-1, len(hashesB)-1) - } + progress <- struct{}{} + pending.Wait() + checkProgress(t, tester.downloader, "final", ethereum.SyncProgress{ + StartingBlock: uint64(testChainBase.len()) - 1, + CurrentBlock: uint64(chainB.len() - 1), + HighestBlock: uint64(chainB.len() - 1), + }) } // Tests that if synchronisation is aborted due to some failure, then the progress @@ -1516,10 +1248,7 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - - // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(blockCacheItems - 15) // Set a sync init hook to catch progress changes starting := make(chan struct{}) @@ -1529,20 +1258,18 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) { starting <- struct{}{} <-progress } - // Retrieve the sync progress and ensure they are zero (pristine sync) - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 { - t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0) - } + checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{}) + // Attempt a full sync with a faulty peer - tester.newPeer("faulty", protocol, hashes, headers, blocks, receipts) - missing := targetBlocks / 2 - delete(tester.peerHeaders["faulty"], hashes[missing]) - delete(tester.peerBlocks["faulty"], hashes[missing]) - delete(tester.peerReceipts["faulty"], hashes[missing]) + brokenChain := chain.shorten(chain.len()) + missing := brokenChain.len() / 2 + delete(brokenChain.headerm, brokenChain.chain[missing]) + delete(brokenChain.blockm, brokenChain.chain[missing]) + delete(brokenChain.receiptm, brokenChain.chain[missing]) + tester.newPeer("faulty", protocol, brokenChain) pending := new(sync.WaitGroup) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("faulty", nil, mode); err == nil { @@ -1550,16 +1277,17 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, targetBlocks) - } + checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{ + HighestBlock: uint64(brokenChain.len() - 1), + }) progress <- struct{}{} pending.Wait() + afterFailedSync := tester.downloader.Progress() - // Synchronise with a good peer and check that the progress origin remind the same after a failure - tester.newPeer("valid", protocol, hashes, headers, blocks, receipts) + // Synchronise with a good peer and check that the progress origin remind the same + // after a failure + tester.newPeer("valid", protocol, chain) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("valid", nil, mode); err != nil { @@ -1567,16 +1295,15 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock > uint64(targetBlocks/2) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/0-%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, targetBlocks/2, targetBlocks) - } - progress <- struct{}{} - pending.Wait() + checkProgress(t, tester.downloader, "completing", afterFailedSync) // Check final progress after successful sync - if progress := tester.downloader.Progress(); progress.StartingBlock > uint64(targetBlocks/2) || progress.CurrentBlock != uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Final progress mismatch: have %v/%v/%v, want 0-%v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks/2, targetBlocks, targetBlocks) - } + progress <- struct{}{} + pending.Wait() + checkProgress(t, tester.downloader, "final", ethereum.SyncProgress{ + CurrentBlock: uint64(chain.len() - 1), + HighestBlock: uint64(chain.len() - 1), + }) } // Tests that if an attacker fakes a chain height, after the attack is detected, @@ -1593,34 +1320,27 @@ func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - - // Create a small block chain - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks+3, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(blockCacheItems - 15) // Set a sync init hook to catch progress changes starting := make(chan struct{}) progress := make(chan struct{}) - tester.downloader.syncInitHook = func(origin, latest uint64) { starting <- struct{}{} <-progress } - // Retrieve the sync progress and ensure they are zero (pristine sync) - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 { - t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0) - } - // Create and sync with an attacker that promises a higher chain than available - tester.newPeer("attack", protocol, hashes, headers, blocks, receipts) - for i := 1; i < 3; i++ { - delete(tester.peerHeaders["attack"], hashes[i]) - delete(tester.peerBlocks["attack"], hashes[i]) - delete(tester.peerReceipts["attack"], hashes[i]) + checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{}) + + // Create and sync with an attacker that promises a higher chain than available. + brokenChain := chain.shorten(chain.len()) + numMissing := 5 + for i := brokenChain.len() - 2; i > brokenChain.len()-numMissing; i-- { + delete(brokenChain.headerm, brokenChain.chain[i]) } + tester.newPeer("attack", protocol, brokenChain) pending := new(sync.WaitGroup) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("attack", nil, mode); err == nil { @@ -1628,14 +1348,17 @@ func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(targetBlocks+3) { - t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, targetBlocks+3) - } + checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{ + HighestBlock: uint64(brokenChain.len() - 1), + }) progress <- struct{}{} pending.Wait() + afterFailedSync := tester.downloader.Progress() - // Synchronise with a good peer and check that the progress height has been reduced to the true value - tester.newPeer("valid", protocol, hashes[3:], headers, blocks, receipts) + // Synchronise with a good peer and check that the progress height has been reduced to + // the true value. + validChain := chain.shorten(chain.len() - numMissing) + tester.newPeer("valid", protocol, validChain) pending.Add(1) go func() { @@ -1645,23 +1368,25 @@ func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock > uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/0-%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, targetBlocks, targetBlocks) - } + checkProgress(t, tester.downloader, "completing", ethereum.SyncProgress{ + CurrentBlock: afterFailedSync.CurrentBlock, + HighestBlock: uint64(validChain.len() - 1), + }) + + // Check final progress after successful sync. progress <- struct{}{} pending.Wait() - - // Check final progress after successful sync - if progress := tester.downloader.Progress(); progress.StartingBlock > uint64(targetBlocks) || progress.CurrentBlock != uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Final progress mismatch: have %v/%v/%v, want 0-%v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks, targetBlocks, targetBlocks) - } + checkProgress(t, tester.downloader, "final", ethereum.SyncProgress{ + CurrentBlock: uint64(validChain.len() - 1), + HighestBlock: uint64(validChain.len() - 1), + }) } // This test reproduces an issue where unexpected deliveries would // block indefinitely if they arrived at the right time. -// We use data driven subtests to manage this so that it will be parallel on its own -// and not with the other tests, avoiding intermittent failures. func TestDeliverHeadersHang(t *testing.T) { + t.Parallel() + testCases := []struct { protocol int syncMode SyncMode @@ -1675,15 +1400,38 @@ func TestDeliverHeadersHang(t *testing.T) { } for _, tc := range testCases { t.Run(fmt.Sprintf("protocol %d mode %v", tc.protocol, tc.syncMode), func(t *testing.T) { + t.Parallel() testDeliverHeadersHang(t, tc.protocol, tc.syncMode) }) } } +func testDeliverHeadersHang(t *testing.T, protocol int, mode SyncMode) { + master := newTester() + defer master.terminate() + chain := testChainBase.shorten(15) + + for i := 0; i < 200; i++ { + tester := newTester() + tester.peerDb = master.peerDb + tester.newPeer("peer", protocol, chain) + + // Whenever the downloader requests headers, flood it with + // a lot of unrequested header deliveries. + tester.downloader.peers.peers["peer"].peer = &floodingTestPeer{ + peer: tester.downloader.peers.peers["peer"].peer, + tester: tester, + } + if err := tester.sync("peer", nil, mode); err != nil { + t.Errorf("test %d: sync failed: %v", i, err) + } + tester.terminate() + } +} + type floodingTestPeer struct { peer Peer tester *downloadTester - pend sync.WaitGroup } func (ftp *floodingTestPeer) Head() (common.Hash, *big.Int) { return ftp.peer.Head() } @@ -1702,54 +1450,32 @@ func (ftp *floodingTestPeer) RequestNodeData(hashes []common.Hash) error { func (ftp *floodingTestPeer) RequestHeadersByNumber(from uint64, count, skip int, reverse bool) error { deliveriesDone := make(chan struct{}, 500) - for i := 0; i < cap(deliveriesDone); i++ { + for i := 0; i < cap(deliveriesDone)-1; i++ { peer := fmt.Sprintf("fake-peer%d", i) - ftp.pend.Add(1) - go func() { ftp.tester.downloader.DeliverHeaders(peer, []*types.Header{{}, {}, {}, {}}) deliveriesDone <- struct{}{} - ftp.pend.Done() }() } - // Deliver the actual requested headers. - go ftp.peer.RequestHeadersByNumber(from, count, skip, reverse) + // None of the extra deliveries should block. timeout := time.After(60 * time.Second) + launched := false for i := 0; i < cap(deliveriesDone); i++ { select { case <-deliveriesDone: + if !launched { + // Start delivering the requested headers + // after one of the flooding responses has arrived. + go func() { + ftp.peer.RequestHeadersByNumber(from, count, skip, reverse) + deliveriesDone <- struct{}{} + }() + launched = true + } case <-timeout: panic("blocked") } } return nil } - -func testDeliverHeadersHang(t *testing.T, protocol int, mode SyncMode) { - t.Parallel() - - master := newTester() - defer master.terminate() - - hashes, headers, blocks, receipts := master.makeChain(5, 0, master.genesis, nil, false) - for i := 0; i < 200; i++ { - tester := newTester() - tester.peerDb = master.peerDb - - tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) - // Whenever the downloader requests headers, flood it with - // a lot of unrequested header deliveries. - tester.downloader.peers.peers["peer"].peer = &floodingTestPeer{ - peer: tester.downloader.peers.peers["peer"].peer, - tester: tester, - } - if err := tester.sync("peer", nil, mode); err != nil { - t.Errorf("test %d: sync failed: %v", i, err) - } - tester.terminate() - - // Flush all goroutines to prevent messing with subsequent tests - tester.downloader.peers.peers["peer"].peer.(*floodingTestPeer).pend.Wait() - } -} diff --git a/eth/downloader/testchain_test.go b/eth/downloader/testchain_test.go new file mode 100644 index 0000000000..0b5a214257 --- /dev/null +++ b/eth/downloader/testchain_test.go @@ -0,0 +1,221 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package downloader + +import ( + "fmt" + "math/big" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/params" +) + +// Test chain parameters. +var ( + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + testAddress = crypto.PubkeyToAddress(testKey.PublicKey) + testDB = ethdb.NewMemDatabase() + testGenesis = core.GenesisBlockForTesting(testDB, testAddress, big.NewInt(1000000000)) +) + +// The common prefix of all test chains: +var testChainBase = newTestChain(blockCacheItems+200, testGenesis) + +// Different forks on top of the base chain: +var testChainForkLightA, testChainForkLightB, testChainForkHeavy *testChain + +func init() { + var forkLen = int(MaxForkAncestry + 50) + var wg sync.WaitGroup + wg.Add(3) + go func() { testChainForkLightA = testChainBase.makeFork(forkLen, false, 1); wg.Done() }() + go func() { testChainForkLightB = testChainBase.makeFork(forkLen, false, 2); wg.Done() }() + go func() { testChainForkHeavy = testChainBase.makeFork(forkLen, true, 3); wg.Done() }() + wg.Wait() +} + +type testChain struct { + genesis *types.Block + chain []common.Hash + headerm map[common.Hash]*types.Header + blockm map[common.Hash]*types.Block + receiptm map[common.Hash][]*types.Receipt + tdm map[common.Hash]*big.Int +} + +// newTestChain creates a blockchain of the given length. +func newTestChain(length int, genesis *types.Block) *testChain { + tc := new(testChain).copy(length) + tc.genesis = genesis + tc.chain = append(tc.chain, genesis.Hash()) + tc.headerm[tc.genesis.Hash()] = tc.genesis.Header() + tc.tdm[tc.genesis.Hash()] = tc.genesis.Difficulty() + tc.blockm[tc.genesis.Hash()] = tc.genesis + tc.generate(length-1, 0, genesis, false) + return tc +} + +// makeFork creates a fork on top of the test chain. +func (tc *testChain) makeFork(length int, heavy bool, seed byte) *testChain { + fork := tc.copy(tc.len() + length) + fork.generate(length, seed, tc.headBlock(), heavy) + return fork +} + +// shorten creates a copy of the chain with the given length. It panics if the +// length is longer than the number of available blocks. +func (tc *testChain) shorten(length int) *testChain { + if length > tc.len() { + panic(fmt.Errorf("can't shorten test chain to %d blocks, it's only %d blocks long", length, tc.len())) + } + return tc.copy(length) +} + +func (tc *testChain) copy(newlen int) *testChain { + cpy := &testChain{ + genesis: tc.genesis, + headerm: make(map[common.Hash]*types.Header, newlen), + blockm: make(map[common.Hash]*types.Block, newlen), + receiptm: make(map[common.Hash][]*types.Receipt, newlen), + tdm: make(map[common.Hash]*big.Int, newlen), + } + for i := 0; i < len(tc.chain) && i < newlen; i++ { + hash := tc.chain[i] + cpy.chain = append(cpy.chain, tc.chain[i]) + cpy.tdm[hash] = tc.tdm[hash] + cpy.blockm[hash] = tc.blockm[hash] + cpy.headerm[hash] = tc.headerm[hash] + cpy.receiptm[hash] = tc.receiptm[hash] + } + return cpy +} + +// generate creates a chain of n blocks starting at and including parent. +// the returned hash chain is ordered head->parent. In addition, every 22th block +// contains a transaction and every 5th an uncle to allow testing correct block +// reassembly. +func (tc *testChain) generate(n int, seed byte, parent *types.Block, heavy bool) { + // start := time.Now() + // defer func() { fmt.Printf("test chain generated in %v\n", time.Since(start)) }() + + blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testDB, n, func(i int, block *core.BlockGen) { + block.SetCoinbase(common.Address{seed}) + // If a heavy chain is requested, delay blocks to raise difficulty + if heavy { + block.OffsetTime(-1) + } + // Include transactions to the miner to make blocks more interesting. + if parent == tc.genesis && i%22 == 0 { + signer := types.MakeSigner(params.TestChainConfig, block.Number()) + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, nil, nil), signer, testKey) + if err != nil { + panic(err) + } + block.AddTx(tx) + } + // if the block number is a multiple of 5, add a bonus uncle to the block + if i > 0 && i%5 == 0 { + block.AddUncle(&types.Header{ + ParentHash: block.PrevBlock(i - 1).Hash(), + Number: big.NewInt(block.Number().Int64() - 1), + }) + } + }) + + // Convert the block-chain into a hash-chain and header/block maps + td := new(big.Int).Set(tc.td(parent.Hash())) + for i, b := range blocks { + td := td.Add(td, b.Difficulty()) + hash := b.Hash() + tc.chain = append(tc.chain, hash) + tc.blockm[hash] = b + tc.headerm[hash] = b.Header() + tc.receiptm[hash] = receipts[i] + tc.tdm[hash] = new(big.Int).Set(td) + } +} + +// len returns the total number of blocks in the chain. +func (tc *testChain) len() int { + return len(tc.chain) +} + +// headBlock returns the head of the chain. +func (tc *testChain) headBlock() *types.Block { + return tc.blockm[tc.chain[len(tc.chain)-1]] +} + +// td returns the total difficulty of the given block. +func (tc *testChain) td(hash common.Hash) *big.Int { + return tc.tdm[hash] +} + +// headersByHash returns headers in ascending order from the given hash. +func (tc *testChain) headersByHash(origin common.Hash, amount int, skip int) []*types.Header { + num, _ := tc.hashToNumber(origin) + return tc.headersByNumber(num, amount, skip) +} + +// headersByNumber returns headers in ascending order from the given number. +func (tc *testChain) headersByNumber(origin uint64, amount int, skip int) []*types.Header { + result := make([]*types.Header, 0, amount) + for num := origin; num < uint64(len(tc.chain)) && len(result) < amount; num += uint64(skip) + 1 { + if header, ok := tc.headerm[tc.chain[int(num)]]; ok { + result = append(result, header) + } + } + return result +} + +// receipts returns the receipts of the given block hashes. +func (tc *testChain) receipts(hashes []common.Hash) [][]*types.Receipt { + results := make([][]*types.Receipt, 0, len(hashes)) + for _, hash := range hashes { + if receipt, ok := tc.receiptm[hash]; ok { + results = append(results, receipt) + } + } + return results +} + +// bodies returns the block bodies of the given block hashes. +func (tc *testChain) bodies(hashes []common.Hash) ([][]*types.Transaction, [][]*types.Header) { + transactions := make([][]*types.Transaction, 0, len(hashes)) + uncles := make([][]*types.Header, 0, len(hashes)) + for _, hash := range hashes { + if block, ok := tc.blockm[hash]; ok { + transactions = append(transactions, block.Transactions()) + uncles = append(uncles, block.Uncles()) + } + } + return transactions, uncles +} + +func (tc *testChain) hashToNumber(target common.Hash) (uint64, bool) { + for num, hash := range tc.chain { + if hash == target { + return uint64(num), true + } + } + return 0, false +} From 81533deae5ee4a7ec08842e2b6647f3affde5a71 Mon Sep 17 00:00:00 2001 From: Mark Vujevits Date: Wed, 7 Nov 2018 20:33:36 +0100 Subject: [PATCH 017/159] swarm/network: light nodes are not dialed, saved and requested from (#17975) * RequestFromPeers does not use peers marked as lightnode * fix warning about variable name * write tests for RequestFromPeers * lightnodes should be omitted from the addressbook * resolve pr comments regarding logging, formatting and comments * resolve pr comments regarding comments and added a missing newline * add assertions to check peers in live connections --- swarm/network/kademlia.go | 20 ++++--- swarm/network/kademlia_test.go | 58 +++++++++++++++++-- swarm/network/stream/delivery.go | 5 +- swarm/network/stream/delivery_test.go | 83 +++++++++++++++++++++++++++ 4 files changed, 153 insertions(+), 13 deletions(-) diff --git a/swarm/network/kademlia.go b/swarm/network/kademlia.go index 55a0c6f13d..cd94741be3 100644 --- a/swarm/network/kademlia.go +++ b/swarm/network/kademlia.go @@ -261,7 +261,7 @@ func (k *Kademlia) On(p *Peer) (uint8, bool) { // found among live peers, do nothing return v }) - if ins { + if ins && !p.BzzPeer.LightNode { a := newEntry(p.BzzAddr) a.conn = p // insert new online peer into addrs @@ -329,14 +329,18 @@ func (k *Kademlia) Off(p *Peer) { k.lock.Lock() defer k.lock.Unlock() var del bool - k.addrs, _, _, _ = pot.Swap(k.addrs, p, pof, func(v pot.Val) pot.Val { - // v cannot be nil, must check otherwise we overwrite entry - if v == nil { - panic(fmt.Sprintf("connected peer not found %v", p)) - } + if !p.BzzPeer.LightNode { + k.addrs, _, _, _ = pot.Swap(k.addrs, p, pof, func(v pot.Val) pot.Val { + // v cannot be nil, must check otherwise we overwrite entry + if v == nil { + panic(fmt.Sprintf("connected peer not found %v", p)) + } + del = true + return newEntry(p.BzzAddr) + }) + } else { del = true - return newEntry(p.BzzAddr) - }) + } if del { k.conns, _, _, _ = pot.Swap(k.conns, p, pof, func(_ pot.Val) pot.Val { diff --git a/swarm/network/kademlia_test.go b/swarm/network/kademlia_test.go index 903c8dbdac..d2e051f457 100644 --- a/swarm/network/kademlia_test.go +++ b/swarm/network/kademlia_test.go @@ -46,19 +46,19 @@ func newTestKademlia(b string) *Kademlia { return NewKademlia(base, params) } -func newTestKadPeer(k *Kademlia, s string) *Peer { - return NewPeer(&BzzPeer{BzzAddr: testKadPeerAddr(s)}, k) +func newTestKadPeer(k *Kademlia, s string, lightNode bool) *Peer { + return NewPeer(&BzzPeer{BzzAddr: testKadPeerAddr(s), LightNode: lightNode}, k) } func On(k *Kademlia, ons ...string) { for _, s := range ons { - k.On(newTestKadPeer(k, s)) + k.On(newTestKadPeer(k, s, false)) } } func Off(k *Kademlia, offs ...string) { for _, s := range offs { - k.Off(newTestKadPeer(k, s)) + k.Off(newTestKadPeer(k, s, false)) } } @@ -254,6 +254,56 @@ func TestSuggestPeerFindPeers(t *testing.T) { } +// a node should stay in the address book if it's removed from the kademlia +func TestOffEffectingAddressBookNormalNode(t *testing.T) { + k := newTestKademlia("00000000") + // peer added to kademlia + k.On(newTestKadPeer(k, "01000000", false)) + // peer should be in the address book + if k.addrs.Size() != 1 { + t.Fatal("known peer addresses should contain 1 entry") + } + // peer should be among live connections + if k.conns.Size() != 1 { + t.Fatal("live peers should contain 1 entry") + } + // remove peer from kademlia + k.Off(newTestKadPeer(k, "01000000", false)) + // peer should be in the address book + if k.addrs.Size() != 1 { + t.Fatal("known peer addresses should contain 1 entry") + } + // peer should not be among live connections + if k.conns.Size() != 0 { + t.Fatal("live peers should contain 0 entry") + } +} + +// a light node should not be in the address book +func TestOffEffectingAddressBookLightNode(t *testing.T) { + k := newTestKademlia("00000000") + // light node peer added to kademlia + k.On(newTestKadPeer(k, "01000000", true)) + // peer should not be in the address book + if k.addrs.Size() != 0 { + t.Fatal("known peer addresses should contain 0 entry") + } + // peer should be among live connections + if k.conns.Size() != 1 { + t.Fatal("live peers should contain 1 entry") + } + // remove peer from kademlia + k.Off(newTestKadPeer(k, "01000000", true)) + // peer should not be in the address book + if k.addrs.Size() != 0 { + t.Fatal("known peer addresses should contain 0 entry") + } + // peer should not be among live connections + if k.conns.Size() != 0 { + t.Fatal("live peers should contain 0 entry") + } +} + func TestSuggestPeerRetries(t *testing.T) { k := newTestKademlia("00000000") k.RetryInterval = int64(300 * time.Millisecond) // cycle diff --git a/swarm/network/stream/delivery.go b/swarm/network/stream/delivery.go index 9092ffe3ed..0109fbdef2 100644 --- a/swarm/network/stream/delivery.go +++ b/swarm/network/stream/delivery.go @@ -245,7 +245,10 @@ func (d *Delivery) RequestFromPeers(ctx context.Context, req *network.Request) ( } else { d.kad.EachConn(req.Addr[:], 255, func(p *network.Peer, po int, nn bool) bool { id := p.ID() - // TODO: skip light nodes that do not accept retrieve requests + if p.LightNode { + // skip light nodes + return true + } if req.SkipPeer(id.String()) { log.Trace("Delivery.RequestFromPeers: skip peer", "peer id", id) return true diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index 29b4f2f69a..c77682e0e1 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -29,10 +29,13 @@ import ( "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/protocols" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" p2ptest "github.com/ethereum/go-ethereum/p2p/testing" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/network" + pq "github.com/ethereum/go-ethereum/swarm/network/priorityqueue" "github.com/ethereum/go-ethereum/swarm/network/simulation" "github.com/ethereum/go-ethereum/swarm/state" "github.com/ethereum/go-ethereum/swarm/storage" @@ -274,6 +277,86 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { } } +// if there is one peer in the Kademlia, RequestFromPeers should return it +func TestRequestFromPeers(t *testing.T) { + dummyPeerID := enode.HexID("3431c3939e1ee2a6345e976a8234f9870152d64879f30bc272a074f6859e75e8") + + addr := network.RandomAddr() + to := network.NewKademlia(addr.OAddr, network.NewKadParams()) + delivery := NewDelivery(to, nil) + protocolsPeer := protocols.NewPeer(p2p.NewPeer(dummyPeerID, "dummy", nil), nil, nil) + peer := network.NewPeer(&network.BzzPeer{ + BzzAddr: network.RandomAddr(), + LightNode: false, + Peer: protocolsPeer, + }, to) + to.On(peer) + r := NewRegistry(addr.ID(), delivery, nil, nil, nil) + + // an empty priorityQueue has to be created to prevent a goroutine being called after the test has finished + sp := &Peer{ + Peer: protocolsPeer, + pq: pq.New(int(PriorityQueue), PriorityQueueCap), + streamer: r, + } + r.setPeer(sp) + req := network.NewRequest( + storage.Address(hash0[:]), + true, + &sync.Map{}, + ) + ctx := context.Background() + id, _, err := delivery.RequestFromPeers(ctx, req) + + if err != nil { + t.Fatal(err) + } + if *id != dummyPeerID { + t.Fatalf("Expected an id, got %v", id) + } +} + +// RequestFromPeers should not return light nodes +func TestRequestFromPeersWithLightNode(t *testing.T) { + dummyPeerID := enode.HexID("3431c3939e1ee2a6345e976a8234f9870152d64879f30bc272a074f6859e75e8") + + addr := network.RandomAddr() + to := network.NewKademlia(addr.OAddr, network.NewKadParams()) + delivery := NewDelivery(to, nil) + + protocolsPeer := protocols.NewPeer(p2p.NewPeer(dummyPeerID, "dummy", nil), nil, nil) + // setting up a lightnode + peer := network.NewPeer(&network.BzzPeer{ + BzzAddr: network.RandomAddr(), + LightNode: true, + Peer: protocolsPeer, + }, to) + to.On(peer) + r := NewRegistry(addr.ID(), delivery, nil, nil, nil) + // an empty priorityQueue has to be created to prevent a goroutine being called after the test has finished + sp := &Peer{ + Peer: protocolsPeer, + pq: pq.New(int(PriorityQueue), PriorityQueueCap), + streamer: r, + } + r.setPeer(sp) + + req := network.NewRequest( + storage.Address(hash0[:]), + true, + &sync.Map{}, + ) + + ctx := context.Background() + // making a request which should return with "no peer found" + _, _, err := delivery.RequestFromPeers(ctx, req) + + expectedError := "no peer found" + if err.Error() != expectedError { + t.Fatalf("expected '%v', got %v", expectedError, err) + } +} + func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { tester, streamer, localStore, teardown, err := newStreamerTester(t, &RegistryOptions{ Retrieval: RetrievalDisabled, From cf3b187bdef59078ba6570a2f5ee046ab87bcefd Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 7 Nov 2018 20:39:08 +0100 Subject: [PATCH 018/159] swarm, cmd/swarm: address ineffectual assignments (#18048) * swarm, cmd/swarm: address ineffectual assignments * swarm/network: remove unused vars from testHandshake * swarm/storage/feed: revert cursor changes --- cmd/swarm/access.go | 6 ++++++ cmd/swarm/fs_test.go | 3 +++ cmd/swarm/upload_test.go | 3 +-- swarm/api/act.go | 3 +++ swarm/api/client/client_test.go | 3 +++ swarm/api/filesystem.go | 4 ++++ swarm/api/http/server.go | 2 +- swarm/api/http/server_test.go | 5 ++++- swarm/api/manifest.go | 1 - swarm/network/hive_test.go | 6 ++++++ swarm/network/protocol_test.go | 12 +----------- swarm/network/simulation/bucket_test.go | 4 ++-- swarm/network/stream/syncer_test.go | 3 +++ swarm/pss/client/client_test.go | 6 ++++++ swarm/pss/notify/notify_test.go | 6 ++++++ swarm/pss/protocol_test.go | 6 ++++++ swarm/state/dbstore.go | 2 +- swarm/state/dbstore_test.go | 3 +++ swarm/state/inmemorystore.go | 2 +- swarm/storage/types.go | 5 +---- 20 files changed, 61 insertions(+), 24 deletions(-) diff --git a/cmd/swarm/access.go b/cmd/swarm/access.go index 629781edd8..072541b659 100644 --- a/cmd/swarm/access.go +++ b/cmd/swarm/access.go @@ -114,6 +114,9 @@ func accessNewPass(ctx *cli.Context) { utils.Fatalf("error getting session key: %v", err) } m, err := api.GenerateAccessControlManifest(ctx, ref, accessKey, ae) + if err != nil { + utils.Fatalf("had an error generating the manifest: %v", err) + } if dryRun { err = printManifests(m, nil) if err != nil { @@ -147,6 +150,9 @@ func accessNewPK(ctx *cli.Context) { utils.Fatalf("error getting session key: %v", err) } m, err := api.GenerateAccessControlManifest(ctx, ref, sessionKey, ae) + if err != nil { + utils.Fatalf("had an error generating the manifest: %v", err) + } if dryRun { err = printManifests(m, nil) if err != nil { diff --git a/cmd/swarm/fs_test.go b/cmd/swarm/fs_test.go index 4f38b094bb..3b722515ee 100644 --- a/cmd/swarm/fs_test.go +++ b/cmd/swarm/fs_test.go @@ -80,6 +80,9 @@ func TestCLISwarmFs(t *testing.T) { t.Fatal(err) } dirPath2, err := createDirInDir(dirPath, "AnotherTestSubDir") + if err != nil { + t.Fatal(err) + } dummyContent := "somerandomtestcontentthatshouldbeasserted" dirs := []string{ diff --git a/cmd/swarm/upload_test.go b/cmd/swarm/upload_test.go index 0ac2456a51..ba4463e8bf 100644 --- a/cmd/swarm/upload_test.go +++ b/cmd/swarm/upload_test.go @@ -243,8 +243,7 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) { } defer os.RemoveAll(tmpDownload) bzzLocator := "bzz:/" + hash - flagss := []string{} - flagss = []string{ + flagss := []string{ "--bzzapi", cluster.Nodes[0].URL, "down", "--recursive", diff --git a/swarm/api/act.go b/swarm/api/act.go index 52d9098271..e54369f9a8 100644 --- a/swarm/api/act.go +++ b/swarm/api/act.go @@ -458,6 +458,9 @@ func DoACT(ctx *cli.Context, privateKey *ecdsa.PrivateKey, salt []byte, grantees return nil, nil, nil, err } sessionKey, err := NewSessionKeyPK(privateKey, granteePub, salt) + if err != nil { + return nil, nil, nil, err + } hasher := sha3.NewKeccak256() hasher.Write(append(sessionKey, 0)) diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go index 03c6cbb28c..c30d699110 100644 --- a/swarm/api/client/client_test.go +++ b/swarm/api/client/client_test.go @@ -457,6 +457,9 @@ func TestClientCreateUpdateFeed(t *testing.T) { } feedManifestHash, err := client.CreateFeedWithManifest(createRequest) + if err != nil { + t.Fatal(err) + } correctManifestAddrHex := "0e9b645ebc3da167b1d56399adc3276f7a08229301b72a03336be0e7d4b71882" if feedManifestHash != correctManifestAddrHex { diff --git a/swarm/api/filesystem.go b/swarm/api/filesystem.go index 43695efc1e..266ef71bec 100644 --- a/swarm/api/filesystem.go +++ b/swarm/api/filesystem.go @@ -122,6 +122,10 @@ func (fs *FileSystem) Upload(lpath, index string, toEncrypt bool) (string, error var wait func(context.Context) error ctx := context.TODO() hash, wait, err = fs.api.fileStore.Store(ctx, f, stat.Size(), toEncrypt) + if err != nil { + errors[i] = err + return + } if hash != nil { list[i].Hash = hash.Hex() } diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 803b789878..3c6735a73e 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -366,7 +366,7 @@ func (s *Server) handleMultipartUpload(r *http.Request, boundary string, mw *api } var size int64 - var reader io.Reader = part + var reader io.Reader if contentLength := part.Header.Get("Content-Length"); contentLength != "" { size, err = strconv.ParseInt(contentLength, 10, 64) if err != nil { diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 159c8a159f..04d0e045aa 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -263,7 +263,7 @@ func TestBzzFeed(t *testing.T) { if resp.StatusCode == http.StatusOK { t.Fatal("Expected error status since feed update does not contain multihash. Received 200 OK") } - b, err = ioutil.ReadAll(resp.Body) + _, err = ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) } @@ -491,6 +491,9 @@ func testBzzGetPath(encrypted bool, t *testing.T) { } defer resp.Body.Close() respbody, err = ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatalf("Error while reading response body: %v", err) + } if string(respbody) != testmanifest[v] { isexpectedfailrequest := false diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index 7c4cc88e47..890ed88bd4 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -557,7 +557,6 @@ func (mt *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *manif if path != entry.Path { return nil, 0 } - pos = epl } } return nil, 0 diff --git a/swarm/network/hive_test.go b/swarm/network/hive_test.go index 059c3dc96d..56adc5a8e9 100644 --- a/swarm/network/hive_test.go +++ b/swarm/network/hive_test.go @@ -70,6 +70,9 @@ func TestHiveStatePersistance(t *testing.T) { defer os.RemoveAll(dir) store, err := state.NewDBStore(dir) //start the hive with an empty dbstore + if err != nil { + t.Fatal(err) + } params := NewHiveParams() s, pp := newHiveTester(t, params, 5, store) @@ -90,6 +93,9 @@ func TestHiveStatePersistance(t *testing.T) { store.Close() persistedStore, err := state.NewDBStore(dir) //start the hive with an empty dbstore + if err != nil { + t.Fatal(err) + } s1, pp := newHiveTester(t, params, 1, persistedStore) diff --git a/swarm/network/protocol_test.go b/swarm/network/protocol_test.go index cdf370f359..f0d2666288 100644 --- a/swarm/network/protocol_test.go +++ b/swarm/network/protocol_test.go @@ -153,17 +153,7 @@ func newBzzHandshakeTester(t *testing.T, n int, addr *BzzAddr, lightNode bool) * // should test handshakes in one exchange? parallelisation func (s *bzzTester) testHandshake(lhs, rhs *HandshakeMsg, disconnects ...*p2ptest.Disconnect) error { - var peers []enode.ID - id := rhs.Addr.ID() - if len(disconnects) > 0 { - for _, d := range disconnects { - peers = append(peers, d.Peer) - } - } else { - peers = []enode.ID{id} - } - - if err := s.TestExchanges(HandshakeMsgExchange(lhs, rhs, id)...); err != nil { + if err := s.TestExchanges(HandshakeMsgExchange(lhs, rhs, rhs.Addr.ID())...); err != nil { return err } diff --git a/swarm/network/simulation/bucket_test.go b/swarm/network/simulation/bucket_test.go index 461d99825c..69df19bfe4 100644 --- a/swarm/network/simulation/bucket_test.go +++ b/swarm/network/simulation/bucket_test.go @@ -94,7 +94,7 @@ func TestServiceBucket(t *testing.T) { t.Fatalf("expected %q, got %q", customValue, s) } - v, ok = sim.NodeItem(id2, customKey) + _, ok = sim.NodeItem(id2, customKey) if ok { t.Fatal("bucket item should not be found") } @@ -119,7 +119,7 @@ func TestServiceBucket(t *testing.T) { t.Fatalf("expected %q, got %q", testValue+id1.String(), s) } - v, ok = items[id2] + _, ok = items[id2] if ok { t.Errorf("node 2 item should not be found") } diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index 113807b98a..b0e35b0db3 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -62,6 +62,9 @@ func createMockStore(globalStore *mockdb.GlobalStore, id enode.ID, addr *network params.Init(datadir) params.BaseKey = addr.Over() lstore, err = storage.NewLocalStore(params, mockStore) + if err != nil { + return nil, "", err + } return lstore, datadir, nil } diff --git a/swarm/pss/client/client_test.go b/swarm/pss/client/client_test.go index cfef3c7947..8f2f0e8059 100644 --- a/swarm/pss/client/client_test.go +++ b/swarm/pss/client/client_test.go @@ -252,7 +252,13 @@ func newServices() adapters.Services { ctxlocal, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() keys, err := wapi.NewKeyPair(ctxlocal) + if err != nil { + return nil, err + } privkey, err := w.GetPrivateKey(keys) + if err != nil { + return nil, err + } psparams := pss.NewPssParams().WithPrivateKey(privkey) pskad := kademlia(ctx.Config.ID) ps, err := pss.NewPss(pskad, psparams) diff --git a/swarm/pss/notify/notify_test.go b/swarm/pss/notify/notify_test.go index 675b41ada2..d4d383a6ba 100644 --- a/swarm/pss/notify/notify_test.go +++ b/swarm/pss/notify/notify_test.go @@ -223,7 +223,13 @@ func newServices(allowRaw bool) adapters.Services { ctxlocal, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() keys, err := wapi.NewKeyPair(ctxlocal) + if err != nil { + return nil, err + } privkey, err := w.GetPrivateKey(keys) + if err != nil { + return nil, err + } pssp := pss.NewPssParams().WithPrivateKey(privkey) pssp.MsgTTL = time.Second * 30 pssp.AllowRaw = allowRaw diff --git a/swarm/pss/protocol_test.go b/swarm/pss/protocol_test.go index f4209fea59..4ef3e90a04 100644 --- a/swarm/pss/protocol_test.go +++ b/swarm/pss/protocol_test.go @@ -93,11 +93,17 @@ func testProtocol(t *testing.T) { lctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() lsub, err := clients[0].Subscribe(lctx, "pss", lmsgC, "receive", topic) + if err != nil { + t.Fatal(err) + } defer lsub.Unsubscribe() rmsgC := make(chan APIMsg) rctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic) + if err != nil { + t.Fatal(err) + } defer rsub.Unsubscribe() // set reciprocal public keys diff --git a/swarm/state/dbstore.go b/swarm/state/dbstore.go index 5e5c172b25..b0aa92e272 100644 --- a/swarm/state/dbstore.go +++ b/swarm/state/dbstore.go @@ -69,7 +69,7 @@ func (s *DBStore) Get(key string, i interface{}) (err error) { // Put stores an object that implements Binary for a specific key. func (s *DBStore) Put(key string, i interface{}) (err error) { - bytes := []byte{} + var bytes []byte marshaler, ok := i.(encoding.BinaryMarshaler) if !ok { diff --git a/swarm/state/dbstore_test.go b/swarm/state/dbstore_test.go index 6683e788f7..f7098956d0 100644 --- a/swarm/state/dbstore_test.go +++ b/swarm/state/dbstore_test.go @@ -112,6 +112,9 @@ func testPersistedStore(t *testing.T, store Store) { as := []string{} err = store.Get("key2", &as) + if err != nil { + t.Fatal(err) + } if len(as) != 3 { t.Fatalf("serialized array did not match expectation") diff --git a/swarm/state/inmemorystore.go b/swarm/state/inmemorystore.go index 1ca25404a1..3ba48592bc 100644 --- a/swarm/state/inmemorystore.go +++ b/swarm/state/inmemorystore.go @@ -59,7 +59,7 @@ func (s *InmemoryStore) Get(key string, i interface{}) (err error) { func (s *InmemoryStore) Put(key string, i interface{}) (err error) { s.mu.Lock() defer s.mu.Unlock() - bytes := []byte{} + var bytes []byte marshaler, ok := i.(encoding.BinaryMarshaler) if !ok { diff --git a/swarm/storage/types.go b/swarm/storage/types.go index ada86831fc..092843db0c 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -244,11 +244,8 @@ func GenerateRandomChunk(dataSize int64) Chunk { } func GenerateRandomChunks(dataSize int64, count int) (chunks []Chunk) { - if dataSize > ch.DefaultSize { - dataSize = ch.DefaultSize - } for i := 0; i < count; i++ { - ch := GenerateRandomChunk(ch.DefaultSize) + ch := GenerateRandomChunk(dataSize) chunks = append(chunks, ch) } return chunks From 503993c819f0a4e7e53f6739e90497106ebdfe09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kurk=C3=B3=20Mih=C3=A1ly?= Date: Thu, 8 Nov 2018 13:11:20 +0200 Subject: [PATCH 019/159] p2p: use enode.ID type in metered connection (#17933) Change the type of the metered connection's id field from string to enode.ID. --- p2p/metrics.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/p2p/metrics.go b/p2p/metrics.go index 6a7c0bad33..d7873f39a5 100644 --- a/p2p/metrics.go +++ b/p2p/metrics.go @@ -74,7 +74,7 @@ const ( type MeteredPeerEvent struct { Type MeteredPeerEventType // Type of peer event IP net.IP // IP address of the peer - ID string // NodeID of the peer + ID enode.ID // NodeID of the peer Elapsed time.Duration // Time elapsed between the connection and the handshake/disconnection Ingress uint64 // Ingress count at the moment of the event Egress uint64 // Egress count at the moment of the event @@ -93,7 +93,7 @@ type meteredConn struct { connected time.Time // Connection time of the peer ip net.IP // IP address of the peer - id string // NodeID of the peer + id enode.ID // NodeID of the peer // trafficMetered denotes if the peer is registered in the traffic registries. // Its value is true if the metered peer count doesn't reach the limit in the @@ -160,8 +160,7 @@ func (c *meteredConn) Write(b []byte) (n int, err error) { // handshakeDone is called when a peer handshake is done. Registers the peer to // the ingress and the egress traffic registries using the peer's IP and node ID, // also emits connect event. -func (c *meteredConn) handshakeDone(nodeID enode.ID) { - id := nodeID.String() +func (c *meteredConn) handshakeDone(id enode.ID) { if atomic.AddInt32(&meteredPeerCount, 1) >= MeteredPeerLimit { // Don't register the peer in the traffic registries. atomic.AddInt32(&meteredPeerCount, -1) @@ -170,7 +169,7 @@ func (c *meteredConn) handshakeDone(nodeID enode.ID) { c.lock.Unlock() log.Warn("Metered peer count reached the limit") } else { - key := fmt.Sprintf("%s/%s", c.ip, id) + key := fmt.Sprintf("%s/%s", c.ip, id.String()) c.lock.Lock() c.id, c.trafficMetered = id, true c.ingressMeter = metrics.NewRegisteredMeter(key, PeerIngressRegistry) @@ -190,7 +189,7 @@ func (c *meteredConn) handshakeDone(nodeID enode.ID) { func (c *meteredConn) Close() error { err := c.Conn.Close() c.lock.RLock() - if c.id == "" { + if c.id == (enode.ID{}) { // If the peer disconnects before the handshake. c.lock.RUnlock() meteredPeerFeed.Send(MeteredPeerEvent{ From 968f6019d00e349e988f720cdaa065def85ea72f Mon Sep 17 00:00:00 2001 From: Corey Lin <514971757@qq.com> Date: Thu, 8 Nov 2018 19:17:01 +0800 Subject: [PATCH 020/159] event, event/filter: minor code cleanup (#18061) --- event/event_test.go | 8 ++++---- event/filter/generic_filter.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/event/event_test.go b/event/event_test.go index a12945a471..2be357ba2d 100644 --- a/event/event_test.go +++ b/event/event_test.go @@ -141,7 +141,7 @@ func TestMuxConcurrent(t *testing.T) { } } -func emptySubscriber(mux *TypeMux, types ...interface{}) { +func emptySubscriber(mux *TypeMux) { s := mux.Subscribe(testEvent(0)) go func() { for range s.Chan() { @@ -182,9 +182,9 @@ func BenchmarkPost1000(b *testing.B) { func BenchmarkPostConcurrent(b *testing.B) { var mux = new(TypeMux) defer mux.Stop() - emptySubscriber(mux, testEvent(0)) - emptySubscriber(mux, testEvent(0)) - emptySubscriber(mux, testEvent(0)) + emptySubscriber(mux) + emptySubscriber(mux) + emptySubscriber(mux) var wg sync.WaitGroup poster := func() { diff --git a/event/filter/generic_filter.go b/event/filter/generic_filter.go index d679b8bfa8..467bf01bec 100644 --- a/event/filter/generic_filter.go +++ b/event/filter/generic_filter.go @@ -25,7 +25,7 @@ type Generic struct { // self = registered, f = incoming func (self Generic) Compare(f Filter) bool { - var strMatch, dataMatch = true, true + var strMatch = true filter := f.(Generic) if (len(self.Str1) > 0 && filter.Str1 != self.Str1) || @@ -40,7 +40,7 @@ func (self Generic) Compare(f Filter) bool { } } - return strMatch && dataMatch + return strMatch } func (self Generic) Trigger(data interface{}) { From c71e4fc4d5e5f97f57f1c52dced187da41b59aaf Mon Sep 17 00:00:00 2001 From: Liang Ma Date: Thu, 8 Nov 2018 11:22:28 +0000 Subject: [PATCH 021/159] p2p: fix comment typo (#18027) --- p2p/rlpx.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/rlpx.go b/p2p/rlpx.go index a105720a44..22a27dd96a 100644 --- a/p2p/rlpx.go +++ b/p2p/rlpx.go @@ -151,7 +151,7 @@ func readProtocolHandshake(rw MsgReader, our *protoHandshake) (*protoHandshake, } if msg.Code == discMsg { // Disconnect before protocol handshake is valid according to the - // spec and we send it ourself if the posthanshake checks fail. + // spec and we send it ourself if the post-handshake checks fail. // We can't return the reason directly, though, because it is echoed // back otherwise. Wrap it in a string instead. var reason [1]DiscReason From 212bf266c55b3737da37e6588142d0c9f7c6e10f Mon Sep 17 00:00:00 2001 From: Corey Lin <514971757@qq.com> Date: Thu, 8 Nov 2018 19:25:14 +0800 Subject: [PATCH 022/159] eth, p2p: fix comment typos (#18014) --- eth/config.go | 2 +- p2p/server.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eth/config.go b/eth/config.go index efbaafb6a2..e32c01a734 100644 --- a/eth/config.go +++ b/eth/config.go @@ -122,7 +122,7 @@ type Config struct { // Miscellaneous options DocRoot string `toml:"-"` - // Type of the EWASM interpreter ("" for detault) + // Type of the EWASM interpreter ("" for default) EWASMInterpreter string // Type of the EVM interpreter ("" for default) EVMInterpreter string diff --git a/p2p/server.go b/p2p/server.go index 38a881f7bd..6678608634 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -228,7 +228,7 @@ type transport interface { MsgReadWriter // transports must provide Close because we use MsgPipe in some of // the tests. Closing the actual network connection doesn't do - // anything in those tests because NsgPipe doesn't use it. + // anything in those tests because MsgPipe doesn't use it. close(err error) } From a5dc087845c90dad126bf203640ac6e876d7bbcb Mon Sep 17 00:00:00 2001 From: Corey Lin <514971757@qq.com> Date: Thu, 8 Nov 2018 20:07:15 +0800 Subject: [PATCH 023/159] core/vm, eth/tracers: use pointer receiver for GetRefund (#18018) --- core/vm/logger_test.go | 2 +- eth/tracers/tracer_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/vm/logger_test.go b/core/vm/logger_test.go index cba7c7a0ef..2ea7535a79 100644 --- a/core/vm/logger_test.go +++ b/core/vm/logger_test.go @@ -46,7 +46,7 @@ type dummyStatedb struct { state.StateDB } -func (dummyStatedb) GetRefund() uint64 { return 1337 } +func (*dummyStatedb) GetRefund() uint64 { return 1337 } func TestStoreCapture(t *testing.T) { var ( diff --git a/eth/tracers/tracer_test.go b/eth/tracers/tracer_test.go index 52f29c83fc..a45a121159 100644 --- a/eth/tracers/tracer_test.go +++ b/eth/tracers/tracer_test.go @@ -48,7 +48,7 @@ type dummyStatedb struct { state.StateDB } -func (dummyStatedb) GetRefund() uint64 { return 1337 } +func (*dummyStatedb) GetRefund() uint64 { return 1337 } func runTrace(tracer *Tracer) (json.RawMessage, error) { env := vm.NewEVM(vm.Context{BlockNumber: big.NewInt(1)}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer}) From 1064e3283db6e388db4fc353c8bd17b57d57fa42 Mon Sep 17 00:00:00 2001 From: JoranHonig Date: Thu, 8 Nov 2018 13:09:25 +0100 Subject: [PATCH 024/159] common/compiler: capture runtime code and source maps (#18020) --- common/compiler/solidity.go | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/common/compiler/solidity.go b/common/compiler/solidity.go index f6e8d2e42a..b7c8ec5638 100644 --- a/common/compiler/solidity.go +++ b/common/compiler/solidity.go @@ -31,14 +31,15 @@ import ( var versionRegexp = regexp.MustCompile(`([0-9]+)\.([0-9]+)\.([0-9]+)`) -// Contract contains information about a compiled contract, alongside its code. +// Contract contains information about a compiled contract, alongside its code and runtime code. type Contract struct { - Code string `json:"code"` - Info ContractInfo `json:"info"` + Code string `json:"code"` + RuntimeCode string `json:"runtime-code"` + Info ContractInfo `json:"info"` } // ContractInfo contains information about a compiled contract, including access -// to the ABI definition, user and developer docs, and metadata. +// to the ABI definition, source mapping, user and developer docs, and metadata. // // Depending on the source, language version, compiler version, and compiler // options will provide information about how the contract was compiled. @@ -48,6 +49,8 @@ type ContractInfo struct { LanguageVersion string `json:"languageVersion"` CompilerVersion string `json:"compilerVersion"` CompilerOptions string `json:"compilerOptions"` + SrcMap string `json:"srcMap"` + SrcMapRuntime string `json:"srcMapRuntime"` AbiDefinition interface{} `json:"abiDefinition"` UserDoc interface{} `json:"userDoc"` DeveloperDoc interface{} `json:"developerDoc"` @@ -63,14 +66,16 @@ type Solidity struct { // --combined-output format type solcOutput struct { Contracts map[string]struct { - Bin, Abi, Devdoc, Userdoc, Metadata string + BinRuntime string `json:"bin-runtime"` + SrcMapRuntime string `json:"srcmap-runtime"` + Bin, SrcMap, Abi, Devdoc, Userdoc, Metadata string } Version string } func (s *Solidity) makeArgs() []string { p := []string{ - "--combined-json", "bin,abi,userdoc,devdoc", + "--combined-json", "bin,bin-runtime,srcmap,srcmap-runtime,abi,userdoc,devdoc", "--optimize", // code optimizer switched on } if s.Major > 0 || s.Minor > 4 || s.Patch > 6 { @@ -157,7 +162,7 @@ func (s *Solidity) run(cmd *exec.Cmd, source string) (map[string]*Contract, erro // provided source, language and compiler version, and compiler options are all // passed through into the Contract structs. // -// The solc output is expected to contain ABI, user docs, and dev docs. +// The solc output is expected to contain ABI, source mapping, user docs, and dev docs. // // Returns an error if the JSON is malformed or missing data, or if the JSON // embedded within the JSON is malformed. @@ -184,13 +189,16 @@ func ParseCombinedJSON(combinedJSON []byte, source string, languageVersion strin return nil, fmt.Errorf("solc: error reading dev doc: %v", err) } contracts[name] = &Contract{ - Code: "0x" + info.Bin, + Code: "0x" + info.Bin, + RuntimeCode: "0x" + info.BinRuntime, Info: ContractInfo{ Source: source, Language: "Solidity", LanguageVersion: languageVersion, CompilerVersion: compilerVersion, CompilerOptions: compilerOptions, + SrcMap: info.SrcMap, + SrcMapRuntime: info.SrcMapRuntime, AbiDefinition: abi, UserDoc: userdoc, DeveloperDoc: devdoc, From bd519ab8aec7c82b28fb2a7e7edc428d8a38e781 Mon Sep 17 00:00:00 2001 From: Ryan Schneider Date: Thu, 8 Nov 2018 04:18:38 -0800 Subject: [PATCH 025/159] internal/web3ext: add eth.getProof (#18052) --- internal/web3ext/web3ext.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index addf3c7660..a5f3196535 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -481,6 +481,12 @@ web3._extend({ params: 2, inputFormatter: [web3._extend.formatters.inputBlockNumberFormatter, web3._extend.utils.toHex] }), + new web3._extend.Method({ + name: 'getProof', + call: 'eth_getProof', + params: 3, + inputFormatter: [web3._extend.formatters.inputAddressFormatter, null, web3._extend.formatters.inputBlockNumberFormatter] + }), ], properties: [ new web3._extend.Property({ From 9313fa63f959f8a5c3609c187120711a484a4c57 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 8 Nov 2018 13:26:29 +0100 Subject: [PATCH 026/159] event/filter: delete unused package (#18063) --- event/filter/filter.go | 95 ---------------------------------- event/filter/filter_test.go | 60 --------------------- event/filter/generic_filter.go | 48 ----------------- 3 files changed, 203 deletions(-) delete mode 100644 event/filter/filter.go delete mode 100644 event/filter/filter_test.go delete mode 100644 event/filter/generic_filter.go diff --git a/event/filter/filter.go b/event/filter/filter.go deleted file mode 100644 index a6fe46d6a0..0000000000 --- a/event/filter/filter.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Package filter implements event filters. -package filter - -import "reflect" - -type Filter interface { - Compare(Filter) bool - Trigger(data interface{}) -} - -type FilterEvent struct { - filter Filter - data interface{} -} - -type Filters struct { - id int - watchers map[int]Filter - ch chan FilterEvent - - quit chan struct{} -} - -func New() *Filters { - return &Filters{ - ch: make(chan FilterEvent), - watchers: make(map[int]Filter), - quit: make(chan struct{}), - } -} - -func (f *Filters) Start() { - go f.loop() -} - -func (f *Filters) Stop() { - close(f.quit) -} - -func (f *Filters) Notify(filter Filter, data interface{}) { - f.ch <- FilterEvent{filter, data} -} - -func (f *Filters) Install(watcher Filter) int { - f.watchers[f.id] = watcher - f.id++ - - return f.id - 1 -} - -func (f *Filters) Uninstall(id int) { - delete(f.watchers, id) -} - -func (f *Filters) loop() { -out: - for { - select { - case <-f.quit: - break out - case event := <-f.ch: - for _, watcher := range f.watchers { - if reflect.TypeOf(watcher) == reflect.TypeOf(event.filter) { - if watcher.Compare(event.filter) { - watcher.Trigger(event.data) - } - } - } - } - } -} - -func (f *Filters) Match(a, b Filter) bool { - return reflect.TypeOf(a) == reflect.TypeOf(b) && a.Compare(b) -} - -func (f *Filters) Get(i int) Filter { - return f.watchers[i] -} diff --git a/event/filter/filter_test.go b/event/filter/filter_test.go deleted file mode 100644 index dcc9112453..0000000000 --- a/event/filter/filter_test.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package filter - -import ( - "testing" - "time" -) - -// Simple test to check if baseline matching/mismatching filtering works. -func TestFilters(t *testing.T) { - fm := New() - fm.Start() - - // Register two filters to catch posted data - first := make(chan struct{}) - fm.Install(Generic{ - Str1: "hello", - Fn: func(data interface{}) { - first <- struct{}{} - }, - }) - second := make(chan struct{}) - fm.Install(Generic{ - Str1: "hello1", - Str2: "hello", - Fn: func(data interface{}) { - second <- struct{}{} - }, - }) - // Post an event that should only match the first filter - fm.Notify(Generic{Str1: "hello"}, true) - fm.Stop() - - // Ensure only the mathcing filters fire - select { - case <-first: - case <-time.After(100 * time.Millisecond): - t.Error("matching filter timed out") - } - select { - case <-second: - t.Error("mismatching filter fired") - case <-time.After(100 * time.Millisecond): - } -} diff --git a/event/filter/generic_filter.go b/event/filter/generic_filter.go deleted file mode 100644 index 467bf01bec..0000000000 --- a/event/filter/generic_filter.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package filter - -type Generic struct { - Str1, Str2, Str3 string - Data map[string]struct{} - - Fn func(data interface{}) -} - -// self = registered, f = incoming -func (self Generic) Compare(f Filter) bool { - var strMatch = true - - filter := f.(Generic) - if (len(self.Str1) > 0 && filter.Str1 != self.Str1) || - (len(self.Str2) > 0 && filter.Str2 != self.Str2) || - (len(self.Str3) > 0 && filter.Str3 != self.Str3) { - strMatch = false - } - - for k := range self.Data { - if _, ok := filter.Data[k]; !ok { - return false - } - } - - return strMatch -} - -func (self Generic) Trigger(data interface{}) { - self.Fn(data) -} From b16cc501a88eb7c36cfd8ee9be1c0c985ef4a7d9 Mon Sep 17 00:00:00 2001 From: tamirms Date: Thu, 8 Nov 2018 13:26:47 +0100 Subject: [PATCH 027/159] ethclient: include block hash from FilterQuery (#17996) ethereum/go-ethereum#16734 introduced BlockHash to the FilterQuery struct. However, ethclient was not updated to include BlockHash in the actual RPC request. --- ethclient/ethclient.go | 36 ++++++++--- ethclient/ethclient_test.go | 120 +++++++++++++++++++++++++++++++++++- 2 files changed, 145 insertions(+), 11 deletions(-) diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index b40837c8cc..f3163e19b3 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -365,26 +365,42 @@ func (ec *Client) NonceAt(ctx context.Context, account common.Address, blockNumb // FilterLogs executes a filter query. func (ec *Client) FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]types.Log, error) { var result []types.Log - err := ec.c.CallContext(ctx, &result, "eth_getLogs", toFilterArg(q)) + arg, err := toFilterArg(q) + if err != nil { + return nil, err + } + err = ec.c.CallContext(ctx, &result, "eth_getLogs", arg) return result, err } // SubscribeFilterLogs subscribes to the results of a streaming filter query. func (ec *Client) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuery, ch chan<- types.Log) (ethereum.Subscription, error) { - return ec.c.EthSubscribe(ctx, ch, "logs", toFilterArg(q)) + arg, err := toFilterArg(q) + if err != nil { + return nil, err + } + return ec.c.EthSubscribe(ctx, ch, "logs", arg) } -func toFilterArg(q ethereum.FilterQuery) interface{} { +func toFilterArg(q ethereum.FilterQuery) (interface{}, error) { arg := map[string]interface{}{ - "fromBlock": toBlockNumArg(q.FromBlock), - "toBlock": toBlockNumArg(q.ToBlock), - "address": q.Addresses, - "topics": q.Topics, + "address": q.Addresses, + "topics": q.Topics, } - if q.FromBlock == nil { - arg["fromBlock"] = "0x0" + if q.BlockHash != nil { + arg["blockHash"] = *q.BlockHash + if q.FromBlock != nil || q.ToBlock != nil { + return nil, fmt.Errorf("cannot specify both BlockHash and FromBlock/ToBlock") + } + } else { + if q.FromBlock == nil { + arg["fromBlock"] = "0x0" + } else { + arg["fromBlock"] = toBlockNumArg(q.FromBlock) + } + arg["toBlock"] = toBlockNumArg(q.ToBlock) } - return arg + return arg, nil } // Pending State diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index 178eb2be98..3e8bf974c2 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -16,7 +16,15 @@ package ethclient -import "github.com/ethereum/go-ethereum" +import ( + "fmt" + "math/big" + "reflect" + "testing" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" +) // Verify that Client implements the ethereum interfaces. var ( @@ -32,3 +40,113 @@ var ( // _ = ethereum.PendingStateEventer(&Client{}) _ = ethereum.PendingContractCaller(&Client{}) ) + +func TestToFilterArg(t *testing.T) { + blockHashErr := fmt.Errorf("cannot specify both BlockHash and FromBlock/ToBlock") + addresses := []common.Address{ + common.HexToAddress("0xD36722ADeC3EdCB29c8e7b5a47f352D701393462"), + } + blockHash := common.HexToHash( + "0xeb94bb7d78b73657a9d7a99792413f50c0a45c51fc62bdcb08a53f18e9a2b4eb", + ) + + for _, testCase := range []struct { + name string + input ethereum.FilterQuery + output interface{} + err error + }{ + { + "without BlockHash", + ethereum.FilterQuery{ + Addresses: addresses, + FromBlock: big.NewInt(1), + ToBlock: big.NewInt(2), + Topics: [][]common.Hash{}, + }, + map[string]interface{}{ + "address": addresses, + "fromBlock": "0x1", + "toBlock": "0x2", + "topics": [][]common.Hash{}, + }, + nil, + }, + { + "with nil fromBlock and nil toBlock", + ethereum.FilterQuery{ + Addresses: addresses, + Topics: [][]common.Hash{}, + }, + map[string]interface{}{ + "address": addresses, + "fromBlock": "0x0", + "toBlock": "latest", + "topics": [][]common.Hash{}, + }, + nil, + }, + { + "with blockhash", + ethereum.FilterQuery{ + Addresses: addresses, + BlockHash: &blockHash, + Topics: [][]common.Hash{}, + }, + map[string]interface{}{ + "address": addresses, + "blockHash": blockHash, + "topics": [][]common.Hash{}, + }, + nil, + }, + { + "with blockhash and from block", + ethereum.FilterQuery{ + Addresses: addresses, + BlockHash: &blockHash, + FromBlock: big.NewInt(1), + Topics: [][]common.Hash{}, + }, + nil, + blockHashErr, + }, + { + "with blockhash and to block", + ethereum.FilterQuery{ + Addresses: addresses, + BlockHash: &blockHash, + ToBlock: big.NewInt(1), + Topics: [][]common.Hash{}, + }, + nil, + blockHashErr, + }, + { + "with blockhash and both from / to block", + ethereum.FilterQuery{ + Addresses: addresses, + BlockHash: &blockHash, + FromBlock: big.NewInt(1), + ToBlock: big.NewInt(2), + Topics: [][]common.Hash{}, + }, + nil, + blockHashErr, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + output, err := toFilterArg(testCase.input) + if (testCase.err == nil) != (err == nil) { + t.Fatalf("expected error %v but got %v", testCase.err, err) + } + if testCase.err != nil { + if testCase.err.Error() != err.Error() { + t.Fatalf("expected error %v but got %v", testCase.err, err) + } + } else if !reflect.DeepEqual(testCase.output, output) { + t.Fatalf("expected filter arg %v but got %v", testCase.output, output) + } + }) + } +} From 144c1c6c52456808428e2b69dbe5c4ebfc3606ca Mon Sep 17 00:00:00 2001 From: gary rong Date: Thu, 8 Nov 2018 16:08:57 +0100 Subject: [PATCH 028/159] consensus: extend getWork API with block number (#18038) --- consensus/ethash/api.go | 11 ++++++----- consensus/ethash/ethash.go | 2 +- consensus/ethash/ethash_test.go | 2 +- consensus/ethash/sealer.go | 5 ++++- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/consensus/ethash/api.go b/consensus/ethash/api.go index a04ea235d9..4d8eed4161 100644 --- a/consensus/ethash/api.go +++ b/consensus/ethash/api.go @@ -37,27 +37,28 @@ type API struct { // result[0] - 32 bytes hex encoded current block header pow-hash // result[1] - 32 bytes hex encoded seed hash used for DAG // result[2] - 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty -func (api *API) GetWork() ([3]string, error) { +// result[3] - hex encoded block number +func (api *API) GetWork() ([4]string, error) { if api.ethash.config.PowMode != ModeNormal && api.ethash.config.PowMode != ModeTest { - return [3]string{}, errors.New("not supported") + return [4]string{}, errors.New("not supported") } var ( - workCh = make(chan [3]string, 1) + workCh = make(chan [4]string, 1) errc = make(chan error, 1) ) select { case api.ethash.fetchWorkCh <- &sealWork{errc: errc, res: workCh}: case <-api.ethash.exitCh: - return [3]string{}, errEthashStopped + return [4]string{}, errEthashStopped } select { case work := <-workCh: return work, nil case err := <-errc: - return [3]string{}, err + return [4]string{}, err } } diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index d124cb1e20..78892e1da8 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -432,7 +432,7 @@ type hashrate struct { // sealWork wraps a seal work package for remote sealer. type sealWork struct { errc chan error - res chan [3]string + res chan [4]string } // Ethash is a consensus engine based on proof-of-work implementing the ethash diff --git a/consensus/ethash/ethash_test.go b/consensus/ethash/ethash_test.go index 8eded2ca81..90cb6470f7 100644 --- a/consensus/ethash/ethash_test.go +++ b/consensus/ethash/ethash_test.go @@ -107,7 +107,7 @@ func TestRemoteSealer(t *testing.T) { ethash.Seal(nil, block, results, nil) var ( - work [3]string + work [4]string err error ) if work, err = api.GetWork(); err != nil || work[0] != sealhash.Hex() { diff --git a/consensus/ethash/sealer.go b/consensus/ethash/sealer.go index 06c98a7811..3a0919ca99 100644 --- a/consensus/ethash/sealer.go +++ b/consensus/ethash/sealer.go @@ -30,6 +30,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" @@ -193,7 +194,7 @@ func (ethash *Ethash) remote(notify []string, noverify bool) { results chan<- *types.Block currentBlock *types.Block - currentWork [3]string + currentWork [4]string notifyTransport = &http.Transport{} notifyClient = &http.Client{ @@ -234,12 +235,14 @@ func (ethash *Ethash) remote(notify []string, noverify bool) { // result[0], 32 bytes hex encoded current block header pow-hash // result[1], 32 bytes hex encoded seed hash used for DAG // result[2], 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty + // result[3], hex encoded block number makeWork := func(block *types.Block) { hash := ethash.SealHash(block.Header()) currentWork[0] = hash.Hex() currentWork[1] = common.BytesToHash(SeedHash(block.NumberU64())).Hex() currentWork[2] = common.BytesToHash(new(big.Int).Div(two256, block.Difficulty()).Bytes()).Hex() + currentWork[3] = hexutil.EncodeBig(block.Number()) // Trace the seal work fetched by remote sealer. currentBlock = block From 870efeef01ad45f1e06bbe4479e5afa9986f3518 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 8 Nov 2018 21:37:19 +0100 Subject: [PATCH 029/159] core/state: remove lock (#18065) The lock in StateDB is useless. It's only held in Copy, but Copy is safe for concurrent use because all it does is read. --- core/state/statedb.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/core/state/statedb.go b/core/state/statedb.go index f0d7cdb6eb..76e67d839d 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -22,7 +22,6 @@ import ( "fmt" "math/big" "sort" - "sync" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -87,8 +86,6 @@ type StateDB struct { journal *journal validRevisions []revision nextRevisionId int - - lock sync.Mutex } // Create a new state from a given trie. @@ -496,9 +493,6 @@ func (db *StateDB) ForEachStorage(addr common.Address, cb func(key, value common // Copy creates a deep, independent copy of the state. // Snapshots of the copied state cannot be applied to the copy. func (self *StateDB) Copy() *StateDB { - self.lock.Lock() - defer self.lock.Unlock() - // Copy all the basic fields, initialize the memory ones state := &StateDB{ db: self.db, From f574c4e74b12d275d24b209aac8dd98ed9b9bb52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kurk=C3=B3=20Mih=C3=A1ly?= Date: Fri, 9 Nov 2018 11:20:51 +0200 Subject: [PATCH 030/159] metrics, p2p: add ephemeral registry (#18067) * metrics, p2p: add ephemeral registry * metrics: fix linter issue --- metrics/registry.go | 5 ++++- p2p/metrics.go | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/metrics/registry.go b/metrics/registry.go index cc34c9dfd2..c1cf7906ce 100644 --- a/metrics/registry.go +++ b/metrics/registry.go @@ -311,7 +311,10 @@ func (r *PrefixedRegistry) UnregisterAll() { r.underlying.UnregisterAll() } -var DefaultRegistry Registry = NewRegistry() +var ( + DefaultRegistry = NewRegistry() + EphemeralRegistry = NewRegistry() +) // Call the given function for each registered metric. func Each(f func(string, interface{})) { diff --git a/p2p/metrics.go b/p2p/metrics.go index d7873f39a5..8df82bb074 100644 --- a/p2p/metrics.go +++ b/p2p/metrics.go @@ -47,8 +47,8 @@ var ( egressConnectMeter = metrics.NewRegisteredMeter(MetricsOutboundConnects, nil) // Meter counting the egress connections egressTrafficMeter = metrics.NewRegisteredMeter(MetricsOutboundTraffic, nil) // Meter metering the cumulative egress traffic - PeerIngressRegistry = metrics.NewPrefixedChildRegistry(metrics.DefaultRegistry, MetricsInboundTraffic+"/") // Registry containing the peer ingress - PeerEgressRegistry = metrics.NewPrefixedChildRegistry(metrics.DefaultRegistry, MetricsOutboundTraffic+"/") // Registry containing the peer egress + PeerIngressRegistry = metrics.NewPrefixedChildRegistry(metrics.EphemeralRegistry, MetricsInboundTraffic+"/") // Registry containing the peer ingress + PeerEgressRegistry = metrics.NewPrefixedChildRegistry(metrics.EphemeralRegistry, MetricsOutboundTraffic+"/") // Registry containing the peer egress meteredPeerFeed event.Feed // Event feed for peer metrics meteredPeerCount int32 // Actually stored peer connection count From 1ff152f3a43e4adf030ac61eb5d8da345554fc5a Mon Sep 17 00:00:00 2001 From: Corey Lin <514971757@qq.com> Date: Fri, 9 Nov 2018 18:51:07 +0800 Subject: [PATCH 031/159] rawdb: remove unused parameter for WritePreimages func (#18059) * rawdb: remove unused parameter for WritePreimages func and modify a spelling mistake * rawdb: update the doc for function WritePreimages --- cmd/utils/cmd.go | 4 ++-- core/blockchain.go | 2 +- core/rawdb/accessors_metadata.go | 5 ++--- core/rawdb/schema.go | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 58d72f32ba..f23aa5775b 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -272,13 +272,13 @@ func ImportPreimages(db *ethdb.LDBDatabase, fn string) error { // Accumulate the preimages and flush when enough ws gathered preimages[crypto.Keccak256Hash(blob)] = common.CopyBytes(blob) if len(preimages) > 1024 { - rawdb.WritePreimages(db, 0, preimages) + rawdb.WritePreimages(db, preimages) preimages = make(map[common.Hash][]byte) } } // Flush the last batch preimage data if len(preimages) > 0 { - rawdb.WritePreimages(db, 0, preimages) + rawdb.WritePreimages(db, preimages) } return nil } diff --git a/core/blockchain.go b/core/blockchain.go index f4a818f4c0..1b0c26ae25 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1002,7 +1002,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. } // Write the positional metadata for transaction/receipt lookups and preimages rawdb.WriteTxLookupEntries(batch, block) - rawdb.WritePreimages(batch, block.NumberU64(), state.Preimages()) + rawdb.WritePreimages(batch, state.Preimages()) status = CanonStatTy } else { diff --git a/core/rawdb/accessors_metadata.go b/core/rawdb/accessors_metadata.go index 514328e87b..3b6e6548d3 100644 --- a/core/rawdb/accessors_metadata.go +++ b/core/rawdb/accessors_metadata.go @@ -77,9 +77,8 @@ func ReadPreimage(db DatabaseReader, hash common.Hash) []byte { return data } -// WritePreimages writes the provided set of preimages to the database. `number` is the -// current block number, and is used for debug messages only. -func WritePreimages(db DatabaseWriter, number uint64, preimages map[common.Hash][]byte) { +// WritePreimages writes the provided set of preimages to the database. +func WritePreimages(db DatabaseWriter, preimages map[common.Hash][]byte) { for hash, preimage := range preimages { if err := db.Put(preimageKey(hash), preimage); err != nil { log.Crit("Failed to store trie preimage", "err", err) diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index ef597ef309..8a9921ef40 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -35,7 +35,7 @@ var ( // headBlockKey tracks the latest know full block's hash. headBlockKey = []byte("LastBlock") - // headFastBlockKey tracks the latest known incomplete block's hash duirng fast sync. + // headFastBlockKey tracks the latest known incomplete block's hash during fast sync. headFastBlockKey = []byte("LastFast") // fastTrieProgressKey tracks the number of trie entries imported during fast sync. From a0876f7433f63276a3d8d4e099b261fd16aada40 Mon Sep 17 00:00:00 2001 From: Andrew Chiw Date: Mon, 12 Nov 2018 13:04:13 +0100 Subject: [PATCH 032/159] Imply that SwarmApiFlag is the API endpoint to connect to, not to listen on (#18071) --- cmd/swarm/flags.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/swarm/flags.go b/cmd/swarm/flags.go index 7e891f3022..0dedca674b 100644 --- a/cmd/swarm/flags.go +++ b/cmd/swarm/flags.go @@ -88,7 +88,7 @@ var ( } SwarmApiFlag = cli.StringFlag{ Name: "bzzapi", - Usage: "Swarm HTTP endpoint", + Usage: "Specifies the Swarm HTTP endpoint to connect to", Value: "http://127.0.0.1:8500", } SwarmRecursiveFlag = cli.BoolFlag{ From 201a0bf18181da8d783f6e7adf3ceaccd159eb73 Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 12 Nov 2018 14:57:17 +0100 Subject: [PATCH 033/159] p2p/simulations, swarm/network: Custom services in snapshot (#17991) * p2p/simulations: Add custom services to simnodes + remove sim down conn objs * p2p/simulation, swarm/network: Add selective services to discovery sim * p2p/simulations, swarm/network: Remove useless comments * p2p/simulations, swarm/network: Clean up mess from rebase * p2p/simulation: Add sleep to prevent connect flakiness in http test * p2p/simulations: added concurrent goroutines to prevent sleeps on simulation connect/disconnect * p2p/simulations, swarm/network/simulations: address pr comments * reinstated dummy service * fixed http snapshot test --- p2p/simulations/http_test.go | 55 ++++++++++++++++++- p2p/simulations/network.go | 44 ++++++++++++++- .../simulations/discovery/discovery_test.go | 31 +++++++++-- 3 files changed, 118 insertions(+), 12 deletions(-) diff --git a/p2p/simulations/http_test.go b/p2p/simulations/http_test.go index 7bdbad1b53..d9513caaa0 100644 --- a/p2p/simulations/http_test.go +++ b/p2p/simulations/http_test.go @@ -18,6 +18,7 @@ package simulations import ( "context" + "flag" "fmt" "math/rand" "net/http/httptest" @@ -28,13 +29,26 @@ import ( "time" "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/rpc" + colorable "github.com/mattn/go-colorable" ) +var ( + loglevel = flag.Int("loglevel", 2, "verbosity of logs") +) + +func init() { + flag.Parse() + + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) +} + // testService implements the node.Service interface and provides protocols // and APIs which are useful for testing nodes in a simulation network type testService struct { @@ -584,9 +598,26 @@ func TestHTTPNodeRPC(t *testing.T) { // TestHTTPSnapshot tests creating and loading network snapshots func TestHTTPSnapshot(t *testing.T) { // start the server - _, s := testHTTPServer(t) + network, s := testHTTPServer(t) defer s.Close() + var eventsDone = make(chan struct{}) + count := 1 + eventsDoneChan := make(chan *Event) + eventSub := network.Events().Subscribe(eventsDoneChan) + go func() { + defer eventSub.Unsubscribe() + for event := range eventsDoneChan { + if event.Type == EventTypeConn && !event.Control { + count-- + if count == 0 { + eventsDone <- struct{}{} + return + } + } + } + }() + // create a two-node network client := NewClient(s.URL) nodeCount := 2 @@ -620,7 +651,7 @@ func TestHTTPSnapshot(t *testing.T) { } states[i] = state } - + <-eventsDone // create a snapshot snap, err := client.CreateSnapshot() if err != nil { @@ -634,9 +665,23 @@ func TestHTTPSnapshot(t *testing.T) { } // create another network - _, s = testHTTPServer(t) + network2, s := testHTTPServer(t) defer s.Close() client = NewClient(s.URL) + count = 1 + eventSub = network2.Events().Subscribe(eventsDoneChan) + go func() { + defer eventSub.Unsubscribe() + for event := range eventsDoneChan { + if event.Type == EventTypeConn && !event.Control { + count-- + if count == 0 { + eventsDone <- struct{}{} + return + } + } + } + }() // subscribe to events so we can check them later events := make(chan *Event, 100) @@ -651,6 +696,7 @@ func TestHTTPSnapshot(t *testing.T) { if err := client.LoadSnapshot(snap); err != nil { t.Fatalf("error loading snapshot: %s", err) } + <-eventsDone // check the nodes and connection exists net, err := client.GetNetwork() @@ -676,6 +722,9 @@ func TestHTTPSnapshot(t *testing.T) { if conn.Other.String() != nodes[1].ID { t.Fatalf("expected connection to have other=%q, got other=%q", nodes[1].ID, conn.Other) } + if !conn.Up { + t.Fatal("should be up") + } // check the node states were restored for i, node := range nodes { diff --git a/p2p/simulations/network.go b/p2p/simulations/network.go index 200015ff39..92ccfde817 100644 --- a/p2p/simulations/network.go +++ b/p2p/simulations/network.go @@ -644,11 +644,18 @@ type NodeSnapshot struct { // Snapshot creates a network snapshot func (net *Network) Snapshot() (*Snapshot, error) { + return net.snapshot(nil, nil) +} + +func (net *Network) SnapshotWithServices(addServices []string, removeServices []string) (*Snapshot, error) { + return net.snapshot(addServices, removeServices) +} + +func (net *Network) snapshot(addServices []string, removeServices []string) (*Snapshot, error) { net.lock.Lock() defer net.lock.Unlock() snap := &Snapshot{ Nodes: make([]NodeSnapshot, len(net.Nodes)), - Conns: make([]Conn, len(net.Conns)), } for i, node := range net.Nodes { snap.Nodes[i] = NodeSnapshot{Node: *node} @@ -660,9 +667,40 @@ func (net *Network) Snapshot() (*Snapshot, error) { return nil, err } snap.Nodes[i].Snapshots = snapshots + for _, addSvc := range addServices { + haveSvc := false + for _, svc := range snap.Nodes[i].Node.Config.Services { + if svc == addSvc { + haveSvc = true + break + } + } + if !haveSvc { + snap.Nodes[i].Node.Config.Services = append(snap.Nodes[i].Node.Config.Services, addSvc) + } + } + if len(removeServices) > 0 { + var cleanedServices []string + for _, svc := range snap.Nodes[i].Node.Config.Services { + haveSvc := false + for _, rmSvc := range removeServices { + if rmSvc == svc { + haveSvc = true + break + } + } + if !haveSvc { + cleanedServices = append(cleanedServices, svc) + } + + } + snap.Nodes[i].Node.Config.Services = cleanedServices + } } - for i, conn := range net.Conns { - snap.Conns[i] = *conn + for _, conn := range net.Conns { + if conn.Up { + snap.Conns = append(snap.Conns, *conn) + } } return snap, nil } diff --git a/swarm/network/simulations/discovery/discovery_test.go b/swarm/network/simulations/discovery/discovery_test.go index 3c3affe58c..cd5456b73e 100644 --- a/swarm/network/simulations/discovery/discovery_test.go +++ b/swarm/network/simulations/discovery/discovery_test.go @@ -85,11 +85,12 @@ func getDbStore(nodeID string) (*state.DBStore, error) { } var ( - nodeCount = flag.Int("nodes", 10, "number of nodes to create (default 10)") - initCount = flag.Int("conns", 1, "number of originally connected peers (default 1)") - snapshotFile = flag.String("snapshot", "", "create snapshot") - loglevel = flag.Int("loglevel", 3, "verbosity of logs") - rawlog = flag.Bool("rawlog", false, "remove terminal formatting from logs") + nodeCount = flag.Int("nodes", 10, "number of nodes to create (default 10)") + initCount = flag.Int("conns", 1, "number of originally connected peers (default 1)") + snapshotFile = flag.String("snapshot", "", "path to create snapshot file in") + loglevel = flag.Int("loglevel", 3, "verbosity of logs") + rawlog = flag.Bool("rawlog", false, "remove terminal formatting from logs") + serviceOverride = flag.String("services", "", "remove or add services to the node snapshot; prefix with \"+\" to add, \"-\" to remove; example: +pss,-discovery") ) func init() { @@ -306,7 +307,25 @@ func discoverySimulation(nodes, conns int, adapter adapters.NodeAdapter) (*simul } if *snapshotFile != "" { - snap, err := net.Snapshot() + var err error + var snap *simulations.Snapshot + if len(*serviceOverride) > 0 { + var addServices []string + var removeServices []string + for _, osvc := range strings.Split(*serviceOverride, ",") { + if strings.Index(osvc, "+") == 0 { + addServices = append(addServices, osvc[1:]) + } else if strings.Index(osvc, "-") == 0 { + removeServices = append(removeServices, osvc[1:]) + } else { + panic("stick to the rules, you know what they are") + } + } + snap, err = net.SnapshotWithServices(addServices, removeServices) + } else { + snap, err = net.Snapshot() + } + if err != nil { return nil, errors.New("no shapshot dude") } From 1212c7b844e3ef13cbb5476b088eae27782535b4 Mon Sep 17 00:00:00 2001 From: gary rong Date: Tue, 13 Nov 2018 00:06:34 +0800 Subject: [PATCH 034/159] core: fix default trie cache limit (#17860) --- core/blockchain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/blockchain.go b/core/blockchain.go index 1b0c26ae25..26ac75b8cf 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -140,7 +140,7 @@ type BlockChain struct { func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(block *types.Block) bool) (*BlockChain, error) { if cacheConfig == nil { cacheConfig = &CacheConfig{ - TrieNodeLimit: 256 * 1024 * 1024, + TrieNodeLimit: 256, TrieTimeLimit: 5 * time.Minute, } } From 8080265f3f591d33e127a924724a8bfe5ced6475 Mon Sep 17 00:00:00 2001 From: Ferenc Szabo Date: Tue, 13 Nov 2018 07:41:01 +0100 Subject: [PATCH 035/159] swarm/storage: fix access count on dbstore after cache hit (#17978) Access count was not incremented when chunk was retrieved from cache. So the garbage collector might have deleted the most frequently accessed chunk from disk. Co-authored-by: Ferenc Szabo --- swarm/storage/ldbstore.go | 48 +++++++++++++++-------- swarm/storage/ldbstore_test.go | 41 +++++++++++++++++++- swarm/storage/localstore.go | 1 + swarm/storage/localstore_test.go | 65 ++++++++++++++++++++++++++++++++ 4 files changed, 138 insertions(+), 17 deletions(-) diff --git a/swarm/storage/ldbstore.go b/swarm/storage/ldbstore.go index 9feb687413..46e0402501 100644 --- a/swarm/storage/ldbstore.go +++ b/swarm/storage/ldbstore.go @@ -186,6 +186,20 @@ func NewLDBStore(params *LDBStoreParams) (s *LDBStore, err error) { return s, nil } +// MarkAccessed increments the access counter as a best effort for a chunk, so +// the chunk won't get garbage collected. +func (s *LDBStore) MarkAccessed(addr Address) { + s.lock.Lock() + defer s.lock.Unlock() + + if s.closed { + return + } + + proximity := s.po(addr) + s.tryAccessIdx(addr, proximity) +} + // initialize and set values for processing of gc round func (s *LDBStore) startGC(c int) { @@ -349,6 +363,7 @@ func (s *LDBStore) collectGarbage() error { s.delete(s.gc.batch.Batch, index, keyIdx, po) singleIterationCount++ s.gc.count++ + log.Trace("garbage collect enqueued chunk for deletion", "key", hash) // break if target is not on max garbage batch boundary if s.gc.count >= s.gc.target { @@ -685,12 +700,7 @@ func (s *LDBStore) Put(ctx context.Context, chunk Chunk) error { idata, err := s.db.Get(ikey) if err != nil { s.doPut(chunk, &index, po) - } else { - log.Debug("ldbstore.put: chunk already exists, only update access", "key", chunk.Address(), "po", po) - decodeIndex(idata, &index) } - index.Access = s.accessCnt - s.accessCnt++ idata = encodeIndex(&index) s.batch.Put(ikey, idata) @@ -723,7 +733,8 @@ func (s *LDBStore) doPut(chunk Chunk, index *dpaDBIndex, po uint8) { s.entryCnt++ dbEntryCount.Inc(1) s.dataIdx++ - + index.Access = s.accessCnt + s.accessCnt++ cntKey := make([]byte, 2) cntKey[0] = keyDistanceCnt cntKey[1] = po @@ -796,18 +807,23 @@ func newMockEncodeDataFunc(mockStore *mock.NodeStore) func(chunk Chunk) []byte { } } -// try to find index; if found, update access cnt and return true -func (s *LDBStore) tryAccessIdx(ikey []byte, po uint8, index *dpaDBIndex) bool { +// tryAccessIdx tries to find index entry. If found then increments the access +// count for garbage collection and returns the index entry and true for found, +// otherwise returns nil and false. +func (s *LDBStore) tryAccessIdx(addr Address, po uint8) (*dpaDBIndex, bool) { + ikey := getIndexKey(addr) idata, err := s.db.Get(ikey) if err != nil { - return false + return nil, false } + + index := new(dpaDBIndex) decodeIndex(idata, index) oldGCIdxKey := getGCIdxKey(index) s.batch.Put(keyAccessCnt, U64ToBytes(s.accessCnt)) - s.accessCnt++ index.Access = s.accessCnt idata = encodeIndex(index) + s.accessCnt++ s.batch.Put(ikey, idata) newGCIdxKey := getGCIdxKey(index) newGCIdxData := getGCIdxValue(index, po, ikey) @@ -817,7 +833,7 @@ func (s *LDBStore) tryAccessIdx(ikey []byte, po uint8, index *dpaDBIndex) bool { case s.batchesC <- struct{}{}: default: } - return true + return index, true } // GetSchema is returning the current named schema of the datastore as read from LevelDB @@ -858,12 +874,12 @@ func (s *LDBStore) Get(_ context.Context, addr Address) (chunk Chunk, err error) // TODO: To conform with other private methods of this object indices should not be updated func (s *LDBStore) get(addr Address) (chunk *chunk, err error) { - var indx dpaDBIndex if s.closed { return nil, ErrDBClosed } proximity := s.po(addr) - if s.tryAccessIdx(getIndexKey(addr), proximity, &indx) { + index, found := s.tryAccessIdx(addr, proximity) + if found { var data []byte if s.getDataFunc != nil { // if getDataFunc is defined, use it to retrieve the chunk data @@ -874,12 +890,12 @@ func (s *LDBStore) get(addr Address) (chunk *chunk, err error) { } } else { // default DbStore functionality to retrieve chunk data - datakey := getDataKey(indx.Idx, proximity) + datakey := getDataKey(index.Idx, proximity) data, err = s.db.Get(datakey) - log.Trace("ldbstore.get retrieve", "key", addr, "indexkey", indx.Idx, "datakey", fmt.Sprintf("%x", datakey), "proximity", proximity) + log.Trace("ldbstore.get retrieve", "key", addr, "indexkey", index.Idx, "datakey", fmt.Sprintf("%x", datakey), "proximity", proximity) if err != nil { log.Trace("ldbstore.get chunk found but could not be accessed", "key", addr, "err", err) - s.deleteNow(&indx, getIndexKey(addr), s.po(addr)) + s.deleteNow(index, getIndexKey(addr), s.po(addr)) return } } diff --git a/swarm/storage/ldbstore_test.go b/swarm/storage/ldbstore_test.go index 48af8c57c0..22213b12d4 100644 --- a/swarm/storage/ldbstore_test.go +++ b/swarm/storage/ldbstore_test.go @@ -31,7 +31,6 @@ import ( ch "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/storage/mock/mem" - ldberrors "github.com/syndtr/goleveldb/leveldb/errors" ) @@ -105,6 +104,46 @@ func testDbStoreCorrect(n int, chunksize int64, mock bool, t *testing.T) { testStoreCorrect(db, n, chunksize, t) } +func TestMarkAccessed(t *testing.T) { + db, cleanup, err := newTestDbStore(false, true) + defer cleanup() + if err != nil { + t.Fatalf("init dbStore failed: %v", err) + } + + h := GenerateRandomChunk(ch.DefaultSize) + + db.Put(context.Background(), h) + + var index dpaDBIndex + addr := h.Address() + idxk := getIndexKey(addr) + + idata, err := db.db.Get(idxk) + if err != nil { + t.Fatal(err) + } + decodeIndex(idata, &index) + + if index.Access != 0 { + t.Fatalf("Expected the access index to be %d, but it is %d", 0, index.Access) + } + + db.MarkAccessed(addr) + db.writeCurrentBatch() + + idata, err = db.db.Get(idxk) + if err != nil { + t.Fatal(err) + } + decodeIndex(idata, &index) + + if index.Access != 1 { + t.Fatalf("Expected the access index to be %d, but it is %d", 1, index.Access) + } + +} + func TestDbStoreRandom_1(t *testing.T) { testDbStoreRandom(1, 0, false, t) } diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index 4fa6fb2f60..6971d759e3 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -153,6 +153,7 @@ func (ls *LocalStore) get(ctx context.Context, addr Address) (chunk Chunk, err e if err == nil { metrics.GetOrRegisterCounter("localstore.get.cachehit", nil).Inc(1) + go ls.DbStore.MarkAccessed(addr) return chunk, nil } diff --git a/swarm/storage/localstore_test.go b/swarm/storage/localstore_test.go index 10f43f30f7..7a07726d1c 100644 --- a/swarm/storage/localstore_test.go +++ b/swarm/storage/localstore_test.go @@ -21,6 +21,7 @@ import ( "io/ioutil" "os" "testing" + "time" ch "github.com/ethereum/go-ethereum/swarm/chunk" ) @@ -144,3 +145,67 @@ func put(store *LocalStore, n int, f func(i int64) Chunk) (hs []Address, errs [] } return hs, errs } + +// TestGetFrequentlyAccessedChunkWontGetGarbageCollected tests that the most +// frequently accessed chunk is not garbage collected from LDBStore, i.e., +// from disk when we are at the capacity and garbage collector runs. For that +// we start putting random chunks into the DB while continuously accessing the +// chunk we care about then check if we can still retrieve it from disk. +func TestGetFrequentlyAccessedChunkWontGetGarbageCollected(t *testing.T) { + ldbCap := defaultGCRatio + store, cleanup := setupLocalStore(t, ldbCap) + defer cleanup() + + var chunks []Chunk + for i := 0; i < ldbCap; i++ { + chunks = append(chunks, GenerateRandomChunk(ch.DefaultSize)) + } + + mostAccessed := chunks[0].Address() + for _, chunk := range chunks { + if err := store.Put(context.Background(), chunk); err != nil { + t.Fatal(err) + } + + if _, err := store.Get(context.Background(), mostAccessed); err != nil { + t.Fatal(err) + } + // Add time for MarkAccessed() to be able to finish in a separate Goroutine + time.Sleep(1 * time.Millisecond) + } + + store.DbStore.collectGarbage() + if _, err := store.DbStore.Get(context.Background(), mostAccessed); err != nil { + t.Logf("most frequntly accessed chunk not found on disk (key: %v)", mostAccessed) + t.Fatal(err) + } + +} + +func setupLocalStore(t *testing.T, ldbCap int) (ls *LocalStore, cleanup func()) { + t.Helper() + + var err error + datadir, err := ioutil.TempDir("", "storage") + if err != nil { + t.Fatal(err) + } + + params := &LocalStoreParams{ + StoreParams: NewStoreParams(uint64(ldbCap), uint(ldbCap), nil, nil), + } + params.Init(datadir) + + store, err := NewLocalStore(params, nil) + if err != nil { + _ = os.RemoveAll(datadir) + t.Fatal(err) + } + + cleanup = func() { + store.Close() + _ = os.RemoveAll(datadir) + } + + return store, cleanup +} From 588aa88121db007b59142ed37e74800c09038a7a Mon Sep 17 00:00:00 2001 From: mr_franklin Date: Tue, 13 Nov 2018 17:02:04 +0800 Subject: [PATCH 036/159] github: format code owners file (#18090) replace tabs by spaces in the code owners file --- .github/CODEOWNERS | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f7f6916c54..9a61d39325 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -9,9 +9,9 @@ les/ @zsfelfoldi light/ @zsfelfoldi mobile/ @karalabe p2p/ @fjl @zsfelfoldi -p2p/simulations @lmars -p2p/protocols @zelig -swarm/api/http @justelad +p2p/simulations @lmars +p2p/protocols @zelig +swarm/api/http @justelad swarm/bmt @zelig swarm/dev @lmars swarm/fuse @jmozah @holisticode From 4fecc7a3b1b9c51efad47ea128abcb7259158487 Mon Sep 17 00:00:00 2001 From: mr_franklin Date: Tue, 13 Nov 2018 17:57:46 +0800 Subject: [PATCH 037/159] eth: fix minor grammar issue in comment (#18091) --- eth/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/handler.go b/eth/handler.go index 1f62d820ec..bd227a84e0 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -653,7 +653,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { trueHead = request.Block.ParentHash() trueTD = new(big.Int).Sub(request.TD, request.Block.Difficulty()) ) - // Update the peers total difficulty if better than the previous + // Update the peer's total difficulty if better than the previous if _, td := p.Head(); trueTD.Cmp(td) > 0 { p.SetHead(trueHead, trueTD) From c41e1bd1ebf8a625dd22a07c31bcd6837705e0d7 Mon Sep 17 00:00:00 2001 From: Ferenc Szabo Date: Tue, 13 Nov 2018 15:22:53 +0100 Subject: [PATCH 038/159] swarm/storage: fix garbage collector index skew (#18080) On file access LDBStore's tryAccessIdx() function created a faulty GC Index Data entry, because not indexing the ikey correctly. That caused the chunk addresses/hashes to start with '00' and the last two digits were dropped. => Incorrect chunk address. Besides the fix, the commit also contains a schema change which will run the CleanGCIndex() function to clean the GC index from erroneous entries. Note: CleanGCIndex() rebuilds the index from scratch which can take a really-really long time with a huge DB (possibly an hour). --- swarm/storage/ldbstore.go | 122 +++++++++++++++++++--------- swarm/storage/ldbstore_test.go | 140 +++++++++++++++++++++++++++++++++ swarm/storage/localstore.go | 53 ++++++++----- swarm/storage/schema.go | 13 ++- 4 files changed, 271 insertions(+), 57 deletions(-) diff --git a/swarm/storage/ldbstore.go b/swarm/storage/ldbstore.go index 46e0402501..fbae59facf 100644 --- a/swarm/storage/ldbstore.go +++ b/swarm/storage/ldbstore.go @@ -57,7 +57,6 @@ var ( var ( keyIndex = byte(0) - keyOldData = byte(1) keyAccessCnt = []byte{2} keyEntryCnt = []byte{3} keyDataIdx = []byte{4} @@ -285,6 +284,10 @@ func getGCIdxValue(index *dpaDBIndex, po uint8, addr Address) []byte { return val } +func parseGCIdxKey(key []byte) (byte, []byte) { + return key[0], key[1:] +} + func parseGCIdxEntry(accessCnt []byte, val []byte) (index *dpaDBIndex, po uint8, addr Address) { index = &dpaDBIndex{ Idx: binary.BigEndian.Uint64(val[1:]), @@ -504,7 +507,7 @@ func (s *LDBStore) Import(in io.Reader) (int64, error) { } } -//Cleanup iterates over the database and deletes chunks if they pass the `f` condition +// Cleanup iterates over the database and deletes chunks if they pass the `f` condition func (s *LDBStore) Cleanup(f func(*chunk) bool) { var errorsFound, removed, total int @@ -569,47 +572,90 @@ func (s *LDBStore) Cleanup(f func(*chunk) bool) { log.Warn(fmt.Sprintf("Found %v errors out of %v entries. Removed %v chunks.", errorsFound, total, removed)) } -func (s *LDBStore) ReIndex() { - //Iterates over the database and checks that there are no faulty chunks +// CleanGCIndex rebuilds the garbage collector index from scratch, while +// removing inconsistent elements, e.g., indices with missing data chunks. +// WARN: it's a pretty heavy, long running function. +func (s *LDBStore) CleanGCIndex() error { + s.lock.Lock() + defer s.lock.Unlock() + + batch := leveldb.Batch{} + + var okEntryCount uint64 + var totalEntryCount uint64 + + // throw out all gc indices, we will rebuild from cleaned index it := s.db.NewIterator() - startPosition := []byte{keyOldData} - it.Seek(startPosition) - var key []byte - var errorsFound, total int + it.Seek([]byte{keyGCIdx}) + var gcDeletes int for it.Valid() { - key = it.Key() - if (key == nil) || (key[0] != keyOldData) { + rowType, _ := parseGCIdxKey(it.Key()) + if rowType != keyGCIdx { break } - data := it.Value() - hasher := s.hashfunc() - hasher.Write(data) - hash := hasher.Sum(nil) - - newKey := make([]byte, 10) - oldCntKey := make([]byte, 2) - newCntKey := make([]byte, 2) - oldCntKey[0] = keyDistanceCnt - newCntKey[0] = keyDistanceCnt - key[0] = keyData - key[1] = s.po(Address(key[1:])) - oldCntKey[1] = key[1] - newCntKey[1] = s.po(Address(newKey[1:])) - copy(newKey[2:], key[1:]) - newValue := append(hash, data...) - - batch := new(leveldb.Batch) - batch.Delete(key) - s.bucketCnt[oldCntKey[1]]-- - batch.Put(oldCntKey, U64ToBytes(s.bucketCnt[oldCntKey[1]])) - batch.Put(newKey, newValue) - s.bucketCnt[newCntKey[1]]++ - batch.Put(newCntKey, U64ToBytes(s.bucketCnt[newCntKey[1]])) - s.db.Write(batch) + batch.Delete(it.Key()) + gcDeletes++ it.Next() } + log.Debug("gc", "deletes", gcDeletes) + if err := s.db.Write(&batch); err != nil { + return err + } + + it.Seek([]byte{keyIndex}) + var idx dpaDBIndex + var poPtrs [256]uint64 + for it.Valid() { + rowType, chunkHash := parseGCIdxKey(it.Key()) + if rowType != keyIndex { + break + } + err := decodeIndex(it.Value(), &idx) + if err != nil { + return fmt.Errorf("corrupt index: %v", err) + } + po := s.po(chunkHash) + + // if we don't find the data key, remove the entry + dataKey := getDataKey(idx.Idx, po) + _, err = s.db.Get(dataKey) + if err != nil { + log.Warn("deleting inconsistent index (missing data)", "key", chunkHash) + batch.Delete(it.Key()) + } else { + gcIdxKey := getGCIdxKey(&idx) + gcIdxData := getGCIdxValue(&idx, po, chunkHash) + batch.Put(gcIdxKey, gcIdxData) + log.Trace("clean ok", "key", chunkHash, "gcKey", gcIdxKey, "gcData", gcIdxData) + okEntryCount++ + if idx.Idx > poPtrs[po] { + poPtrs[po] = idx.Idx + } + } + totalEntryCount++ + it.Next() + } + it.Release() - log.Warn(fmt.Sprintf("Found %v errors out of %v entries", errorsFound, total)) + log.Debug("gc cleanup entries", "ok", okEntryCount, "total", totalEntryCount, "batchlen", batch.Len()) + + var entryCount [8]byte + binary.BigEndian.PutUint64(entryCount[:], okEntryCount) + batch.Put(keyEntryCnt, entryCount[:]) + var poKey [2]byte + poKey[0] = keyDistanceCnt + for i, poPtr := range poPtrs { + poKey[1] = uint8(i) + if poPtr == 0 { + batch.Delete(poKey[:]) + } else { + var idxCount [8]byte + binary.BigEndian.PutUint64(idxCount[:], poPtr) + batch.Put(poKey[:], idxCount[:]) + } + } + + return s.db.Write(&batch) } // Delete is removes a chunk and updates indices. @@ -826,7 +872,7 @@ func (s *LDBStore) tryAccessIdx(addr Address, po uint8) (*dpaDBIndex, bool) { s.accessCnt++ s.batch.Put(ikey, idata) newGCIdxKey := getGCIdxKey(index) - newGCIdxData := getGCIdxValue(index, po, ikey) + newGCIdxData := getGCIdxValue(index, po, ikey[1:]) s.batch.Delete(oldGCIdxKey) s.batch.Put(newGCIdxKey, newGCIdxData) select { @@ -844,7 +890,7 @@ func (s *LDBStore) GetSchema() (string, error) { data, err := s.db.Get(keySchema) if err != nil { if err == leveldb.ErrNotFound { - return "", nil + return DbSchemaNone, nil } return "", err } diff --git a/swarm/storage/ldbstore_test.go b/swarm/storage/ldbstore_test.go index 22213b12d4..07557980c6 100644 --- a/swarm/storage/ldbstore_test.go +++ b/swarm/storage/ldbstore_test.go @@ -19,6 +19,7 @@ package storage import ( "bytes" "context" + "encoding/binary" "fmt" "io/ioutil" "os" @@ -623,6 +624,145 @@ func TestLDBStoreCollectGarbageAccessUnlikeIndex(t *testing.T) { log.Info("ldbstore", "total", n, "missing", missing, "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt) } +func TestCleanIndex(t *testing.T) { + capacity := 5000 + n := 3 + + ldb, cleanup := newLDBStore(t) + ldb.setCapacity(uint64(capacity)) + defer cleanup() + + chunks, err := mputRandomChunks(ldb, n, 4096) + if err != nil { + t.Fatal(err) + } + + // remove the data of the first chunk + po := ldb.po(chunks[0].Address()[:]) + dataKey := make([]byte, 10) + dataKey[0] = keyData + dataKey[1] = byte(po) + // dataKey[2:10] = first chunk has storageIdx 0 on [2:10] + if _, err := ldb.db.Get(dataKey); err != nil { + t.Fatal(err) + } + if err := ldb.db.Delete(dataKey); err != nil { + t.Fatal(err) + } + + // remove the gc index row for the first chunk + gcFirstCorrectKey := make([]byte, 9) + gcFirstCorrectKey[0] = keyGCIdx + if err := ldb.db.Delete(gcFirstCorrectKey); err != nil { + t.Fatal(err) + } + + // warp the gc data of the second chunk + // this data should be correct again after the clean + gcSecondCorrectKey := make([]byte, 9) + gcSecondCorrectKey[0] = keyGCIdx + binary.BigEndian.PutUint64(gcSecondCorrectKey[1:], uint64(1)) + gcSecondCorrectVal, err := ldb.db.Get(gcSecondCorrectKey) + if err != nil { + t.Fatal(err) + } + warpedGCVal := make([]byte, len(gcSecondCorrectVal)+1) + copy(warpedGCVal[1:], gcSecondCorrectVal) + if err := ldb.db.Delete(gcSecondCorrectKey); err != nil { + t.Fatal(err) + } + if err := ldb.db.Put(gcSecondCorrectKey, warpedGCVal); err != nil { + t.Fatal(err) + } + + if err := ldb.CleanGCIndex(); err != nil { + t.Fatal(err) + } + + // the index without corresponding data should have been deleted + idxKey := make([]byte, 33) + idxKey[0] = keyIndex + copy(idxKey[1:], chunks[0].Address()) + if _, err := ldb.db.Get(idxKey); err == nil { + t.Fatalf("expected chunk 0 idx to be pruned: %v", idxKey) + } + + // the two other indices should be present + copy(idxKey[1:], chunks[1].Address()) + if _, err := ldb.db.Get(idxKey); err != nil { + t.Fatalf("expected chunk 1 idx to be present: %v", idxKey) + } + + copy(idxKey[1:], chunks[2].Address()) + if _, err := ldb.db.Get(idxKey); err != nil { + t.Fatalf("expected chunk 2 idx to be present: %v", idxKey) + } + + // first gc index should still be gone + if _, err := ldb.db.Get(gcFirstCorrectKey); err == nil { + t.Fatalf("expected gc 0 idx to be pruned: %v", idxKey) + } + + // second gc index should still be fixed + if _, err := ldb.db.Get(gcSecondCorrectKey); err != nil { + t.Fatalf("expected gc 1 idx to be present: %v", idxKey) + } + + // third gc index should be unchanged + binary.BigEndian.PutUint64(gcSecondCorrectKey[1:], uint64(2)) + if _, err := ldb.db.Get(gcSecondCorrectKey); err != nil { + t.Fatalf("expected gc 2 idx to be present: %v", idxKey) + } + + c, err := ldb.db.Get(keyEntryCnt) + if err != nil { + t.Fatalf("expected gc 2 idx to be present: %v", idxKey) + } + + // entrycount should now be one less + entryCount := binary.BigEndian.Uint64(c) + if entryCount != 2 { + t.Fatalf("expected entrycnt to be 2, was %d", c) + } + + // the chunks might accidentally be in the same bin + // if so that bin counter will now be 2 - the highest added index. + // if not, the total of them will be 3 + poBins := []uint8{ldb.po(chunks[1].Address()), ldb.po(chunks[2].Address())} + if poBins[0] == poBins[1] { + poBins = poBins[:1] + } + + var binTotal uint64 + var currentBin [2]byte + currentBin[0] = keyDistanceCnt + if len(poBins) == 1 { + currentBin[1] = poBins[0] + c, err := ldb.db.Get(currentBin[:]) + if err != nil { + t.Fatalf("expected gc 2 idx to be present: %v", idxKey) + } + binCount := binary.BigEndian.Uint64(c) + if binCount != 2 { + t.Fatalf("expected entrycnt to be 2, was %d", binCount) + } + } else { + for _, bin := range poBins { + currentBin[1] = bin + c, err := ldb.db.Get(currentBin[:]) + if err != nil { + t.Fatalf("expected gc 2 idx to be present: %v", idxKey) + } + binCount := binary.BigEndian.Uint64(c) + binTotal += binCount + + } + if binTotal != 3 { + t.Fatalf("expected sum of bin indices to be 3, was %d", binTotal) + } + } +} + func waitGc(ctx context.Context, ldb *LDBStore) { <-ldb.gc.runC ldb.gc.runC <- struct{}{} diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index 6971d759e3..fa98848ddc 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -196,31 +196,48 @@ func (ls *LocalStore) Close() { // Migrate checks the datastore schema vs the runtime schema, and runs migrations if they don't match func (ls *LocalStore) Migrate() error { - schema, err := ls.DbStore.GetSchema() + actualDbSchema, err := ls.DbStore.GetSchema() if err != nil { log.Error(err.Error()) return err } - log.Debug("found schema", "schema", schema, "runtime-schema", CurrentDbSchema) - if schema != CurrentDbSchema { - // run migrations + log.Debug("running migrations for", "schema", actualDbSchema, "runtime-schema", CurrentDbSchema) - if schema == "" { - log.Debug("running migrations for", "schema", schema, "runtime-schema", CurrentDbSchema) - - // delete chunks that are not valid, i.e. chunks that do not pass any of the ls.Validators - ls.DbStore.Cleanup(func(c *chunk) bool { - return !ls.isValid(c) - }) - - err := ls.DbStore.PutSchema(DbSchemaPurity) - if err != nil { - log.Error(err.Error()) - return err - } - } + if actualDbSchema == CurrentDbSchema { + return nil } + if actualDbSchema == DbSchemaNone { + ls.migrateFromNoneToPurity() + actualDbSchema = DbSchemaPurity + } + + if err := ls.DbStore.PutSchema(actualDbSchema); err != nil { + return err + } + + if actualDbSchema == DbSchemaPurity { + if err := ls.migrateFromPurityToHalloween(); err != nil { + return err + } + actualDbSchema = DbSchemaHalloween + } + + if err := ls.DbStore.PutSchema(actualDbSchema); err != nil { + return err + } return nil } + +func (ls *LocalStore) migrateFromNoneToPurity() { + // delete chunks that are not valid, i.e. chunks that do not pass + // any of the ls.Validators + ls.DbStore.Cleanup(func(c *chunk) bool { + return !ls.isValid(c) + }) +} + +func (ls *LocalStore) migrateFromPurityToHalloween() error { + return ls.DbStore.CleanGCIndex() +} diff --git a/swarm/storage/schema.go b/swarm/storage/schema.go index fb8498a293..91847ca0f9 100644 --- a/swarm/storage/schema.go +++ b/swarm/storage/schema.go @@ -1,6 +1,17 @@ package storage +// The DB schema we want to use. The actual/current DB schema might differ +// until migrations are run. +const CurrentDbSchema = DbSchemaHalloween + +// There was a time when we had no schema at all. +const DbSchemaNone = "" + // "purity" is the first formal schema of LevelDB we release together with Swarm 0.3.5 const DbSchemaPurity = "purity" -const CurrentDbSchema = DbSchemaPurity +// "halloween" is here because we had a screw in the garbage collector index. +// Because of that we had to rebuild the GC index to get rid of erroneous +// entries and that takes a long time. This schema is used for bookkeeping, +// so rebuild index will run just once. +const DbSchemaHalloween = "halloween" From cef7ed53bdec8d3b43b30ac33e8dfc2a92d3ea3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 14 Nov 2018 10:16:28 +0200 Subject: [PATCH 039/159] params: update CHTs --- params/config.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/params/config.go b/params/config.go index ba719b873b..007e4a66d8 100644 --- a/params/config.go +++ b/params/config.go @@ -49,10 +49,10 @@ var ( // MainnetTrustedCheckpoint contains the light client trusted checkpoint for the main network. MainnetTrustedCheckpoint = &TrustedCheckpoint{ Name: "mainnet", - SectionIndex: 195, - SectionHead: common.HexToHash("0x1cdd2a84cf6c1261ffccc88f6bcefb513abd7934a96c1e909fbf74767560f16b"), - CHTRoot: common.HexToHash("0xe453333c20391d16b91b6fe11c104704f62c8dba15f69db73b4cdf7e100105eb"), - BloomRoot: common.HexToHash("0x47f30069473072e00d2cdca146dce40f0aad243dfc8221bf810822c091674efe"), + SectionIndex: 203, + SectionHead: common.HexToHash("0xc9e05fc67c6a9815adc8072eb18805b53da53a9a6a273e05541e1b7542cf937a"), + CHTRoot: common.HexToHash("0xb85f42447d59f7c3e6679b9a37ed983593fd52efd6251b883592662e95769d5b"), + BloomRoot: common.HexToHash("0xf93d50cb4c49b403c6fd33cd60896d3b36184275be0a51bae4df5e8844ac624c"), } // TestnetChainConfig contains the chain parameters to run a node on the Ropsten test network. @@ -73,10 +73,10 @@ var ( // TestnetTrustedCheckpoint contains the light client trusted checkpoint for the Ropsten test network. TestnetTrustedCheckpoint = &TrustedCheckpoint{ Name: "testnet", - SectionIndex: 126, - SectionHead: common.HexToHash("0x48f7dd4c9c60be04bf15fd4d0bcac46ddd8caf6b01d6fb8f8e1f7955cdd1337a"), - CHTRoot: common.HexToHash("0x6e54cb80a1884881ea1a114243af9012c95e0296b47f103b5ab124313968508e"), - BloomRoot: common.HexToHash("0xb55accf6dce6455b47db8510d15eff38d0ed7378829f3036d26b48e7d15da3f6"), + SectionIndex: 134, + SectionHead: common.HexToHash("0x17053ecbe045bebefaa01e7716cc85a4e22647e181416cc1098ccbb73a088931"), + CHTRoot: common.HexToHash("0x4d2b86422e46ed76f0e3f50f06632c409f809c8375e53c8bc0f782bcb93dd49a"), + BloomRoot: common.HexToHash("0xccba62232ee56c2967afc58f136a47ba7dc545ae586e6be666430d94516306c7"), } // RinkebyChainConfig contains the chain parameters to run a node on the Rinkeby test network. @@ -100,10 +100,10 @@ var ( // RinkebyTrustedCheckpoint contains the light client trusted checkpoint for the Rinkeby test network. RinkebyTrustedCheckpoint = &TrustedCheckpoint{ Name: "rinkeby", - SectionIndex: 93, - SectionHead: common.HexToHash("0xdefb94aa217ab38f2919f7318d1d5476bd2aabf1ec9148047fe03e555615e0b4"), - CHTRoot: common.HexToHash("0x52c98c2fe508a8332c27dc10538f3fead43306e2b22b597587763c2fe6586da6"), - BloomRoot: common.HexToHash("0x93d83be0c1b12f732b1a027ecdfb16f39b0d020b8c10bfb90e76f3b01adfc5b6"), + SectionIndex: 100, + SectionHead: common.HexToHash("0xf18f9b43e16f37b12e68818536ffe455ff18d676274ffdd856a8520ed61bb514"), + CHTRoot: common.HexToHash("0x473f5d603b1fedad75d97fd58692130b9ac9ade1aca01eb9363d79bd1c43c791"), + BloomRoot: common.HexToHash("0xa39ced3ddbb87e909c7531df2afb6414bea9c9a60ab94da9c6b467535f05326e"), } // AllEthashProtocolChanges contains every protocol change (EIPs) introduced From eb8fa3cc89ae3a3247c649486839b1c250554d2d Mon Sep 17 00:00:00 2001 From: Alexey Sharov Date: Wed, 14 Nov 2018 15:21:14 +0700 Subject: [PATCH 040/159] cmd/swarm, swarm/api/http, swarm/bmt, swarm/fuse, swarm/network/stream, swarm/storage, swarm/storage/encryption, swarm/testutil: use pseudo-random instead of crypto-random for test files content generation (#18083) - Replace "crypto/rand" to "math/rand" for files content generation - Remove swarm/network_test.go.Shuffle and swarm/btm/btm_test.go.Shuffle - because go1.9 support dropped (see https://github.com/ethereum/go-ethereum/pull/17807 and comments to swarm/network_test.go.Shuffle) --- cmd/swarm/access_test.go | 3 +- cmd/swarm/export_test.go | 36 +--- cmd/swarm/feeds_test.go | 37 ++-- cmd/swarm/manifest_test.go | 6 +- cmd/swarm/run_test.go | 3 +- cmd/swarm/upload_test.go | 23 +-- swarm/api/client/client_test.go | 17 +- swarm/api/http/response_test.go | 12 +- swarm/api/http/server_test.go | 32 ++-- .../http.go => api/http/test_server.go} | 2 +- swarm/bmt/bmt_test.go | 71 ++------ swarm/fuse/swarmfs_test.go | 163 +++++++----------- swarm/network/stream/common_test.go | 9 +- swarm/network/stream/delivery_test.go | 7 +- swarm/network/stream/intervals_test.go | 6 +- swarm/network/stream/snapshot_sync_test.go | 5 +- swarm/network/stream/syncer_test.go | 5 +- swarm/network_test.go | 33 +--- swarm/storage/chunker_test.go | 34 ++-- swarm/storage/common_test.go | 7 +- swarm/storage/encryption/encryption_test.go | 5 +- swarm/storage/filestore_test.go | 16 +- swarm/storage/types.go | 11 -- swarm/testutil/file.go | 21 +++ 24 files changed, 202 insertions(+), 362 deletions(-) rename swarm/{testutil/http.go => api/http/test_server.go} (99%) diff --git a/cmd/swarm/access_test.go b/cmd/swarm/access_test.go index b4d2e1dbc5..e812cd8fd1 100644 --- a/cmd/swarm/access_test.go +++ b/cmd/swarm/access_test.go @@ -38,6 +38,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" swarm "github.com/ethereum/go-ethereum/swarm/api/client" + swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" "github.com/ethereum/go-ethereum/swarm/testutil" ) @@ -54,7 +55,7 @@ var DefaultCurve = crypto.S256() // is then fetched through 2nd node. since the tested code is not key-aware - we can just // fetch from the 2nd node using HTTP BasicAuth func TestAccessPassword(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() dataFilename := testutil.TempFileWithContent(t, data) diff --git a/cmd/swarm/export_test.go b/cmd/swarm/export_test.go index c533511af0..f1bc2f2658 100644 --- a/cmd/swarm/export_test.go +++ b/cmd/swarm/export_test.go @@ -19,9 +19,7 @@ package main import ( "bytes" "crypto/md5" - "crypto/rand" "io" - "io/ioutil" "net/http" "os" "runtime" @@ -29,6 +27,7 @@ import ( "testing" "github.com/ethereum/go-ethereum/swarm" + "github.com/ethereum/go-ethereum/swarm/testutil" ) // TestCLISwarmExportImport perform the following test: @@ -45,11 +44,12 @@ func TestCLISwarmExportImport(t *testing.T) { cluster := newTestCluster(t, 1) // generate random 10mb file - f, cleanup := generateRandomFile(t, 10000000) - defer cleanup() + content := testutil.RandomBytes(1, 10000000) + fileName := testutil.TempFileWithContent(t, string(content)) + defer os.Remove(fileName) // upload the file with 'swarm up' and expect a hash - up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", f.Name()) + up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", fileName) _, matches := up.ExpectRegexp(`[a-f\d]{64}`) up.ExpectExit() hash := matches[0] @@ -96,7 +96,7 @@ func TestCLISwarmExportImport(t *testing.T) { } // compare downloaded file with the generated random file - mustEqualFiles(t, f, res.Body) + mustEqualFiles(t, bytes.NewReader(content), res.Body) } func mustEqualFiles(t *testing.T, up io.Reader, down io.Reader) { @@ -117,27 +117,3 @@ func mustEqualFiles(t *testing.T, up io.Reader, down io.Reader) { t.Fatalf("downloaded imported file md5=%x (length %v) is not the same as the generated one mp5=%x (length %v)", downHash, downLen, upHash, upLen) } } - -func generateRandomFile(t *testing.T, size int) (f *os.File, teardown func()) { - // create a tmp file - tmp, err := ioutil.TempFile("", "swarm-test") - if err != nil { - t.Fatal(err) - } - - // callback for tmp file cleanup - teardown = func() { - tmp.Close() - os.Remove(tmp.Name()) - } - - // write 10mb random data to file - buf := make([]byte, 10000000) - _, err = rand.Read(buf) - if err != nil { - t.Fatal(err) - } - ioutil.WriteFile(tmp.Name(), buf, 0755) - - return tmp, teardown -} diff --git a/cmd/swarm/feeds_test.go b/cmd/swarm/feeds_test.go index 46727c21d3..fc3f72ab15 100644 --- a/cmd/swarm/feeds_test.go +++ b/cmd/swarm/feeds_test.go @@ -20,49 +20,37 @@ import ( "bytes" "encoding/json" "fmt" - "io" "io/ioutil" "os" "testing" - "github.com/ethereum/go-ethereum/swarm/api" - "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" - "github.com/ethereum/go-ethereum/swarm/testutil" - - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/swarm/storage/feed" - "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/api" swarm "github.com/ethereum/go-ethereum/swarm/api/client" swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" + "github.com/ethereum/go-ethereum/swarm/storage/feed" + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" + "github.com/ethereum/go-ethereum/swarm/testutil" ) func TestCLIFeedUpdate(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, func(api *api.API) testutil.TestServer { + srv := swarmhttp.NewTestSwarmServer(t, func(api *api.API) swarmhttp.TestServer { return swarmhttp.NewServer(api, "") }, nil) log.Info("starting a test swarm server") defer srv.Close() // create a private key file for signing - pkfile, err := ioutil.TempFile("", "swarm-test") - if err != nil { - t.Fatal(err) - } - defer pkfile.Close() - defer os.Remove(pkfile.Name()) privkeyHex := "0000000000000000000000000000000000000000000000000000000000001979" privKey, _ := crypto.HexToECDSA(privkeyHex) address := crypto.PubkeyToAddress(privKey.PublicKey) - // save the private key to a file - _, err = io.WriteString(pkfile, privkeyHex) - if err != nil { - t.Fatal(err) - } + pkFileName := testutil.TempFileWithContent(t, privkeyHex) + defer os.Remove(pkFileName) // compose a topic. We'll be doing quotes about Miguel de Cervantes var topic feed.Topic @@ -76,7 +64,7 @@ func TestCLIFeedUpdate(t *testing.T) { flags := []string{ "--bzzapi", srv.URL, - "--bzzaccount", pkfile.Name(), + "--bzzaccount", pkFileName, "feed", "update", "--topic", topic.Hex(), "--name", name, @@ -89,13 +77,10 @@ func TestCLIFeedUpdate(t *testing.T) { // now try to get the update using the client client := swarm.NewClient(srv.URL) - if err != nil { - t.Fatal(err) - } // build the same topic as before, this time // we use NewTopic to create a topic automatically. - topic, err = feed.NewTopic(name, subject) + topic, err := feed.NewTopic(name, subject) if err != nil { t.Fatal(err) } @@ -153,7 +138,7 @@ func TestCLIFeedUpdate(t *testing.T) { // test publishing a manifest flags = []string{ "--bzzapi", srv.URL, - "--bzzaccount", pkfile.Name(), + "--bzzaccount", pkFileName, "feed", "create", "--topic", topic.Hex(), } diff --git a/cmd/swarm/manifest_test.go b/cmd/swarm/manifest_test.go index 7ea4e0c45b..01d982aa7a 100644 --- a/cmd/swarm/manifest_test.go +++ b/cmd/swarm/manifest_test.go @@ -26,7 +26,7 @@ import ( "github.com/ethereum/go-ethereum/swarm/api" swarm "github.com/ethereum/go-ethereum/swarm/api/client" - "github.com/ethereum/go-ethereum/swarm/testutil" + swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" ) // TestManifestChange tests manifest add, update and remove @@ -58,7 +58,7 @@ func TestManifestChangeEncrypted(t *testing.T) { // Argument encrypt controls whether to use encryption or not. func testManifestChange(t *testing.T, encrypt bool) { t.Parallel() - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() tmp, err := ioutil.TempDir("", "swarm-manifest-test") @@ -430,7 +430,7 @@ func TestNestedDefaultEntryUpdateEncrypted(t *testing.T) { func testNestedDefaultEntryUpdate(t *testing.T, encrypt bool) { t.Parallel() - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() tmp, err := ioutil.TempDir("", "swarm-manifest-test") diff --git a/cmd/swarm/run_test.go b/cmd/swarm/run_test.go index 55199e955e..416fa7a503 100644 --- a/cmd/swarm/run_test.go +++ b/cmd/swarm/run_test.go @@ -42,7 +42,6 @@ import ( "github.com/ethereum/go-ethereum/swarm" "github.com/ethereum/go-ethereum/swarm/api" swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" - "github.com/ethereum/go-ethereum/swarm/testutil" ) var loglevel = flag.Int("loglevel", 3, "verbosity of logs") @@ -58,7 +57,7 @@ func init() { }) } -func serverFunc(api *api.API) testutil.TestServer { +func serverFunc(api *api.API) swarmhttp.TestServer { return swarmhttp.NewServer(api, "") } func TestMain(m *testing.M) { diff --git a/cmd/swarm/upload_test.go b/cmd/swarm/upload_test.go index ba4463e8bf..5f9844950e 100644 --- a/cmd/swarm/upload_test.go +++ b/cmd/swarm/upload_test.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/log" swarm "github.com/ethereum/go-ethereum/swarm/api/client" + swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" "github.com/ethereum/go-ethereum/swarm/testutil" "github.com/mattn/go-colorable" ) @@ -77,33 +78,22 @@ func testCLISwarmUp(toEncrypt bool, t *testing.T) { cluster := newTestCluster(t, 3) defer cluster.Shutdown() - // create a tmp file - tmp, err := ioutil.TempFile("", "swarm-test") - if err != nil { - t.Fatal(err) - } - defer tmp.Close() - defer os.Remove(tmp.Name()) + tmpFileName := testutil.TempFileWithContent(t, data) + defer os.Remove(tmpFileName) // write data to file - data := "notsorandomdata" - _, err = io.WriteString(tmp, data) - if err != nil { - t.Fatal(err) - } - hashRegexp := `[a-f\d]{64}` flags := []string{ "--bzzapi", cluster.Nodes[0].URL, "up", - tmp.Name()} + tmpFileName} if toEncrypt { hashRegexp = `[a-f\d]{128}` flags = []string{ "--bzzapi", cluster.Nodes[0].URL, "up", "--encrypt", - tmp.Name()} + tmpFileName} } // upload the file with 'swarm up' and expect a hash log.Info(fmt.Sprintf("uploading file with 'swarm up'")) @@ -203,7 +193,6 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) { } defer os.RemoveAll(tmpUploadDir) // create tmp files - data := "notsorandomdata" for _, path := range []string{"tmp1", "tmp2"} { if err := ioutil.WriteFile(filepath.Join(tmpUploadDir, path), bytes.NewBufferString(data).Bytes(), 0644); err != nil { t.Fatal(err) @@ -298,7 +287,7 @@ func TestCLISwarmUpDefaultPath(t *testing.T) { } func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() tmp, err := ioutil.TempDir("", "swarm-defaultpath-test") diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go index c30d699110..76b3493978 100644 --- a/swarm/api/client/client_test.go +++ b/swarm/api/client/client_test.go @@ -33,10 +33,9 @@ import ( swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" "github.com/ethereum/go-ethereum/swarm/multihash" "github.com/ethereum/go-ethereum/swarm/storage/feed" - "github.com/ethereum/go-ethereum/swarm/testutil" ) -func serverFunc(api *api.API) testutil.TestServer { +func serverFunc(api *api.API) swarmhttp.TestServer { return swarmhttp.NewServer(api, "") } @@ -49,7 +48,7 @@ func TestClientUploadDownloadRawEncrypted(t *testing.T) { } func testClientUploadDownloadRaw(toEncrypt bool, t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() client := NewClient(srv.URL) @@ -90,7 +89,7 @@ func TestClientUploadDownloadFilesEncrypted(t *testing.T) { } func testClientUploadDownloadFiles(toEncrypt bool, t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() client := NewClient(srv.URL) @@ -188,7 +187,7 @@ func newTestDirectory(t *testing.T) string { // TestClientUploadDownloadDirectory tests uploading and downloading a // directory of files to a swarm manifest func TestClientUploadDownloadDirectory(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() dir := newTestDirectory(t) @@ -254,7 +253,7 @@ func TestClientFileListEncrypted(t *testing.T) { } func testClientFileList(toEncrypt bool, t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() dir := newTestDirectory(t) @@ -312,7 +311,7 @@ func testClientFileList(toEncrypt bool, t *testing.T) { // TestClientMultipartUpload tests uploading files to swarm using a multipart // upload func TestClientMultipartUpload(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() // define an uploader which uploads testDirFiles with some data @@ -378,7 +377,7 @@ func TestClientCreateFeedMultihash(t *testing.T) { signer, _ := newTestSigner() - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) client := NewClient(srv.URL) defer srv.Close() @@ -440,7 +439,7 @@ func TestClientCreateUpdateFeed(t *testing.T) { signer, _ := newTestSigner() - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) client := NewClient(srv.URL) defer srv.Close() diff --git a/swarm/api/http/response_test.go b/swarm/api/http/response_test.go index 50a704be6d..486c19ab0e 100644 --- a/swarm/api/http/response_test.go +++ b/swarm/api/http/response_test.go @@ -24,12 +24,10 @@ import ( "testing" "golang.org/x/net/html" - - "github.com/ethereum/go-ethereum/swarm/testutil" ) func TestError(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() var resp *http.Response @@ -55,7 +53,7 @@ func TestError(t *testing.T) { } func Test404Page(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() var resp *http.Response @@ -81,7 +79,7 @@ func Test404Page(t *testing.T) { } func Test500Page(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() var resp *http.Response @@ -106,7 +104,7 @@ func Test500Page(t *testing.T) { } } func Test500PageWith0xHashPrefix(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() var resp *http.Response @@ -136,7 +134,7 @@ func Test500PageWith0xHashPrefix(t *testing.T) { } func TestJsonResponse(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() var resp *http.Response diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 04d0e045aa..1ef3deece2 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -20,7 +20,6 @@ import ( "archive/tar" "bytes" "context" - "crypto/rand" "encoding/json" "errors" "flag" @@ -58,7 +57,7 @@ func init() { log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) } -func serverFunc(api *api.API) testutil.TestServer { +func serverFunc(api *api.API) TestServer { return NewServer(api, "") } @@ -79,7 +78,7 @@ func TestBzzFeedMultihash(t *testing.T) { signer, _ := newTestSigner() - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() // add the data our multihash aliased manifest will point to @@ -167,26 +166,19 @@ func TestBzzFeedMultihash(t *testing.T) { // Test Swarm feeds using the raw update methods func TestBzzFeed(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) signer, _ := newTestSigner() defer srv.Close() // data of update 1 - update1Data := make([]byte, 666) + update1Data := testutil.RandomBytes(1, 666) update1Timestamp := srv.CurrentTime - _, err := rand.Read(update1Data) - if err != nil { - t.Fatal(err) - } //data for update 2 update2Data := []byte("foo") topic, _ := feed.NewTopic("foo.eth", nil) updateRequest := feed.NewFirstRequest(topic) - if err != nil { - t.Fatal(err) - } updateRequest.SetData(update1Data) if err := updateRequest.Sign(signer); err != nil { @@ -450,7 +442,7 @@ func testBzzGetPath(encrypted bool, t *testing.T) { addr := [3]storage.Address{} - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() for i, mf := range testmanifest { @@ -688,7 +680,7 @@ func TestBzzTar(t *testing.T) { } func testBzzTar(encrypted bool, t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() fileNames := []string{"tmp1.txt", "tmp2.lock", "tmp3.rtf"} fileContents := []string{"tmp1textfilevalue", "tmp2lockfilelocked", "tmp3isjustaplaintextfile"} @@ -823,7 +815,7 @@ func TestBzzRootRedirectEncrypted(t *testing.T) { } func testBzzRootRedirect(toEncrypt bool, t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() // create a manifest with some data at the root path @@ -878,7 +870,7 @@ func testBzzRootRedirect(toEncrypt bool, t *testing.T) { } func TestMethodsNotAllowed(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() databytes := "bar" for _, c := range []struct { @@ -937,7 +929,7 @@ func httpDo(httpMethod string, url string, reqBody io.Reader, headers map[string } func TestGet(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() for _, testCase := range []struct { @@ -1020,7 +1012,7 @@ func TestGet(t *testing.T) { } func TestModify(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() swarmClient := swarm.NewClient(srv.URL) @@ -1121,7 +1113,7 @@ func TestMultiPartUpload(t *testing.T) { // POST /bzz:/ Content-Type: multipart/form-data verbose := false // Setup Swarm - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() url := fmt.Sprintf("%s/bzz:/", srv.URL) @@ -1152,7 +1144,7 @@ func TestMultiPartUpload(t *testing.T) { // TestBzzGetFileWithResolver tests fetching a file using a mocked ENS resolver func TestBzzGetFileWithResolver(t *testing.T) { resolver := newTestResolveValidator("") - srv := testutil.NewTestSwarmServer(t, serverFunc, resolver) + srv := NewTestSwarmServer(t, serverFunc, resolver) defer srv.Close() fileNames := []string{"dir1/tmp1.txt", "dir2/tmp2.lock", "dir3/tmp3.rtf"} fileContents := []string{"tmp1textfilevalue", "tmp2lockfilelocked", "tmp3isjustaplaintextfile"} diff --git a/swarm/testutil/http.go b/swarm/api/http/test_server.go similarity index 99% rename from swarm/testutil/http.go rename to swarm/api/http/test_server.go index cdf9239d06..9245c9c5b8 100644 --- a/swarm/testutil/http.go +++ b/swarm/api/http/test_server.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package testutil +package http import ( "io/ioutil" diff --git a/swarm/bmt/bmt_test.go b/swarm/bmt/bmt_test.go index 760aa11d8b..683ba4f5b2 100644 --- a/swarm/bmt/bmt_test.go +++ b/swarm/bmt/bmt_test.go @@ -18,10 +18,8 @@ package bmt import ( "bytes" - crand "crypto/rand" "encoding/binary" "fmt" - "io" "math/rand" "sync" "sync/atomic" @@ -29,6 +27,7 @@ import ( "time" "github.com/ethereum/go-ethereum/crypto/sha3" + "github.com/ethereum/go-ethereum/swarm/testutil" ) // the actual data length generated (could be longer than max datalength of the BMT) @@ -116,14 +115,11 @@ func TestRefHasher(t *testing.T) { }) // run the tests - for _, x := range tests { + for i, x := range tests { for segmentCount := x.from; segmentCount <= x.to; segmentCount++ { for length := 1; length <= segmentCount*32; length++ { t.Run(fmt.Sprintf("%d_segments_%d_bytes", segmentCount, length), func(t *testing.T) { - data := make([]byte, length) - if _, err := io.ReadFull(crand.Reader, data); err != nil && err != io.EOF { - t.Fatal(err) - } + data := testutil.RandomBytes(i, length) expected := x.expected(data) actual := NewRefHasher(sha3.NewKeccak256, segmentCount).Hash(data) if !bytes.Equal(actual, expected) { @@ -156,7 +152,7 @@ func TestHasherEmptyData(t *testing.T) { // tests sequential write with entire max size written in one go func TestSyncHasherCorrectness(t *testing.T) { - data := newData(BufferSize) + data := testutil.RandomBytes(1, BufferSize) hasher := sha3.NewKeccak256 size := hasher().Size() @@ -182,7 +178,7 @@ func TestSyncHasherCorrectness(t *testing.T) { // tests order-neutral concurrent writes with entire max size written in one go func TestAsyncCorrectness(t *testing.T) { - data := newData(BufferSize) + data := testutil.RandomBytes(1, BufferSize) hasher := sha3.NewKeccak256 size := hasher().Size() whs := []whenHash{first, last, random} @@ -236,7 +232,7 @@ func testHasherReuse(poolsize int, t *testing.T) { bmt := New(pool) for i := 0; i < 100; i++ { - data := newData(BufferSize) + data := testutil.RandomBytes(1, BufferSize) n := rand.Intn(bmt.Size()) err := testHasherCorrectness(bmt, hasher, data, n, segmentCount) if err != nil { @@ -256,7 +252,7 @@ func TestBMTConcurrentUse(t *testing.T) { for i := 0; i < cycles; i++ { go func() { bmt := New(pool) - data := newData(BufferSize) + data := testutil.RandomBytes(1, BufferSize) n := rand.Intn(bmt.Size()) errc <- testHasherCorrectness(bmt, hasher, data, n, 128) }() @@ -290,7 +286,7 @@ func TestBMTWriterBuffers(t *testing.T) { defer pool.Drain(0) n := count * 32 bmt := New(pool) - data := newData(n) + data := testutil.RandomBytes(1, n) rbmt := NewRefHasher(hasher, count) refHash := rbmt.Hash(data) expHash := syncHash(bmt, nil, data) @@ -413,7 +409,7 @@ func BenchmarkPool(t *testing.B) { // benchmarks simple sha3 hash on chunks func benchmarkSHA3(t *testing.B, n int) { - data := newData(n) + data := testutil.RandomBytes(1, n) hasher := sha3.NewKeccak256 h := hasher() @@ -432,7 +428,7 @@ func benchmarkSHA3(t *testing.B, n int) { func benchmarkBMTBaseline(t *testing.B, n int) { hasher := sha3.NewKeccak256 hashSize := hasher().Size() - data := newData(hashSize) + data := testutil.RandomBytes(1, hashSize) t.ReportAllocs() t.ResetTimer() @@ -456,7 +452,7 @@ func benchmarkBMTBaseline(t *testing.B, n int) { // benchmarks BMT Hasher func benchmarkBMT(t *testing.B, n int) { - data := newData(n) + data := testutil.RandomBytes(1, n) hasher := sha3.NewKeccak256 pool := NewTreePool(hasher, segmentCount, PoolSize) bmt := New(pool) @@ -470,12 +466,12 @@ func benchmarkBMT(t *testing.B, n int) { // benchmarks BMT hasher with asynchronous concurrent segment/section writes func benchmarkBMTAsync(t *testing.B, n int, wh whenHash, double bool) { - data := newData(n) + data := testutil.RandomBytes(1, n) hasher := sha3.NewKeccak256 pool := NewTreePool(hasher, segmentCount, PoolSize) bmt := New(pool).NewAsyncWriter(double) idxs, segments := splitAndShuffle(bmt.SectionSize(), data) - shuffle(len(idxs), func(i int, j int) { + rand.Shuffle(len(idxs), func(i int, j int) { idxs[i], idxs[j] = idxs[j], idxs[i] }) @@ -488,7 +484,7 @@ func benchmarkBMTAsync(t *testing.B, n int, wh whenHash, double bool) { // benchmarks 100 concurrent bmt hashes with pool capacity func benchmarkPool(t *testing.B, poolsize, n int) { - data := newData(n) + data := testutil.RandomBytes(1, n) hasher := sha3.NewKeccak256 pool := NewTreePool(hasher, segmentCount, poolsize) cycles := 100 @@ -511,7 +507,7 @@ func benchmarkPool(t *testing.B, poolsize, n int) { // benchmarks the reference hasher func benchmarkRefHasher(t *testing.B, n int) { - data := newData(n) + data := testutil.RandomBytes(1, n) hasher := sha3.NewKeccak256 rbmt := NewRefHasher(hasher, 128) @@ -522,15 +518,6 @@ func benchmarkRefHasher(t *testing.B, n int) { } } -func newData(bufferSize int) []byte { - data := make([]byte, bufferSize) - _, err := io.ReadFull(crand.Reader, data) - if err != nil { - panic(err.Error()) - } - return data -} - // Hash hashes the data and the span using the bmt hasher func syncHash(h *Hasher, span, data []byte) []byte { h.ResetWithLength(span) @@ -553,7 +540,7 @@ func splitAndShuffle(secsize int, data []byte) (idxs []int, segments [][]byte) { section := data[i*secsize : end] segments = append(segments, section) } - shuffle(n, func(i int, j int) { + rand.Shuffle(n, func(i int, j int) { idxs[i], idxs[j] = idxs[j], idxs[i] }) return idxs, segments @@ -594,29 +581,3 @@ func asyncHash(bmt SectionWriter, span []byte, l int, wh whenHash, idxs []int, s } return <-c } - -// this is also in swarm/network_test.go -// shuffle pseudo-randomizes the order of elements. -// n is the number of elements. Shuffle panics if n < 0. -// swap swaps the elements with indexes i and j. -func shuffle(n int, swap func(i, j int)) { - if n < 0 { - panic("invalid argument to Shuffle") - } - - // Fisher-Yates shuffle: https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle - // Shuffle really ought not be called with n that doesn't fit in 32 bits. - // Not only will it take a very long time, but with 2³¹! possible permutations, - // there's no way that any PRNG can have a big enough internal state to - // generate even a minuscule percentage of the possible permutations. - // Nevertheless, the right API signature accepts an int n, so handle it as best we can. - i := n - 1 - for ; i > 1<<31-1-1; i-- { - j := int(rand.Int63n(int64(i + 1))) - swap(i, j) - } - for ; i > 0; i-- { - j := int(rand.Int31n(int32(i + 1))) - swap(i, j) - } -} diff --git a/swarm/fuse/swarmfs_test.go b/swarm/fuse/swarmfs_test.go index 87d9185505..460e31c4e9 100644 --- a/swarm/fuse/swarmfs_test.go +++ b/swarm/fuse/swarmfs_test.go @@ -20,20 +20,19 @@ package fuse import ( "bytes" - "crypto/rand" "flag" "fmt" "io" "io/ioutil" + "math/rand" "os" "path/filepath" "testing" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" "github.com/ethereum/go-ethereum/swarm/storage" - - "github.com/ethereum/go-ethereum/log" - + "github.com/ethereum/go-ethereum/swarm/testutil" colorable "github.com/mattn/go-colorable" ) @@ -229,12 +228,6 @@ func checkFile(t *testing.T, testMountDir, fname string, contents []byte) { } } -func getRandomBytes(size int) []byte { - contents := make([]byte, size) - rand.Read(contents) - return contents -} - func isDirEmpty(name string) bool { f, err := os.Open(name) if err != nil { @@ -328,22 +321,22 @@ func (ta *testAPI) mountListAndUnmount(t *testing.T, toEncrypt bool) { dat.testMountDir = filepath.Join(dat.testDir, "testMountDir") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["2.txt"] = fileInfo{0711, 333, 444, getRandomBytes(10)} - dat.files["3.txt"] = fileInfo{0622, 333, 444, getRandomBytes(100)} - dat.files["4.txt"] = fileInfo{0533, 333, 444, getRandomBytes(1024)} - dat.files["5.txt"] = fileInfo{0544, 333, 444, getRandomBytes(10)} - dat.files["6.txt"] = fileInfo{0555, 333, 444, getRandomBytes(10)} - dat.files["7.txt"] = fileInfo{0666, 333, 444, getRandomBytes(10)} - dat.files["8.txt"] = fileInfo{0777, 333, 333, getRandomBytes(10)} - dat.files["11.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)} - dat.files["111.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)} - dat.files["two/2.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)} - dat.files["two/2/2.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)} - dat.files["two/2./2.txt"] = fileInfo{0777, 444, 444, getRandomBytes(10)} - dat.files["twice/2.txt"] = fileInfo{0777, 444, 333, getRandomBytes(200)} - dat.files["one/two/three/four/five/six/seven/eight/nine/10.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10240)} - dat.files["one/two/three/four/five/six/six"] = fileInfo{0777, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} + dat.files["2.txt"] = fileInfo{0711, 333, 444, testutil.RandomBytes(2, 10)} + dat.files["3.txt"] = fileInfo{0622, 333, 444, testutil.RandomBytes(3, 100)} + dat.files["4.txt"] = fileInfo{0533, 333, 444, testutil.RandomBytes(4, 1024)} + dat.files["5.txt"] = fileInfo{0544, 333, 444, testutil.RandomBytes(5, 10)} + dat.files["6.txt"] = fileInfo{0555, 333, 444, testutil.RandomBytes(6, 10)} + dat.files["7.txt"] = fileInfo{0666, 333, 444, testutil.RandomBytes(7, 10)} + dat.files["8.txt"] = fileInfo{0777, 333, 333, testutil.RandomBytes(8, 10)} + dat.files["11.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(9, 10)} + dat.files["111.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(10, 10)} + dat.files["two/2.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(11, 10)} + dat.files["two/2/2.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(12, 10)} + dat.files["two/2./2.txt"] = fileInfo{0777, 444, 444, testutil.RandomBytes(13, 10)} + dat.files["twice/2.txt"] = fileInfo{0777, 444, 333, testutil.RandomBytes(14, 200)} + dat.files["one/two/three/four/five/six/seven/eight/nine/10.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(15, 10240)} + dat.files["one/two/three/four/five/six/six"] = fileInfo{0777, 333, 444, testutil.RandomBytes(16, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -386,7 +379,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "max-upload1") dat.testMountDir = filepath.Join(dat.testDir, "max-mount1") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -396,7 +389,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "max-upload2") dat.testMountDir = filepath.Join(dat.testDir, "max-mount2") - dat.files["2.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["2.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -405,7 +398,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "max-upload3") dat.testMountDir = filepath.Join(dat.testDir, "max-mount3") - dat.files["3.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["3.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -414,7 +407,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "max-upload4") dat.testMountDir = filepath.Join(dat.testDir, "max-mount4") - dat.files["4.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["4.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -423,7 +416,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "max-upload5") dat.testMountDir = filepath.Join(dat.testDir, "max-mount5") - dat.files["5.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["5.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -436,7 +429,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) { if err != nil { t.Fatalf("Couldn't create upload dir 6: %v", err) } - dat.files["6.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["6.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} testMountDir6 := filepath.Join(dat.testDir, "max-mount6") err = os.MkdirAll(testMountDir6, 0777) if err != nil { @@ -475,7 +468,7 @@ func (ta *testAPI) remount(t *testing.T, toEncrypt bool) { dat.testMountDir = filepath.Join(dat.testDir, "remount-mount1") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -494,7 +487,7 @@ func (ta *testAPI) remount(t *testing.T, toEncrypt bool) { } // mount a different hash in already mounted point - dat.files["2.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["2.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} testUploadDir2, err3 := addDir(dat.testDir, "remount-upload2") if err3 != nil { t.Fatalf("Error creating second upload dir: %v", err3) @@ -543,7 +536,7 @@ func (ta *testAPI) unmount(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "ex-upload1") dat.testMountDir = filepath.Join(dat.testDir, "ex-mount1") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -591,7 +584,7 @@ func (ta *testAPI) unmountWhenResourceBusy(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "ex-upload1") dat.testMountDir = filepath.Join(dat.testDir, "ex-mount1") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -609,7 +602,7 @@ func (ta *testAPI) unmountWhenResourceBusy(t *testing.T, toEncrypt bool) { //we need to manually close the file before mount for this test //but let's defer too in case of errors defer d.Close() - _, err = d.Write(getRandomBytes(10)) + _, err = d.Write(testutil.RandomBytes(1, 10)) if err != nil { t.Fatalf("Couldn't write to file: %v", err) } @@ -667,7 +660,7 @@ func (ta *testAPI) seekInMultiChunkFile(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "seek-upload1") dat.testMountDir = filepath.Join(dat.testDir, "seek-mount") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10240)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10240)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -733,9 +726,9 @@ func (ta *testAPI) createNewFile(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "create-upload1") dat.testMountDir = filepath.Join(dat.testDir, "create-mount") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} + dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)} + dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -751,11 +744,7 @@ func (ta *testAPI) createNewFile(t *testing.T, toEncrypt bool) { } defer d.Close() log.Debug("Opened file") - contents := make([]byte, 11) - _, err = rand.Read(contents) - if err != nil { - t.Fatalf("Could not rand read contents %v", err) - } + contents := testutil.RandomBytes(1, 11) log.Debug("content read") _, err = d.Write(contents) if err != nil { @@ -815,7 +804,7 @@ func (ta *testAPI) createNewFileInsideDirectory(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "createinsidedir-upload") dat.testMountDir = filepath.Join(dat.testDir, "createinsidedir-mount") dat.files = make(map[string]fileInfo) - dat.files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["one/1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -832,11 +821,7 @@ func (ta *testAPI) createNewFileInsideDirectory(t *testing.T, toEncrypt bool) { } defer d.Close() log.Debug("File opened") - contents := make([]byte, 11) - _, err = rand.Read(contents) - if err != nil { - t.Fatalf("Error filling random bytes into byte array %v", err) - } + contents := testutil.RandomBytes(1, 11) log.Debug("Content read") _, err = d.Write(contents) if err != nil { @@ -896,7 +881,7 @@ func (ta *testAPI) createNewFileInsideNewDirectory(t *testing.T, toEncrypt bool) dat.testUploadDir = filepath.Join(dat.testDir, "createinsidenewdir-upload") dat.testMountDir = filepath.Join(dat.testDir, "createinsidenewdir-mount") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -916,11 +901,7 @@ func (ta *testAPI) createNewFileInsideNewDirectory(t *testing.T, toEncrypt bool) } defer d.Close() log.Debug("File opened") - contents := make([]byte, 11) - _, err = rand.Read(contents) - if err != nil { - t.Fatalf("Error writing random bytes to byte array: %v", err) - } + contents := testutil.RandomBytes(1, 11) log.Debug("content read") _, err = d.Write(contents) if err != nil { @@ -976,9 +957,9 @@ func (ta *testAPI) removeExistingFile(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "remove-upload") dat.testMountDir = filepath.Join(dat.testDir, "remove-mount") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} + dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)} + dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -1036,9 +1017,9 @@ func (ta *testAPI) removeExistingFileInsideDir(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "remove-upload") dat.testMountDir = filepath.Join(dat.testDir, "remove-mount") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["one/five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["one/six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} + dat.files["one/five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)} + dat.files["one/six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -1104,9 +1085,9 @@ func (ta *testAPI) removeNewlyAddedFile(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "removenew-upload") dat.testMountDir = filepath.Join(dat.testDir, "removenew-mount") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} + dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)} + dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -1127,11 +1108,7 @@ func (ta *testAPI) removeNewlyAddedFile(t *testing.T, toEncrypt bool) { } defer d.Close() log.Debug("file opened") - contents := make([]byte, 11) - _, err = rand.Read(contents) - if err != nil { - t.Fatalf("Error writing random bytes to byte array: %v", err) - } + contents := testutil.RandomBytes(1, 11) log.Debug("content read") _, err = d.Write(contents) if err != nil { @@ -1201,9 +1178,9 @@ func (ta *testAPI) addNewFileAndModifyContents(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "modifyfile-upload") dat.testMountDir = filepath.Join(dat.testDir, "modifyfile-mount") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} + dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)} + dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -1357,9 +1334,9 @@ func (ta *testAPI) removeEmptyDir(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "rmdir-upload") dat.testMountDir = filepath.Join(dat.testDir, "rmdir-mount") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} + dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)} + dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -1406,9 +1383,9 @@ func (ta *testAPI) removeDirWhichHasFiles(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "rmdir-upload") dat.testMountDir = filepath.Join(dat.testDir, "rmdir-mount") dat.files = make(map[string]fileInfo) - dat.files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["two/five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["two/six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["one/1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} + dat.files["two/five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)} + dat.files["two/six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -1480,12 +1457,12 @@ func (ta *testAPI) removeDirWhichHasSubDirs(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "rmsubdir-upload") dat.testMountDir = filepath.Join(dat.testDir, "rmsubdir-mount") dat.files = make(map[string]fileInfo) - dat.files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["two/three/2.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["two/three/3.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["two/four/5.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["two/four/6.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["two/four/six/7.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["one/1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} + dat.files["two/three/2.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)} + dat.files["two/three/3.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)} + dat.files["two/four/5.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(4, 10)} + dat.files["two/four/6.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(5, 10)} + dat.files["two/four/six/7.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(6, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -1567,11 +1544,7 @@ func (ta *testAPI) appendFileContentsToEnd(t *testing.T, toEncrypt bool) { dat.testMountDir = filepath.Join(dat.testDir, "appendlargefile-mount") dat.files = make(map[string]fileInfo) - line1 := make([]byte, 10) - _, err = rand.Read(line1) - if err != nil { - t.Fatalf("Error writing random bytes to byte array: %v", err) - } + line1 := testutil.RandomBytes(1, 10) dat.files["1.txt"] = fileInfo{0700, 333, 444, line1} @@ -1588,11 +1561,7 @@ func (ta *testAPI) appendFileContentsToEnd(t *testing.T, toEncrypt bool) { } defer fd.Close() log.Debug("file opened") - line2 := make([]byte, 5) - _, err = rand.Read(line2) - if err != nil { - t.Fatalf("Error writing random bytes to byte array: %v", err) - } + line2 := testutil.RandomBytes(1, 5) log.Debug("line read") _, err = fd.Seek(int64(len(line1)), 0) if err != nil { diff --git a/swarm/network/stream/common_test.go b/swarm/network/stream/common_test.go index 72fdb2bd96..721b873b78 100644 --- a/swarm/network/stream/common_test.go +++ b/swarm/network/stream/common_test.go @@ -18,7 +18,6 @@ package stream import ( "context" - crand "crypto/rand" "errors" "flag" "fmt" @@ -40,6 +39,7 @@ import ( "github.com/ethereum/go-ethereum/swarm/state" "github.com/ethereum/go-ethereum/swarm/storage" mockdb "github.com/ethereum/go-ethereum/swarm/storage/mock/db" + "github.com/ethereum/go-ethereum/swarm/testutil" colorable "github.com/mattn/go-colorable" ) @@ -230,12 +230,7 @@ func generateRandomFile() (string, error) { //generate a random file size between minFileSize and maxFileSize fileSize := rand.Intn(maxFileSize-minFileSize) + minFileSize log.Debug(fmt.Sprintf("Generated file with filesize %d kB", fileSize)) - b := make([]byte, fileSize*1024) - _, err := crand.Read(b) - if err != nil { - log.Error("Error generating random file.", "err", err) - return "", err - } + b := testutil.RandomBytes(1, fileSize*1024) return string(b), nil } diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index c77682e0e1..c9a530115c 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -19,9 +19,7 @@ package stream import ( "bytes" "context" - crand "crypto/rand" "fmt" - "io" "os" "sync" "testing" @@ -39,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/swarm/network/simulation" "github.com/ethereum/go-ethereum/swarm/state" "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/ethereum/go-ethereum/swarm/testutil" ) //Tests initializing a retrieve request @@ -530,7 +529,7 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck //now we can actually upload a (random) file to the round-robin store size := chunkCount * chunkSize log.Debug("Storing data to file store") - fileHash, wait, err := roundRobinFileStore.Store(ctx, io.LimitReader(crand.Reader, int64(size)), int64(size), false) + fileHash, wait, err := roundRobinFileStore.Store(ctx, testutil.RandomReader(1, size), int64(size), false) // wait until all chunks stored if err != nil { return err @@ -719,7 +718,7 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skip for i := 0; i < chunkCount; i++ { // create actual size real chunks ctx := context.TODO() - hash, wait, err := remoteFileStore.Store(ctx, io.LimitReader(crand.Reader, int64(chunkSize)), int64(chunkSize), false) + hash, wait, err := remoteFileStore.Store(ctx, testutil.RandomReader(i, chunkSize), int64(chunkSize), false) if err != nil { b.Fatalf("expected no error. got %v", err) } diff --git a/swarm/network/stream/intervals_test.go b/swarm/network/stream/intervals_test.go index 0c95fabb70..037984f220 100644 --- a/swarm/network/stream/intervals_test.go +++ b/swarm/network/stream/intervals_test.go @@ -18,10 +18,8 @@ package stream import ( "context" - crand "crypto/rand" "encoding/binary" "fmt" - "io" "os" "sync" "testing" @@ -36,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/swarm/network/simulation" "github.com/ethereum/go-ethereum/swarm/state" "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/ethereum/go-ethereum/swarm/testutil" ) func TestIntervalsLive(t *testing.T) { @@ -130,7 +129,8 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { fileStore := item.(*storage.FileStore) size := chunkCount * chunkSize - _, wait, err := fileStore.Store(ctx, io.LimitReader(crand.Reader, int64(size)), int64(size), false) + + _, wait, err := fileStore.Store(ctx, testutil.RandomReader(1, size), int64(size), false) if err != nil { log.Error("Store error: %v", "err", err) t.Fatal(err) diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go index 2ddbed9368..4bd7f38f5b 100644 --- a/swarm/network/stream/snapshot_sync_test.go +++ b/swarm/network/stream/snapshot_sync_test.go @@ -17,9 +17,7 @@ package stream import ( "context" - crand "crypto/rand" "fmt" - "io" "os" "runtime" "sync" @@ -39,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/swarm/state" "github.com/ethereum/go-ethereum/swarm/storage" mockdb "github.com/ethereum/go-ethereum/swarm/storage/mock/db" + "github.com/ethereum/go-ethereum/swarm/testutil" ) const MaxTimeout = 600 @@ -603,7 +602,7 @@ func uploadFileToSingleNodeStore(id enode.ID, chunkCount int, lstore *storage.Lo size := chunkSize var rootAddrs []storage.Address for i := 0; i < chunkCount; i++ { - rk, wait, err := fileStore.Store(context.TODO(), io.LimitReader(crand.Reader, int64(size)), int64(size), false) + rk, wait, err := fileStore.Store(context.TODO(), testutil.RandomReader(i, size), int64(size), false) if err != nil { return nil, err } diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index b0e35b0db3..a543cae058 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -18,9 +18,7 @@ package stream import ( "context" - crand "crypto/rand" "fmt" - "io" "io/ioutil" "math" "os" @@ -39,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/swarm/state" "github.com/ethereum/go-ethereum/swarm/storage" mockdb "github.com/ethereum/go-ethereum/swarm/storage/mock/db" + "github.com/ethereum/go-ethereum/swarm/testutil" ) const dataChunkCount = 200 @@ -183,7 +182,7 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck } fileStore := item.(*storage.FileStore) size := chunkCount * chunkSize - _, wait, err := fileStore.Store(ctx, io.LimitReader(crand.Reader, int64(size)), int64(size), false) + _, wait, err := fileStore.Store(ctx, testutil.RandomReader(j, size), int64(size), false) if err != nil { t.Fatal(err.Error()) } diff --git a/swarm/network_test.go b/swarm/network_test.go index aab1c05099..d84f28147f 100644 --- a/swarm/network_test.go +++ b/swarm/network_test.go @@ -335,7 +335,7 @@ func testSwarmNetwork(t *testing.T, o *testSwarmNetworkOptions, steps ...testSwa result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error { nodeIDs := sim.UpNodeIDs() - shuffle(len(nodeIDs), func(i, j int) { + rand.Shuffle(len(nodeIDs), func(i, j int) { nodeIDs[i], nodeIDs[j] = nodeIDs[j], nodeIDs[i] }) for _, id := range nodeIDs { @@ -404,7 +404,7 @@ func retrieve( nodeStatusM *sync.Map, totalFoundCount *uint64, ) (missing uint64) { - shuffle(len(files), func(i, j int) { + rand.Shuffle(len(files), func(i, j int) { files[i], files[j] = files[j], files[i] }) @@ -499,32 +499,3 @@ func retrieve( return uint64(totalCheckCount) - atomic.LoadUint64(totalFoundCount) } - -// Backported from stdlib https://golang.org/src/math/rand/rand.go?s=11175:11215#L333 -// -// Replace with rand.Shuffle from go 1.10 when go 1.9 support is dropped. -// -// shuffle pseudo-randomizes the order of elements. -// n is the number of elements. Shuffle panics if n < 0. -// swap swaps the elements with indexes i and j. -func shuffle(n int, swap func(i, j int)) { - if n < 0 { - panic("invalid argument to Shuffle") - } - - // Fisher-Yates shuffle: https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle - // Shuffle really ought not be called with n that doesn't fit in 32 bits. - // Not only will it take a very long time, but with 2³¹! possible permutations, - // there's no way that any PRNG can have a big enough internal state to - // generate even a minuscule percentage of the possible permutations. - // Nevertheless, the right API signature accepts an int n, so handle it as best we can. - i := n - 1 - for ; i > 1<<31-1-1; i-- { - j := int(rand.Int63n(int64(i + 1))) - swap(i, j) - } - for ; i > 0; i-- { - j := int(rand.Int31n(int32(i + 1))) - swap(i, j) - } -} diff --git a/swarm/storage/chunker_test.go b/swarm/storage/chunker_test.go index 6172d8a092..1f847edcb9 100644 --- a/swarm/storage/chunker_test.go +++ b/swarm/storage/chunker_test.go @@ -19,13 +19,13 @@ package storage import ( "bytes" "context" - "crypto/rand" "encoding/binary" "fmt" "io" "testing" "github.com/ethereum/go-ethereum/crypto/sha3" + "github.com/ethereum/go-ethereum/swarm/testutil" ) /* @@ -47,7 +47,7 @@ func newTestHasherStore(store ChunkStore, hash string) *hasherStore { } func testRandomBrokenData(n int, tester *chunkerTester) { - data := io.LimitReader(rand.Reader, int64(n)) + data := testutil.RandomReader(1, n) brokendata := brokenLimitReader(data, n, n/2) buf := make([]byte, n) @@ -56,7 +56,7 @@ func testRandomBrokenData(n int, tester *chunkerTester) { tester.t.Fatalf("Broken reader is not broken, hence broken. Returns: %v", err) } - data = io.LimitReader(rand.Reader, int64(n)) + data = testutil.RandomReader(2, n) brokendata = brokenLimitReader(data, n, n/2) putGetter := newTestHasherStore(NewMapChunkStore(), SHA3Hash) @@ -77,7 +77,8 @@ func testRandomData(usePyramid bool, hash string, n int, tester *chunkerTester) input, found := tester.inputs[uint64(n)] var data io.Reader if !found { - data, input = GenerateRandomData(n) + input = testutil.RandomBytes(1, n) + data = bytes.NewReader(input) tester.inputs[uint64(n)] = input } else { data = io.LimitReader(bytes.NewReader(input), int64(n)) @@ -118,14 +119,13 @@ func testRandomData(usePyramid bool, hash string, n int, tester *chunkerTester) // testing partial read for i := 1; i < n; i += 10000 { readableLength := n - i - output := make([]byte, readableLength) r, err := reader.ReadAt(output, int64(i)) if r != readableLength || err != io.EOF { tester.t.Fatalf("readAt error with offset %v read: %v n = %v err = %v\n", i, r, readableLength, err) } if input != nil { - if !bytes.Equal(output, input[i:]) { - tester.t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", input[i:], output) + if !bytes.Equal(output[:readableLength], input[i:]) { + tester.t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", input[i:], output[:readableLength]) } } } @@ -177,7 +177,8 @@ func TestDataAppend(t *testing.T) { input, found := tester.inputs[uint64(n)] var data io.Reader if !found { - data, input = GenerateRandomData(n) + input = testutil.RandomBytes(i, n) + data = bytes.NewReader(input) tester.inputs[uint64(n)] = input } else { data = io.LimitReader(bytes.NewReader(input), int64(n)) @@ -199,7 +200,8 @@ func TestDataAppend(t *testing.T) { appendInput, found := tester.inputs[uint64(m)] var appendData io.Reader if !found { - appendData, appendInput = GenerateRandomData(m) + appendInput = testutil.RandomBytes(i, m) + appendData = bytes.NewReader(appendInput) tester.inputs[uint64(m)] = appendInput } else { appendData = io.LimitReader(bytes.NewReader(appendInput), int64(m)) @@ -272,7 +274,7 @@ func benchReadAll(reader LazySectionReader) { func benchmarkSplitJoin(n int, t *testing.B) { t.ReportAllocs() for i := 0; i < t.N; i++ { - data := testDataReader(n) + data := testutil.RandomReader(i, n) putGetter := newTestHasherStore(NewMapChunkStore(), SHA3Hash) ctx := context.TODO() @@ -292,7 +294,7 @@ func benchmarkSplitJoin(n int, t *testing.B) { func benchmarkSplitTreeSHA3(n int, t *testing.B) { t.ReportAllocs() for i := 0; i < t.N; i++ { - data := testDataReader(n) + data := testutil.RandomReader(i, n) putGetter := newTestHasherStore(&FakeChunkStore{}, SHA3Hash) ctx := context.Background() @@ -311,7 +313,7 @@ func benchmarkSplitTreeSHA3(n int, t *testing.B) { func benchmarkSplitTreeBMT(n int, t *testing.B) { t.ReportAllocs() for i := 0; i < t.N; i++ { - data := testDataReader(n) + data := testutil.RandomReader(i, n) putGetter := newTestHasherStore(&FakeChunkStore{}, BMTHash) ctx := context.Background() @@ -329,7 +331,7 @@ func benchmarkSplitTreeBMT(n int, t *testing.B) { func benchmarkSplitPyramidBMT(n int, t *testing.B) { t.ReportAllocs() for i := 0; i < t.N; i++ { - data := testDataReader(n) + data := testutil.RandomReader(i, n) putGetter := newTestHasherStore(&FakeChunkStore{}, BMTHash) ctx := context.Background() @@ -347,7 +349,7 @@ func benchmarkSplitPyramidBMT(n int, t *testing.B) { func benchmarkSplitPyramidSHA3(n int, t *testing.B) { t.ReportAllocs() for i := 0; i < t.N; i++ { - data := testDataReader(n) + data := testutil.RandomReader(i, n) putGetter := newTestHasherStore(&FakeChunkStore{}, SHA3Hash) ctx := context.Background() @@ -365,8 +367,8 @@ func benchmarkSplitPyramidSHA3(n int, t *testing.B) { func benchmarkSplitAppendPyramid(n, m int, t *testing.B) { t.ReportAllocs() for i := 0; i < t.N; i++ { - data := testDataReader(n) - data1 := testDataReader(m) + data := testutil.RandomReader(i, n) + data1 := testutil.RandomReader(t.N+i, m) store := NewMapChunkStore() putGetter := newTestHasherStore(store, SHA3Hash) diff --git a/swarm/storage/common_test.go b/swarm/storage/common_test.go index 600be164a9..af104a5aeb 100644 --- a/swarm/storage/common_test.go +++ b/swarm/storage/common_test.go @@ -19,7 +19,6 @@ package storage import ( "bytes" "context" - "crypto/rand" "flag" "fmt" "io" @@ -31,7 +30,7 @@ import ( "github.com/ethereum/go-ethereum/log" ch "github.com/ethereum/go-ethereum/swarm/chunk" - colorable "github.com/mattn/go-colorable" + "github.com/mattn/go-colorable" ) var ( @@ -151,10 +150,6 @@ func mget(store ChunkStore, hs []Address, f func(h Address, chunk Chunk) error) return err } -func testDataReader(l int) (r io.Reader) { - return io.LimitReader(rand.Reader, int64(l)) -} - func (r *brokenLimitedReader) Read(buf []byte) (int, error) { if r.off+len(buf) > r.errAt { return 0, fmt.Errorf("Broken reader") diff --git a/swarm/storage/encryption/encryption_test.go b/swarm/storage/encryption/encryption_test.go index c3abefdcec..0c0d0508c3 100644 --- a/swarm/storage/encryption/encryption_test.go +++ b/swarm/storage/encryption/encryption_test.go @@ -18,12 +18,12 @@ package encryption import ( "bytes" - "crypto/rand" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto/sha3" + "github.com/ethereum/go-ethereum/swarm/testutil" ) var expectedTransformedHex = "352187af3a843decc63ceca6cb01ea39dbcf77caf0a8f705f5c30d557044ceec9392b94a79376f1e5c10cd0c0f2a98e5353bf22b3ea4fdac6677ee553dec192e3db64e179d0474e96088fb4abd2babd67de123fb398bdf84d818f7bda2c1ab60b3ea0e0569ae54aa969658eb4844e6960d2ff44d7c087ee3aaffa1c0ee5df7e50b615f7ad90190f022934ad5300c7d1809bfe71a11cc04cece5274eb97a5f20350630522c1dbb7cebaf4f97f84e03f5cfd88f2b48880b25d12f4d5e75c150f704ef6b46c72e07db2b705ac3644569dccd22fd8f964f6ef787fda63c46759af334e6f665f70eac775a7017acea49f3c7696151cb1b9434fa4ac27fb803921ffb5ec58dafa168098d7d5b97e384be3384cf5bc235c3d887fef89fe76c0065f9b8d6ad837b442340d9e797b46ef5709ea3358bc415df11e4830de986ef0f1c418ffdcc80e9a3cda9bea0ab5676c0d4240465c43ba527e3b4ea50b4f6255b510e5d25774a75449b0bd71e56c537ade4fcf0f4d63c99ae1dbb5a844971e2c19941b8facfcfc8ee3056e7cb3c7114c5357e845b52f7103cb6e00d2308c37b12baa5b769e1cc7b00fc06f2d16e70cc27a82cb9c1a4e40cb0d43907f73df2c9db44f1b51a6b0bc6d09f77ac3be14041fae3f9df2da42df43ae110904f9ecee278030185254d7c6e918a5512024d047f77a992088cb3190a6587aa54d0c7231c1cd2e455e0d4c07f74bece68e29cd8ba0190c0bcfb26d24634af5d91a81ef5d4dd3d614836ce942ddbf7bb1399317f4c03faa675f325f18324bf9433844bfe5c4cc04130c8d5c329562b7cd66e72f7355de8f5375a72202971613c32bd7f3fcdcd51080758cd1d0a46dbe8f0374381dbc359f5864250c63dde8131cbd7c98ae2b0147d6ea4bf65d1443d511b18e6d608bbb46ac036353b4c51df306a10a6f6939c38629a5c18aaf89cac04bd3ad5156e6b92011c88341cb08551bab0a89e6a46538f5af33b86121dba17e3a434c273f385cd2e8cb90bdd32747d8425d929ccbd9b0815c73325988855549a8489dfd047daf777aaa3099e54cf997175a5d9e1edfe363e3b68c70e02f6bf4fcde6a0f3f7d0e7e98bde1a72ae8b6cd27b32990680cc4a04fc467f41c5adcaddabfc71928a3f6872c360c1d765260690dd28b269864c8e380d9c92ef6b89b0094c8f9bb22608b4156381b19b920e9583c9616ce5693b4d2a6c689f02e6a91584a8e501e107403d2689dd0045269dd9946c0e969fb656a3b39d84a798831f5f9290f163eb2f97d3ae25071324e95e2256d9c1e56eb83c26397855323edc202d56ad05894333b7f0ed3c1e4734782eb8bd5477242fd80d7a89b12866f85cfae476322f032465d6b1253993033fccd4723530630ab97a1566460af9c90c9da843c229406e65f3fa578bd6bf04dee9b6153807ddadb8ceefc5c601a8ab26023c67b1ab1e8e0f29ce94c78c308005a781853e7a2e0e51738939a657c987b5e611f32f47b5ff461c52e63e0ea390515a8e1f5393dae54ea526934b5f310b76e3fa050e40718cb4c8a20e58946d6ee1879f08c52764422fe542b3240e75eccb7aa75b1f8a651e37a3bc56b0932cdae0e985948468db1f98eb4b77b82081ea25d8a762db00f7898864984bd80e2f3f35f236bf57291dec28f550769943bcfb6f884b7687589b673642ef7fe5d7d5a87d3eca5017f83ccb9a3310520474479464cb3f433440e7e2f1e28c0aef700a45848573409e7ab66e0cfd4fe5d2147ace81bc65fd8891f6245cd69246bbf5c27830e5ab882dd1d02aba34ff6ca9af88df00fd602892f02fedbdc65dedec203faf3f8ff4a97314e0ddb58b9ab756a61a562597f4088b445fcc3b28a708ca7b1485dcd791b779fbf2b3ef1ec5c6205f595fbe45a02105034147e5a146089c200a49dae33ae051a08ea5f974a21540aaeffa7f9d9e3d35478016fb27b871036eb27217a5b834b461f535752fb5f1c8dded3ae14ce3a2ef6639e2fe41939e3509e46e347a95d50b2080f1ba42c804b290ddc912c952d1cec3f2661369f738feacc0dbf1ea27429c644e45f9e26f30c341acd34c7519b2a1663e334621691e810767e9918c2c547b2e23cce915f97d26aac8d0d2fcd3edb7986ad4e2b8a852edebad534cb6c0e9f0797d3563e5409d7e068e48356c67ce519246cd9c560e881453df97cbba562018811e6cf8c327f399d1d1253ab47a19f4a0ccc7c6d86a9603e0551da310ea595d71305c4aad96819120a92cdbaf1f77ec8df9cc7c838c0d4de1e8692dd81da38268d1d71324bcffdafbe5122e4b81828e021e936d83ae8021eac592aa52cd296b5ce392c7173d622f8e07d18f59bb1b08ba15211af6703463b09b593af3c37735296816d9f2e7a369354a5374ea3955e14ca8ac56d5bfe4aef7a21bd825d6ae85530bee5d2aaaa4914981b3dfdb2e92ec2a27c83d74b59e84ff5c056f7d8945745f2efc3dcf28f288c6cd8383700fb2312f7001f24dd40015e436ae23e052fe9070ea9535b9c989898a9bda3d5382cf10e432fae6ccf0c825b3e6436edd3a9f8846e5606f8563931b5f29ba407c5236e5730225dda211a8504ec1817bc935e1fd9a532b648c502df302ed2063aed008fd5676131ac9e95998e9447b02bd29d77e38fcfd2959f2de929b31970335eb2a74348cc6918bc35b9bf749eab0fe304c946cd9e1ca284e6853c42646e60b6b39e0d3fb3c260abfc5c1b4ca3c3770f344118ca7c7f5c1ad1f123f8f369cd60afc3cdb3e9e81968c5c9fa7c8b014ffe0508dd4f0a2a976d5d1ca8fc9ad7a237d92cfe7b41413d934d6e142824b252699397e48e4bac4e91ebc10602720684bd0863773c548f9a2f9724245e47b129ecf65afd7252aac48c8a8d6fd3d888af592a01fb02dc71ed7538a700d3d16243e4621e0fcf9f8ed2b4e11c9fa9a95338bb1dac74a7d9bc4eb8cbf900b634a2a56469c00f5994e4f0934bdb947640e6d67e47d0b621aacd632bfd3c800bd7d93bd329f494a90e06ed51535831bd6e07ac1b4b11434ef3918fa9511813a002913f33f836454798b8d1787fea9a4c4743ba091ed192ed92f4d33e43a226bf9503e1a83a16dd340b3cbbf38af6db0d99201da8de529b4225f3d2fa2aad6621afc6c79ef3537720591edfc681ae6d00ede53ed724fc71b23b90d2e9b7158aaee98d626a4fe029107df2cb5f90147e07ebe423b1519d848af18af365c71bfd0665db46be493bbe99b79a188de0cf3594aef2299f0324075bdce9eb0b87bc29d62401ba4fd6ae48b1ba33261b5b845279becf38ee03e3dc5c45303321c5fac96fd02a3ad8c9e3b02127b320501333c9e6360440d1ad5e64a6239501502dde1a49c9abe33b66098458eee3d611bb06ffcd234a1b9aef4af5021cd61f0de6789f822ee116b5078aae8c129e8391d8987500d322b58edd1595dc570b57341f2df221b94a96ab7fbcf32a8ca9684196455694024623d7ed49f7d66e8dd453c0bae50e0d8b34377b22d0ece059e2c385dfc70b9089fcd27577c51f4d870b5738ee2b68c361a67809c105c7848b68860a829f29930857a9f9d40b14fd2384ac43bafdf43c0661103794c4bd07d1cfdd4681b6aeaefad53d4c1473359bcc5a83b09189352e5bb9a7498dd0effb89c35aad26954551f8b0621374b449bf515630bd3974dca982279733470fdd059aa9c3df403d8f22b38c4709c82d8f12b888e22990350490e16179caf406293cc9e65f116bafcbe96af132f679877061107a2f690a82a8cb46eea57a90abd23798c5937c6fe6b17be3f9bfa01ce117d2c268181b9095bf49f395fea07ca03838de0588c5e2db633e836d64488c1421e653ea52d810d096048c092d0da6e02fa6613890219f51a76148c8588c2487b171a28f17b7a299204874af0131725d793481333be5f08e86ca837a226850b0c1060891603bfecf9e55cddd22c0dbb28d495342d9cc3de8409f72e52a0115141cffe755c74f061c1a770428ccb0ae59536ee6fc074fbfc6cacb51a549d327527e20f8407477e60355863f1153f9ce95641198663c968874e7fdb29407bd771d94fdda8180cbb0358f5874738db705924b8cbe0cd5e1484aeb64542fe8f38667b7c34baf818c63b1e18440e9fba575254d063fd49f24ef26432f4eb323f3836972dca87473e3e9bb26dc3be236c3aae6bc8a6da567442309da0e8450e242fc9db836e2964f2c76a3b80a2c677979882dda7d7ebf62c93664018bcf4ec431fe6b403d49b3b36618b9c07c2d0d4569cb8d52223903debc72ec113955b206c34f1ae5300990ccfc0180f47d91afdb542b6312d12aeff7e19c645dc0b9fe6e3288e9539f6d5870f99882df187bfa6d24d179dfd1dac22212c8b5339f7171a3efc15b760fed8f68538bc5cbd845c2d1ab41f3a6c692820653eaef7930c02fbe6061d93805d73decdbb945572a7c44ed0241982a6e4d2d730898f82b3d9877cb7bca41cc6dcee67aa0c3d6db76f0b0a708ace0031113e48429de5d886c10e9200f68f32263a2fbf44a5992c2459fda7b8796ba796e3a0804fc25992ed2c9a5fe0580a6b809200ecde6caa0364b58be11564dcb9a616766dd7906db5636ee708b0204f38d309466d8d4a162965dd727e29f5a6c133e9b4ed5bafe803e479f9b2a7640c942c4a40b14ac7dc9828546052761a070f6404008f1ec3605836339c3da95a00b4fd81b2cabf88b51d2087d5b83e8c5b69bf96d8c72cbd278dad3bbb42b404b436f84ad688a22948adf60a81090f1e904291503c16e9f54b05fc76c881a5f95f0e732949e95d3f1bae2d3652a14fe0dda2d68879604657171856ef72637def2a96ac47d7b3fe86eb3198f5e0e626f06be86232305f2ae79ffcd2725e48208f9d8d63523f81915acc957563ab627cd6bc68c2a37d59fb0ed77a90aa9d085d6914a8ebada22a2c2d471b5163aeddd799d90fbb10ed6851ace2c4af504b7d572686700a59d6db46d5e42bb83f8e0c0ffe1dfa6582cc0b34c921ff6e85e83188d24906d5c08bb90069639e713051b3102b53e6f703e8210017878add5df68e6f2b108de279c5490e9eef5590185c4a1c744d4e00d244e1245a8805bd30407b1bc488db44870ccfd75a8af104df78efa2fb7ba31f048a263efdb3b63271fff4922bece9a71187108f65744a24f4947dc556b7440cb4fa45d296bb7f724588d1f245125b21ea063500029bd49650237f53899daf1312809552c81c5827341263cc807a29fe84746170cdfa1ff3838399a5645319bcaff674bb70efccdd88b3d3bb2f2d98111413585dc5d5bd5168f43b3f55e58972a5b2b9b3733febf02f931bd436648cb617c3794841aab961fe41277ab07812e1d3bc4ff6f4350a3e615bfba08c3b9480ef57904d3a16f7e916345202e3f93d11f7a7305170cb8c4eb9ac88ace8bbd1f377bdd5855d3162d6723d4435e84ce529b8f276a8927915ac759a0d04e5ca4a9d3da6291f0333b475df527e99fe38f7a4082662e8125936640c26dd1d17cf284ce6e2b17777a05aa0574f7793a6a062cc6f7263f7ab126b4528a17becfdec49ac0f7d8705aa1704af97fb861faa8a466161b2b5c08a5bacc79fe8500b913d65c8d3c52d1fd52d2ab2c9f52196e712455619c1cd3e0f391b274487944240e2ed8858dd0823c801094310024ae3fe4dd1cf5a2b6487b42cc5937bbafb193ee331d87e378258963d49b9da90899bbb4b88e79f78e866b0213f4719f67da7bcc2fce073c01e87c62ea3cdbcd589cfc41281f2f4c757c742d6d1e" @@ -125,8 +125,7 @@ func testEncryptDecryptIsIdentity(t *testing.T, padding int, initCtr uint32, dat key := GenerateRandomKey(keyLength) enc := New(key, padding, initCtr, hashFunc) - data := make([]byte, dataLength) - rand.Read(data) + data := testutil.RandomBytes(1, dataLength) encrypted, err := enc.Encrypt(data) if err != nil { diff --git a/swarm/storage/filestore_test.go b/swarm/storage/filestore_test.go index 9fe58f60c8..fb0f761a4a 100644 --- a/swarm/storage/filestore_test.go +++ b/swarm/storage/filestore_test.go @@ -23,6 +23,8 @@ import ( "io/ioutil" "os" "testing" + + "github.com/ethereum/go-ethereum/swarm/testutil" ) const testDataSize = 0x0001000 @@ -49,9 +51,9 @@ func testFileStoreRandom(toEncrypt bool, t *testing.T) { fileStore := NewFileStore(localStore, NewFileStoreParams()) defer os.RemoveAll("/tmp/bzz") - reader, slice := GenerateRandomData(testDataSize) + slice := testutil.RandomBytes(1, testDataSize) ctx := context.TODO() - key, wait, err := fileStore.Store(ctx, reader, testDataSize, toEncrypt) + key, wait, err := fileStore.Store(ctx, bytes.NewReader(slice), testDataSize, toEncrypt) if err != nil { t.Fatalf("Store error: %v", err) } @@ -63,13 +65,13 @@ func testFileStoreRandom(toEncrypt bool, t *testing.T) { if isEncrypted != toEncrypt { t.Fatalf("isEncrypted expected %v got %v", toEncrypt, isEncrypted) } - resultSlice := make([]byte, len(slice)) + resultSlice := make([]byte, testDataSize) n, err := resultReader.ReadAt(resultSlice, 0) if err != io.EOF { t.Fatalf("Retrieve error: %v", err) } - if n != len(slice) { - t.Fatalf("Slice size error got %d, expected %d.", n, len(slice)) + if n != testDataSize { + t.Fatalf("Slice size error got %d, expected %d.", n, testDataSize) } if !bytes.Equal(slice, resultSlice) { t.Fatalf("Comparison error.") @@ -114,9 +116,9 @@ func testFileStoreCapacity(toEncrypt bool, t *testing.T) { DbStore: db, } fileStore := NewFileStore(localStore, NewFileStoreParams()) - reader, slice := GenerateRandomData(testDataSize) + slice := testutil.RandomBytes(1, testDataSize) ctx := context.TODO() - key, wait, err := fileStore.Store(ctx, reader, testDataSize, toEncrypt) + key, wait, err := fileStore.Store(ctx, bytes.NewReader(slice), testDataSize, toEncrypt) if err != nil { t.Errorf("Store error: %v", err) } diff --git a/swarm/storage/types.go b/swarm/storage/types.go index 092843db0c..42557766ef 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -25,7 +25,6 @@ import ( "fmt" "hash" "io" - "io/ioutil" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto/sha3" @@ -251,16 +250,6 @@ func GenerateRandomChunks(dataSize int64, count int) (chunks []Chunk) { return chunks } -func GenerateRandomData(l int) (r io.Reader, slice []byte) { - slice, err := ioutil.ReadAll(io.LimitReader(rand.Reader, int64(l))) - if err != nil { - panic("rand error") - } - // log.Warn("generate random data", "len", len(slice), "data", common.Bytes2Hex(slice)) - r = io.LimitReader(bytes.NewReader(slice), int64(l)) - return r, slice -} - // Size, Seek, Read, ReadAt type LazySectionReader interface { Context() context.Context diff --git a/swarm/testutil/file.go b/swarm/testutil/file.go index ecb0d971ed..70732aa92e 100644 --- a/swarm/testutil/file.go +++ b/swarm/testutil/file.go @@ -17,8 +17,10 @@ package testutil import ( + "bytes" "io" "io/ioutil" + "math/rand" "os" "strings" "testing" @@ -42,3 +44,22 @@ func TempFileWithContent(t *testing.T, content string) string { } return tempFile.Name() } + +// RandomBytes returns pseudo-random deterministic result +// because test fails must be reproducible +func RandomBytes(seed, length int) []byte { + b := make([]byte, length) + reader := rand.New(rand.NewSource(int64(seed))) + for n := 0; n < length; { + read, err := reader.Read(b[n:]) + if err != nil { + panic(err) + } + n += read + } + return b +} + +func RandomReader(seed, length int) *bytes.Reader { + return bytes.NewReader(RandomBytes(seed, length)) +} From 58632d44021bf095b43a1bb2443e6e3690a94739 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 14 Nov 2018 10:22:10 +0200 Subject: [PATCH 041/159] params, swarm: release Geth v1.8.18 and Swarm v0.3.6 --- params/version.go | 8 ++++---- swarm/version/version.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/params/version.go b/params/version.go index 9e2f540c32..1abbd1a748 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 18 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 8 // Minor version component of the current release + VersionPatch = 18 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. diff --git a/swarm/version/version.go b/swarm/version/version.go index c70c30943d..0f5a453daa 100644 --- a/swarm/version/version.go +++ b/swarm/version/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 0 // Major version component of the current release - VersionMinor = 3 // Minor version component of the current release - VersionPatch = 6 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 0 // Major version component of the current release + VersionMinor = 3 // Minor version component of the current release + VersionPatch = 6 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From 48b4e8069cc1505a68bf69e8209dc17a9ecbfc96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 14 Nov 2018 10:27:30 +0200 Subject: [PATCH 042/159] params, swarm: begin Geth v1.8.19 and Swarm v0.3.7 cycle --- params/version.go | 8 ++++---- swarm/version/version.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/params/version.go b/params/version.go index 1abbd1a748..b9dcc2a842 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 18 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 8 // Minor version component of the current release + VersionPatch = 19 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. diff --git a/swarm/version/version.go b/swarm/version/version.go index 0f5a453daa..17ef34f5f4 100644 --- a/swarm/version/version.go +++ b/swarm/version/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 0 // Major version component of the current release - VersionMinor = 3 // Minor version component of the current release - VersionPatch = 6 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 0 // Major version component of the current release + VersionMinor = 3 // Minor version component of the current release + VersionPatch = 7 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From 698843b45f0dddd6f862c0733838272e6b4aa7f8 Mon Sep 17 00:00:00 2001 From: Kenso Trabing Date: Wed, 14 Nov 2018 05:21:10 -0500 Subject: [PATCH 043/159] rpc: fix example typo (#18100) whishes --> wishes --- rpc/client_example_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpc/client_example_test.go b/rpc/client_example_test.go index 9c21c12d5d..3bb8717b80 100644 --- a/rpc/client_example_test.go +++ b/rpc/client_example_test.go @@ -25,7 +25,7 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) -// In this example, our client whishes to track the latest 'block number' +// In this example, our client wishes to track the latest 'block number' // known to the server. The server supports two methods: // // eth_getBlockByNumber("latest", {}) From 23de6197f9104a17ece1dc370c862bd23dcae273 Mon Sep 17 00:00:00 2001 From: Kenso Trabing Date: Wed, 14 Nov 2018 05:21:52 -0500 Subject: [PATCH 044/159] rpc: fix package doc typo (#18101) Changed "send" to "send," in two places --- rpc/doc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpc/doc.go b/rpc/doc.go index 9a6c4abbce..c60381b5ae 100644 --- a/rpc/doc.go +++ b/rpc/doc.go @@ -32,7 +32,7 @@ An example method: func (s *CalcService) Add(a, b int) (int, error) When the returned error isn't nil the returned integer is ignored and the error is -send back to the client. Otherwise the returned integer is send back to the client. +sent back to the client. Otherwise the returned integer is sent back to the client. Optional arguments are supported by accepting pointer values as arguments. E.g. if we want to do the addition in an optional finite field we can accept a mod From 9a000601c6c4e4f8134caedba1957ffe28d2b659 Mon Sep 17 00:00:00 2001 From: mr_franklin Date: Wed, 14 Nov 2018 20:50:30 +0800 Subject: [PATCH 045/159] consensus/clique: fix comment typo (#18103) --- consensus/clique/clique.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index eae09f91df..0cb72c35c7 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -696,7 +696,7 @@ func (c *Clique) SealHash(header *types.Header) common.Hash { return sigHash(header) } -// Close implements consensus.Engine. It's a noop for clique as there is are no background threads. +// Close implements consensus.Engine. It's a noop for clique as there are no background threads. func (c *Clique) Close() error { return nil } From b8a2ac3fcf4c8249bd997969821feb4f9d0a428a Mon Sep 17 00:00:00 2001 From: Sheldon <11510383@mail.sustc.edu.cn> Date: Thu, 15 Nov 2018 17:10:45 +0800 Subject: [PATCH 046/159] les: fix pubkey index typo (#18093) --- les/serverpool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/les/serverpool.go b/les/serverpool.go index 0fe6e49b61..52b54b371f 100644 --- a/les/serverpool.go +++ b/les/serverpool.go @@ -683,7 +683,7 @@ func (e *poolEntry) DecodeRLP(s *rlp.Stream) error { } func encodePubkey64(pub *ecdsa.PublicKey) []byte { - return crypto.FromECDSAPub(pub)[:1] + return crypto.FromECDSAPub(pub)[1:] } func decodePubkey64(b []byte) (*ecdsa.PublicKey, error) { From 14346e4ef97ca812fb9f1d5d2cd87021c0155cf6 Mon Sep 17 00:00:00 2001 From: Kenso Trabing Date: Thu, 15 Nov 2018 04:11:14 -0500 Subject: [PATCH 047/159] internal: fix typo in comments (#18106) Changed "signTransactions" to "signTransaction" --- internal/ethapi/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 0c751c3282..43a33e9921 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -339,7 +339,7 @@ func (s *PrivateAccountAPI) LockAccount(addr common.Address) bool { return fetchKeystore(s.am).Lock(addr) == nil } -// signTransactions sets defaults and signs the given transaction +// signTransaction sets defaults and signs the given transaction // NOTE: the caller needs to ensure that the nonceLock is held, if applicable, // and release it after the transaction has been submitted to the tx pool func (s *PrivateAccountAPI) signTransaction(ctx context.Context, args *SendTxArgs, passwd string) (*types.Transaction, error) { From 434dd5bc0067cdf604d84426df9086015721dd36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 12 Nov 2018 18:47:34 +0200 Subject: [PATCH 048/159] cmd, core, eth, light, trie: add trie read caching layer --- cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 24 +- core/blockchain.go | 21 +- core/state/database.go | 14 +- eth/api.go | 6 +- eth/api_tracer.go | 4 +- eth/backend.go | 2 +- eth/config.go | 22 +- eth/gen_config.go | 28 ++- light/postprocess.go | 4 +- tests/block_test_util.go | 2 +- trie/database.go | 205 +++++++++------ trie/iterator_test.go | 14 +- trie/trie_test.go | 9 +- vendor/github.com/allegro/bigcache/LICENSE | 201 +++++++++++++++ vendor/github.com/allegro/bigcache/README.md | 150 +++++++++++ .../github.com/allegro/bigcache/bigcache.go | 202 +++++++++++++++ vendor/github.com/allegro/bigcache/bytes.go | 14 ++ .../allegro/bigcache/bytes_appengine.go | 7 + vendor/github.com/allegro/bigcache/clock.go | 14 ++ vendor/github.com/allegro/bigcache/config.go | 86 +++++++ .../github.com/allegro/bigcache/encoding.go | 62 +++++ .../allegro/bigcache/entry_not_found_error.go | 17 ++ vendor/github.com/allegro/bigcache/fnv.go | 28 +++ vendor/github.com/allegro/bigcache/hash.go | 8 + .../github.com/allegro/bigcache/iterator.go | 122 +++++++++ vendor/github.com/allegro/bigcache/logger.go | 30 +++ .../allegro/bigcache/queue/bytes_queue.go | 210 ++++++++++++++++ vendor/github.com/allegro/bigcache/shard.go | 236 ++++++++++++++++++ vendor/github.com/allegro/bigcache/stats.go | 15 ++ vendor/github.com/allegro/bigcache/utils.go | 16 ++ vendor/vendor.json | 12 + 33 files changed, 1660 insertions(+), 127 deletions(-) create mode 100644 vendor/github.com/allegro/bigcache/LICENSE create mode 100644 vendor/github.com/allegro/bigcache/README.md create mode 100644 vendor/github.com/allegro/bigcache/bigcache.go create mode 100644 vendor/github.com/allegro/bigcache/bytes.go create mode 100644 vendor/github.com/allegro/bigcache/bytes_appengine.go create mode 100644 vendor/github.com/allegro/bigcache/clock.go create mode 100644 vendor/github.com/allegro/bigcache/config.go create mode 100644 vendor/github.com/allegro/bigcache/encoding.go create mode 100644 vendor/github.com/allegro/bigcache/entry_not_found_error.go create mode 100644 vendor/github.com/allegro/bigcache/fnv.go create mode 100644 vendor/github.com/allegro/bigcache/hash.go create mode 100644 vendor/github.com/allegro/bigcache/iterator.go create mode 100644 vendor/github.com/allegro/bigcache/logger.go create mode 100644 vendor/github.com/allegro/bigcache/queue/bytes_queue.go create mode 100644 vendor/github.com/allegro/bigcache/shard.go create mode 100644 vendor/github.com/allegro/bigcache/stats.go create mode 100644 vendor/github.com/allegro/bigcache/utils.go diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 0288b33806..69802a48a7 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -89,6 +89,7 @@ var ( utils.LightKDFFlag, utils.CacheFlag, utils.CacheDatabaseFlag, + utils.CacheTrieFlag, utils.CacheGCFlag, utils.TrieCacheGenFlag, utils.ListenPortFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 8b0491ce3c..82f17e0ee8 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -132,6 +132,7 @@ var AppHelpFlagGroups = []flagGroup{ Flags: []cli.Flag{ utils.CacheFlag, utils.CacheDatabaseFlag, + utils.CacheTrieFlag, utils.CacheGCFlag, utils.TrieCacheGenFlag, }, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 429c2bbb9b..d7b698c7e2 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -295,7 +295,12 @@ var ( CacheDatabaseFlag = cli.IntFlag{ Name: "cache.database", Usage: "Percentage of cache memory allowance to use for database io", - Value: 75, + Value: 50, + } + CacheTrieFlag = cli.IntFlag{ + Name: "cache.trie", + Usage: "Percentage of cache memory allowance to use for trie caching", + Value: 25, } CacheGCFlag = cli.IntFlag{ Name: "cache.gc", @@ -1157,8 +1162,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { } cfg.NoPruning = ctx.GlobalString(GCModeFlag.Name) == "archive" + if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) { + cfg.TrieCleanCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100 + } if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) { - cfg.TrieCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100 + cfg.TrieDirtyCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100 } if ctx.GlobalIsSet(MinerNotifyFlag.Name) { cfg.MinerNotify = strings.Split(ctx.GlobalString(MinerNotifyFlag.Name), ",") @@ -1393,12 +1401,16 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name) } cache := &core.CacheConfig{ - Disabled: ctx.GlobalString(GCModeFlag.Name) == "archive", - TrieNodeLimit: eth.DefaultConfig.TrieCache, - TrieTimeLimit: eth.DefaultConfig.TrieTimeout, + Disabled: ctx.GlobalString(GCModeFlag.Name) == "archive", + TrieCleanLimit: eth.DefaultConfig.TrieCleanCache, + TrieDirtyLimit: eth.DefaultConfig.TrieDirtyCache, + TrieTimeLimit: eth.DefaultConfig.TrieTimeout, + } + if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) { + cache.TrieCleanLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100 } if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) { - cache.TrieNodeLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100 + cache.TrieDirtyLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100 } vmcfg := vm.Config{EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name)} chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg, nil) diff --git a/core/blockchain.go b/core/blockchain.go index 26ac75b8cf..22f130ce67 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -68,9 +68,10 @@ const ( // CacheConfig contains the configuration values for the trie caching/pruning // that's resident in a blockchain. type CacheConfig struct { - Disabled bool // Whether to disable trie write caching (archive node) - TrieNodeLimit int // Memory limit (MB) at which to flush the current in-memory trie to disk - TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk + Disabled bool // Whether to disable trie write caching (archive node) + TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory + TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk + TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk } // BlockChain represents the canonical chain given a database with a genesis @@ -140,8 +141,9 @@ type BlockChain struct { func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(block *types.Block) bool) (*BlockChain, error) { if cacheConfig == nil { cacheConfig = &CacheConfig{ - TrieNodeLimit: 256, - TrieTimeLimit: 5 * time.Minute, + TrieCleanLimit: 256, + TrieDirtyLimit: 256, + TrieTimeLimit: 5 * time.Minute, } } bodyCache, _ := lru.New(bodyCacheLimit) @@ -156,7 +158,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par cacheConfig: cacheConfig, db: db, triegc: prque.New(nil), - stateCache: state.NewDatabase(db), + stateCache: state.NewDatabaseWithCache(db, cacheConfig.TrieCleanLimit), quit: make(chan struct{}), shouldPreserve: shouldPreserve, bodyCache: bodyCache, @@ -393,6 +395,11 @@ func (bc *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) { return state.New(root, bc.stateCache) } +// StateCache returns the caching database underpinning the blockchain instance. +func (bc *BlockChain) StateCache() state.Database { + return bc.stateCache +} + // Reset purges the entire blockchain, restoring it to its genesis state. func (bc *BlockChain) Reset() error { return bc.ResetWithGenesisBlock(bc.genesisBlock) @@ -938,7 +945,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. // If we exceeded our memory allowance, flush matured singleton nodes to disk var ( nodes, imgs = triedb.Size() - limit = common.StorageSize(bc.cacheConfig.TrieNodeLimit) * 1024 * 1024 + limit = common.StorageSize(bc.cacheConfig.TrieDirtyLimit) * 1024 * 1024 ) if nodes > limit || imgs > 4*1024*1024 { triedb.Cap(limit - ethdb.IdealBatchSize) diff --git a/core/state/database.go b/core/state/database.go index c1b630991c..f6ea144b9b 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -72,13 +72,19 @@ type Trie interface { } // NewDatabase creates a backing store for state. The returned database is safe for -// concurrent use and retains cached trie nodes in memory. The pool is an optional -// intermediate trie-node memory pool between the low level storage layer and the -// high level trie abstraction. +// concurrent use and retains a few recent expanded trie nodes in memory. To keep +// more historical state in memory, use the NewDatabaseWithCache constructor. func NewDatabase(db ethdb.Database) Database { + return NewDatabaseWithCache(db, 0) +} + +// NewDatabase creates a backing store for state. The returned database is safe for +// concurrent use and retains both a few recent expanded trie nodes in memory, as +// well as a lot of collapsed RLP trie nodes in a large memory cache. +func NewDatabaseWithCache(db ethdb.Database, cache int) Database { csc, _ := lru.New(codeSizeCacheSize) return &cachingDB{ - db: trie.NewDatabase(db), + db: trie.NewDatabaseWithCache(db, cache), codeSizeCache: csc, } } diff --git a/eth/api.go b/eth/api.go index 3ec3afb81e..816b9cd335 100644 --- a/eth/api.go +++ b/eth/api.go @@ -444,16 +444,16 @@ func (api *PrivateDebugAPI) getModifiedAccounts(startBlock, endBlock *types.Bloc if startBlock.Number().Uint64() >= endBlock.Number().Uint64() { return nil, fmt.Errorf("start block height (%d) must be less than end block height (%d)", startBlock.Number().Uint64(), endBlock.Number().Uint64()) } + triedb := api.eth.BlockChain().StateCache().TrieDB() - oldTrie, err := trie.NewSecure(startBlock.Root(), trie.NewDatabase(api.eth.chainDb), 0) + oldTrie, err := trie.NewSecure(startBlock.Root(), triedb, 0) if err != nil { return nil, err } - newTrie, err := trie.NewSecure(endBlock.Root(), trie.NewDatabase(api.eth.chainDb), 0) + newTrie, err := trie.NewSecure(endBlock.Root(), triedb, 0) if err != nil { return nil, err } - diff, _ := trie.NewDifferenceIterator(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{})) iter := trie.NewIterator(diff) diff --git a/eth/api_tracer.go b/eth/api_tracer.go index 80552ada8c..2ebbcc5fd9 100644 --- a/eth/api_tracer.go +++ b/eth/api_tracer.go @@ -138,7 +138,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl // Ensure we have a valid starting state before doing any work origin := start.NumberU64() - database := state.NewDatabase(api.eth.ChainDb()) + database := state.NewDatabaseWithCache(api.eth.ChainDb(), 16) // Chain tracing will probably start at genesis if number := start.NumberU64(); number > 0 { start = api.eth.blockchain.GetBlock(start.ParentHash(), start.NumberU64()-1) @@ -492,7 +492,7 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (* } // Otherwise try to reexec blocks until we find a state or reach our limit origin := block.NumberU64() - database := state.NewDatabase(api.eth.ChainDb()) + database := state.NewDatabaseWithCache(api.eth.ChainDb(), 16) for i := uint64(0); i < reexec; i++ { block = api.eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) diff --git a/eth/backend.go b/eth/backend.go index b555b064ad..4721408425 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -154,7 +154,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { EWASMInterpreter: config.EWASMInterpreter, EVMInterpreter: config.EVMInterpreter, } - cacheConfig = &core.CacheConfig{Disabled: config.NoPruning, TrieNodeLimit: config.TrieCache, TrieTimeLimit: config.TrieTimeout} + cacheConfig = &core.CacheConfig{Disabled: config.NoPruning, TrieCleanLimit: config.TrieCleanCache, TrieDirtyLimit: config.TrieDirtyCache, TrieTimeLimit: config.TrieTimeout} ) eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, eth.chainConfig, eth.engine, vmConfig, eth.shouldPreserve) if err != nil { diff --git a/eth/config.go b/eth/config.go index e32c01a734..601f4735e3 100644 --- a/eth/config.go +++ b/eth/config.go @@ -43,15 +43,16 @@ var DefaultConfig = Config{ DatasetsInMem: 1, DatasetsOnDisk: 2, }, - NetworkId: 1, - LightPeers: 100, - DatabaseCache: 768, - TrieCache: 256, - TrieTimeout: 60 * time.Minute, - MinerGasFloor: 8000000, - MinerGasCeil: 8000000, - MinerGasPrice: big.NewInt(params.GWei), - MinerRecommit: 3 * time.Second, + NetworkId: 1, + LightPeers: 100, + DatabaseCache: 512, + TrieCleanCache: 256, + TrieDirtyCache: 256, + TrieTimeout: 60 * time.Minute, + MinerGasFloor: 8000000, + MinerGasCeil: 8000000, + MinerGasPrice: big.NewInt(params.GWei), + MinerRecommit: 3 * time.Second, TxPool: core.DefaultTxPoolConfig, GPO: gasprice.Config{ @@ -94,7 +95,8 @@ type Config struct { SkipBcVersionCheck bool `toml:"-"` DatabaseHandles int `toml:"-"` DatabaseCache int - TrieCache int + TrieCleanCache int + TrieDirtyCache int TrieTimeout time.Duration // Mining-related options diff --git a/eth/gen_config.go b/eth/gen_config.go index d401a917d2..2777aa9e83 100644 --- a/eth/gen_config.go +++ b/eth/gen_config.go @@ -28,7 +28,8 @@ func (c Config) MarshalTOML() (interface{}, error) { SkipBcVersionCheck bool `toml:"-"` DatabaseHandles int `toml:"-"` DatabaseCache int - TrieCache int + TrieCleanCache int + TrieDirtyCache int TrieTimeout time.Duration Etherbase common.Address `toml:",omitempty"` MinerNotify []string `toml:",omitempty"` @@ -43,6 +44,8 @@ func (c Config) MarshalTOML() (interface{}, error) { GPO gasprice.Config EnablePreimageRecording bool DocRoot string `toml:"-"` + EWASMInterpreter string + EVMInterpreter string } var enc Config enc.Genesis = c.Genesis @@ -54,7 +57,8 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.SkipBcVersionCheck = c.SkipBcVersionCheck enc.DatabaseHandles = c.DatabaseHandles enc.DatabaseCache = c.DatabaseCache - enc.TrieCache = c.TrieCache + enc.TrieCleanCache = c.TrieCleanCache + enc.TrieDirtyCache = c.TrieDirtyCache enc.TrieTimeout = c.TrieTimeout enc.Etherbase = c.Etherbase enc.MinerNotify = c.MinerNotify @@ -69,6 +73,8 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.GPO = c.GPO enc.EnablePreimageRecording = c.EnablePreimageRecording enc.DocRoot = c.DocRoot + enc.EWASMInterpreter = c.EWASMInterpreter + enc.EVMInterpreter = c.EVMInterpreter return &enc, nil } @@ -84,7 +90,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { SkipBcVersionCheck *bool `toml:"-"` DatabaseHandles *int `toml:"-"` DatabaseCache *int - TrieCache *int + TrieCleanCache *int + TrieDirtyCache *int TrieTimeout *time.Duration Etherbase *common.Address `toml:",omitempty"` MinerNotify []string `toml:",omitempty"` @@ -99,6 +106,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { GPO *gasprice.Config EnablePreimageRecording *bool DocRoot *string `toml:"-"` + EWASMInterpreter *string + EVMInterpreter *string } var dec Config if err := unmarshal(&dec); err != nil { @@ -131,8 +140,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.DatabaseCache != nil { c.DatabaseCache = *dec.DatabaseCache } - if dec.TrieCache != nil { - c.TrieCache = *dec.TrieCache + if dec.TrieCleanCache != nil { + c.TrieCleanCache = *dec.TrieCleanCache + } + if dec.TrieDirtyCache != nil { + c.TrieDirtyCache = *dec.TrieDirtyCache } if dec.TrieTimeout != nil { c.TrieTimeout = *dec.TrieTimeout @@ -176,5 +188,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.DocRoot != nil { c.DocRoot = *dec.DocRoot } + if dec.EWASMInterpreter != nil { + c.EWASMInterpreter = *dec.EWASMInterpreter + } + if dec.EVMInterpreter != nil { + c.EVMInterpreter = *dec.EVMInterpreter + } return nil } diff --git a/light/postprocess.go b/light/postprocess.go index 1cfd7535e7..dd1b74a7be 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -159,7 +159,7 @@ func NewChtIndexer(db ethdb.Database, odr OdrBackend, size, confirms uint64) *co diskdb: db, odr: odr, trieTable: trieTable, - triedb: trie.NewDatabase(trieTable), + triedb: trie.NewDatabaseWithCache(trieTable, 1), // Use a tiny cache only to keep memory down sectionSize: size, } return core.NewChainIndexer(db, ethdb.NewTable(db, "chtIndex-"), backend, size, confirms, time.Millisecond*100, "cht") @@ -281,7 +281,7 @@ func NewBloomTrieIndexer(db ethdb.Database, odr OdrBackend, parentSize, size uin diskdb: db, odr: odr, trieTable: trieTable, - triedb: trie.NewDatabase(trieTable), + triedb: trie.NewDatabaseWithCache(trieTable, 1), // Use a tiny cache only to keep memory down parentSize: parentSize, size: size, } diff --git a/tests/block_test_util.go b/tests/block_test_util.go index 12cba3244f..9fa69bf4ed 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -118,7 +118,7 @@ func (t *BlockTest) Run() error { } else { engine = ethash.NewShared() } - chain, err := core.NewBlockChain(db, nil, config, engine, vm.Config{}, nil) + chain, err := core.NewBlockChain(db, &core.CacheConfig{TrieCleanLimit: 0}, config, engine, vm.Config{}, nil) if err != nil { return err } diff --git a/trie/database.go b/trie/database.go index d0691b637e..71190b3f3f 100644 --- a/trie/database.go +++ b/trie/database.go @@ -22,6 +22,7 @@ import ( "sync" "time" + "github.com/allegro/bigcache" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" @@ -30,6 +31,11 @@ import ( ) var ( + memcacheCleanHitMeter = metrics.NewRegisteredMeter("trie/memcache/clean/hit", nil) + memcacheCleanMissMeter = metrics.NewRegisteredMeter("trie/memcache/clean/miss", nil) + memcacheCleanReadMeter = metrics.NewRegisteredMeter("trie/memcache/clean/read", nil) + memcacheCleanWriteMeter = metrics.NewRegisteredMeter("trie/memcache/clean/write", nil) + memcacheFlushTimeTimer = metrics.NewRegisteredResettingTimer("trie/memcache/flush/time", nil) memcacheFlushNodesMeter = metrics.NewRegisteredMeter("trie/memcache/flush/nodes", nil) memcacheFlushSizeMeter = metrics.NewRegisteredMeter("trie/memcache/flush/size", nil) @@ -64,9 +70,10 @@ type DatabaseReader interface { type Database struct { diskdb ethdb.Database // Persistent storage for matured trie nodes - nodes map[common.Hash]*cachedNode // Data and references relationships of a node - oldest common.Hash // Oldest tracked node, flush-list head - newest common.Hash // Newest tracked node, flush-list tail + cleans *bigcache.BigCache // GC friendly memory cache of clean node RLPs + dirties map[common.Hash]*cachedNode // Data and references relationships of dirty nodes + oldest common.Hash // Oldest tracked node, flush-list head + newest common.Hash // Newest tracked node, flush-list tail preimages map[common.Hash][]byte // Preimages of nodes from the secure trie seckeybuf [secureKeyLength]byte // Ephemeral buffer for calculating preimage keys @@ -79,7 +86,7 @@ type Database struct { flushnodes uint64 // Nodes flushed since last commit flushsize common.StorageSize // Data storage flushed since last commit - nodesSize common.StorageSize // Storage size of the nodes cache (exc. flushlist) + dirtiesSize common.StorageSize // Storage size of the dirty node cache (exc. flushlist) preimagesSize common.StorageSize // Storage size of the preimages cache lock sync.RWMutex @@ -262,11 +269,30 @@ func expandNode(hash hashNode, n node, cachegen uint16) node { } // NewDatabase creates a new trie database to store ephemeral trie content before -// its written out to disk or garbage collected. +// its written out to disk or garbage collected. No read cache is created, so all +// data retrievals will hit the underlying disk database. func NewDatabase(diskdb ethdb.Database) *Database { + return NewDatabaseWithCache(diskdb, 0) +} + +// NewDatabaseWithCache creates a new trie database to store ephemeral trie content +// before its written out to disk or garbage collected. It also acts as a read cache +// for nodes loaded from disk. +func NewDatabaseWithCache(diskdb ethdb.Database, cache int) *Database { + var cleans *bigcache.BigCache + if cache > 0 { + cleans, _ = bigcache.NewBigCache(bigcache.Config{ + Shards: 1024, + LifeWindow: time.Hour, + MaxEntriesInWindow: cache * 1024, + MaxEntrySize: 512, + HardMaxCacheSize: cache, + }) + } return &Database{ diskdb: diskdb, - nodes: map[common.Hash]*cachedNode{{}: {}}, + cleans: cleans, + dirties: map[common.Hash]*cachedNode{{}: {}}, preimages: make(map[common.Hash][]byte), } } @@ -293,7 +319,7 @@ func (db *Database) InsertBlob(hash common.Hash, blob []byte) { // size tracking. func (db *Database) insert(hash common.Hash, blob []byte, node node) { // If the node's already cached, skip - if _, ok := db.nodes[hash]; ok { + if _, ok := db.dirties[hash]; ok { return } // Create the cached entry for this node @@ -303,19 +329,19 @@ func (db *Database) insert(hash common.Hash, blob []byte, node node) { flushPrev: db.newest, } for _, child := range entry.childs() { - if c := db.nodes[child]; c != nil { + if c := db.dirties[child]; c != nil { c.parents++ } } - db.nodes[hash] = entry + db.dirties[hash] = entry // Update the flush-list endpoints if db.oldest == (common.Hash{}) { db.oldest, db.newest = hash, hash } else { - db.nodes[db.newest].flushNext, db.newest = hash, hash + db.dirties[db.newest].flushNext, db.newest = hash, hash } - db.nodesSize += common.StorageSize(common.HashLength + entry.size) + db.dirtiesSize += common.StorageSize(common.HashLength + entry.size) } // insertPreimage writes a new trie node pre-image to the memory database if it's @@ -333,35 +359,64 @@ func (db *Database) insertPreimage(hash common.Hash, preimage []byte) { // node retrieves a cached trie node from memory, or returns nil if none can be // found in the memory cache. func (db *Database) node(hash common.Hash, cachegen uint16) node { - // Retrieve the node from cache if available + // Retrieve the node from the clean cache if available + if db.cleans != nil { + if enc, err := db.cleans.Get(string(hash[:])); err == nil && enc != nil { + memcacheCleanHitMeter.Mark(1) + memcacheCleanReadMeter.Mark(int64(len(enc))) + return mustDecodeNode(hash[:], enc, cachegen) + } + } + // Retrieve the node from the dirty cache if available db.lock.RLock() - node := db.nodes[hash] + dirty := db.dirties[hash] db.lock.RUnlock() - if node != nil { - return node.obj(hash, cachegen) + if dirty != nil { + return dirty.obj(hash, cachegen) } // Content unavailable in memory, attempt to retrieve from disk enc, err := db.diskdb.Get(hash[:]) if err != nil || enc == nil { return nil } + if db.cleans != nil { + db.cleans.Set(string(hash[:]), enc) + memcacheCleanMissMeter.Mark(1) + memcacheCleanWriteMeter.Mark(int64(len(enc))) + } return mustDecodeNode(hash[:], enc, cachegen) } // Node retrieves an encoded cached trie node from memory. If it cannot be found // cached, the method queries the persistent database for the content. func (db *Database) Node(hash common.Hash) ([]byte, error) { - // Retrieve the node from cache if available + // Retrieve the node from the clean cache if available + if db.cleans != nil { + if enc, err := db.cleans.Get(string(hash[:])); err == nil && enc != nil { + memcacheCleanHitMeter.Mark(1) + memcacheCleanReadMeter.Mark(int64(len(enc))) + return enc, nil + } + } + // Retrieve the node from the dirty cache if available db.lock.RLock() - node := db.nodes[hash] + dirty := db.dirties[hash] db.lock.RUnlock() - if node != nil { - return node.rlp(), nil + if dirty != nil { + return dirty.rlp(), nil } // Content unavailable in memory, attempt to retrieve from disk - return db.diskdb.Get(hash[:]) + enc, err := db.diskdb.Get(hash[:]) + if err == nil && enc != nil { + if db.cleans != nil { + db.cleans.Set(string(hash[:]), enc) + memcacheCleanMissMeter.Mark(1) + memcacheCleanWriteMeter.Mark(int64(len(enc))) + } + } + return enc, err } // preimage retrieves a cached trie node pre-image from memory. If it cannot be @@ -395,8 +450,8 @@ func (db *Database) Nodes() []common.Hash { db.lock.RLock() defer db.lock.RUnlock() - var hashes = make([]common.Hash, 0, len(db.nodes)) - for hash := range db.nodes { + var hashes = make([]common.Hash, 0, len(db.dirties)) + for hash := range db.dirties { if hash != (common.Hash{}) { // Special case for "root" references/nodes hashes = append(hashes, hash) } @@ -415,18 +470,18 @@ func (db *Database) Reference(child common.Hash, parent common.Hash) { // reference is the private locked version of Reference. func (db *Database) reference(child common.Hash, parent common.Hash) { // If the node does not exist, it's a node pulled from disk, skip - node, ok := db.nodes[child] + node, ok := db.dirties[child] if !ok { return } // If the reference already exists, only duplicate for roots - if db.nodes[parent].children == nil { - db.nodes[parent].children = make(map[common.Hash]uint16) - } else if _, ok = db.nodes[parent].children[child]; ok && parent != (common.Hash{}) { + if db.dirties[parent].children == nil { + db.dirties[parent].children = make(map[common.Hash]uint16) + } else if _, ok = db.dirties[parent].children[child]; ok && parent != (common.Hash{}) { return } node.parents++ - db.nodes[parent].children[child]++ + db.dirties[parent].children[child]++ } // Dereference removes an existing reference from a root node. @@ -439,25 +494,25 @@ func (db *Database) Dereference(root common.Hash) { db.lock.Lock() defer db.lock.Unlock() - nodes, storage, start := len(db.nodes), db.nodesSize, time.Now() + nodes, storage, start := len(db.dirties), db.dirtiesSize, time.Now() db.dereference(root, common.Hash{}) - db.gcnodes += uint64(nodes - len(db.nodes)) - db.gcsize += storage - db.nodesSize + db.gcnodes += uint64(nodes - len(db.dirties)) + db.gcsize += storage - db.dirtiesSize db.gctime += time.Since(start) memcacheGCTimeTimer.Update(time.Since(start)) - memcacheGCSizeMeter.Mark(int64(storage - db.nodesSize)) - memcacheGCNodesMeter.Mark(int64(nodes - len(db.nodes))) + memcacheGCSizeMeter.Mark(int64(storage - db.dirtiesSize)) + memcacheGCNodesMeter.Mark(int64(nodes - len(db.dirties))) - log.Debug("Dereferenced trie from memory database", "nodes", nodes-len(db.nodes), "size", storage-db.nodesSize, "time", time.Since(start), - "gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", db.gctime, "livenodes", len(db.nodes), "livesize", db.nodesSize) + log.Debug("Dereferenced trie from memory database", "nodes", nodes-len(db.dirties), "size", storage-db.dirtiesSize, "time", time.Since(start), + "gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", db.gctime, "livenodes", len(db.dirties), "livesize", db.dirtiesSize) } // dereference is the private locked version of Dereference. func (db *Database) dereference(child common.Hash, parent common.Hash) { // Dereference the parent-child - node := db.nodes[parent] + node := db.dirties[parent] if node.children != nil && node.children[child] > 0 { node.children[child]-- @@ -466,7 +521,7 @@ func (db *Database) dereference(child common.Hash, parent common.Hash) { } } // If the child does not exist, it's a previously committed node. - node, ok := db.nodes[child] + node, ok := db.dirties[child] if !ok { return } @@ -483,20 +538,20 @@ func (db *Database) dereference(child common.Hash, parent common.Hash) { switch child { case db.oldest: db.oldest = node.flushNext - db.nodes[node.flushNext].flushPrev = common.Hash{} + db.dirties[node.flushNext].flushPrev = common.Hash{} case db.newest: db.newest = node.flushPrev - db.nodes[node.flushPrev].flushNext = common.Hash{} + db.dirties[node.flushPrev].flushNext = common.Hash{} default: - db.nodes[node.flushPrev].flushNext = node.flushNext - db.nodes[node.flushNext].flushPrev = node.flushPrev + db.dirties[node.flushPrev].flushNext = node.flushNext + db.dirties[node.flushNext].flushPrev = node.flushPrev } // Dereference all children and delete the node for _, hash := range node.childs() { db.dereference(hash, child) } - delete(db.nodes, child) - db.nodesSize -= common.StorageSize(common.HashLength + int(node.size)) + delete(db.dirties, child) + db.dirtiesSize -= common.StorageSize(common.HashLength + int(node.size)) } } @@ -509,13 +564,13 @@ func (db *Database) Cap(limit common.StorageSize) error { // by only uncaching existing data when the database write finalizes. db.lock.RLock() - nodes, storage, start := len(db.nodes), db.nodesSize, time.Now() + nodes, storage, start := len(db.dirties), db.dirtiesSize, time.Now() batch := db.diskdb.NewBatch() - // db.nodesSize only contains the useful data in the cache, but when reporting + // db.dirtiesSize only contains the useful data in the cache, but when reporting // the total memory consumption, the maintenance metadata is also needed to be // counted. For every useful node, we track 2 extra hashes as the flushlist. - size := db.nodesSize + common.StorageSize((len(db.nodes)-1)*2*common.HashLength) + size := db.dirtiesSize + common.StorageSize((len(db.dirties)-1)*2*common.HashLength) // If the preimage cache got large enough, push to disk. If it's still small // leave for later to deduplicate writes. @@ -540,7 +595,7 @@ func (db *Database) Cap(limit common.StorageSize) error { oldest := db.oldest for size > limit && oldest != (common.Hash{}) { // Fetch the oldest referenced node and push into the batch - node := db.nodes[oldest] + node := db.dirties[oldest] if err := batch.Put(oldest[:], node.rlp()); err != nil { db.lock.RUnlock() return err @@ -578,25 +633,25 @@ func (db *Database) Cap(limit common.StorageSize) error { db.preimagesSize = 0 } for db.oldest != oldest { - node := db.nodes[db.oldest] - delete(db.nodes, db.oldest) + node := db.dirties[db.oldest] + delete(db.dirties, db.oldest) db.oldest = node.flushNext - db.nodesSize -= common.StorageSize(common.HashLength + int(node.size)) + db.dirtiesSize -= common.StorageSize(common.HashLength + int(node.size)) } if db.oldest != (common.Hash{}) { - db.nodes[db.oldest].flushPrev = common.Hash{} + db.dirties[db.oldest].flushPrev = common.Hash{} } - db.flushnodes += uint64(nodes - len(db.nodes)) - db.flushsize += storage - db.nodesSize + db.flushnodes += uint64(nodes - len(db.dirties)) + db.flushsize += storage - db.dirtiesSize db.flushtime += time.Since(start) memcacheFlushTimeTimer.Update(time.Since(start)) - memcacheFlushSizeMeter.Mark(int64(storage - db.nodesSize)) - memcacheFlushNodesMeter.Mark(int64(nodes - len(db.nodes))) + memcacheFlushSizeMeter.Mark(int64(storage - db.dirtiesSize)) + memcacheFlushNodesMeter.Mark(int64(nodes - len(db.dirties))) - log.Debug("Persisted nodes from memory database", "nodes", nodes-len(db.nodes), "size", storage-db.nodesSize, "time", time.Since(start), - "flushnodes", db.flushnodes, "flushsize", db.flushsize, "flushtime", db.flushtime, "livenodes", len(db.nodes), "livesize", db.nodesSize) + log.Debug("Persisted nodes from memory database", "nodes", nodes-len(db.dirties), "size", storage-db.dirtiesSize, "time", time.Since(start), + "flushnodes", db.flushnodes, "flushsize", db.flushsize, "flushtime", db.flushtime, "livenodes", len(db.dirties), "livesize", db.dirtiesSize) return nil } @@ -630,7 +685,7 @@ func (db *Database) Commit(node common.Hash, report bool) error { } } // Move the trie itself into the batch, flushing if enough data is accumulated - nodes, storage := len(db.nodes), db.nodesSize + nodes, storage := len(db.dirties), db.dirtiesSize if err := db.commit(node, batch); err != nil { log.Error("Failed to commit trie from trie database", "err", err) db.lock.RUnlock() @@ -654,15 +709,15 @@ func (db *Database) Commit(node common.Hash, report bool) error { db.uncache(node) memcacheCommitTimeTimer.Update(time.Since(start)) - memcacheCommitSizeMeter.Mark(int64(storage - db.nodesSize)) - memcacheCommitNodesMeter.Mark(int64(nodes - len(db.nodes))) + memcacheCommitSizeMeter.Mark(int64(storage - db.dirtiesSize)) + memcacheCommitNodesMeter.Mark(int64(nodes - len(db.dirties))) logger := log.Info if !report { logger = log.Debug } - logger("Persisted trie from memory database", "nodes", nodes-len(db.nodes)+int(db.flushnodes), "size", storage-db.nodesSize+db.flushsize, "time", time.Since(start)+db.flushtime, - "gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", db.gctime, "livenodes", len(db.nodes), "livesize", db.nodesSize) + logger("Persisted trie from memory database", "nodes", nodes-len(db.dirties)+int(db.flushnodes), "size", storage-db.dirtiesSize+db.flushsize, "time", time.Since(start)+db.flushtime, + "gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", db.gctime, "livenodes", len(db.dirties), "livesize", db.dirtiesSize) // Reset the garbage collection statistics db.gcnodes, db.gcsize, db.gctime = 0, 0, 0 @@ -674,7 +729,7 @@ func (db *Database) Commit(node common.Hash, report bool) error { // commit is the private locked version of Commit. func (db *Database) commit(hash common.Hash, batch ethdb.Batch) error { // If the node does not exist, it's a previously committed node - node, ok := db.nodes[hash] + node, ok := db.dirties[hash] if !ok { return nil } @@ -702,7 +757,7 @@ func (db *Database) commit(hash common.Hash, batch ethdb.Batch) error { // to disk. func (db *Database) uncache(hash common.Hash) { // If the node does not exist, we're done on this path - node, ok := db.nodes[hash] + node, ok := db.dirties[hash] if !ok { return } @@ -710,20 +765,20 @@ func (db *Database) uncache(hash common.Hash) { switch hash { case db.oldest: db.oldest = node.flushNext - db.nodes[node.flushNext].flushPrev = common.Hash{} + db.dirties[node.flushNext].flushPrev = common.Hash{} case db.newest: db.newest = node.flushPrev - db.nodes[node.flushPrev].flushNext = common.Hash{} + db.dirties[node.flushPrev].flushNext = common.Hash{} default: - db.nodes[node.flushPrev].flushNext = node.flushNext - db.nodes[node.flushNext].flushPrev = node.flushPrev + db.dirties[node.flushPrev].flushNext = node.flushNext + db.dirties[node.flushNext].flushPrev = node.flushPrev } // Uncache the node's subtries and remove the node itself too for _, child := range node.childs() { db.uncache(child) } - delete(db.nodes, hash) - db.nodesSize -= common.StorageSize(common.HashLength + int(node.size)) + delete(db.dirties, hash) + db.dirtiesSize -= common.StorageSize(common.HashLength + int(node.size)) } // Size returns the current storage size of the memory cache in front of the @@ -732,11 +787,11 @@ func (db *Database) Size() (common.StorageSize, common.StorageSize) { db.lock.RLock() defer db.lock.RUnlock() - // db.nodesSize only contains the useful data in the cache, but when reporting + // db.dirtiesSize only contains the useful data in the cache, but when reporting // the total memory consumption, the maintenance metadata is also needed to be // counted. For every useful node, we track 2 extra hashes as the flushlist. - var flushlistSize = common.StorageSize((len(db.nodes) - 1) * 2 * common.HashLength) - return db.nodesSize + flushlistSize, db.preimagesSize + var flushlistSize = common.StorageSize((len(db.dirties) - 1) * 2 * common.HashLength) + return db.dirtiesSize + flushlistSize, db.preimagesSize } // verifyIntegrity is a debug method to iterate over the entire trie stored in @@ -749,12 +804,12 @@ func (db *Database) verifyIntegrity() { // Iterate over all the cached nodes and accumulate them into a set reachable := map[common.Hash]struct{}{{}: {}} - for child := range db.nodes[common.Hash{}].children { + for child := range db.dirties[common.Hash{}].children { db.accumulate(child, reachable) } // Find any unreachable but cached nodes unreachable := []string{} - for hash, node := range db.nodes { + for hash, node := range db.dirties { if _, ok := reachable[hash]; !ok { unreachable = append(unreachable, fmt.Sprintf("%x: {Node: %v, Parents: %d, Prev: %x, Next: %x}", hash, node.node, node.parents, node.flushPrev, node.flushNext)) @@ -769,7 +824,7 @@ func (db *Database) verifyIntegrity() { // cached children found in memory. func (db *Database) accumulate(hash common.Hash, reachable map[common.Hash]struct{}) { // Mark the node reachable if present in the memory cache - node, ok := db.nodes[hash] + node, ok := db.dirties[hash] if !ok { return } diff --git a/trie/iterator_test.go b/trie/iterator_test.go index 2a510b1c2d..4f633b195f 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -113,7 +113,7 @@ func TestNodeIteratorCoverage(t *testing.T) { t.Errorf("failed to retrieve reported node %x: %v", hash, err) } } - for hash, obj := range db.nodes { + for hash, obj := range db.dirties { if obj != nil && hash != (common.Hash{}) { if _, ok := hashes[hash]; !ok { t.Errorf("state entry not reported %x", hash) @@ -333,8 +333,8 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) { } } if memonly { - robj = triedb.nodes[rkey] - delete(triedb.nodes, rkey) + robj = triedb.dirties[rkey] + delete(triedb.dirties, rkey) } else { rval, _ = diskdb.Get(rkey[:]) diskdb.Delete(rkey[:]) @@ -350,7 +350,7 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) { // Add the node back and continue iteration. if memonly { - triedb.nodes[rkey] = robj + triedb.dirties[rkey] = robj } else { diskdb.Put(rkey[:], rval) } @@ -393,8 +393,8 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) { barNodeObj *cachedNode ) if memonly { - barNodeObj = triedb.nodes[barNodeHash] - delete(triedb.nodes, barNodeHash) + barNodeObj = triedb.dirties[barNodeHash] + delete(triedb.dirties, barNodeHash) } else { barNodeBlob, _ = diskdb.Get(barNodeHash[:]) diskdb.Delete(barNodeHash[:]) @@ -411,7 +411,7 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) { } // Reinsert the missing node. if memonly { - triedb.nodes[barNodeHash] = barNodeObj + triedb.dirties[barNodeHash] = barNodeObj } else { diskdb.Put(barNodeHash[:], barNodeBlob) } diff --git a/trie/trie_test.go b/trie/trie_test.go index f8e5fd12a1..f9d6029c99 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -119,7 +119,7 @@ func testMissingNode(t *testing.T, memonly bool) { hash := common.HexToHash("0xe1d943cc8f061a0c0b98162830b970395ac9315654824bf21b73b891365262f9") if memonly { - delete(triedb.nodes, hash) + delete(triedb.dirties, hash) } else { diskdb.Delete(hash[:]) } @@ -342,15 +342,16 @@ func TestCacheUnload(t *testing.T) { // Commit the trie repeatedly and access key1. // The branch containing it is loaded from DB exactly two times: // in the 0th and 6th iteration. - db := &countingDB{Database: trie.db.diskdb, gets: make(map[string]int)} - trie, _ = New(root, NewDatabase(db)) + diskdb := &countingDB{Database: trie.db.diskdb, gets: make(map[string]int)} + triedb := NewDatabase(diskdb) + trie, _ = New(root, triedb) trie.SetCacheLimit(5) for i := 0; i < 12; i++ { getString(trie, key1) trie.Commit(nil) } // Check that it got loaded two times. - for dbkey, count := range db.gets { + for dbkey, count := range diskdb.gets { if count != 2 { t.Errorf("db key %x loaded %d times, want %d times", []byte(dbkey), count, 2) } diff --git a/vendor/github.com/allegro/bigcache/LICENSE b/vendor/github.com/allegro/bigcache/LICENSE new file mode 100644 index 0000000000..8dada3edaf --- /dev/null +++ b/vendor/github.com/allegro/bigcache/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/allegro/bigcache/README.md b/vendor/github.com/allegro/bigcache/README.md new file mode 100644 index 0000000000..c23f7f36ce --- /dev/null +++ b/vendor/github.com/allegro/bigcache/README.md @@ -0,0 +1,150 @@ +# BigCache [![Build Status](https://travis-ci.org/allegro/bigcache.svg?branch=master)](https://travis-ci.org/allegro/bigcache) [![Coverage Status](https://coveralls.io/repos/github/allegro/bigcache/badge.svg?branch=master)](https://coveralls.io/github/allegro/bigcache?branch=master) [![GoDoc](https://godoc.org/github.com/allegro/bigcache?status.svg)](https://godoc.org/github.com/allegro/bigcache) [![Go Report Card](https://goreportcard.com/badge/github.com/allegro/bigcache)](https://goreportcard.com/report/github.com/allegro/bigcache) + +Fast, concurrent, evicting in-memory cache written to keep big number of entries without impact on performance. +BigCache keeps entries on heap but omits GC for them. To achieve that operations on bytes arrays take place, +therefore entries (de)serialization in front of the cache will be needed in most use cases. + +## Usage + +### Simple initialization + +```go +import "github.com/allegro/bigcache" + +cache, _ := bigcache.NewBigCache(bigcache.DefaultConfig(10 * time.Minute)) + +cache.Set("my-unique-key", []byte("value")) + +entry, _ := cache.Get("my-unique-key") +fmt.Println(string(entry)) +``` + +### Custom initialization + +When cache load can be predicted in advance then it is better to use custom initialization because additional memory +allocation can be avoided in that way. + +```go +import ( + "log" + + "github.com/allegro/bigcache" +) + +config := bigcache.Config { + // number of shards (must be a power of 2) + Shards: 1024, + // time after which entry can be evicted + LifeWindow: 10 * time.Minute, + // rps * lifeWindow, used only in initial memory allocation + MaxEntriesInWindow: 1000 * 10 * 60, + // max entry size in bytes, used only in initial memory allocation + MaxEntrySize: 500, + // prints information about additional memory allocation + Verbose: true, + // cache will not allocate more memory than this limit, value in MB + // if value is reached then the oldest entries can be overridden for the new ones + // 0 value means no size limit + HardMaxCacheSize: 8192, + // callback fired when the oldest entry is removed because of its expiration time or no space left + // for the new entry, or because delete was called. A bitmask representing the reason will be returned. + // Default value is nil which means no callback and it prevents from unwrapping the oldest entry. + OnRemove: nil, + // OnRemoveWithReason is a callback fired when the oldest entry is removed because of its expiration time or no space left + // for the new entry, or because delete was called. A constant representing the reason will be passed through. + // Default value is nil which means no callback and it prevents from unwrapping the oldest entry. + // Ignored if OnRemove is specified. + OnRemoveWithReason: nil, + } + +cache, initErr := bigcache.NewBigCache(config) +if initErr != nil { + log.Fatal(initErr) +} + +cache.Set("my-unique-key", []byte("value")) + +if entry, err := cache.Get("my-unique-key"); err == nil { + fmt.Println(string(entry)) +} +``` + +## Benchmarks + +Three caches were compared: bigcache, [freecache](https://github.com/coocood/freecache) and map. +Benchmark tests were made using an i7-6700K with 32GB of RAM on Windows 10. + +### Writes and reads + +```bash +cd caches_bench; go test -bench=. -benchtime=10s ./... -timeout 30m + +BenchmarkMapSet-8 3000000 569 ns/op 202 B/op 3 allocs/op +BenchmarkConcurrentMapSet-8 1000000 1592 ns/op 347 B/op 8 allocs/op +BenchmarkFreeCacheSet-8 3000000 775 ns/op 355 B/op 2 allocs/op +BenchmarkBigCacheSet-8 3000000 640 ns/op 303 B/op 2 allocs/op +BenchmarkMapGet-8 5000000 407 ns/op 24 B/op 1 allocs/op +BenchmarkConcurrentMapGet-8 3000000 558 ns/op 24 B/op 2 allocs/op +BenchmarkFreeCacheGet-8 2000000 682 ns/op 136 B/op 2 allocs/op +BenchmarkBigCacheGet-8 3000000 512 ns/op 152 B/op 4 allocs/op +BenchmarkBigCacheSetParallel-8 10000000 225 ns/op 313 B/op 3 allocs/op +BenchmarkFreeCacheSetParallel-8 10000000 218 ns/op 341 B/op 3 allocs/op +BenchmarkConcurrentMapSetParallel-8 5000000 318 ns/op 200 B/op 6 allocs/op +BenchmarkBigCacheGetParallel-8 20000000 178 ns/op 152 B/op 4 allocs/op +BenchmarkFreeCacheGetParallel-8 20000000 295 ns/op 136 B/op 3 allocs/op +BenchmarkConcurrentMapGetParallel-8 10000000 237 ns/op 24 B/op 2 allocs/op +``` + +Writes and reads in bigcache are faster than in freecache. +Writes to map are the slowest. + +### GC pause time + +```bash +cd caches_bench; go run caches_gc_overhead_comparison.go + +Number of entries: 20000000 +GC pause for bigcache: 5.8658ms +GC pause for freecache: 32.4341ms +GC pause for map: 52.9661ms +``` + +Test shows how long are the GC pauses for caches filled with 20mln of entries. +Bigcache and freecache have very similar GC pause time. +It is clear that both reduce GC overhead in contrast to map +which GC pause time took more than 10 seconds. + +## How it works + +BigCache relies on optimization presented in 1.5 version of Go ([issue-9477](https://github.com/golang/go/issues/9477)). +This optimization states that if map without pointers in keys and values is used then GC will omit its content. +Therefore BigCache uses `map[uint64]uint32` where keys are hashed and values are offsets of entries. + +Entries are kept in bytes array, to omit GC again. +Bytes array size can grow to gigabytes without impact on performance +because GC will only see single pointer to it. + +## Bigcache vs Freecache + +Both caches provide the same core features but they reduce GC overhead in different ways. +Bigcache relies on `map[uint64]uint32`, freecache implements its own mapping built on +slices to reduce number of pointers. + +Results from benchmark tests are presented above. +One of the advantage of bigcache over freecache is that you don’t need to know +the size of the cache in advance, because when bigcache is full, +it can allocate additional memory for new entries instead of +overwriting existing ones as freecache does currently. +However hard max size in bigcache also can be set, check [HardMaxCacheSize](https://godoc.org/github.com/allegro/bigcache#Config). + +## HTTP Server + +This package also includes an easily deployable HTTP implementation of BigCache, which can be found in the [server](/server) package. + +## More + +Bigcache genesis is described in allegro.tech blog post: [writing a very fast cache service in Go](http://allegro.tech/2016/03/writing-fast-cache-service-in-go.html) + +## License + +BigCache is released under the Apache 2.0 license (see [LICENSE](LICENSE)) diff --git a/vendor/github.com/allegro/bigcache/bigcache.go b/vendor/github.com/allegro/bigcache/bigcache.go new file mode 100644 index 0000000000..e6aee8663b --- /dev/null +++ b/vendor/github.com/allegro/bigcache/bigcache.go @@ -0,0 +1,202 @@ +package bigcache + +import ( + "fmt" + "time" +) + +const ( + minimumEntriesInShard = 10 // Minimum number of entries in single shard +) + +// BigCache is fast, concurrent, evicting cache created to keep big number of entries without impact on performance. +// It keeps entries on heap but omits GC for them. To achieve that, operations take place on byte arrays, +// therefore entries (de)serialization in front of the cache will be needed in most use cases. +type BigCache struct { + shards []*cacheShard + lifeWindow uint64 + clock clock + hash Hasher + config Config + shardMask uint64 + maxShardSize uint32 + close chan struct{} +} + +// RemoveReason is a value used to signal to the user why a particular key was removed in the OnRemove callback. +type RemoveReason uint32 + +const ( + // Expired means the key is past its LifeWindow. + Expired RemoveReason = iota + // NoSpace means the key is the oldest and the cache size was at its maximum when Set was called, or the + // entry exceeded the maximum shard size. + NoSpace + // Deleted means Delete was called and this key was removed as a result. + Deleted +) + +// NewBigCache initialize new instance of BigCache +func NewBigCache(config Config) (*BigCache, error) { + return newBigCache(config, &systemClock{}) +} + +func newBigCache(config Config, clock clock) (*BigCache, error) { + + if !isPowerOfTwo(config.Shards) { + return nil, fmt.Errorf("Shards number must be power of two") + } + + if config.Hasher == nil { + config.Hasher = newDefaultHasher() + } + + cache := &BigCache{ + shards: make([]*cacheShard, config.Shards), + lifeWindow: uint64(config.LifeWindow.Seconds()), + clock: clock, + hash: config.Hasher, + config: config, + shardMask: uint64(config.Shards - 1), + maxShardSize: uint32(config.maximumShardSize()), + close: make(chan struct{}), + } + + var onRemove func(wrappedEntry []byte, reason RemoveReason) + if config.OnRemove != nil { + onRemove = cache.providedOnRemove + } else if config.OnRemoveWithReason != nil { + onRemove = cache.providedOnRemoveWithReason + } else { + onRemove = cache.notProvidedOnRemove + } + + for i := 0; i < config.Shards; i++ { + cache.shards[i] = initNewShard(config, onRemove, clock) + } + + if config.CleanWindow > 0 { + go func() { + ticker := time.NewTicker(config.CleanWindow) + defer ticker.Stop() + for { + select { + case t := <-ticker.C: + cache.cleanUp(uint64(t.Unix())) + case <-cache.close: + return + } + } + }() + } + + return cache, nil +} + +// Close is used to signal a shutdown of the cache when you are done with it. +// This allows the cleaning goroutines to exit and ensures references are not +// kept to the cache preventing GC of the entire cache. +func (c *BigCache) Close() error { + close(c.close) + return nil +} + +// Get reads entry for the key. +// It returns an EntryNotFoundError when +// no entry exists for the given key. +func (c *BigCache) Get(key string) ([]byte, error) { + hashedKey := c.hash.Sum64(key) + shard := c.getShard(hashedKey) + return shard.get(key, hashedKey) +} + +// Set saves entry under the key +func (c *BigCache) Set(key string, entry []byte) error { + hashedKey := c.hash.Sum64(key) + shard := c.getShard(hashedKey) + return shard.set(key, hashedKey, entry) +} + +// Delete removes the key +func (c *BigCache) Delete(key string) error { + hashedKey := c.hash.Sum64(key) + shard := c.getShard(hashedKey) + return shard.del(key, hashedKey) +} + +// Reset empties all cache shards +func (c *BigCache) Reset() error { + for _, shard := range c.shards { + shard.reset(c.config) + } + return nil +} + +// Len computes number of entries in cache +func (c *BigCache) Len() int { + var len int + for _, shard := range c.shards { + len += shard.len() + } + return len +} + +// Capacity returns amount of bytes store in the cache. +func (c *BigCache) Capacity() int { + var len int + for _, shard := range c.shards { + len += shard.capacity() + } + return len +} + +// Stats returns cache's statistics +func (c *BigCache) Stats() Stats { + var s Stats + for _, shard := range c.shards { + tmp := shard.getStats() + s.Hits += tmp.Hits + s.Misses += tmp.Misses + s.DelHits += tmp.DelHits + s.DelMisses += tmp.DelMisses + s.Collisions += tmp.Collisions + } + return s +} + +// Iterator returns iterator function to iterate over EntryInfo's from whole cache. +func (c *BigCache) Iterator() *EntryInfoIterator { + return newIterator(c) +} + +func (c *BigCache) onEvict(oldestEntry []byte, currentTimestamp uint64, evict func(reason RemoveReason) error) bool { + oldestTimestamp := readTimestampFromEntry(oldestEntry) + if currentTimestamp-oldestTimestamp > c.lifeWindow { + evict(Expired) + return true + } + return false +} + +func (c *BigCache) cleanUp(currentTimestamp uint64) { + for _, shard := range c.shards { + shard.cleanUp(currentTimestamp) + } +} + +func (c *BigCache) getShard(hashedKey uint64) (shard *cacheShard) { + return c.shards[hashedKey&c.shardMask] +} + +func (c *BigCache) providedOnRemove(wrappedEntry []byte, reason RemoveReason) { + c.config.OnRemove(readKeyFromEntry(wrappedEntry), readEntry(wrappedEntry)) +} + +func (c *BigCache) providedOnRemoveWithReason(wrappedEntry []byte, reason RemoveReason) { + if c.config.onRemoveFilter == 0 || (1< 0 { + c.config.OnRemoveWithReason(readKeyFromEntry(wrappedEntry), readEntry(wrappedEntry), reason) + } +} + +func (c *BigCache) notProvidedOnRemove(wrappedEntry []byte, reason RemoveReason) { +} diff --git a/vendor/github.com/allegro/bigcache/bytes.go b/vendor/github.com/allegro/bigcache/bytes.go new file mode 100644 index 0000000000..3944bfe135 --- /dev/null +++ b/vendor/github.com/allegro/bigcache/bytes.go @@ -0,0 +1,14 @@ +// +build !appengine + +package bigcache + +import ( + "reflect" + "unsafe" +) + +func bytesToString(b []byte) string { + bytesHeader := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + strHeader := reflect.StringHeader{Data: bytesHeader.Data, Len: bytesHeader.Len} + return *(*string)(unsafe.Pointer(&strHeader)) +} diff --git a/vendor/github.com/allegro/bigcache/bytes_appengine.go b/vendor/github.com/allegro/bigcache/bytes_appengine.go new file mode 100644 index 0000000000..3892f3b54f --- /dev/null +++ b/vendor/github.com/allegro/bigcache/bytes_appengine.go @@ -0,0 +1,7 @@ +// +build appengine + +package bigcache + +func bytesToString(b []byte) string { + return string(b) +} diff --git a/vendor/github.com/allegro/bigcache/clock.go b/vendor/github.com/allegro/bigcache/clock.go new file mode 100644 index 0000000000..f8b535e133 --- /dev/null +++ b/vendor/github.com/allegro/bigcache/clock.go @@ -0,0 +1,14 @@ +package bigcache + +import "time" + +type clock interface { + epoch() int64 +} + +type systemClock struct { +} + +func (c systemClock) epoch() int64 { + return time.Now().Unix() +} diff --git a/vendor/github.com/allegro/bigcache/config.go b/vendor/github.com/allegro/bigcache/config.go new file mode 100644 index 0000000000..9654143ab1 --- /dev/null +++ b/vendor/github.com/allegro/bigcache/config.go @@ -0,0 +1,86 @@ +package bigcache + +import "time" + +// Config for BigCache +type Config struct { + // Number of cache shards, value must be a power of two + Shards int + // Time after which entry can be evicted + LifeWindow time.Duration + // Interval between removing expired entries (clean up). + // If set to <= 0 then no action is performed. Setting to < 1 second is counterproductive — bigcache has a one second resolution. + CleanWindow time.Duration + // Max number of entries in life window. Used only to calculate initial size for cache shards. + // When proper value is set then additional memory allocation does not occur. + MaxEntriesInWindow int + // Max size of entry in bytes. Used only to calculate initial size for cache shards. + MaxEntrySize int + // Verbose mode prints information about new memory allocation + Verbose bool + // Hasher used to map between string keys and unsigned 64bit integers, by default fnv64 hashing is used. + Hasher Hasher + // HardMaxCacheSize is a limit for cache size in MB. Cache will not allocate more memory than this limit. + // It can protect application from consuming all available memory on machine, therefore from running OOM Killer. + // Default value is 0 which means unlimited size. When the limit is higher than 0 and reached then + // the oldest entries are overridden for the new ones. + HardMaxCacheSize int + // OnRemove is a callback fired when the oldest entry is removed because of its expiration time or no space left + // for the new entry, or because delete was called. + // Default value is nil which means no callback and it prevents from unwrapping the oldest entry. + OnRemove func(key string, entry []byte) + // OnRemoveWithReason is a callback fired when the oldest entry is removed because of its expiration time or no space left + // for the new entry, or because delete was called. A constant representing the reason will be passed through. + // Default value is nil which means no callback and it prevents from unwrapping the oldest entry. + // Ignored if OnRemove is specified. + OnRemoveWithReason func(key string, entry []byte, reason RemoveReason) + + onRemoveFilter int + + // Logger is a logging interface and used in combination with `Verbose` + // Defaults to `DefaultLogger()` + Logger Logger +} + +// DefaultConfig initializes config with default values. +// When load for BigCache can be predicted in advance then it is better to use custom config. +func DefaultConfig(eviction time.Duration) Config { + return Config{ + Shards: 1024, + LifeWindow: eviction, + CleanWindow: 0, + MaxEntriesInWindow: 1000 * 10 * 60, + MaxEntrySize: 500, + Verbose: true, + Hasher: newDefaultHasher(), + HardMaxCacheSize: 0, + Logger: DefaultLogger(), + } +} + +// initialShardSize computes initial shard size +func (c Config) initialShardSize() int { + return max(c.MaxEntriesInWindow/c.Shards, minimumEntriesInShard) +} + +// maximumShardSize computes maximum shard size +func (c Config) maximumShardSize() int { + maxShardSize := 0 + + if c.HardMaxCacheSize > 0 { + maxShardSize = convertMBToBytes(c.HardMaxCacheSize) / c.Shards + } + + return maxShardSize +} + +// OnRemoveFilterSet sets which remove reasons will trigger a call to OnRemoveWithReason. +// Filtering out reasons prevents bigcache from unwrapping them, which saves cpu. +func (c Config) OnRemoveFilterSet(reasons ...RemoveReason) Config { + c.onRemoveFilter = 0 + for i := range reasons { + c.onRemoveFilter |= 1 << uint(reasons[i]) + } + + return c +} diff --git a/vendor/github.com/allegro/bigcache/encoding.go b/vendor/github.com/allegro/bigcache/encoding.go new file mode 100644 index 0000000000..4d434e5dce --- /dev/null +++ b/vendor/github.com/allegro/bigcache/encoding.go @@ -0,0 +1,62 @@ +package bigcache + +import ( + "encoding/binary" +) + +const ( + timestampSizeInBytes = 8 // Number of bytes used for timestamp + hashSizeInBytes = 8 // Number of bytes used for hash + keySizeInBytes = 2 // Number of bytes used for size of entry key + headersSizeInBytes = timestampSizeInBytes + hashSizeInBytes + keySizeInBytes // Number of bytes used for all headers +) + +func wrapEntry(timestamp uint64, hash uint64, key string, entry []byte, buffer *[]byte) []byte { + keyLength := len(key) + blobLength := len(entry) + headersSizeInBytes + keyLength + + if blobLength > len(*buffer) { + *buffer = make([]byte, blobLength) + } + blob := *buffer + + binary.LittleEndian.PutUint64(blob, timestamp) + binary.LittleEndian.PutUint64(blob[timestampSizeInBytes:], hash) + binary.LittleEndian.PutUint16(blob[timestampSizeInBytes+hashSizeInBytes:], uint16(keyLength)) + copy(blob[headersSizeInBytes:], key) + copy(blob[headersSizeInBytes+keyLength:], entry) + + return blob[:blobLength] +} + +func readEntry(data []byte) []byte { + length := binary.LittleEndian.Uint16(data[timestampSizeInBytes+hashSizeInBytes:]) + + // copy on read + dst := make([]byte, len(data)-int(headersSizeInBytes+length)) + copy(dst, data[headersSizeInBytes+length:]) + + return dst +} + +func readTimestampFromEntry(data []byte) uint64 { + return binary.LittleEndian.Uint64(data) +} + +func readKeyFromEntry(data []byte) string { + length := binary.LittleEndian.Uint16(data[timestampSizeInBytes+hashSizeInBytes:]) + + // copy on read + dst := make([]byte, length) + copy(dst, data[headersSizeInBytes:headersSizeInBytes+length]) + + return bytesToString(dst) +} + +func readHashFromEntry(data []byte) uint64 { + return binary.LittleEndian.Uint64(data[timestampSizeInBytes:]) +} + +func resetKeyFromEntry(data []byte) { + binary.LittleEndian.PutUint64(data[timestampSizeInBytes:], 0) +} diff --git a/vendor/github.com/allegro/bigcache/entry_not_found_error.go b/vendor/github.com/allegro/bigcache/entry_not_found_error.go new file mode 100644 index 0000000000..883bdc29eb --- /dev/null +++ b/vendor/github.com/allegro/bigcache/entry_not_found_error.go @@ -0,0 +1,17 @@ +package bigcache + +import "fmt" + +// EntryNotFoundError is an error type struct which is returned when entry was not found for provided key +type EntryNotFoundError struct { + key string +} + +func notFound(key string) error { + return &EntryNotFoundError{key} +} + +// Error returned when entry does not exist. +func (e EntryNotFoundError) Error() string { + return fmt.Sprintf("Entry %q not found", e.key) +} diff --git a/vendor/github.com/allegro/bigcache/fnv.go b/vendor/github.com/allegro/bigcache/fnv.go new file mode 100644 index 0000000000..188c9aa6dc --- /dev/null +++ b/vendor/github.com/allegro/bigcache/fnv.go @@ -0,0 +1,28 @@ +package bigcache + +// newDefaultHasher returns a new 64-bit FNV-1a Hasher which makes no memory allocations. +// Its Sum64 method will lay the value out in big-endian byte order. +// See https://en.wikipedia.org/wiki/Fowler–Noll–Vo_hash_function +func newDefaultHasher() Hasher { + return fnv64a{} +} + +type fnv64a struct{} + +const ( + // offset64 FNVa offset basis. See https://en.wikipedia.org/wiki/Fowler–Noll–Vo_hash_function#FNV-1a_hash + offset64 = 14695981039346656037 + // prime64 FNVa prime value. See https://en.wikipedia.org/wiki/Fowler–Noll–Vo_hash_function#FNV-1a_hash + prime64 = 1099511628211 +) + +// Sum64 gets the string and returns its uint64 hash value. +func (f fnv64a) Sum64(key string) uint64 { + var hash uint64 = offset64 + for i := 0; i < len(key); i++ { + hash ^= uint64(key[i]) + hash *= prime64 + } + + return hash +} diff --git a/vendor/github.com/allegro/bigcache/hash.go b/vendor/github.com/allegro/bigcache/hash.go new file mode 100644 index 0000000000..5f8ade774d --- /dev/null +++ b/vendor/github.com/allegro/bigcache/hash.go @@ -0,0 +1,8 @@ +package bigcache + +// Hasher is responsible for generating unsigned, 64 bit hash of provided string. Hasher should minimize collisions +// (generating same hash for different strings) and while performance is also important fast functions are preferable (i.e. +// you can use FarmHash family). +type Hasher interface { + Sum64(string) uint64 +} diff --git a/vendor/github.com/allegro/bigcache/iterator.go b/vendor/github.com/allegro/bigcache/iterator.go new file mode 100644 index 0000000000..70b98d9004 --- /dev/null +++ b/vendor/github.com/allegro/bigcache/iterator.go @@ -0,0 +1,122 @@ +package bigcache + +import "sync" + +type iteratorError string + +func (e iteratorError) Error() string { + return string(e) +} + +// ErrInvalidIteratorState is reported when iterator is in invalid state +const ErrInvalidIteratorState = iteratorError("Iterator is in invalid state. Use SetNext() to move to next position") + +// ErrCannotRetrieveEntry is reported when entry cannot be retrieved from underlying +const ErrCannotRetrieveEntry = iteratorError("Could not retrieve entry from cache") + +var emptyEntryInfo = EntryInfo{} + +// EntryInfo holds informations about entry in the cache +type EntryInfo struct { + timestamp uint64 + hash uint64 + key string + value []byte +} + +// Key returns entry's underlying key +func (e EntryInfo) Key() string { + return e.key +} + +// Hash returns entry's hash value +func (e EntryInfo) Hash() uint64 { + return e.hash +} + +// Timestamp returns entry's timestamp (time of insertion) +func (e EntryInfo) Timestamp() uint64 { + return e.timestamp +} + +// Value returns entry's underlying value +func (e EntryInfo) Value() []byte { + return e.value +} + +// EntryInfoIterator allows to iterate over entries in the cache +type EntryInfoIterator struct { + mutex sync.Mutex + cache *BigCache + currentShard int + currentIndex int + elements []uint32 + elementsCount int + valid bool +} + +// SetNext moves to next element and returns true if it exists. +func (it *EntryInfoIterator) SetNext() bool { + it.mutex.Lock() + + it.valid = false + it.currentIndex++ + + if it.elementsCount > it.currentIndex { + it.valid = true + it.mutex.Unlock() + return true + } + + for i := it.currentShard + 1; i < it.cache.config.Shards; i++ { + it.elements, it.elementsCount = it.cache.shards[i].copyKeys() + + // Non empty shard - stick with it + if it.elementsCount > 0 { + it.currentIndex = 0 + it.currentShard = i + it.valid = true + it.mutex.Unlock() + return true + } + } + it.mutex.Unlock() + return false +} + +func newIterator(cache *BigCache) *EntryInfoIterator { + elements, count := cache.shards[0].copyKeys() + + return &EntryInfoIterator{ + cache: cache, + currentShard: 0, + currentIndex: -1, + elements: elements, + elementsCount: count, + } +} + +// Value returns current value from the iterator +func (it *EntryInfoIterator) Value() (EntryInfo, error) { + it.mutex.Lock() + + if !it.valid { + it.mutex.Unlock() + return emptyEntryInfo, ErrInvalidIteratorState + } + + entry, err := it.cache.shards[it.currentShard].getEntry(int(it.elements[it.currentIndex])) + + if err != nil { + it.mutex.Unlock() + return emptyEntryInfo, ErrCannotRetrieveEntry + } + it.mutex.Unlock() + + return EntryInfo{ + timestamp: readTimestampFromEntry(entry), + hash: readHashFromEntry(entry), + key: readKeyFromEntry(entry), + value: readEntry(entry), + }, nil +} diff --git a/vendor/github.com/allegro/bigcache/logger.go b/vendor/github.com/allegro/bigcache/logger.go new file mode 100644 index 0000000000..50e84abc80 --- /dev/null +++ b/vendor/github.com/allegro/bigcache/logger.go @@ -0,0 +1,30 @@ +package bigcache + +import ( + "log" + "os" +) + +// Logger is invoked when `Config.Verbose=true` +type Logger interface { + Printf(format string, v ...interface{}) +} + +// this is a safeguard, breaking on compile time in case +// `log.Logger` does not adhere to our `Logger` interface. +// see https://golang.org/doc/faq#guarantee_satisfies_interface +var _ Logger = &log.Logger{} + +// DefaultLogger returns a `Logger` implementation +// backed by stdlib's log +func DefaultLogger() *log.Logger { + return log.New(os.Stdout, "", log.LstdFlags) +} + +func newLogger(custom Logger) Logger { + if custom != nil { + return custom + } + + return DefaultLogger() +} diff --git a/vendor/github.com/allegro/bigcache/queue/bytes_queue.go b/vendor/github.com/allegro/bigcache/queue/bytes_queue.go new file mode 100644 index 0000000000..0285c72cd5 --- /dev/null +++ b/vendor/github.com/allegro/bigcache/queue/bytes_queue.go @@ -0,0 +1,210 @@ +package queue + +import ( + "encoding/binary" + "log" + "time" +) + +const ( + // Number of bytes used to keep information about entry size + headerEntrySize = 4 + // Bytes before left margin are not used. Zero index means element does not exist in queue, useful while reading slice from index + leftMarginIndex = 1 + // Minimum empty blob size in bytes. Empty blob fills space between tail and head in additional memory allocation. + // It keeps entries indexes unchanged + minimumEmptyBlobSize = 32 + headerEntrySize +) + +// BytesQueue is a non-thread safe queue type of fifo based on bytes array. +// For every push operation index of entry is returned. It can be used to read the entry later +type BytesQueue struct { + array []byte + capacity int + maxCapacity int + head int + tail int + count int + rightMargin int + headerBuffer []byte + verbose bool + initialCapacity int +} + +type queueError struct { + message string +} + +// NewBytesQueue initialize new bytes queue. +// Initial capacity is used in bytes array allocation +// When verbose flag is set then information about memory allocation are printed +func NewBytesQueue(initialCapacity int, maxCapacity int, verbose bool) *BytesQueue { + return &BytesQueue{ + array: make([]byte, initialCapacity), + capacity: initialCapacity, + maxCapacity: maxCapacity, + headerBuffer: make([]byte, headerEntrySize), + tail: leftMarginIndex, + head: leftMarginIndex, + rightMargin: leftMarginIndex, + verbose: verbose, + initialCapacity: initialCapacity, + } +} + +// Reset removes all entries from queue +func (q *BytesQueue) Reset() { + // Just reset indexes + q.tail = leftMarginIndex + q.head = leftMarginIndex + q.rightMargin = leftMarginIndex + q.count = 0 +} + +// Push copies entry at the end of queue and moves tail pointer. Allocates more space if needed. +// Returns index for pushed data or error if maximum size queue limit is reached. +func (q *BytesQueue) Push(data []byte) (int, error) { + dataLen := len(data) + + if q.availableSpaceAfterTail() < dataLen+headerEntrySize { + if q.availableSpaceBeforeHead() >= dataLen+headerEntrySize { + q.tail = leftMarginIndex + } else if q.capacity+headerEntrySize+dataLen >= q.maxCapacity && q.maxCapacity > 0 { + return -1, &queueError{"Full queue. Maximum size limit reached."} + } else { + q.allocateAdditionalMemory(dataLen + headerEntrySize) + } + } + + index := q.tail + + q.push(data, dataLen) + + return index, nil +} + +func (q *BytesQueue) allocateAdditionalMemory(minimum int) { + start := time.Now() + if q.capacity < minimum { + q.capacity += minimum + } + q.capacity = q.capacity * 2 + if q.capacity > q.maxCapacity && q.maxCapacity > 0 { + q.capacity = q.maxCapacity + } + + oldArray := q.array + q.array = make([]byte, q.capacity) + + if leftMarginIndex != q.rightMargin { + copy(q.array, oldArray[:q.rightMargin]) + + if q.tail < q.head { + emptyBlobLen := q.head - q.tail - headerEntrySize + q.push(make([]byte, emptyBlobLen), emptyBlobLen) + q.head = leftMarginIndex + q.tail = q.rightMargin + } + } + + if q.verbose { + log.Printf("Allocated new queue in %s; Capacity: %d \n", time.Since(start), q.capacity) + } +} + +func (q *BytesQueue) push(data []byte, len int) { + binary.LittleEndian.PutUint32(q.headerBuffer, uint32(len)) + q.copy(q.headerBuffer, headerEntrySize) + + q.copy(data, len) + + if q.tail > q.head { + q.rightMargin = q.tail + } + + q.count++ +} + +func (q *BytesQueue) copy(data []byte, len int) { + q.tail += copy(q.array[q.tail:], data[:len]) +} + +// Pop reads the oldest entry from queue and moves head pointer to the next one +func (q *BytesQueue) Pop() ([]byte, error) { + data, size, err := q.peek(q.head) + if err != nil { + return nil, err + } + + q.head += headerEntrySize + size + q.count-- + + if q.head == q.rightMargin { + q.head = leftMarginIndex + if q.tail == q.rightMargin { + q.tail = leftMarginIndex + } + q.rightMargin = q.tail + } + + return data, nil +} + +// Peek reads the oldest entry from list without moving head pointer +func (q *BytesQueue) Peek() ([]byte, error) { + data, _, err := q.peek(q.head) + return data, err +} + +// Get reads entry from index +func (q *BytesQueue) Get(index int) ([]byte, error) { + data, _, err := q.peek(index) + return data, err +} + +// Capacity returns number of allocated bytes for queue +func (q *BytesQueue) Capacity() int { + return q.capacity +} + +// Len returns number of entries kept in queue +func (q *BytesQueue) Len() int { + return q.count +} + +// Error returns error message +func (e *queueError) Error() string { + return e.message +} + +func (q *BytesQueue) peek(index int) ([]byte, int, error) { + + if q.count == 0 { + return nil, 0, &queueError{"Empty queue"} + } + + if index <= 0 { + return nil, 0, &queueError{"Index must be grater than zero. Invalid index."} + } + + if index+headerEntrySize >= len(q.array) { + return nil, 0, &queueError{"Index out of range"} + } + + blockSize := int(binary.LittleEndian.Uint32(q.array[index : index+headerEntrySize])) + return q.array[index+headerEntrySize : index+headerEntrySize+blockSize], blockSize, nil +} + +func (q *BytesQueue) availableSpaceAfterTail() int { + if q.tail >= q.head { + return q.capacity - q.tail + } + return q.head - q.tail - minimumEmptyBlobSize +} + +func (q *BytesQueue) availableSpaceBeforeHead() int { + if q.tail >= q.head { + return q.head - leftMarginIndex - minimumEmptyBlobSize + } + return q.head - q.tail - minimumEmptyBlobSize +} diff --git a/vendor/github.com/allegro/bigcache/shard.go b/vendor/github.com/allegro/bigcache/shard.go new file mode 100644 index 0000000000..56a9fb0895 --- /dev/null +++ b/vendor/github.com/allegro/bigcache/shard.go @@ -0,0 +1,236 @@ +package bigcache + +import ( + "fmt" + "sync" + "sync/atomic" + + "github.com/allegro/bigcache/queue" +) + +type onRemoveCallback func(wrappedEntry []byte, reason RemoveReason) + +type cacheShard struct { + hashmap map[uint64]uint32 + entries queue.BytesQueue + lock sync.RWMutex + entryBuffer []byte + onRemove onRemoveCallback + + isVerbose bool + logger Logger + clock clock + lifeWindow uint64 + + stats Stats +} + +func (s *cacheShard) get(key string, hashedKey uint64) ([]byte, error) { + s.lock.RLock() + itemIndex := s.hashmap[hashedKey] + + if itemIndex == 0 { + s.lock.RUnlock() + s.miss() + return nil, notFound(key) + } + + wrappedEntry, err := s.entries.Get(int(itemIndex)) + if err != nil { + s.lock.RUnlock() + s.miss() + return nil, err + } + if entryKey := readKeyFromEntry(wrappedEntry); key != entryKey { + if s.isVerbose { + s.logger.Printf("Collision detected. Both %q and %q have the same hash %x", key, entryKey, hashedKey) + } + s.lock.RUnlock() + s.collision() + return nil, notFound(key) + } + s.lock.RUnlock() + s.hit() + return readEntry(wrappedEntry), nil +} + +func (s *cacheShard) set(key string, hashedKey uint64, entry []byte) error { + currentTimestamp := uint64(s.clock.epoch()) + + s.lock.Lock() + + if previousIndex := s.hashmap[hashedKey]; previousIndex != 0 { + if previousEntry, err := s.entries.Get(int(previousIndex)); err == nil { + resetKeyFromEntry(previousEntry) + } + } + + if oldestEntry, err := s.entries.Peek(); err == nil { + s.onEvict(oldestEntry, currentTimestamp, s.removeOldestEntry) + } + + w := wrapEntry(currentTimestamp, hashedKey, key, entry, &s.entryBuffer) + + for { + if index, err := s.entries.Push(w); err == nil { + s.hashmap[hashedKey] = uint32(index) + s.lock.Unlock() + return nil + } + if s.removeOldestEntry(NoSpace) != nil { + s.lock.Unlock() + return fmt.Errorf("entry is bigger than max shard size") + } + } +} + +func (s *cacheShard) del(key string, hashedKey uint64) error { + s.lock.RLock() + itemIndex := s.hashmap[hashedKey] + + if itemIndex == 0 { + s.lock.RUnlock() + s.delmiss() + return notFound(key) + } + + wrappedEntry, err := s.entries.Get(int(itemIndex)) + if err != nil { + s.lock.RUnlock() + s.delmiss() + return err + } + s.lock.RUnlock() + + s.lock.Lock() + { + delete(s.hashmap, hashedKey) + s.onRemove(wrappedEntry, Deleted) + resetKeyFromEntry(wrappedEntry) + } + s.lock.Unlock() + + s.delhit() + return nil +} + +func (s *cacheShard) onEvict(oldestEntry []byte, currentTimestamp uint64, evict func(reason RemoveReason) error) bool { + oldestTimestamp := readTimestampFromEntry(oldestEntry) + if currentTimestamp-oldestTimestamp > s.lifeWindow { + evict(Expired) + return true + } + return false +} + +func (s *cacheShard) cleanUp(currentTimestamp uint64) { + s.lock.Lock() + for { + if oldestEntry, err := s.entries.Peek(); err != nil { + break + } else if evicted := s.onEvict(oldestEntry, currentTimestamp, s.removeOldestEntry); !evicted { + break + } + } + s.lock.Unlock() +} + +func (s *cacheShard) getOldestEntry() ([]byte, error) { + return s.entries.Peek() +} + +func (s *cacheShard) getEntry(index int) ([]byte, error) { + return s.entries.Get(index) +} + +func (s *cacheShard) copyKeys() (keys []uint32, next int) { + keys = make([]uint32, len(s.hashmap)) + + s.lock.RLock() + + for _, index := range s.hashmap { + keys[next] = index + next++ + } + + s.lock.RUnlock() + return keys, next +} + +func (s *cacheShard) removeOldestEntry(reason RemoveReason) error { + oldest, err := s.entries.Pop() + if err == nil { + hash := readHashFromEntry(oldest) + delete(s.hashmap, hash) + s.onRemove(oldest, reason) + return nil + } + return err +} + +func (s *cacheShard) reset(config Config) { + s.lock.Lock() + s.hashmap = make(map[uint64]uint32, config.initialShardSize()) + s.entryBuffer = make([]byte, config.MaxEntrySize+headersSizeInBytes) + s.entries.Reset() + s.lock.Unlock() +} + +func (s *cacheShard) len() int { + s.lock.RLock() + res := len(s.hashmap) + s.lock.RUnlock() + return res +} + +func (s *cacheShard) capacity() int { + s.lock.RLock() + res := s.entries.Capacity() + s.lock.RUnlock() + return res +} + +func (s *cacheShard) getStats() Stats { + var stats = Stats{ + Hits: atomic.LoadInt64(&s.stats.Hits), + Misses: atomic.LoadInt64(&s.stats.Misses), + DelHits: atomic.LoadInt64(&s.stats.DelHits), + DelMisses: atomic.LoadInt64(&s.stats.DelMisses), + Collisions: atomic.LoadInt64(&s.stats.Collisions), + } + return stats +} + +func (s *cacheShard) hit() { + atomic.AddInt64(&s.stats.Hits, 1) +} + +func (s *cacheShard) miss() { + atomic.AddInt64(&s.stats.Misses, 1) +} + +func (s *cacheShard) delhit() { + atomic.AddInt64(&s.stats.DelHits, 1) +} + +func (s *cacheShard) delmiss() { + atomic.AddInt64(&s.stats.DelMisses, 1) +} + +func (s *cacheShard) collision() { + atomic.AddInt64(&s.stats.Collisions, 1) +} + +func initNewShard(config Config, callback onRemoveCallback, clock clock) *cacheShard { + return &cacheShard{ + hashmap: make(map[uint64]uint32, config.initialShardSize()), + entries: *queue.NewBytesQueue(config.initialShardSize()*config.MaxEntrySize, config.maximumShardSize(), config.Verbose), + entryBuffer: make([]byte, config.MaxEntrySize+headersSizeInBytes), + onRemove: callback, + + isVerbose: config.Verbose, + logger: newLogger(config.Logger), + clock: clock, + lifeWindow: uint64(config.LifeWindow.Seconds()), + } +} diff --git a/vendor/github.com/allegro/bigcache/stats.go b/vendor/github.com/allegro/bigcache/stats.go new file mode 100644 index 0000000000..07157132a2 --- /dev/null +++ b/vendor/github.com/allegro/bigcache/stats.go @@ -0,0 +1,15 @@ +package bigcache + +// Stats stores cache statistics +type Stats struct { + // Hits is a number of successfully found keys + Hits int64 `json:"hits"` + // Misses is a number of not found keys + Misses int64 `json:"misses"` + // DelHits is a number of successfully deleted keys + DelHits int64 `json:"delete_hits"` + // DelMisses is a number of not deleted keys + DelMisses int64 `json:"delete_misses"` + // Collisions is a number of happened key-collisions + Collisions int64 `json:"collisions"` +} diff --git a/vendor/github.com/allegro/bigcache/utils.go b/vendor/github.com/allegro/bigcache/utils.go new file mode 100644 index 0000000000..ca1df79b93 --- /dev/null +++ b/vendor/github.com/allegro/bigcache/utils.go @@ -0,0 +1,16 @@ +package bigcache + +func max(a, b int) int { + if a > b { + return a + } + return b +} + +func convertMBToBytes(value int) int { + return value * 1024 * 1024 +} + +func isPowerOfTwo(number int) bool { + return (number & (number - 1)) == 0 +} diff --git a/vendor/vendor.json b/vendor/vendor.json index 06268535e9..1bfe09da72 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -38,6 +38,18 @@ "revision": "5d049714c4a64225c3c79a7cf7d02f7fb5b96338", "revisionTime": "2018-01-16T20:38:02Z" }, + { + "checksumSHA1": "9Niiu1GNhWUrXnGZrl8AU4EzbVE=", + "path": "github.com/allegro/bigcache", + "revision": "bff00e20c68d9f136477d62d182a7dc917bae0ca", + "revisionTime": "2018-10-22T20:06:25Z" + }, + { + "checksumSHA1": "zqToN+R6KybEskp1D4G/lAOKXU4=", + "path": "github.com/allegro/bigcache/queue", + "revision": "bff00e20c68d9f136477d62d182a7dc917bae0ca", + "revisionTime": "2018-10-22T20:06:25Z" + }, { "checksumSHA1": "USkefO0g1U9mr+8hagv3fpSkrxg=", "path": "github.com/aristanetworks/goarista/monotime", From a6942b9f25bd6c67a5ab9e80ab95aae25a08da6d Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 15 Nov 2018 14:57:03 +0100 Subject: [PATCH 049/159] swarm/storage: Batched database migration (#18113) --- swarm/storage/ldbstore.go | 139 ++++++++++++++++++++++++--------- swarm/storage/ldbstore_test.go | 32 ++++++++ 2 files changed, 135 insertions(+), 36 deletions(-) diff --git a/swarm/storage/ldbstore.go b/swarm/storage/ldbstore.go index fbae59facf..bd4f6b9162 100644 --- a/swarm/storage/ldbstore.go +++ b/swarm/storage/ldbstore.go @@ -284,7 +284,7 @@ func getGCIdxValue(index *dpaDBIndex, po uint8, addr Address) []byte { return val } -func parseGCIdxKey(key []byte) (byte, []byte) { +func parseIdxKey(key []byte) (byte, []byte) { return key[0], key[1:] } @@ -589,7 +589,7 @@ func (s *LDBStore) CleanGCIndex() error { it.Seek([]byte{keyGCIdx}) var gcDeletes int for it.Valid() { - rowType, _ := parseGCIdxKey(it.Key()) + rowType, _ := parseIdxKey(it.Key()) if rowType != keyGCIdx { break } @@ -601,47 +601,113 @@ func (s *LDBStore) CleanGCIndex() error { if err := s.db.Write(&batch); err != nil { return err } - - it.Seek([]byte{keyIndex}) - var idx dpaDBIndex - var poPtrs [256]uint64 - for it.Valid() { - rowType, chunkHash := parseGCIdxKey(it.Key()) - if rowType != keyIndex { - break - } - err := decodeIndex(it.Value(), &idx) - if err != nil { - return fmt.Errorf("corrupt index: %v", err) - } - po := s.po(chunkHash) - - // if we don't find the data key, remove the entry - dataKey := getDataKey(idx.Idx, po) - _, err = s.db.Get(dataKey) - if err != nil { - log.Warn("deleting inconsistent index (missing data)", "key", chunkHash) - batch.Delete(it.Key()) - } else { - gcIdxKey := getGCIdxKey(&idx) - gcIdxData := getGCIdxValue(&idx, po, chunkHash) - batch.Put(gcIdxKey, gcIdxData) - log.Trace("clean ok", "key", chunkHash, "gcKey", gcIdxKey, "gcData", gcIdxData) - okEntryCount++ - if idx.Idx > poPtrs[po] { - poPtrs[po] = idx.Idx - } - } - totalEntryCount++ - it.Next() - } + batch.Reset() it.Release() + + // corrected po index pointer values + var poPtrs [256]uint64 + + // set to true if chunk count not on 4096 iteration boundary + var doneIterating bool + + // last key index in previous iteration + lastIdxKey := []byte{keyIndex} + + // counter for debug output + var cleanBatchCount int + + // go through all key index entries + for !doneIterating { + cleanBatchCount++ + var idxs []dpaDBIndex + var chunkHashes [][]byte + var pos []uint8 + it := s.db.NewIterator() + + it.Seek(lastIdxKey) + + // 4096 is just a nice number, don't look for any hidden meaning here... + var i int + for i = 0; i < 4096; i++ { + + // this really shouldn't happen unless database is empty + // but let's keep it to be safe + if !it.Valid() { + doneIterating = true + break + } + + // if it's not keyindex anymore we're done iterating + rowType, chunkHash := parseIdxKey(it.Key()) + if rowType != keyIndex { + doneIterating = true + break + } + + // decode the retrieved index + var idx dpaDBIndex + err := decodeIndex(it.Value(), &idx) + if err != nil { + return fmt.Errorf("corrupt index: %v", err) + } + po := s.po(chunkHash) + lastIdxKey = it.Key() + + // if we don't find the data key, remove the entry + // if we find it, add to the array of new gc indices to create + dataKey := getDataKey(idx.Idx, po) + _, err = s.db.Get(dataKey) + if err != nil { + log.Warn("deleting inconsistent index (missing data)", "key", chunkHash) + batch.Delete(it.Key()) + } else { + idxs = append(idxs, idx) + chunkHashes = append(chunkHashes, chunkHash) + pos = append(pos, po) + okEntryCount++ + if idx.Idx > poPtrs[po] { + poPtrs[po] = idx.Idx + } + } + totalEntryCount++ + it.Next() + } + it.Release() + + // flush the key index corrections + err := s.db.Write(&batch) + if err != nil { + return err + } + batch.Reset() + + // add correct gc indices + for i, okIdx := range idxs { + gcIdxKey := getGCIdxKey(&okIdx) + gcIdxData := getGCIdxValue(&okIdx, pos[i], chunkHashes[i]) + batch.Put(gcIdxKey, gcIdxData) + log.Trace("clean ok", "key", chunkHashes[i], "gcKey", gcIdxKey, "gcData", gcIdxData) + } + + // flush them + err = s.db.Write(&batch) + if err != nil { + return err + } + batch.Reset() + + log.Debug("clean gc index pass", "batch", cleanBatchCount, "checked", i, "kept", len(idxs)) + } + log.Debug("gc cleanup entries", "ok", okEntryCount, "total", totalEntryCount, "batchlen", batch.Len()) + // lastly add updated entry count var entryCount [8]byte binary.BigEndian.PutUint64(entryCount[:], okEntryCount) batch.Put(keyEntryCnt, entryCount[:]) + + // and add the new po index pointers var poKey [2]byte poKey[0] = keyDistanceCnt for i, poPtr := range poPtrs { @@ -655,6 +721,7 @@ func (s *LDBStore) CleanGCIndex() error { } } + // if you made it this far your harddisk has survived. Congratulations return s.db.Write(&batch) } diff --git a/swarm/storage/ldbstore_test.go b/swarm/storage/ldbstore_test.go index 07557980c6..9c10b36290 100644 --- a/swarm/storage/ldbstore_test.go +++ b/swarm/storage/ldbstore_test.go @@ -761,6 +761,38 @@ func TestCleanIndex(t *testing.T) { t.Fatalf("expected sum of bin indices to be 3, was %d", binTotal) } } + + // check that the iterator quits properly + chunks, err = mputRandomChunks(ldb, 4100, 4096) + if err != nil { + t.Fatal(err) + } + + po = ldb.po(chunks[4099].Address()[:]) + dataKey = make([]byte, 10) + dataKey[0] = keyData + dataKey[1] = byte(po) + binary.BigEndian.PutUint64(dataKey[2:], 4099+3) + if _, err := ldb.db.Get(dataKey); err != nil { + t.Fatal(err) + } + if err := ldb.db.Delete(dataKey); err != nil { + t.Fatal(err) + } + + if err := ldb.CleanGCIndex(); err != nil { + t.Fatal(err) + } + + // entrycount should now be one less of added chunks + c, err = ldb.db.Get(keyEntryCnt) + if err != nil { + t.Fatalf("expected gc 2 idx to be present: %v", idxKey) + } + entryCount = binary.BigEndian.Uint64(c) + if entryCount != 4099+2 { + t.Fatalf("expected entrycnt to be 2, was %d", c) + } } func waitGc(ctx context.Context, ldb *LDBStore) { From b91766fe6db6e484f2fd75f617307a48ff85600e Mon Sep 17 00:00:00 2001 From: mr_franklin Date: Thu, 15 Nov 2018 22:31:24 +0800 Subject: [PATCH 050/159] eth: fix comment typo (#18114) * consensus/clique: fix comment typo * eth,eth/downloader: fix comment typo --- eth/downloader/queue.go | 2 +- eth/handler.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index 863cc8de14..7c33953811 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -325,7 +325,7 @@ func (q *queue) Schedule(headers []*types.Header, from uint64) []*types.Header { } // Make sure no duplicate requests are executed if _, ok := q.blockTaskPool[hash]; ok { - log.Warn("Header already scheduled for block fetch", "number", header.Number, "hash", hash) + log.Warn("Header already scheduled for block fetch", "number", header.Number, "hash", hash) continue } if _, ok := q.receiptTaskPool[hash]; ok { diff --git a/eth/handler.go b/eth/handler.go index bd227a84e0..741fc9d5a5 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -658,7 +658,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { p.SetHead(trueHead, trueTD) // Schedule a sync if above ours. Note, this will not fire a sync for a gap of - // a singe block (as the true TD is below the propagated block), however this + // a single block (as the true TD is below the propagated block), however this // scenario should easily be covered by the fetcher. currentBlock := pm.blockchain.CurrentBlock() if trueTD.Cmp(pm.blockchain.GetTd(currentBlock.Hash(), currentBlock.NumberU64())) > 0 { From 324027640bcaf137b8c9e96bc26f0833711497af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Thu, 15 Nov 2018 21:06:27 +0100 Subject: [PATCH 051/159] swarm/network/simulation: use simulations.Event instead p2p.PeerEvent (#18098) --- swarm/network/simulation/events.go | 114 +++++++++++++++------ swarm/network/simulation/example_test.go | 13 ++- swarm/network/stream/delivery_test.go | 8 +- swarm/network/stream/intervals_test.go | 5 +- swarm/network/stream/snapshot_sync_test.go | 11 +- swarm/network/stream/syncer_test.go | 5 +- 6 files changed, 101 insertions(+), 55 deletions(-) diff --git a/swarm/network/simulation/events.go b/swarm/network/simulation/events.go index 594d36225c..d73c3af4ee 100644 --- a/swarm/network/simulation/events.go +++ b/swarm/network/simulation/events.go @@ -20,16 +20,18 @@ import ( "context" "sync" - "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/simulations" ) // PeerEvent is the type of the channel returned by Simulation.PeerEvents. type PeerEvent struct { // NodeID is the ID of node that the event is caught on. NodeID enode.ID + // PeerID is the ID of the peer node that the event is caught on. + PeerID enode.ID // Event is the event that is caught. - Event *p2p.PeerEvent + Event *simulations.Event // Error is the error that may have happened during event watching. Error error } @@ -37,9 +39,13 @@ type PeerEvent struct { // PeerEventsFilter defines a filter on PeerEvents to exclude messages with // defined properties. Use PeerEventsFilter methods to set required options. type PeerEventsFilter struct { - t *p2p.PeerEventType - protocol *string - msgCode *uint64 + eventType simulations.EventType + + connUp *bool + + msgReceive *bool + protocol *string + msgCode *uint64 } // NewPeerEventsFilter returns a new PeerEventsFilter instance. @@ -47,20 +53,48 @@ func NewPeerEventsFilter() *PeerEventsFilter { return &PeerEventsFilter{} } -// Type sets the filter to only one peer event type. -func (f *PeerEventsFilter) Type(t p2p.PeerEventType) *PeerEventsFilter { - f.t = &t +// Connect sets the filter to events when two nodes connect. +func (f *PeerEventsFilter) Connect() *PeerEventsFilter { + f.eventType = simulations.EventTypeConn + b := true + f.connUp = &b + return f +} + +// Drop sets the filter to events when two nodes disconnect. +func (f *PeerEventsFilter) Drop() *PeerEventsFilter { + f.eventType = simulations.EventTypeConn + b := false + f.connUp = &b + return f +} + +// ReceivedMessages sets the filter to only messages that are received. +func (f *PeerEventsFilter) ReceivedMessages() *PeerEventsFilter { + f.eventType = simulations.EventTypeMsg + b := true + f.msgReceive = &b + return f +} + +// SentMessages sets the filter to only messages that are sent. +func (f *PeerEventsFilter) SentMessages() *PeerEventsFilter { + f.eventType = simulations.EventTypeMsg + b := false + f.msgReceive = &b return f } // Protocol sets the filter to only one message protocol. func (f *PeerEventsFilter) Protocol(p string) *PeerEventsFilter { + f.eventType = simulations.EventTypeMsg f.protocol = &p return f } // MsgCode sets the filter to only one msg code. func (f *PeerEventsFilter) MsgCode(c uint64) *PeerEventsFilter { + f.eventType = simulations.EventTypeMsg f.msgCode = &c return f } @@ -80,19 +114,8 @@ func (s *Simulation) PeerEvents(ctx context.Context, ids []enode.ID, filters ... go func(id enode.ID) { defer s.shutdownWG.Done() - client, err := s.Net.GetNode(id).Client() - if err != nil { - subsWG.Done() - eventC <- PeerEvent{NodeID: id, Error: err} - return - } - events := make(chan *p2p.PeerEvent) - sub, err := client.Subscribe(ctx, "admin", events, "peerEvents") - if err != nil { - subsWG.Done() - eventC <- PeerEvent{NodeID: id, Error: err} - return - } + events := make(chan *simulations.Event) + sub := s.Net.Events().Subscribe(events) defer sub.Unsubscribe() subsWG.Done() @@ -110,28 +133,55 @@ func (s *Simulation) PeerEvents(ctx context.Context, ids []enode.ID, filters ... case <-s.Done(): return case e := <-events: + // ignore control events + if e.Control { + continue + } match := len(filters) == 0 // if there are no filters match all events for _, f := range filters { - if f.t != nil && *f.t != e.Type { - continue + if f.eventType == simulations.EventTypeConn && e.Conn != nil { + if *f.connUp != e.Conn.Up { + continue + } + // all connection filter parameters matched, break the loop + match = true + break } - if f.protocol != nil && *f.protocol != e.Protocol { - continue + if f.eventType == simulations.EventTypeMsg && e.Msg != nil { + if f.msgReceive != nil && *f.msgReceive != e.Msg.Received { + continue + } + if f.protocol != nil && *f.protocol != e.Msg.Protocol { + continue + } + if f.msgCode != nil && *f.msgCode != e.Msg.Code { + continue + } + // all message filter parameters matched, break the loop + match = true + break } - if f.msgCode != nil && e.MsgCode != nil && *f.msgCode != *e.MsgCode { - continue + } + var peerID enode.ID + switch e.Type { + case simulations.EventTypeConn: + peerID = e.Conn.One + if peerID == id { + peerID = e.Conn.Other + } + case simulations.EventTypeMsg: + peerID = e.Msg.One + if peerID == id { + peerID = e.Msg.Other } - // all filter parameters matched, break the loop - match = true - break } if match { select { - case eventC <- PeerEvent{NodeID: id, Event: e}: + case eventC <- PeerEvent{NodeID: id, PeerID: peerID, Event: e}: case <-ctx.Done(): if err := ctx.Err(); err != nil { select { - case eventC <- PeerEvent{NodeID: id, Error: err}: + case eventC <- PeerEvent{NodeID: id, PeerID: peerID, Error: err}: case <-s.Done(): } } diff --git a/swarm/network/simulation/example_test.go b/swarm/network/simulation/example_test.go index 84b0634b45..bacc64d53a 100644 --- a/swarm/network/simulation/example_test.go +++ b/swarm/network/simulation/example_test.go @@ -24,7 +24,6 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/network/simulation" @@ -87,7 +86,7 @@ func ExampleSimulation_PeerEvents() { log.Error("peer event", "err", e.Error) continue } - log.Info("peer event", "node", e.NodeID, "peer", e.Event.Peer, "msgcode", e.Event.MsgCode) + log.Info("peer event", "node", e.NodeID, "peer", e.PeerID, "type", e.Event.Type) } }() } @@ -100,7 +99,7 @@ func ExampleSimulation_PeerEvents_disconnections() { disconnections := sim.PeerEvents( context.Background(), sim.NodeIDs(), - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop), + simulation.NewPeerEventsFilter().Drop(), ) go func() { @@ -109,7 +108,7 @@ func ExampleSimulation_PeerEvents_disconnections() { log.Error("peer drop", "err", d.Error) continue } - log.Warn("peer drop", "node", d.NodeID, "peer", d.Event.Peer) + log.Warn("peer drop", "node", d.NodeID, "peer", d.PeerID) } }() } @@ -124,8 +123,8 @@ func ExampleSimulation_PeerEvents_multipleFilters() { context.Background(), sim.NodeIDs(), // Watch when bzz messages 1 and 4 are received. - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeMsgRecv).Protocol("bzz").MsgCode(1), - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeMsgRecv).Protocol("bzz").MsgCode(4), + simulation.NewPeerEventsFilter().ReceivedMessages().Protocol("bzz").MsgCode(1), + simulation.NewPeerEventsFilter().ReceivedMessages().Protocol("bzz").MsgCode(4), ) go func() { @@ -134,7 +133,7 @@ func ExampleSimulation_PeerEvents_multipleFilters() { log.Error("bzz message", "err", m.Error) continue } - log.Info("bzz message", "node", m.NodeID, "peer", m.Event.Peer) + log.Info("bzz message", "node", m.NodeID, "peer", m.PeerID) } }() } diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index c9a530115c..6b6025115e 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -565,13 +565,13 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck disconnections := sim.PeerEvents( context.Background(), sim.NodeIDs(), - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop), + simulation.NewPeerEventsFilter().Drop(), ) go func() { for d := range disconnections { if d.Error != nil { - log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer) + log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) t.Fatal(d.Error) } } @@ -697,13 +697,13 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skip disconnections := sim.PeerEvents( context.Background(), sim.NodeIDs(), - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop), + simulation.NewPeerEventsFilter().Drop(), ) go func() { for d := range disconnections { if d.Error != nil { - log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer) + log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) b.Fatal(d.Error) } } diff --git a/swarm/network/stream/intervals_test.go b/swarm/network/stream/intervals_test.go index 037984f220..b9525d4a49 100644 --- a/swarm/network/stream/intervals_test.go +++ b/swarm/network/stream/intervals_test.go @@ -27,7 +27,6 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/swarm/network" @@ -154,7 +153,7 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { disconnections := sim.PeerEvents( context.Background(), sim.NodeIDs(), - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop), + simulation.NewPeerEventsFilter().Drop(), ) err = registry.Subscribe(storer, NewStream(externalStreamName, "", live), history, Top) @@ -165,7 +164,7 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { go func() { for d := range disconnections { if d.Error != nil { - log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer) + log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) t.Fatal(d.Error) } } diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go index 4bd7f38f5b..96c37bddcb 100644 --- a/swarm/network/stream/snapshot_sync_test.go +++ b/swarm/network/stream/snapshot_sync_test.go @@ -27,7 +27,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/simulations" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" @@ -210,12 +209,12 @@ func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) { disconnections := sim.PeerEvents( context.Background(), sim.NodeIDs(), - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop), + simulation.NewPeerEventsFilter().Drop(), ) go func() { for d := range disconnections { - log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer) + log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) t.Fatal("unexpected disconnect") cancelSimRun() } @@ -402,12 +401,12 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int) disconnections := sim.PeerEvents( context.Background(), sim.NodeIDs(), - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop), + simulation.NewPeerEventsFilter().Drop(), ) go func() { for d := range disconnections { - log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer) + log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) t.Fatal("unexpected disconnect") cancelSimRun() } @@ -428,7 +427,7 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int) var subscriptionCount int - filter := simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeMsgRecv).Protocol("stream").MsgCode(4) + filter := simulation.NewPeerEventsFilter().ReceivedMessages().Protocol("stream").MsgCode(4) eventC := sim.PeerEvents(ctx, nodeIDs, filter) for j, node := range nodeIDs { diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index a543cae058..f4e0554511 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -28,7 +28,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/swarm/log" @@ -151,13 +150,13 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck disconnections := sim.PeerEvents( context.Background(), sim.NodeIDs(), - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop), + simulation.NewPeerEventsFilter().Drop(), ) go func() { for d := range disconnections { if d.Error != nil { - log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer) + log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) t.Fatal(d.Error) } } From ffe2fc3bc4d77ad3f503d2bc1cdd62eac8d03c5b Mon Sep 17 00:00:00 2001 From: holisticode Date: Thu, 15 Nov 2018 17:41:19 -0500 Subject: [PATCH 052/159] Swarm accounting (#18050) * swarm: completed 1st phase of swap accounting * swarm: swap accounting for swarm with p2p accounting * swarm/swap: addressed PR comments * swarm/swap: ignore ErrNotFound on stateStore.Get() * swarm/swap: GetPeerBalance test; add TODO for chequebook API check * swarm/network/stream: fix NewRegistry calls with new arguments * swarm/swap: address @justelad's PR comments --- swarm/network/stream/common_test.go | 2 +- swarm/network/stream/delivery_test.go | 8 +- swarm/network/stream/intervals_test.go | 2 +- .../network/stream/snapshot_retrieval_test.go | 2 +- swarm/network/stream/snapshot_sync_test.go | 4 +- swarm/network/stream/stream.go | 76 +++++--- swarm/network/stream/syncer_test.go | 2 +- swarm/swap/swap.go | 93 +++++++++ swarm/swap/swap_test.go | 184 ++++++++++++++++++ swarm/swarm.go | 18 +- 10 files changed, 353 insertions(+), 38 deletions(-) create mode 100644 swarm/swap/swap.go create mode 100644 swarm/swap/swap_test.go diff --git a/swarm/network/stream/common_test.go b/swarm/network/stream/common_test.go index 721b873b78..c5f1fa176a 100644 --- a/swarm/network/stream/common_test.go +++ b/swarm/network/stream/common_test.go @@ -114,7 +114,7 @@ func newStreamerTester(t *testing.T, registryOptions *RegistryOptions) (*p2ptest delivery := NewDelivery(to, netStore) netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New - streamer := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), registryOptions) + streamer := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), registryOptions, nil) teardown := func() { streamer.Close() removeDataDir() diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index 6b6025115e..a6173a3892 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -290,7 +290,7 @@ func TestRequestFromPeers(t *testing.T) { Peer: protocolsPeer, }, to) to.On(peer) - r := NewRegistry(addr.ID(), delivery, nil, nil, nil) + r := NewRegistry(addr.ID(), delivery, nil, nil, nil, nil) // an empty priorityQueue has to be created to prevent a goroutine being called after the test has finished sp := &Peer{ @@ -331,7 +331,7 @@ func TestRequestFromPeersWithLightNode(t *testing.T) { Peer: protocolsPeer, }, to) to.On(peer) - r := NewRegistry(addr.ID(), delivery, nil, nil, nil) + r := NewRegistry(addr.ID(), delivery, nil, nil, nil, nil) // an empty priorityQueue has to be created to prevent a goroutine being called after the test has finished sp := &Peer{ Peer: protocolsPeer, @@ -480,7 +480,7 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck SkipCheck: skipCheck, Syncing: SyncingDisabled, Retrieval: RetrievalEnabled, - }) + }, nil) bucket.Store(bucketKeyRegistry, r) fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) @@ -655,7 +655,7 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skip Syncing: SyncingDisabled, Retrieval: RetrievalDisabled, SyncUpdateDelay: 0, - }) + }, nil) fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) bucket.Store(bucketKeyFileStore, fileStore) diff --git a/swarm/network/stream/intervals_test.go b/swarm/network/stream/intervals_test.go index b9525d4a49..defb6df50a 100644 --- a/swarm/network/stream/intervals_test.go +++ b/swarm/network/stream/intervals_test.go @@ -84,7 +84,7 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { Retrieval: RetrievalDisabled, Syncing: SyncingRegisterOnly, SkipCheck: skipCheck, - }) + }, nil) bucket.Store(bucketKeyRegistry, r) r.RegisterClientFunc(externalStreamName, func(p *Peer, t string, live bool) (Client, error) { diff --git a/swarm/network/stream/snapshot_retrieval_test.go b/swarm/network/stream/snapshot_retrieval_test.go index ad15193414..5ea0b1511b 100644 --- a/swarm/network/stream/snapshot_retrieval_test.go +++ b/swarm/network/stream/snapshot_retrieval_test.go @@ -130,7 +130,7 @@ func retrievalStreamerFunc(ctx *adapters.ServiceContext, bucket *sync.Map) (s no Retrieval: RetrievalEnabled, Syncing: SyncingAutoSubscribe, SyncUpdateDelay: 3 * time.Second, - }) + }, nil) fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) bucket.Store(bucketKeyFileStore, fileStore) diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go index 96c37bddcb..6b92c32ae9 100644 --- a/swarm/network/stream/snapshot_sync_test.go +++ b/swarm/network/stream/snapshot_sync_test.go @@ -166,7 +166,7 @@ func streamerFunc(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Servic Retrieval: RetrievalDisabled, Syncing: SyncingAutoSubscribe, SyncUpdateDelay: 3 * time.Second, - }) + }, nil) bucket.Store(bucketKeyRegistry, r) @@ -360,7 +360,7 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int) r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ Retrieval: RetrievalDisabled, Syncing: SyncingRegisterOnly, - }) + }, nil) bucket.Store(bucketKeyRegistry, r) fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) diff --git a/swarm/network/stream/stream.go b/swarm/network/stream/stream.go index 695ff0c502..32e107823b 100644 --- a/swarm/network/stream/stream.go +++ b/swarm/network/stream/stream.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "math" + "reflect" "sync" "time" @@ -87,6 +88,9 @@ type Registry struct { intervalsStore state.Store autoRetrieval bool //automatically subscribe to retrieve request stream maxPeerServers int + spec *protocols.Spec //this protocol's spec + balance protocols.Balance //implements protocols.Balance, for accounting + prices protocols.Prices //implements protocols.Prices, provides prices to accounting } // RegistryOptions holds optional values for NewRegistry constructor. @@ -99,7 +103,7 @@ type RegistryOptions struct { } // NewRegistry is Streamer constructor -func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.SyncChunkStore, intervalsStore state.Store, options *RegistryOptions) *Registry { +func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.SyncChunkStore, intervalsStore state.Store, options *RegistryOptions, balance protocols.Balance) *Registry { if options == nil { options = &RegistryOptions{} } @@ -119,7 +123,10 @@ func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.Sy intervalsStore: intervalsStore, autoRetrieval: retrieval, maxPeerServers: options.MaxPeerServers, + balance: balance, } + streamer.setupSpec() + streamer.api = NewAPI(streamer) delivery.getPeer = streamer.getPeer @@ -228,6 +235,17 @@ func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.Sy return streamer } +//we need to construct a spec instance per node instance +func (r *Registry) setupSpec() { + //first create the "bare" spec + r.createSpec() + //if balance is nil, this node has been started without swap support (swapEnabled flag is false) + if r.balance != nil && !reflect.ValueOf(r.balance).IsNil() { + //swap is enabled, so setup the hook + r.spec.Hook = protocols.NewAccounting(r.balance, r.prices) + } +} + // RegisterClient registers an incoming streamer constructor func (r *Registry) RegisterClientFunc(stream string, f func(*Peer, string, bool) (Client, error)) { r.clientMu.Lock() @@ -492,7 +510,7 @@ func (r *Registry) updateSyncing() { } func (r *Registry) runProtocol(p *p2p.Peer, rw p2p.MsgReadWriter) error { - peer := protocols.NewPeer(p, rw, Spec) + peer := protocols.NewPeer(p, rw, r.spec) bp := network.NewBzzPeer(peer) np := network.NewPeer(bp, r.delivery.kad) r.delivery.kad.On(np) @@ -716,35 +734,43 @@ func (c *clientParams) clientCreated() { close(c.clientCreatedC) } -// Spec is the spec of the streamer protocol -var Spec = &protocols.Spec{ - Name: "stream", - Version: 8, - MaxMsgSize: 10 * 1024 * 1024, - Messages: []interface{}{ - UnsubscribeMsg{}, - OfferedHashesMsg{}, - WantedHashesMsg{}, - TakeoverProofMsg{}, - SubscribeMsg{}, - RetrieveRequestMsg{}, - ChunkDeliveryMsgRetrieval{}, - SubscribeErrorMsg{}, - RequestSubscriptionMsg{}, - QuitMsg{}, - ChunkDeliveryMsgSyncing{}, - }, +//GetSpec returns the streamer spec to callers +//This used to be a global variable but for simulations with +//multiple nodes its fields (notably the Hook) would be overwritten +func (r *Registry) GetSpec() *protocols.Spec { + return r.spec +} + +func (r *Registry) createSpec() { + // Spec is the spec of the streamer protocol + var spec = &protocols.Spec{ + Name: "stream", + Version: 8, + MaxMsgSize: 10 * 1024 * 1024, + Messages: []interface{}{ + UnsubscribeMsg{}, + OfferedHashesMsg{}, + WantedHashesMsg{}, + TakeoverProofMsg{}, + SubscribeMsg{}, + RetrieveRequestMsg{}, + ChunkDeliveryMsgRetrieval{}, + SubscribeErrorMsg{}, + RequestSubscriptionMsg{}, + QuitMsg{}, + ChunkDeliveryMsgSyncing{}, + }, + } + r.spec = spec } func (r *Registry) Protocols() []p2p.Protocol { return []p2p.Protocol{ { - Name: Spec.Name, - Version: Spec.Version, - Length: Spec.Length(), + Name: r.spec.Name, + Version: r.spec.Version, + Length: r.spec.Length(), Run: r.runProtocol, - // NodeInfo: , - // PeerInfo: , }, } } diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index f4e0554511..fe20bab266 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -118,7 +118,7 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck Retrieval: RetrievalDisabled, Syncing: SyncingAutoSubscribe, SkipCheck: skipCheck, - }) + }, nil) fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) bucket.Store(bucketKeyFileStore, fileStore) diff --git a/swarm/swap/swap.go b/swarm/swap/swap.go new file mode 100644 index 0000000000..137eb141d9 --- /dev/null +++ b/swarm/swap/swap.go @@ -0,0 +1,93 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package swap + +import ( + "errors" + "fmt" + "strconv" + "sync" + + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/protocols" + "github.com/ethereum/go-ethereum/swarm/log" + "github.com/ethereum/go-ethereum/swarm/state" +) + +// SwAP Swarm Accounting Protocol +// a peer to peer micropayment system +// A node maintains an individual balance with every peer +// Only messages which have a price will be accounted for +type Swap struct { + stateStore state.Store //stateStore is needed in order to keep balances across sessions + lock sync.RWMutex //lock the balances + balances map[enode.ID]int64 //map of balances for each peer +} + +// New - swap constructor +func New(stateStore state.Store) (swap *Swap) { + swap = &Swap{ + stateStore: stateStore, + balances: make(map[enode.ID]int64), + } + return +} + +//Swap implements the protocols.Balance interface +//Add is the (sole) accounting function +func (s *Swap) Add(amount int64, peer *protocols.Peer) (err error) { + s.lock.Lock() + defer s.lock.Unlock() + + //load existing balances from the state store + err = s.loadState(peer) + if err != nil && err != state.ErrNotFound { + return + } + //adjust the balance + //if amount is negative, it will decrease, otherwise increase + s.balances[peer.ID()] += amount + //save the new balance to the state store + peerBalance := s.balances[peer.ID()] + err = s.stateStore.Put(peer.ID().String(), &peerBalance) + + log.Debug(fmt.Sprintf("balance for peer %s: %s", peer.ID().String(), strconv.FormatInt(peerBalance, 10))) + return err +} + +//GetPeerBalance returns the balance for a given peer +func (swap *Swap) GetPeerBalance(peer enode.ID) (int64, error) { + swap.lock.RLock() + defer swap.lock.RUnlock() + if p, ok := swap.balances[peer]; ok { + return p, nil + } + return 0, errors.New("Peer not found") +} + +//load balances from the state store (persisted) +func (s *Swap) loadState(peer *protocols.Peer) (err error) { + var peerBalance int64 + peerID := peer.ID() + //only load if the current instance doesn't already have this peer's + //balance in memory + if _, ok := s.balances[peerID]; !ok { + err = s.stateStore.Get(peerID.String(), &peerBalance) + s.balances[peerID] = peerBalance + } + return +} diff --git a/swarm/swap/swap_test.go b/swarm/swap/swap_test.go new file mode 100644 index 0000000000..f2e3ba168a --- /dev/null +++ b/swarm/swap/swap_test.go @@ -0,0 +1,184 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package swap + +import ( + "flag" + "fmt" + "io/ioutil" + mrand "math/rand" + "os" + "testing" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/protocols" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + "github.com/ethereum/go-ethereum/swarm/state" + colorable "github.com/mattn/go-colorable" +) + +var ( + loglevel = flag.Int("loglevel", 2, "verbosity of logs") +) + +func init() { + flag.Parse() + mrand.Seed(time.Now().UnixNano()) + + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) +} + +//Test getting a peer's balance +func TestGetPeerBalance(t *testing.T) { + //create a test swap account + swap, testDir := createTestSwap(t) + defer os.RemoveAll(testDir) + + //test for correct value + testPeer := newDummyPeer() + swap.balances[testPeer.ID()] = 888 + b, err := swap.GetPeerBalance(testPeer.ID()) + if err != nil { + t.Fatal(err) + } + if b != 888 { + t.Fatalf("Expected peer's balance to be %d, but is %d", 888, b) + } + + //test for inexistent node + id := adapters.RandomNodeConfig().ID + _, err = swap.GetPeerBalance(id) + if err == nil { + t.Fatal("Expected call to fail, but it didn't!") + } + if err.Error() != "Peer not found" { + t.Fatalf("Expected test to fail with %s, but is %s", "Peer not found", err.Error()) + } +} + +//Test that repeated bookings do correct accounting +func TestRepeatedBookings(t *testing.T) { + //create a test swap account + swap, testDir := createTestSwap(t) + defer os.RemoveAll(testDir) + + testPeer := newDummyPeer() + amount := mrand.Intn(100) + cnt := 1 + mrand.Intn(10) + for i := 0; i < cnt; i++ { + swap.Add(int64(amount), testPeer.Peer) + } + expectedBalance := int64(cnt * amount) + realBalance := swap.balances[testPeer.ID()] + if expectedBalance != realBalance { + t.Fatal(fmt.Sprintf("After %d credits of %d, expected balance to be: %d, but is: %d", cnt, amount, expectedBalance, realBalance)) + } + + testPeer2 := newDummyPeer() + amount = mrand.Intn(100) + cnt = 1 + mrand.Intn(10) + for i := 0; i < cnt; i++ { + swap.Add(0-int64(amount), testPeer2.Peer) + } + expectedBalance = int64(0 - (cnt * amount)) + realBalance = swap.balances[testPeer2.ID()] + if expectedBalance != realBalance { + t.Fatal(fmt.Sprintf("After %d debits of %d, expected balance to be: %d, but is: %d", cnt, amount, expectedBalance, realBalance)) + } + + //mixed debits and credits + amount1 := mrand.Intn(100) + amount2 := mrand.Intn(55) + amount3 := mrand.Intn(999) + swap.Add(int64(amount1), testPeer2.Peer) + swap.Add(int64(0-amount2), testPeer2.Peer) + swap.Add(int64(0-amount3), testPeer2.Peer) + + expectedBalance = expectedBalance + int64(amount1-amount2-amount3) + realBalance = swap.balances[testPeer2.ID()] + + if expectedBalance != realBalance { + t.Fatal(fmt.Sprintf("After mixed debits and credits, expected balance to be: %d, but is: %d", expectedBalance, realBalance)) + } +} + +//try restoring a balance from state store +//this is simulated by creating a node, +//assigning it an arbitrary balance, +//then closing the state store. +//Then we re-open the state store and check that +//the balance is still the same +func TestRestoreBalanceFromStateStore(t *testing.T) { + //create a test swap account + swap, testDir := createTestSwap(t) + defer os.RemoveAll(testDir) + + testPeer := newDummyPeer() + swap.balances[testPeer.ID()] = -8888 + + tmpBalance := swap.balances[testPeer.ID()] + swap.stateStore.Put(testPeer.ID().String(), &tmpBalance) + + swap.stateStore.Close() + swap.stateStore = nil + + stateStore, err := state.NewDBStore(testDir) + if err != nil { + t.Fatal(err) + } + + var newBalance int64 + stateStore.Get(testPeer.ID().String(), &newBalance) + + //compare the balances + if tmpBalance != newBalance { + t.Fatal(fmt.Sprintf("Unexpected balance value after sending cheap message test. Expected balance: %d, balance is: %d", + tmpBalance, newBalance)) + } +} + +//create a test swap account +//creates a stateStore for persistence and a Swap account +func createTestSwap(t *testing.T) (*Swap, string) { + dir, err := ioutil.TempDir("", "swap_test_store") + if err != nil { + t.Fatal(err) + } + stateStore, err2 := state.NewDBStore(dir) + if err2 != nil { + t.Fatal(err2) + } + swap := New(stateStore) + return swap, dir +} + +type dummyPeer struct { + *protocols.Peer +} + +//creates a dummy protocols.Peer with dummy MsgReadWriter +func newDummyPeer() *dummyPeer { + id := adapters.RandomNodeConfig().ID + protoPeer := protocols.NewPeer(p2p.NewPeer(id, "testPeer", nil), nil, nil) + dummy := &dummyPeer{ + Peer: protoPeer, + } + return dummy +} diff --git a/swarm/swarm.go b/swarm/swarm.go index 1fb5443fd4..dc3756d3af 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -51,6 +51,7 @@ import ( "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/storage/feed" "github.com/ethereum/go-ethereum/swarm/storage/mock" + "github.com/ethereum/go-ethereum/swarm/swap" "github.com/ethereum/go-ethereum/swarm/tracing" ) @@ -78,6 +79,7 @@ type Swarm struct { netStore *storage.NetStore sfs *fuse.SwarmFS // need this to cleanup all the active mounts on node exit ps *pss.Pss + swap *swap.Swap tracerClose io.Closer } @@ -171,6 +173,14 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e delivery := stream.NewDelivery(to, self.netStore) self.netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, config.DeliverySkipCheck).New + if config.SwapEnabled { + balancesStore, err := state.NewDBStore(filepath.Join(config.Path, "balances.db")) + if err != nil { + return nil, err + } + self.swap = swap.New(balancesStore) + } + var nodeID enode.ID if err := nodeID.UnmarshalText([]byte(config.NodeID)); err != nil { return nil, err @@ -193,7 +203,7 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e SyncUpdateDelay: config.SyncUpdateDelay, MaxPeerServers: config.MaxStreamPeerServers, } - self.streamer = stream.NewRegistry(nodeID, delivery, self.netStore, stateStore, registryOptions) + self.streamer = stream.NewRegistry(nodeID, delivery, self.netStore, stateStore, registryOptions, self.swap) // Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage self.fileStore = storage.NewFileStore(self.netStore, self.config.FileStoreParams) @@ -216,7 +226,7 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e log.Debug("Setup local storage") - self.bzz = network.NewBzz(bzzconfig, to, stateStore, stream.Spec, self.streamer.Run) + self.bzz = network.NewBzz(bzzconfig, to, stateStore, self.streamer.GetSpec(), self.streamer.Run) // Pss = postal service over swarm (devp2p over bzz) self.ps, err = pss.NewPss(to, config.Pss) @@ -353,7 +363,9 @@ func (self *Swarm) Start(srv *p2p.Server) error { newaddr := self.bzz.UpdateLocalAddr([]byte(srv.Self().String())) log.Info("Updated bzz local addr", "oaddr", fmt.Sprintf("%x", newaddr.OAddr), "uaddr", fmt.Sprintf("%s", newaddr.UAddr)) // set chequebook - if self.config.SwapEnabled { + //TODO: Currently if swap is enabled and no chequebook (or inexistent) contract is provided, the node would crash. + //Once we integrate back the contracts, this check MUST be revisited + if self.config.SwapEnabled && self.config.SwapAPI != "" { ctx := context.Background() // The initial setup has no deadline. err := self.SetChequebook(ctx) if err != nil { From 68be45e5f89a5bde33376a8e103c133392f18516 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Kurowski?= Date: Fri, 16 Nov 2018 10:50:48 +0100 Subject: [PATCH 053/159] trie: return hasher to pool (#18116) * trie: return hasher to pool * trie: minor code formatting fix --- trie/iterator.go | 2 ++ trie/proof.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/trie/iterator.go b/trie/iterator.go index 00b890eb89..77f1681665 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -181,6 +181,8 @@ func (it *nodeIterator) LeafProof() [][]byte { if len(it.stack) > 0 { if _, ok := it.stack[len(it.stack)-1].node.(valueNode); ok { hasher := newHasher(0, 0, nil) + defer returnHasherToPool(hasher) + proofs := make([][]byte, 0, len(it.stack)) for i, item := range it.stack[:len(it.stack)-1] { diff --git a/trie/proof.go b/trie/proof.go index 6cb8f4d5f7..62c47beda2 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -66,6 +66,8 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error { } } hasher := newHasher(0, 0, nil) + defer returnHasherToPool(hasher) + for i, n := range nodes { // Don't bother checking for errors here since hasher panics // if encoding doesn't work and we're not writing to any database. From 51b2f1620cc22aa410dee24a6021bb767e0ff998 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 12 Nov 2018 14:18:56 +0100 Subject: [PATCH 054/159] downloader: different sync strategy --- eth/downloader/downloader.go | 122 ++++++++++++++++++++---------- eth/downloader/downloader_test.go | 76 +++++++++++++++++++ 2 files changed, 157 insertions(+), 41 deletions(-) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 56c54c8ed5..1f52fc6db4 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -430,7 +430,7 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.I } height := latest.Number.Uint64() - origin, err := d.findAncestor(p, height) + origin, err := d.findAncestor(p, latest) if err != nil { return err } @@ -587,41 +587,85 @@ func (d *Downloader) fetchHeight(p *peerConnection) (*types.Header, error) { } } +// calculateRequestSpan calculates what headers to request from a peer when trying to determine the +// common ancestor. +// It returns parameters to be used for peer.RequestHeadersByNumber: +// from - starting block number +// count - number of headers to request +// skip - number of headers to skip +// and also returns 'max', the last block which is expected to be returned by the remote peers, +// given the (from,count,skip) +func calculateRequestSpan(remoteHeight, localHeight uint64) (int64, int, int, uint64) { + var ( + from int + count int + MaxCount = MaxHeaderFetch / 16 + ) + // requestHead is the highest block that we will ask for. If requestHead is not offset, + // the highest block that we will get is 16 blocks back from head, which means we + // will fetch 14 or 15 blocks unnecessarily in the case the height difference + // between us and the peer is 1-2 blocks, which is most common + requestHead := int(remoteHeight) - 1 + if requestHead < 0 { + requestHead = 0 + } + // requestBottom is the lowest block we want included in the query + // Ideally, we want to include just below own head + requestBottom := int(localHeight - 1) + if requestBottom < 0 { + requestBottom = 0 + } + totalSpan := requestHead - requestBottom + span := 1 + totalSpan/MaxCount + if span < 2 { + span = 2 + } + if span > 16 { + span = 16 + } + + count = 1 + totalSpan/span + if count > MaxCount { + count = MaxCount + } + if count < 2 { + count = 2 + } + from = requestHead - (count-1)*span + if from < 0 { + from = 0 + } + max := from + (count-1)*span + return int64(from), count, span - 1, uint64(max) +} + // findAncestor tries to locate the common ancestor link of the local chain and // a remote peers blockchain. In the general case when our node was in sync and // on the correct chain, checking the top N links should already get us a match. // In the rare scenario when we ended up on a long reorganisation (i.e. none of // the head links match), we do a binary search to find the common ancestor. -func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, error) { +func (d *Downloader) findAncestor(p *peerConnection, remoteHeader *types.Header) (uint64, error) { // Figure out the valid ancestor range to prevent rewrite attacks - floor, ceil := int64(-1), d.lightchain.CurrentHeader().Number.Uint64() + var ( + floor = int64(-1) + localHeight uint64 + remoteHeight = remoteHeader.Number.Uint64() + ) + switch d.mode { + case FullSync: + localHeight = d.blockchain.CurrentBlock().NumberU64() + case FastSync: + localHeight = d.blockchain.CurrentFastBlock().NumberU64() + default: + localHeight = d.lightchain.CurrentHeader().Number.Uint64() + } + p.log.Debug("Looking for common ancestor", "local", localHeight, "remote", remoteHeight) + if localHeight >= MaxForkAncestry { + floor = int64(localHeight - MaxForkAncestry) + } - if d.mode == FullSync { - ceil = d.blockchain.CurrentBlock().NumberU64() - } else if d.mode == FastSync { - ceil = d.blockchain.CurrentFastBlock().NumberU64() - } - if ceil >= MaxForkAncestry { - floor = int64(ceil - MaxForkAncestry) - } - p.log.Debug("Looking for common ancestor", "local", ceil, "remote", height) - - // Request the topmost blocks to short circuit binary ancestor lookup - head := ceil - if head > height { - head = height - } - from := int64(head) - int64(MaxHeaderFetch) - if from < 0 { - from = 0 - } - // Span out with 15 block gaps into the future to catch bad head reports - limit := 2 * MaxHeaderFetch / 16 - count := 1 + int((int64(ceil)-from)/16) - if count > limit { - count = limit - } - go p.peer.RequestHeadersByNumber(uint64(from), count, 15, false) + from, count, skip, max := calculateRequestSpan(remoteHeight, localHeight) + go p.peer.RequestHeadersByNumber(uint64(from), count, skip, false) // Wait for the remote response to the head fetch number, hash := uint64(0), common.Hash{} @@ -647,9 +691,10 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err return 0, errEmptyHeaderSet } // Make sure the peer's reply conforms to the request - for i := 0; i < len(headers); i++ { - if number := headers[i].Number.Int64(); number != from+int64(i)*16 { - p.log.Warn("Head headers broke chain ordering", "index", i, "requested", from+int64(i)*16, "received", number) + for i, header := range headers { + expectNumber := from + int64(i)*int64((skip+1)) + if number := header.Number.Int64(); number != expectNumber { + p.log.Warn("Head headers broke chain ordering", "index", i, "requested", expectNumber, "received", number) return 0, errInvalidChain } } @@ -657,20 +702,15 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err finished = true for i := len(headers) - 1; i >= 0; i-- { // Skip any headers that underflow/overflow our requested set - if headers[i].Number.Int64() < from || headers[i].Number.Uint64() > ceil { + if headers[i].Number.Int64() < from || headers[i].Number.Uint64() > max { continue } // Otherwise check if we already know the header or not h := headers[i].Hash() n := headers[i].Number.Uint64() - if (d.mode == FullSync && d.blockchain.HasBlock(h, n)) || (d.mode != FullSync && d.lightchain.HasHeader(h, n)) { + if (d.mode == FullSync && d.blockchain.HasBlock(h, n)) || + (d.mode != FullSync && d.lightchain.HasHeader(h, n)) { number, hash = n, h - - // If every header is known, even future ones, the peer straight out lied about its head - if number > height && i == limit-1 { - p.log.Warn("Lied about chain head", "reported", height, "found", number) - return 0, errStallingPeer - } break } } @@ -694,7 +734,7 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err return number, nil } // Ancestor not found, we need to binary search over our chain - start, end := uint64(0), head + start, end := uint64(0), remoteHeight if floor > 0 { start = uint64(floor) } diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 1fe02d8849..b30976d721 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "math/big" + "strings" "sync" "sync/atomic" "testing" @@ -1479,3 +1480,78 @@ func (ftp *floodingTestPeer) RequestHeadersByNumber(from uint64, count, skip int } return nil } + +func TestRemoteHeaderRequestSpan(t *testing.T) { + testCases := []struct { + remoteHeight uint64 + localHeight uint64 + expected []int + }{ + // Remote is way higher. We should ask for the remote head and go backwards + {1500, 1000, + []int{1323, 1339, 1355, 1371, 1387, 1403, 1419, 1435, 1451, 1467, 1483, 1499}, + }, + {15000, 13006, + []int{14823, 14839, 14855, 14871, 14887, 14903, 14919, 14935, 14951, 14967, 14983, 14999}, + }, + //Remote is pretty close to us. We don't have to fetch as many + {1200, 1150, + []int{1149, 1154, 1159, 1164, 1169, 1174, 1179, 1184, 1189, 1194, 1199}, + }, + // Remote is equal to us (so on a fork with higher td) + // We should get the closest couple of ancestors + {1500, 1500, + []int{1497, 1499}, + }, + // We're higher than the remote! Odd + {1000, 1500, + []int{997, 999}, + }, + // Check some weird edgecases that it behaves somewhat rationally + {0, 1500, + []int{0, 2}, + }, + {6000000, 0, + []int{5999823, 5999839, 5999855, 5999871, 5999887, 5999903, 5999919, 5999935, 5999951, 5999967, 5999983, 5999999}, + }, + {0, 0, + []int{0, 2}, + }, + } + reqs := func(from, count, span int) []int { + var r []int + num := from + for len(r) < count { + r = append(r, num) + num += span + 1 + } + return r + } + for i, tt := range testCases { + from, count, span, max := calculateRequestSpan(tt.remoteHeight, tt.localHeight) + data := reqs(int(from), count, span) + + if max != uint64(data[len(data)-1]) { + t.Errorf("test %d: wrong last value %d != %d", i, data[len(data)-1], max) + } + failed := false + if len(data) != len(tt.expected) { + failed = true + t.Errorf("test %d: length wrong, expected %d got %d", i, len(tt.expected), len(data)) + } else { + for j, n := range data { + if n != tt.expected[j] { + failed = true + break + } + } + } + if failed { + res := strings.Replace(fmt.Sprint(data), " ", ",", -1) + exp := strings.Replace(fmt.Sprint(tt.expected), " ", ",", -1) + fmt.Printf("got: %v\n", res) + fmt.Printf("exp: %v\n", exp) + t.Errorf("test %d: wrong values", i) + } + } +} From accc0fab4f407eaeab428127bd5395a28f371f9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 16 Nov 2018 13:15:05 +0200 Subject: [PATCH 055/159] core, eth/downloader: fix ancestor lookup for fast sync --- core/blockchain.go | 13 ++++++++++-- core/rawdb/accessors_chain.go | 9 +++++++++ eth/downloader/downloader.go | 33 +++++++++++++++++++++++++++---- eth/downloader/downloader_test.go | 32 ++++++++++++++++++++---------- 4 files changed, 71 insertions(+), 16 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 22f130ce67..d173b2de29 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -561,6 +561,17 @@ func (bc *BlockChain) HasBlock(hash common.Hash, number uint64) bool { return rawdb.HasBody(bc.db, hash, number) } +// HasFastBlock checks if a fast block is fully present in the database or not. +func (bc *BlockChain) HasFastBlock(hash common.Hash, number uint64) bool { + if !bc.HasBlock(hash, number) { + return false + } + if bc.receiptsCache.Contains(hash) { + return true + } + return rawdb.HasReceipts(bc.db, hash, number) +} + // HasState checks if state trie is fully present in the database or not. func (bc *BlockChain) HasState(hash common.Hash) bool { _, err := bc.stateCache.OpenTrie(hash) @@ -618,12 +629,10 @@ func (bc *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts { if receipts, ok := bc.receiptsCache.Get(hash); ok { return receipts.(types.Receipts) } - number := rawdb.ReadHeaderNumber(bc.db, hash) if number == nil { return nil } - receipts := rawdb.ReadReceipts(bc.db, hash, *number) bc.receiptsCache.Add(hash, receipts) return receipts diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 6660e17de1..491a125c65 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -271,6 +271,15 @@ func DeleteTd(db DatabaseDeleter, hash common.Hash, number uint64) { } } +// HasReceipts verifies the existence of all the transaction receipts belonging +// to a block. +func HasReceipts(db DatabaseReader, hash common.Hash, number uint64) bool { + if has, err := db.Has(blockReceiptsKey(number, hash)); !has || err != nil { + return false + } + return true +} + // ReadReceipts retrieves all the transaction receipts belonging to a block. func ReadReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Receipts { // Retrieve the flattened receipt slice diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 1f52fc6db4..f81a5cbac2 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -181,6 +181,9 @@ type BlockChain interface { // HasBlock verifies a block's presence in the local chain. HasBlock(common.Hash, uint64) bool + // HasFastBlock verifies a fast block's presence in the local chain. + HasFastBlock(common.Hash, uint64) bool + // GetBlockByHash retrieves a block from the local chain. GetBlockByHash(common.Hash) *types.Block @@ -663,8 +666,9 @@ func (d *Downloader) findAncestor(p *peerConnection, remoteHeader *types.Header) if localHeight >= MaxForkAncestry { floor = int64(localHeight - MaxForkAncestry) } - from, count, skip, max := calculateRequestSpan(remoteHeight, localHeight) + + p.log.Trace("Span searching for common ancestor", "count", count, "from", from, "skip", skip) go p.peer.RequestHeadersByNumber(uint64(from), count, skip, false) // Wait for the remote response to the head fetch @@ -708,8 +712,17 @@ func (d *Downloader) findAncestor(p *peerConnection, remoteHeader *types.Header) // Otherwise check if we already know the header or not h := headers[i].Hash() n := headers[i].Number.Uint64() - if (d.mode == FullSync && d.blockchain.HasBlock(h, n)) || - (d.mode != FullSync && d.lightchain.HasHeader(h, n)) { + + var known bool + switch d.mode { + case FullSync: + known = d.blockchain.HasBlock(h, n) + case FastSync: + known = d.blockchain.HasFastBlock(h, n) + default: + known = d.lightchain.HasHeader(h, n) + } + if known { number, hash = n, h break } @@ -738,6 +751,8 @@ func (d *Downloader) findAncestor(p *peerConnection, remoteHeader *types.Header) if floor > 0 { start = uint64(floor) } + p.log.Trace("Binary searching for common ancestor", "start", start, "end", end) + for start+1 < end { // Split our chain interval in two, and request the hash to cross check check := (start + end) / 2 @@ -770,7 +785,17 @@ func (d *Downloader) findAncestor(p *peerConnection, remoteHeader *types.Header) // Modify the search interval based on the response h := headers[0].Hash() n := headers[0].Number.Uint64() - if (d.mode == FullSync && !d.blockchain.HasBlock(h, n)) || (d.mode != FullSync && !d.lightchain.HasHeader(h, n)) { + + var known bool + switch d.mode { + case FullSync: + known = d.blockchain.HasBlock(h, n) + case FastSync: + known = d.blockchain.HasFastBlock(h, n) + default: + known = d.lightchain.HasHeader(h, n) + } + if !known { end = check break } diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index b30976d721..1a42965d39 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -115,6 +115,15 @@ func (dl *downloadTester) HasBlock(hash common.Hash, number uint64) bool { return dl.GetBlockByHash(hash) != nil } +// HasFastBlock checks if a block is present in the testers canonical chain. +func (dl *downloadTester) HasFastBlock(hash common.Hash, number uint64) bool { + dl.lock.RLock() + defer dl.lock.RUnlock() + + _, ok := dl.ownReceipts[hash] + return ok +} + // GetHeader retrieves a header from the testers canonical chain. func (dl *downloadTester) GetHeaderByHash(hash common.Hash) *types.Header { dl.lock.RLock() @@ -235,6 +244,7 @@ func (dl *downloadTester) InsertChain(blocks types.Blocks) (i int, err error) { dl.ownHeaders[block.Hash()] = block.Header() } dl.ownBlocks[block.Hash()] = block + dl.ownReceipts[block.Hash()] = make(types.Receipts, 0) dl.stateDb.Put(block.Root().Bytes(), []byte{0x00}) dl.ownChainTd[block.Hash()] = new(big.Int).Add(dl.ownChainTd[block.ParentHash()], block.Difficulty()) } @@ -375,28 +385,28 @@ func (dlp *downloadTesterPeer) RequestNodeData(hashes []common.Hash) error { // assertOwnChain checks if the local chain contains the correct number of items // of the various chain components. func assertOwnChain(t *testing.T, tester *downloadTester, length int) { + // Mark this method as a helper to report errors at callsite, not in here + t.Helper() + assertOwnForkedChain(t, tester, 1, []int{length}) } // assertOwnForkedChain checks if the local forked chain contains the correct // number of items of the various chain components. func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, lengths []int) { - // Initialize the counters for the first fork - headers, blocks, receipts := lengths[0], lengths[0], lengths[0]-fsMinFullBlocks + // Mark this method as a helper to report errors at callsite, not in here + t.Helper() + + // Initialize the counters for the first fork + headers, blocks, receipts := lengths[0], lengths[0], lengths[0] - if receipts < 0 { - receipts = 1 - } // Update the counters for each subsequent fork for _, length := range lengths[1:] { headers += length - common blocks += length - common - receipts += length - common - fsMinFullBlocks + receipts += length - common } - switch tester.downloader.mode { - case FullSync: - receipts = 1 - case LightSync: + if tester.downloader.mode == LightSync { blocks, receipts = 1, 1 } if hs := len(tester.ownHeaders); hs != headers { @@ -1150,7 +1160,9 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { } func checkProgress(t *testing.T, d *Downloader, stage string, want ethereum.SyncProgress) { + // Mark this method as a helper to report errors at callsite, not in here t.Helper() + p := d.Progress() p.KnownStates, p.PulledStates = 0, 0 want.KnownStates, want.PulledStates = 0, 0 From d136e985e80e4c3dc27678b0f54d6e34dd671850 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 16 Nov 2018 16:35:39 +0200 Subject: [PATCH 056/159] trie: go fmt package --- trie/proof.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trie/proof.go b/trie/proof.go index 62c47beda2..f90ecd7d88 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -67,7 +67,7 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error { } hasher := newHasher(0, 0, nil) defer returnHasherToPool(hasher) - + for i, n := range nodes { // Don't bother checking for errors here since hasher panics // if encoding doesn't work and we're not writing to any database. From 51e2e78d26b9e8c7b6297cf51662cf22debbb7c6 Mon Sep 17 00:00:00 2001 From: Elad Date: Sun, 18 Nov 2018 15:24:03 +0530 Subject: [PATCH 057/159] swarm/api/http: change request served msg log level (#18127) --- swarm/api/http/middleware.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/api/http/middleware.go b/swarm/api/http/middleware.go index ccc040c54d..f5f70138b3 100644 --- a/swarm/api/http/middleware.go +++ b/swarm/api/http/middleware.go @@ -75,7 +75,7 @@ func InitLoggingResponseWriter(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { writer := newLoggingResponseWriter(w) h.ServeHTTP(writer, r) - log.Debug("request served", "ruid", GetRUID(r.Context()), "code", writer.statusCode) + log.Info("request served", "ruid", GetRUID(r.Context()), "code", writer.statusCode) }) } From 3333fe660f00b519a9d16398ca3801ab9313be16 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Mon, 19 Nov 2018 12:26:45 +0100 Subject: [PATCH 058/159] swarm/storage: speed up garbage collection and rpc tests (#18128) --- swarm/storage/ldbstore_test.go | 9 +++++---- swarm/storage/mock/rpc/rpc_test.go | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/swarm/storage/ldbstore_test.go b/swarm/storage/ldbstore_test.go index 9c10b36290..e8b9ae39bc 100644 --- a/swarm/storage/ldbstore_test.go +++ b/swarm/storage/ldbstore_test.go @@ -344,17 +344,18 @@ func TestLDBStoreWithoutCollectGarbage(t *testing.T) { func TestLDBStoreCollectGarbage(t *testing.T) { // below max ronud - cap := defaultMaxGCRound / 2 + initialCap := defaultMaxGCRound / 100 + cap := initialCap / 2 t.Run(fmt.Sprintf("A/%d/%d", cap, cap*4), testLDBStoreCollectGarbage) t.Run(fmt.Sprintf("B/%d/%d", cap, cap*4), testLDBStoreRemoveThenCollectGarbage) // at max round - cap = defaultMaxGCRound + cap = initialCap t.Run(fmt.Sprintf("A/%d/%d", cap, cap*4), testLDBStoreCollectGarbage) t.Run(fmt.Sprintf("B/%d/%d", cap, cap*4), testLDBStoreRemoveThenCollectGarbage) // more than max around, not on threshold - cap = defaultMaxGCRound * 1.1 + cap = initialCap + 500 t.Run(fmt.Sprintf("A/%d/%d", cap, cap*4), testLDBStoreCollectGarbage) t.Run(fmt.Sprintf("B/%d/%d", cap, cap*4), testLDBStoreRemoveThenCollectGarbage) @@ -578,7 +579,7 @@ func testLDBStoreRemoveThenCollectGarbage(t *testing.T) { // TestLDBStoreCollectGarbageAccessUnlikeIndex tests garbage collection where accesscount differs from indexcount func TestLDBStoreCollectGarbageAccessUnlikeIndex(t *testing.T) { - capacity := defaultMaxGCRound * 2 + capacity := defaultMaxGCRound / 100 * 2 n := capacity - 1 ldb, cleanup := newLDBStore(t) diff --git a/swarm/storage/mock/rpc/rpc_test.go b/swarm/storage/mock/rpc/rpc_test.go index 52b634a445..f62340edec 100644 --- a/swarm/storage/mock/rpc/rpc_test.go +++ b/swarm/storage/mock/rpc/rpc_test.go @@ -37,5 +37,5 @@ func TestRPCStore(t *testing.T) { store := NewGlobalStore(rpc.DialInProc(server)) defer store.Close() - test.MockStore(t, store, 100) + test.MockStore(t, store, 30) } From 6b6c4d1c2754f8dd70172ab58d7ee33cf9058c7d Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Mon, 19 Nov 2018 14:57:22 +0100 Subject: [PATCH 059/159] cmd/swarm: speed up tests - use global cluster (#18129) --- cmd/swarm/access_test.go | 79 ++++++++++++++++-------------- cmd/swarm/export_test.go | 4 +- cmd/swarm/feeds_test.go | 2 - cmd/swarm/fs_test.go | 6 --- cmd/swarm/run_test.go | 11 +++++ cmd/swarm/upload_test.go | 101 +++++++++++++++++++-------------------- 6 files changed, 106 insertions(+), 97 deletions(-) diff --git a/cmd/swarm/access_test.go b/cmd/swarm/access_test.go index e812cd8fd1..9357c577e3 100644 --- a/cmd/swarm/access_test.go +++ b/cmd/swarm/access_test.go @@ -14,8 +14,6 @@ // You should have received a copy of the GNU General Public License // along with go-ethereum. If not, see . -// +build !windows - package main import ( @@ -28,6 +26,7 @@ import ( gorand "math/rand" "net/http" "os" + "runtime" "strings" "testing" "time" @@ -37,8 +36,7 @@ import ( "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" - swarm "github.com/ethereum/go-ethereum/swarm/api/client" - swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" + swarmapi "github.com/ethereum/go-ethereum/swarm/api/client" "github.com/ethereum/go-ethereum/swarm/testutil" ) @@ -49,22 +47,41 @@ const ( var DefaultCurve = crypto.S256() -// TestAccessPassword tests for the correct creation of an ACT manifest protected by a password. +func TestACT(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + + initCluster(t) + + cases := []struct { + name string + f func(t *testing.T) + }{ + {"Password", testPassword}, + {"PK", testPK}, + {"ACTWithoutBogus", testACTWithoutBogus}, + {"ACTWithBogus", testACTWithBogus}, + } + + for _, tc := range cases { + t.Run(tc.name, tc.f) + } +} + +// testPassword tests for the correct creation of an ACT manifest protected by a password. // The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry // The parties participating - node (publisher), uploads to second node then disappears. Content which was uploaded // is then fetched through 2nd node. since the tested code is not key-aware - we can just // fetch from the 2nd node using HTTP BasicAuth -func TestAccessPassword(t *testing.T) { - srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) - defer srv.Close() - +func testPassword(t *testing.T) { dataFilename := testutil.TempFileWithContent(t, data) defer os.RemoveAll(dataFilename) // upload the file with 'swarm up' and expect a hash up := runSwarm(t, "--bzzapi", - srv.URL, //it doesn't matter through which node we upload content + cluster.Nodes[0].URL, "up", "--encrypt", dataFilename) @@ -138,16 +155,17 @@ func TestAccessPassword(t *testing.T) { if a.Publisher != "" { t.Fatal("should be empty") } - client := swarm.NewClient(srv.URL) + + client := swarmapi.NewClient(cluster.Nodes[0].URL) hash, err := client.UploadManifest(&m, false) if err != nil { t.Fatal(err) } - httpClient := &http.Client{} + url := cluster.Nodes[0].URL + "/" + "bzz:/" + hash - url := srv.URL + "/" + "bzz:/" + hash + httpClient := &http.Client{} response, err := httpClient.Get(url) if err != nil { t.Fatal(err) @@ -189,7 +207,7 @@ func TestAccessPassword(t *testing.T) { //download file with 'swarm down' with wrong password up = runSwarm(t, "--bzzapi", - srv.URL, + cluster.Nodes[0].URL, "down", "bzz:/"+hash, tmp, @@ -203,16 +221,12 @@ func TestAccessPassword(t *testing.T) { up.ExpectExit() } -// TestAccessPK tests for the correct creation of an ACT manifest between two parties (publisher and grantee). +// testPK tests for the correct creation of an ACT manifest between two parties (publisher and grantee). // The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry // The parties participating - node (publisher), uploads to second node (which is also the grantee) then disappears. // Content which was uploaded is then fetched through the grantee's http proxy. Since the tested code is private-key aware, // the test will fail if the proxy's given private key is not granted on the ACT. -func TestAccessPK(t *testing.T) { - // Setup Swarm and upload a test file to it - cluster := newTestCluster(t, 2) - defer cluster.Shutdown() - +func testPK(t *testing.T) { dataFilename := testutil.TempFileWithContent(t, data) defer os.RemoveAll(dataFilename) @@ -318,7 +332,7 @@ func TestAccessPK(t *testing.T) { if a.Publisher != pkComp { t.Fatal("publisher key did not match") } - client := swarm.NewClient(cluster.Nodes[0].URL) + client := swarmapi.NewClient(cluster.Nodes[0].URL) hash, err := client.UploadManifest(&m, false) if err != nil { @@ -344,29 +358,24 @@ func TestAccessPK(t *testing.T) { } } -// TestAccessACT tests the creation of the ACT manifest end-to-end, without any bogus entries (i.e. default scenario = 3 nodes 1 unauthorized) -func TestAccessACT(t *testing.T) { - testAccessACT(t, 0) +// testACTWithoutBogus tests the creation of the ACT manifest end-to-end, without any bogus entries (i.e. default scenario = 3 nodes 1 unauthorized) +func testACTWithoutBogus(t *testing.T) { + testACT(t, 0) } -// TestAccessACTScale tests the creation of the ACT manifest end-to-end, with 1000 bogus entries (i.e. 1000 EC keys + default scenario = 3 nodes 1 unauthorized = 1003 keys in the ACT manifest) -func TestAccessACTScale(t *testing.T) { - testAccessACT(t, 1000) +// testACTWithBogus tests the creation of the ACT manifest end-to-end, with 100 bogus entries (i.e. 100 EC keys + default scenario = 3 nodes 1 unauthorized = 103 keys in the ACT manifest) +func testACTWithBogus(t *testing.T) { + testACT(t, 100) } -// TestAccessACT tests the e2e creation, uploading and downloading of an ACT access control with both EC keys AND password protection +// testACT tests the e2e creation, uploading and downloading of an ACT access control with both EC keys AND password protection // the test fires up a 3 node cluster, then randomly picks 2 nodes which will be acting as grantees to the data // set and also protects the ACT with a password. the third node should fail decoding the reference as it will not be granted access. // the third node then then tries to download using a correct password (and succeeds) then uses a wrong password and fails. // the publisher uploads through one of the nodes then disappears. -func testAccessACT(t *testing.T, bogusEntries int) { - // Setup Swarm and upload a test file to it - const clusterSize = 3 - cluster := newTestCluster(t, clusterSize) - defer cluster.Shutdown() - +func testACT(t *testing.T, bogusEntries int) { var uploadThroughNode = cluster.Nodes[0] - client := swarm.NewClient(uploadThroughNode.URL) + client := swarmapi.NewClient(uploadThroughNode.URL) r1 := gorand.New(gorand.NewSource(time.Now().UnixNano())) nodeToSkip := r1.Intn(clusterSize) // a number between 0 and 2 (node indices in `cluster`) diff --git a/cmd/swarm/export_test.go b/cmd/swarm/export_test.go index f1bc2f2658..e8671eea79 100644 --- a/cmd/swarm/export_test.go +++ b/cmd/swarm/export_test.go @@ -43,8 +43,8 @@ func TestCLISwarmExportImport(t *testing.T) { } cluster := newTestCluster(t, 1) - // generate random 10mb file - content := testutil.RandomBytes(1, 10000000) + // generate random 1mb file + content := testutil.RandomBytes(1, 1000000) fileName := testutil.TempFileWithContent(t, string(content)) defer os.Remove(fileName) diff --git a/cmd/swarm/feeds_test.go b/cmd/swarm/feeds_test.go index fc3f72ab15..a0cedf0d36 100644 --- a/cmd/swarm/feeds_test.go +++ b/cmd/swarm/feeds_test.go @@ -36,7 +36,6 @@ import ( ) func TestCLIFeedUpdate(t *testing.T) { - srv := swarmhttp.NewTestSwarmServer(t, func(api *api.API) swarmhttp.TestServer { return swarmhttp.NewServer(api, "") }, nil) @@ -44,7 +43,6 @@ func TestCLIFeedUpdate(t *testing.T) { defer srv.Close() // create a private key file for signing - privkeyHex := "0000000000000000000000000000000000000000000000000000000000001979" privKey, _ := crypto.HexToECDSA(privkeyHex) address := crypto.PubkeyToAddress(privKey.PublicKey) diff --git a/cmd/swarm/fs_test.go b/cmd/swarm/fs_test.go index 3b722515ee..ac4223b661 100644 --- a/cmd/swarm/fs_test.go +++ b/cmd/swarm/fs_test.go @@ -29,14 +29,8 @@ import ( "time" "github.com/ethereum/go-ethereum/log" - colorable "github.com/mattn/go-colorable" ) -func init() { - log.PrintOrigins(true) - log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) -} - type testFile struct { filePath string content string diff --git a/cmd/swarm/run_test.go b/cmd/swarm/run_test.go index 416fa7a503..680d238d06 100644 --- a/cmd/swarm/run_test.go +++ b/cmd/swarm/run_test.go @@ -57,6 +57,17 @@ func init() { }) } +const clusterSize = 3 + +var clusteronce sync.Once +var cluster *testCluster + +func initCluster(t *testing.T) { + clusteronce.Do(func() { + cluster = newTestCluster(t, clusterSize) + }) +} + func serverFunc(api *api.API) swarmhttp.TestServer { return swarmhttp.NewServer(api, "") } diff --git a/cmd/swarm/upload_test.go b/cmd/swarm/upload_test.go index 5f9844950e..616486e37c 100644 --- a/cmd/swarm/upload_test.go +++ b/cmd/swarm/upload_test.go @@ -31,8 +31,7 @@ import ( "time" "github.com/ethereum/go-ethereum/log" - swarm "github.com/ethereum/go-ethereum/swarm/api/client" - swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" + swarmapi "github.com/ethereum/go-ethereum/swarm/api/client" "github.com/ethereum/go-ethereum/swarm/testutil" "github.com/mattn/go-colorable" ) @@ -42,42 +41,50 @@ func init() { log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) } -// TestCLISwarmUp tests that running 'swarm up' makes the resulting file +func TestSwarmUp(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + + initCluster(t) + + cases := []struct { + name string + f func(t *testing.T) + }{ + {"NoEncryption", testNoEncryption}, + {"Encrypted", testEncrypted}, + {"RecursiveNoEncryption", testRecursiveNoEncryption}, + {"RecursiveEncrypted", testRecursiveEncrypted}, + {"DefaultPathAll", testDefaultPathAll}, + } + + for _, tc := range cases { + t.Run(tc.name, tc.f) + } +} + +// testNoEncryption tests that running 'swarm up' makes the resulting file // available from all nodes via the HTTP API -func TestCLISwarmUp(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip() - } - - testCLISwarmUp(false, t) -} -func TestCLISwarmUpRecursive(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip() - } - testCLISwarmUpRecursive(false, t) +func testNoEncryption(t *testing.T) { + testDefault(false, t) } -// TestCLISwarmUpEncrypted tests that running 'swarm encrypted-up' makes the resulting file +// testEncrypted tests that running 'swarm up --encrypted' makes the resulting file // available from all nodes via the HTTP API -func TestCLISwarmUpEncrypted(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip() - } - testCLISwarmUp(true, t) -} -func TestCLISwarmUpEncryptedRecursive(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip() - } - testCLISwarmUpRecursive(true, t) +func testEncrypted(t *testing.T) { + testDefault(true, t) } -func testCLISwarmUp(toEncrypt bool, t *testing.T) { - log.Info("starting 3 node cluster") - cluster := newTestCluster(t, 3) - defer cluster.Shutdown() +func testRecursiveNoEncryption(t *testing.T) { + testRecursive(false, t) +} +func testRecursiveEncrypted(t *testing.T) { + testRecursive(true, t) +} + +func testDefault(toEncrypt bool, t *testing.T) { tmpFileName := testutil.TempFileWithContent(t, data) defer os.Remove(tmpFileName) @@ -182,11 +189,7 @@ func testCLISwarmUp(toEncrypt bool, t *testing.T) { } } -func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) { - fmt.Println("starting 3 node cluster") - cluster := newTestCluster(t, 3) - defer cluster.Shutdown() - +func testRecursive(toEncrypt bool, t *testing.T) { tmpUploadDir, err := ioutil.TempDir("", "swarm-test") if err != nil { t.Fatal(err) @@ -253,7 +256,7 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) { switch mode := fi.Mode(); { case mode.IsRegular(): - if file, err := swarm.Open(path.Join(tmpDownload, v.Name())); err != nil { + if file, err := swarmapi.Open(path.Join(tmpDownload, v.Name())); err != nil { t.Fatalf("encountered an error opening the file returned from the CLI: %v", err) } else { ff := make([]byte, len(data)) @@ -274,22 +277,16 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) { } } -// TestCLISwarmUpDefaultPath tests swarm recursive upload with relative and absolute +// testDefaultPathAll tests swarm recursive upload with relative and absolute // default paths and with encryption. -func TestCLISwarmUpDefaultPath(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip() - } - testCLISwarmUpDefaultPath(false, false, t) - testCLISwarmUpDefaultPath(false, true, t) - testCLISwarmUpDefaultPath(true, false, t) - testCLISwarmUpDefaultPath(true, true, t) +func testDefaultPathAll(t *testing.T) { + testDefaultPath(false, false, t) + testDefaultPath(false, true, t) + testDefaultPath(true, false, t) + testDefaultPath(true, true, t) } -func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T) { - srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) - defer srv.Close() - +func testDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T) { tmp, err := ioutil.TempDir("", "swarm-defaultpath-test") if err != nil { t.Fatal(err) @@ -312,7 +309,7 @@ func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T args := []string{ "--bzzapi", - srv.URL, + cluster.Nodes[0].URL, "--recursive", "--defaultpath", defaultPath, @@ -329,7 +326,7 @@ func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T up.ExpectExit() hash := matches[0] - client := swarm.NewClient(srv.URL) + client := swarmapi.NewClient(cluster.Nodes[0].URL) m, isEncrypted, err := client.DownloadManifest(hash) if err != nil { From d31f1f4fdb905acb3d7c766683e584bb17f8df0d Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Mon, 19 Nov 2018 14:58:10 +0100 Subject: [PATCH 060/159] cmd/swarm/swarm-smoke: update smoke tests to fit the new scheme for the k8s cluster (#18104) --- cmd/swarm/swarm-smoke/main.go | 4 ++-- cmd/swarm/swarm-smoke/upload_and_sync.go | 23 +++++++++++++---------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index 97054dd479..4ff17fd5b7 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -45,8 +45,8 @@ func main() { app.Flags = []cli.Flag{ cli.StringFlag{ Name: "cluster-endpoint", - Value: "testing", - Usage: "cluster to point to (local, open or testing)", + Value: "prod", + Usage: "cluster to point to (prod or a given namespace)", Destination: &cluster, }, cli.IntFlag{ diff --git a/cmd/swarm/swarm-smoke/upload_and_sync.go b/cmd/swarm/swarm-smoke/upload_and_sync.go index 358141c7cc..7872421d3d 100644 --- a/cmd/swarm/swarm-smoke/upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/upload_and_sync.go @@ -20,6 +20,7 @@ import ( "bytes" "crypto/md5" crand "crypto/rand" + "crypto/tls" "errors" "fmt" "io" @@ -32,6 +33,7 @@ import ( "time" "github.com/ethereum/go-ethereum/log" + colorable "github.com/mattn/go-colorable" "github.com/pborman/uuid" cli "gopkg.in/urfave/cli.v1" @@ -39,18 +41,13 @@ import ( func generateEndpoints(scheme string, cluster string, from int, to int) { if cluster == "prod" { - cluster = "" - } else if cluster == "local" { for port := from; port <= to; port++ { - endpoints = append(endpoints, fmt.Sprintf("%s://localhost:%v", scheme, port)) + endpoints = append(endpoints, fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, port)) } - return } else { - cluster = cluster + "." - } - - for port := from; port <= to; port++ { - endpoints = append(endpoints, fmt.Sprintf("%s://%v.%sswarm-gateways.net", scheme, port, cluster)) + for port := from; port <= to; port++ { + endpoints = append(endpoints, fmt.Sprintf("%s://swarm-%v-%s.stg.swarm-gateways.net", scheme, port, cluster)) + } } if includeLocalhost { @@ -59,6 +56,9 @@ func generateEndpoints(scheme string, cluster string, from int, to int) { } func cliUploadAndSync(c *cli.Context) error { + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) + defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now()) generateEndpoints(scheme, cluster, from, to) @@ -112,7 +112,10 @@ func fetch(hash string, endpoint string, original []byte, ruid string) error { time.Sleep(3 * time.Second) log.Trace("http get request", "ruid", ruid, "api", endpoint, "hash", hash) - res, err := http.Get(endpoint + "/bzz:/" + hash + "/") + client := &http.Client{Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }} + res, err := client.Get(endpoint + "/bzz:/" + hash + "/") if err != nil { log.Warn(err.Error(), "ruid", ruid) return err From 7bf7bd2f5088d8abb21daef42b8dfce27b194bc7 Mon Sep 17 00:00:00 2001 From: Javier Peletier Date: Tue, 20 Nov 2018 08:23:43 +0100 Subject: [PATCH 061/159] internal/cmdtest: Expose process exit status and errors (#18046) --- internal/cmdtest/test_cmd.go | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/internal/cmdtest/test_cmd.go b/internal/cmdtest/test_cmd.go index 20e82ec2a9..f8b1b43c01 100644 --- a/internal/cmdtest/test_cmd.go +++ b/internal/cmdtest/test_cmd.go @@ -27,6 +27,7 @@ import ( "regexp" "strings" "sync" + "syscall" "testing" "text/template" "time" @@ -50,6 +51,8 @@ type TestCmd struct { stdout *bufio.Reader stdin io.WriteCloser stderr *testlogger + // Err will contain the process exit error or interrupt signal error + Err error } // Run exec's the current binary using name as argv[0] which will trigger the @@ -182,11 +185,25 @@ func (tt *TestCmd) ExpectExit() { } func (tt *TestCmd) WaitExit() { - tt.cmd.Wait() + tt.Err = tt.cmd.Wait() } func (tt *TestCmd) Interrupt() { - tt.cmd.Process.Signal(os.Interrupt) + tt.Err = tt.cmd.Process.Signal(os.Interrupt) +} + +// ExitStatus exposes the process' OS exit code +// It will only return a valid value after the process has finished. +func (tt *TestCmd) ExitStatus() int { + if tt.Err != nil { + exitErr := tt.Err.(*exec.ExitError) + if exitErr != nil { + if status, ok := exitErr.Sys().(syscall.WaitStatus); ok { + return status.ExitStatus() + } + } + } + return 0 } // StderrText returns any stderr output written so far. From d876f214e5500962d6acc1f99a6f2f7c5f63db8b Mon Sep 17 00:00:00 2001 From: Ferenc Szabo Date: Tue, 20 Nov 2018 08:30:38 +0100 Subject: [PATCH 062/159] swarm/storage: move 'running migrations for' log line (#18120) So that we only see the log message when we actually have to migrate. --- swarm/storage/localstore.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index fa98848ddc..111821ff65 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -194,7 +194,8 @@ func (ls *LocalStore) Close() { ls.DbStore.Close() } -// Migrate checks the datastore schema vs the runtime schema, and runs migrations if they don't match +// Migrate checks the datastore schema vs the runtime schema and runs +// migrations if they don't match func (ls *LocalStore) Migrate() error { actualDbSchema, err := ls.DbStore.GetSchema() if err != nil { @@ -202,12 +203,12 @@ func (ls *LocalStore) Migrate() error { return err } - log.Debug("running migrations for", "schema", actualDbSchema, "runtime-schema", CurrentDbSchema) - if actualDbSchema == CurrentDbSchema { return nil } + log.Debug("running migrations for", "schema", actualDbSchema, "runtime-schema", CurrentDbSchema) + if actualDbSchema == DbSchemaNone { ls.migrateFromNoneToPurity() actualDbSchema = DbSchemaPurity From 3d997b6decfaa42e37521ae20bf58886c8b2de8f Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 20 Nov 2018 10:08:02 +0100 Subject: [PATCH 063/159] whisper: log errors on failed tests (#18134) Debug traces to investigate a travis issue on MacOS --- whisper/whisperv5/peer_test.go | 2 +- whisper/whisperv6/peer_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/whisper/whisperv5/peer_test.go b/whisper/whisperv5/peer_test.go index 35616aaaf6..2449532070 100644 --- a/whisper/whisperv5/peer_test.go +++ b/whisper/whisperv5/peer_test.go @@ -139,7 +139,7 @@ func initialize(t *testing.T) { err = node.server.Start() if err != nil { - t.Fatalf("failed to start server %d.", i) + t.Fatalf("failed to start server %d. err: %v", i, err) } for j := 0; j < i; j++ { diff --git a/whisper/whisperv6/peer_test.go b/whisper/whisperv6/peer_test.go index 65e62d96cc..c5b044e1a6 100644 --- a/whisper/whisperv6/peer_test.go +++ b/whisper/whisperv6/peer_test.go @@ -232,7 +232,7 @@ func initialize(t *testing.T) { func startServer(t *testing.T, s *p2p.Server) { err := s.Start() if err != nil { - t.Fatalf("failed to start the fisrt server.") + t.Fatalf("failed to start the first server. err: %v", err) } atomic.AddInt64(&result.started, 1) From 493903eedecaae3d14966cd99aa84d146ea0ce13 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Sat, 20 Oct 2018 10:43:59 +0200 Subject: [PATCH 064/159] core: better side-chain importing --- core/blockchain.go | 367 +++++++++++++++++++++++++++++++----------- node/node.go | 2 +- p2p/discover/table.go | 2 +- 3 files changed, 274 insertions(+), 97 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index d173b2de29..74ac30e709 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1048,6 +1048,80 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { return n, err } +// addFutureBlock checks if the block is within the max allowed window to get accepted for future processing, and +// returns an error if the block is too far ahead and was not added. +func (bc *BlockChain) addFutureBlock(block *types.Block) error { + max := big.NewInt(time.Now().Unix() + maxTimeFutureBlocks) + if block.Time().Cmp(max) > 0 { + return fmt.Errorf("future block: %v > %v", block.Time(), max) + } + bc.futureBlocks.Add(block.Hash(), block) + return nil +} + +// importBatch is a helper function to assist during chain import +type importBatch struct { + chain types.Blocks + results <-chan error + index int + validator Validator +} + +// newBatch creates a new batch based on the given blocks, which are assumed to be a contiguous chain +func newBatch(chain types.Blocks, results <-chan error, validator Validator) *importBatch { + return &importBatch{ + chain: chain, + results: results, + index: -1, + validator: validator, + } +} + +// next returns the next block in the batch, along with any potential validation error for that block +// When the end is reached, it will return (nil, nil), but Current() will always return the last element. +func (batch *importBatch) next() (*types.Block, error) { + if batch.index+1 >= len(batch.chain) { + return nil, nil + } + batch.index++ + if err := <-batch.results; err != nil { + return batch.chain[batch.index], err + } + return batch.chain[batch.index], batch.validator.ValidateBody(batch.chain[batch.index]) +} + +// current returns the current block that's being processed. Even after the next() has progressed the entire +// chain, current will always return the last element +func (batch *importBatch) current() *types.Block { + if batch.index < 0 { + return nil + } + return batch.chain[batch.index] +} + +// previous returns the previous block was being processed, or nil +func (batch *importBatch) previous() *types.Block { + if batch.index < 1 { + return nil + } + return batch.chain[batch.index-1] +} + +// first returns the first block in the batch +func (batch *importBatch) first() *types.Block { + return batch.chain[0] +} + +// remaining returns the number of remaining blocks +func (batch *importBatch) remaining() int { + return len(batch.chain) - batch.index +} + +// processed returns the number of processed blocks +func (batch *importBatch) processed() int { + return batch.index + 1 +} + // insertChain will execute the actual chain insertion and event aggregation. The // only reason this method exists as a separate one is to make locking cleaner // with deferred statements. @@ -1067,12 +1141,27 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty chain[i-1].Hash().Bytes()[:4], i, chain[i].NumberU64(), chain[i].Hash().Bytes()[:4], chain[i].ParentHash().Bytes()[:4]) } } + log.Info("insertChain", "from", chain[0].NumberU64(), "to", chain[len(chain)-1].NumberU64()) + // Pre-checks passed, start the full block imports bc.wg.Add(1) defer bc.wg.Done() - bc.chainmu.Lock() defer bc.chainmu.Unlock() + return bc.insertChainInternal(chain, true) +} + +//insertChainInternal is the internal implementation of insertChain, which assumes that +// 1. chains are contiguous, and +// 2. The `chainMu` lock is held +// This method is split out so that import batches that require re-injecting historical blocks can do +// so without releasing the lock, which could lead to racey behaviour. If a sidechain import is in progress, +// and the historic state is imported, but then new canon-head is added before the actual sidechain completes, +// then the historic state could be pruned again +func (bc *BlockChain) insertChainInternal(chain types.Blocks, verifySeals bool) (int, []interface{}, []*types.Log, error) { + + // Start a parallel signature recovery (signer will fluke on fork transition, minimal perf loss) + senderCacher.recoverFromBlocks(types.MakeSigner(bc.chainConfig, chain[0].Number()), chain) // A queued approach to delivering events. This is generally // faster than direct delivery and requires much less mutex @@ -1082,6 +1171,8 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty events = make([]interface{}, 0, len(chain)) lastCanon *types.Block coalescedLogs []*types.Log + block *types.Block + err error ) // Start the parallel header verifier headers := make([]*types.Header, len(chain)) @@ -1089,16 +1180,57 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty for i, block := range chain { headers[i] = block.Header() - seals[i] = true + seals[i] = verifySeals } abort, results := bc.engine.VerifyHeaders(bc, headers, seals) defer close(abort) - // Start a parallel signature recovery (signer will fluke on fork transition, minimal perf loss) - senderCacher.recoverFromBlocks(types.MakeSigner(bc.chainConfig, chain[0].Number()), chain) + // Peek the error for the first block + batch := newBatch(chain, results, bc.Validator()) + if block, err = batch.next(); err != nil { + if err == consensus.ErrPrunedAncestor { + return bc.insertSidechainInternal(batch, err) + } else if err == consensus.ErrFutureBlock || + (err == consensus.ErrUnknownAncestor && bc.futureBlocks.Contains(batch.first().ParentHash())) { - // Iterate over the blocks and insert when the verifier permits - for i, block := range chain { + // The first block is a future block + // We can shove that one and any child blocks (that fail because of UnknownAncestor) into the future-queue + for block != nil && (batch.index == 0 || err == consensus.ErrUnknownAncestor) { + block := batch.current() + if futureError := bc.addFutureBlock(block); futureError != nil { + return batch.index, events, coalescedLogs, futureError + } + block, err = batch.next() + } + stats.queued += batch.processed() + stats.ignored += batch.remaining() + + // If there are any still remaining, mark as ignored + return batch.index, events, coalescedLogs, err + } else if err == ErrKnownBlock { + + // Block and state both already known -- there can be two explanations. + // 1. We did a roll-back, and should now do a re-import + // 2. The block is stored as a sidechain, and is lying about it's stateroot, and passes a stateroot + // from the canonical chain, which has not been verified. + + // Skip all known blocks that are blocks behind us + currentNum := bc.CurrentBlock().NumberU64() + for block != nil && err == ErrKnownBlock && currentNum >= block.NumberU64() { + // We ignore these + stats.ignored++ + block, err = batch.next() + } + // Falls through to the block import + } else { + // Some other error + stats.ignored += len(batch.chain) + bc.reportBlock(block, nil, err) + return batch.index, events, coalescedLogs, err + } + } + // No validation errors + for ; block != nil && err == nil; block, err = batch.next() { // If the chain is terminating, stop processing blocks if atomic.LoadInt32(&bc.procInterrupt) == 1 { log.Debug("Premature abort during blocks processing") @@ -1107,115 +1239,45 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty // If the header is a banned one, straight out abort if BadHashes[block.Hash()] { bc.reportBlock(block, nil, ErrBlacklistedHash) - return i, events, coalescedLogs, ErrBlacklistedHash + return batch.index, events, coalescedLogs, ErrBlacklistedHash } - // Wait for the block's verification to complete bstart := time.Now() - - err := <-results - if err == nil { - err = bc.Validator().ValidateBody(block) - } - switch { - case err == ErrKnownBlock: - // Block and state both already known. However if the current block is below - // this number we did a rollback and we should reimport it nonetheless. - if bc.CurrentBlock().NumberU64() >= block.NumberU64() { - stats.ignored++ - continue - } - - case err == consensus.ErrFutureBlock: - // Allow up to MaxFuture second in the future blocks. If this limit is exceeded - // the chain is discarded and processed at a later time if given. - max := big.NewInt(time.Now().Unix() + maxTimeFutureBlocks) - if block.Time().Cmp(max) > 0 { - return i, events, coalescedLogs, fmt.Errorf("future block: %v > %v", block.Time(), max) - } - bc.futureBlocks.Add(block.Hash(), block) - stats.queued++ - continue - - case err == consensus.ErrUnknownAncestor && bc.futureBlocks.Contains(block.ParentHash()): - bc.futureBlocks.Add(block.Hash(), block) - stats.queued++ - continue - - case err == consensus.ErrPrunedAncestor: - // Block competing with the canonical chain, store in the db, but don't process - // until the competitor TD goes above the canonical TD - currentBlock := bc.CurrentBlock() - localTd := bc.GetTd(currentBlock.Hash(), currentBlock.NumberU64()) - externTd := new(big.Int).Add(bc.GetTd(block.ParentHash(), block.NumberU64()-1), block.Difficulty()) - if localTd.Cmp(externTd) > 0 { - if err = bc.WriteBlockWithoutState(block, externTd); err != nil { - return i, events, coalescedLogs, err - } - continue - } - // Competitor chain beat canonical, gather all blocks from the common ancestor - var winner []*types.Block - - parent := bc.GetBlock(block.ParentHash(), block.NumberU64()-1) - for !bc.HasState(parent.Root()) { - winner = append(winner, parent) - parent = bc.GetBlock(parent.ParentHash(), parent.NumberU64()-1) - } - for j := 0; j < len(winner)/2; j++ { - winner[j], winner[len(winner)-1-j] = winner[len(winner)-1-j], winner[j] - } - // Import all the pruned blocks to make the state available - bc.chainmu.Unlock() - _, evs, logs, err := bc.insertChain(winner) - bc.chainmu.Lock() - events, coalescedLogs = evs, logs - - if err != nil { - return i, events, coalescedLogs, err - } - - case err != nil: - bc.reportBlock(block, nil, err) - return i, events, coalescedLogs, err - } - // Create a new statedb using the parent block and report an - // error if it fails. var parent *types.Block - if i == 0 { + parent = batch.previous() + if parent == nil { parent = bc.GetBlock(block.ParentHash(), block.NumberU64()-1) - } else { - parent = chain[i-1] } + state, err := state.New(parent.Root(), bc.stateCache) if err != nil { - return i, events, coalescedLogs, err + return batch.index, events, coalescedLogs, err } // Process block using the parent state as reference point. receipts, logs, usedGas, err := bc.processor.Process(block, state, bc.vmConfig) if err != nil { bc.reportBlock(block, receipts, err) - return i, events, coalescedLogs, err + return batch.index, events, coalescedLogs, err } // Validate the state using the default validator - err = bc.Validator().ValidateState(block, parent, state, receipts, usedGas) - if err != nil { + if err := bc.Validator().ValidateState(block, parent, state, receipts, usedGas); err != nil { bc.reportBlock(block, receipts, err) - return i, events, coalescedLogs, err + return batch.index, events, coalescedLogs, err } proctime := time.Since(bstart) // Write the block to the chain and get the status. status, err := bc.WriteBlockWithState(block, receipts, state) if err != nil { - return i, events, coalescedLogs, err + return batch.index, events, coalescedLogs, err } switch status { case CanonStatTy: - log.Debug("Inserted new block", "number", block.Number(), "hash", block.Hash(), "uncles", len(block.Uncles()), - "txs", len(block.Transactions()), "gas", block.GasUsed(), "elapsed", common.PrettyDuration(time.Since(bstart))) + log.Debug("Inserted new block", "number", block.Number(), "hash", block.Hash(), + "uncles", len(block.Uncles()), "txs", len(block.Transactions()), "gas", block.GasUsed(), + "elapsed", common.PrettyDuration(time.Since(bstart)), + "root", block.Root().String()) coalescedLogs = append(coalescedLogs, logs...) - blockInsertTimer.UpdateSince(bstart) events = append(events, ChainEvent{block, block.Hash(), logs}) lastCanon = block @@ -1223,23 +1285,138 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty bc.gcproc += proctime case SideStatTy: - log.Debug("Inserted forked block", "number", block.Number(), "hash", block.Hash(), "diff", block.Difficulty(), "elapsed", - common.PrettyDuration(time.Since(bstart)), "txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles())) - - blockInsertTimer.UpdateSince(bstart) + log.Debug("Inserted forked block", "number", block.Number(), "hash", block.Hash(), + "diff", block.Difficulty(), "elapsed", common.PrettyDuration(time.Since(bstart)), + "txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()), + "root", block.Root().String()) events = append(events, ChainSideEvent{block}) } + blockInsertTimer.UpdateSince(bstart) stats.processed++ stats.usedGas += usedGas cache, _ := bc.stateCache.TrieDB().Size() - stats.report(chain, i, cache) + stats.report(chain, batch.index, cache) } + + // Any blocks remaining here? If so, the only ones we need to care about are + // shoving future blocks into queue + if block != nil && err == consensus.ErrFutureBlock { + if futureErr := bc.addFutureBlock(block); futureErr != nil { + return batch.index, events, coalescedLogs, futureErr + } + for block, err = batch.next(); block != nil && err == consensus.ErrUnknownAncestor; { + if futureErr := bc.addFutureBlock(block); futureErr != nil { + return batch.index, events, coalescedLogs, futureErr + } + stats.queued++ + block, err = batch.next() + } + } + stats.ignored += batch.remaining() // Append a single chain head event if we've progressed the chain if lastCanon != nil && bc.CurrentBlock().Hash() == lastCanon.Hash() { events = append(events, ChainHeadEvent{lastCanon}) } - return 0, events, coalescedLogs, nil + return 0, events, coalescedLogs, err +} + +// insertSidechainInternal should be called when an import batch hits upon a pruned ancestor error, which happens when +// an sidechain with a sufficiently old fork-block is found. It writes all (header-and-body-valid) blocks to disk, then +// tries to switch over to the new chain if the TD exceeded the current chain. +// It assumes that relevant locks are held already (hence 'Internal') +func (bc *BlockChain) insertSidechainInternal(batch *importBatch, err error) (int, []interface{}, []*types.Log, error) { + // If we're given a chain of blocks, and the first one is pruned, that means we're getting a + // sidechain imported. On the sidechain, we validate headers, but do not validate body and state + // (and actually import them) until the sidechain reaches a higher TD. + // Until then, we store them in the database (assuming that the header PoW check works out) + var ( + externTd *big.Int + canonHeadNumber = bc.CurrentBlock().NumberU64() + events = make([]interface{}, 0) + coalescedLogs []*types.Log + ) + // The first sidechain block error is already verified to be ErrPrunedAncestor. Since we don't import + // them here, we expect ErrUnknownAncestor for the remaining ones. Any other errors means that + // the block is invalid, and should not be written to disk. + block := batch.current() + for block != nil && (err == consensus.ErrPrunedAncestor) { + // Check the canonical stateroot for that number + if remoteNum := block.NumberU64(); canonHeadNumber >= remoteNum { + canonBlock := bc.GetBlockByNumber(remoteNum) + if canonBlock != nil && canonBlock.Root() == block.Root() { + // This is most likely a shadow-state attack. + // When a fork is imported into the database, and it eventually reaches a block height which is + // not pruned, we just found that the state already exist! This means that the sidechain block + // refers to a state which already exists in our canon chain. + // If left unchecked, we would now proceed importing the blocks, without actually having verified + // the state of the previous blocks. + log.Warn("Sidechain ghost-state attack detected", "blocknum", block.NumberU64(), + "sidechain root", block.Root(), "canon root", canonBlock.Root()) + // If someone legitimately side-mines blocks, they would still be imported as usual. However, + // we cannot risk writing unverified blocks to disk when they obviously target the pruning + // mechanism. + return batch.index, events, coalescedLogs, fmt.Errorf("sidechain ghost-state attack detected") + } + } + if externTd == nil { + externTd = bc.GetTd(block.ParentHash(), block.NumberU64()-1) + } + externTd = new(big.Int).Add(externTd, block.Difficulty()) + if !bc.HasBlock(block.Hash(), block.NumberU64()) { + if err := bc.WriteBlockWithoutState(block, externTd); err != nil { + return batch.index, events, coalescedLogs, err + } + } + block, err = batch.next() + } + // At this point, we've written all sidechain blocks to database. Loop ended either on some other error, + // or all were processed. If there was some other error, we can ignore the rest of those blocks. + // + // If the externTd was larger than our local TD, we now need to reimport the previous + // blocks to regenerate the required state + currentBlock := bc.CurrentBlock() + localTd := bc.GetTd(currentBlock.Hash(), currentBlock.NumberU64()) + // don't process until the competitor TD goes above the canonical TD + if localTd.Cmp(externTd) > 0 { + // If we have hit a sidechain, we may have to reimport pruned blocks + log.Info("Sidechain stored", "start", batch.first().NumberU64(), "end", batch.current().NumberU64(), "sidechain TD", externTd, "local TD", localTd) + return batch.index, events, coalescedLogs, err + } + // Competitor chain beat canonical. Before we reprocess to get the common ancestor, investigate if + // any blocks in the chain are 'known bad' blocks. + for index, b := range batch.chain { + if bc.badBlocks.Contains(b.Hash()) { + log.Info("Sidechain import aborted, bad block found", "index", index, "hash", b.Hash()) + return index, events, coalescedLogs, fmt.Errorf("known bad block %d 0x%x", b.NumberU64(), b.Hash()) + } + } + // gather all blocks from the common ancestor + var parents []*types.Block + // Import all the pruned blocks to make the state available + parent := bc.GetBlock(batch.first().ParentHash(), batch.first().NumberU64()-1) + for !bc.HasState(parent.Root()) { + if bc.badBlocks.Contains(parent.Hash()) { + log.Info("Sidechain parent processing aborted, bad block found", "number", parent.NumberU64(), "hash", parent.Hash()) + return 0, events, coalescedLogs, fmt.Errorf("known bad block %d 0x%x", parent.NumberU64(), parent.Hash()) + } + parents = append(parents, parent) + parent = bc.GetBlock(parent.ParentHash(), parent.NumberU64()-1) + } + for j := 0; j < len(parents)/2; j++ { + parents[j], parents[len(parents)-1-j] = parents[len(parents)-1-j], parents[j] + } + // Import all the pruned blocks to make the state available + // During re-import, we can disable PoW-verification, since these are already verified + log.Info("Inserting parent blocks for reprocessing", "first", parents[0].NumberU64(), "count", len(parents), "last", parents[len(parents)-1].NumberU64) + _, evs, logs, err := bc.insertChainInternal(parents, false) + events, coalescedLogs = evs, logs + if err != nil { + return 0, events, coalescedLogs, err + } + log.Info("Inserting sidechain blocks for processing") + errindex, events, coalescedLogs, err := bc.insertChainInternal(batch.chain[0:batch.index], false) + return errindex, events, coalescedLogs, err } // insertStats tracks and reports on block insertion. diff --git a/node/node.go b/node/node.go index 85299dba74..8461008391 100644 --- a/node/node.go +++ b/node/node.go @@ -287,7 +287,7 @@ func (n *Node) startInProc(apis []rpc.API) error { if err := handler.RegisterName(api.Namespace, api.Service); err != nil { return err } - n.log.Debug("InProc registered", "service", api.Service, "namespace", api.Namespace) + n.log.Debug("InProc registered", "namespace", api.Namespace) } n.inprocHandler = handler return nil diff --git a/p2p/discover/table.go b/p2p/discover/table.go index afd4c9a272..9f7f1d41b9 100644 --- a/p2p/discover/table.go +++ b/p2p/discover/table.go @@ -434,7 +434,7 @@ func (tab *Table) loadSeedNodes() { for i := range seeds { seed := seeds[i] age := log.Lazy{Fn: func() interface{} { return time.Since(tab.db.LastPongReceived(seed.ID())) }} - log.Debug("Found seed node in database", "id", seed.ID(), "addr", seed.addr(), "age", age) + log.Trace("Found seed node in database", "id", seed.ID(), "addr", seed.addr(), "age", age) tab.add(seed) } } From 21dd59bd047d1f291c1a773b690825a3654ea969 Mon Sep 17 00:00:00 2001 From: Franklin Date: Tue, 20 Nov 2018 22:16:40 +0800 Subject: [PATCH 065/159] . --- core/blockchain.go | 6 +++++- node/node.go | 2 +- rpc/ipc.go | 6 +++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index d173b2de29..ee1c0702b5 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -445,7 +445,11 @@ func (bc *BlockChain) repair(head **types.Block) error { return nil } // Otherwise rewind one block and recheck state availability there - (*head) = bc.GetBlock((*head).ParentHash(), (*head).NumberU64()-1) + block := bc.GetBlock((*head).ParentHash(), (*head).NumberU64()-1) + if block == nil { + return fmt.Errorf("failed to repair block, can not get block at height %d", (*head).NumberU64()) + } + (*head) = block } } diff --git a/node/node.go b/node/node.go index 85299dba74..0a931a9dd5 100644 --- a/node/node.go +++ b/node/node.go @@ -322,7 +322,7 @@ func (n *Node) stopIPC() { n.ipcListener.Close() n.ipcListener = nil - n.log.Info("IPC endpoint closed", "endpoint", n.ipcEndpoint) + n.log.Info("IPC endpoint closed", "url", n.ipcEndpoint) } if n.ipcHandler != nil { n.ipcHandler.Stop() diff --git a/rpc/ipc.go b/rpc/ipc.go index b05e503d74..1467f7a0c9 100644 --- a/rpc/ipc.go +++ b/rpc/ipc.go @@ -24,17 +24,17 @@ import ( "github.com/ethereum/go-ethereum/p2p/netutil" ) -// ServeListener accepts connections on l, serving JSON-RPC on them. +// ServeListener accepts connections on l, serving IPC-RPC on them. func (srv *Server) ServeListener(l net.Listener) error { for { conn, err := l.Accept() if netutil.IsTemporaryError(err) { - log.Warn("RPC accept error", "err", err) + log.Warn("IPC accept error", "err", err) continue } else if err != nil { return err } - log.Trace("Accepted connection", "addr", conn.RemoteAddr()) + log.Trace("IPC accepted connection") go srv.ServeCodec(NewJSONCodec(conn), OptionMethodInvocation|OptionSubscriptions) } } From 5d80a1b6652b1c5eb50b73e9582d9000829d7c9a Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Tue, 20 Nov 2018 20:14:37 +0100 Subject: [PATCH 066/159] whisper/mailserver: reduce the max number of opened files (#18142) This should reduce the occurences of travis failures on MacOS Also fix some linter warnings --- whisper/mailserver/mailserver.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/whisper/mailserver/mailserver.go b/whisper/mailserver/mailserver.go index af9418d9f8..d7af4baae3 100644 --- a/whisper/mailserver/mailserver.go +++ b/whisper/mailserver/mailserver.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +// Package mailserver provides a naive, example mailserver implementation package mailserver import ( @@ -26,9 +27,11 @@ import ( "github.com/ethereum/go-ethereum/rlp" whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/opt" "github.com/syndtr/goleveldb/leveldb/util" ) +// WMailServer represents the state data of the mailserver. type WMailServer struct { db *leveldb.DB w *whisper.Whisper @@ -42,6 +45,8 @@ type DBKey struct { raw []byte } +// NewDbKey is a helper function that creates a levelDB +// key from a hash and an integer. func NewDbKey(t uint32, h common.Hash) *DBKey { const sz = common.HashLength + 4 var k DBKey @@ -53,6 +58,7 @@ func NewDbKey(t uint32, h common.Hash) *DBKey { return &k } +// Init initializes the mail server. func (s *WMailServer) Init(shh *whisper.Whisper, path string, password string, pow float64) error { var err error if len(path) == 0 { @@ -63,7 +69,7 @@ func (s *WMailServer) Init(shh *whisper.Whisper, path string, password string, p return fmt.Errorf("password is not specified") } - s.db, err = leveldb.OpenFile(path, nil) + s.db, err = leveldb.OpenFile(path, &opt.Options{OpenFilesCacheCapacity: 32}) if err != nil { return fmt.Errorf("open DB file: %s", err) } @@ -82,12 +88,14 @@ func (s *WMailServer) Init(shh *whisper.Whisper, path string, password string, p return nil } +// Close cleans up before shutdown. func (s *WMailServer) Close() { if s.db != nil { s.db.Close() } } +// Archive stores the func (s *WMailServer) Archive(env *whisper.Envelope) { key := NewDbKey(env.Expiry-env.TTL, env.Hash()) rawEnvelope, err := rlp.EncodeToBytes(env) @@ -101,6 +109,8 @@ func (s *WMailServer) Archive(env *whisper.Envelope) { } } +// DeliverMail responds with saved messages upon request by the +// messages' owner. func (s *WMailServer) DeliverMail(peer *whisper.Peer, request *whisper.Envelope) { if peer == nil { log.Error("Whisper peer is nil") From c7e522fd17040485a5c2966c45a5c45c0558945e Mon Sep 17 00:00:00 2001 From: a-sklyarov Date: Wed, 21 Nov 2018 10:38:49 +0000 Subject: [PATCH 067/159] Update minimum required Go version in README.md (#18151) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f308fb1011..4b62bfde03 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ For prerequisites and detailed build instructions please read the [Installation Instructions](https://github.com/ethereum/go-ethereum/wiki/Building-Ethereum) on the wiki. -Building geth requires both a Go (version 1.7 or later) and a C compiler. +Building geth requires both a Go (version 1.9 or later) and a C compiler. You can install them using your favourite package manager. Once the dependencies are installed, run From 3fd87f219342ca97a6595263a2aa28bec65fee04 Mon Sep 17 00:00:00 2001 From: mr_franklin Date: Wed, 21 Nov 2018 18:52:02 +0800 Subject: [PATCH 068/159] core: fix comment typo (#18144) --- core/tx_pool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/tx_pool.go b/core/tx_pool.go index f6da5da2a7..fc35d1f243 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -825,7 +825,7 @@ func (pool *TxPool) addTxs(txs []*types.Transaction, local bool) []error { // addTxsLocked attempts to queue a batch of transactions if they are valid, // whilst assuming the transaction pool lock is already held. func (pool *TxPool) addTxsLocked(txs []*types.Transaction, local bool) []error { - // Add the batch of transaction, tracking the accepted ones + // Add the batch of transactions, tracking the accepted ones dirty := make(map[common.Address]struct{}) errs := make([]error, len(txs)) From 333b5fb123a497efa1a9ef54a437b24dfb3936cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 20 Nov 2018 14:15:26 +0200 Subject: [PATCH 069/159] core: polish side chain importer a bit --- core/blockchain.go | 479 +++++++++++++++----------------------- core/blockchain_insert.go | 143 ++++++++++++ core/blockchain_test.go | 8 +- 3 files changed, 332 insertions(+), 298 deletions(-) create mode 100644 core/blockchain_insert.go diff --git a/core/blockchain.go b/core/blockchain.go index 74ac30e709..d789467916 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1036,6 +1036,18 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. return status, nil } +// addFutureBlock checks if the block is within the max allowed window to get +// accepted for future processing, and returns an error if the block is too far +// ahead and was not added. +func (bc *BlockChain) addFutureBlock(block *types.Block) error { + max := big.NewInt(time.Now().Unix() + maxTimeFutureBlocks) + if block.Time().Cmp(max) > 0 { + return fmt.Errorf("future block timestamp %v > allowed %v", block.Time(), max) + } + bc.futureBlocks.Add(block.Hash(), block) + return nil +} + // InsertChain attempts to insert the given batch of blocks in to the canonical // chain or, otherwise, create a fork. If an error is returned it will return // the index number of the failing block as well an error describing what went @@ -1043,92 +1055,9 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. // // After insertion is done, all accumulated events will be fired. func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { - n, events, logs, err := bc.insertChain(chain) - bc.PostChainEvents(events, logs) - return n, err -} - -// addFutureBlock checks if the block is within the max allowed window to get accepted for future processing, and -// returns an error if the block is too far ahead and was not added. -func (bc *BlockChain) addFutureBlock(block *types.Block) error { - max := big.NewInt(time.Now().Unix() + maxTimeFutureBlocks) - if block.Time().Cmp(max) > 0 { - return fmt.Errorf("future block: %v > %v", block.Time(), max) - } - bc.futureBlocks.Add(block.Hash(), block) - return nil -} - -// importBatch is a helper function to assist during chain import -type importBatch struct { - chain types.Blocks - results <-chan error - index int - validator Validator -} - -// newBatch creates a new batch based on the given blocks, which are assumed to be a contiguous chain -func newBatch(chain types.Blocks, results <-chan error, validator Validator) *importBatch { - return &importBatch{ - chain: chain, - results: results, - index: -1, - validator: validator, - } -} - -// next returns the next block in the batch, along with any potential validation error for that block -// When the end is reached, it will return (nil, nil), but Current() will always return the last element. -func (batch *importBatch) next() (*types.Block, error) { - if batch.index+1 >= len(batch.chain) { - return nil, nil - } - batch.index++ - if err := <-batch.results; err != nil { - return batch.chain[batch.index], err - } - return batch.chain[batch.index], batch.validator.ValidateBody(batch.chain[batch.index]) -} - -// current returns the current block that's being processed. Even after the next() has progressed the entire -// chain, current will always return the last element -func (batch *importBatch) current() *types.Block { - if batch.index < 0 { - return nil - } - return batch.chain[batch.index] -} - -// previous returns the previous block was being processed, or nil -func (batch *importBatch) previous() *types.Block { - if batch.index < 1 { - return nil - } - return batch.chain[batch.index-1] -} - -// first returns the first block in the batch -func (batch *importBatch) first() *types.Block { - return batch.chain[0] -} - -// remaining returns the number of remaining blocks -func (batch *importBatch) remaining() int { - return len(batch.chain) - batch.index -} - -// processed returns the number of processed blocks -func (batch *importBatch) processed() int { - return batch.index + 1 -} - -// insertChain will execute the actual chain insertion and event aggregation. The -// only reason this method exists as a separate one is to make locking cleaner -// with deferred statements. -func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*types.Log, error) { // Sanity check that we have something meaningful to import if len(chain) == 0 { - return 0, nil, nil, nil + return 0, nil } // Do a sanity check that the provided chain is actually ordered and linked for i := 1; i < len(chain); i++ { @@ -1137,29 +1066,34 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty log.Error("Non contiguous block insert", "number", chain[i].Number(), "hash", chain[i].Hash(), "parent", chain[i].ParentHash(), "prevnumber", chain[i-1].Number(), "prevhash", chain[i-1].Hash()) - return 0, nil, nil, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, chain[i-1].NumberU64(), + return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, chain[i-1].NumberU64(), chain[i-1].Hash().Bytes()[:4], i, chain[i].NumberU64(), chain[i].Hash().Bytes()[:4], chain[i].ParentHash().Bytes()[:4]) } } - log.Info("insertChain", "from", chain[0].NumberU64(), "to", chain[len(chain)-1].NumberU64()) - // Pre-checks passed, start the full block imports bc.wg.Add(1) - defer bc.wg.Done() bc.chainmu.Lock() - defer bc.chainmu.Unlock() - return bc.insertChainInternal(chain, true) + n, events, logs, err := bc.insertChain(chain, true) + bc.chainmu.Unlock() + bc.wg.Done() + + bc.PostChainEvents(events, logs) + return n, err } -//insertChainInternal is the internal implementation of insertChain, which assumes that -// 1. chains are contiguous, and -// 2. The `chainMu` lock is held -// This method is split out so that import batches that require re-injecting historical blocks can do -// so without releasing the lock, which could lead to racey behaviour. If a sidechain import is in progress, -// and the historic state is imported, but then new canon-head is added before the actual sidechain completes, -// then the historic state could be pruned again -func (bc *BlockChain) insertChainInternal(chain types.Blocks, verifySeals bool) (int, []interface{}, []*types.Log, error) { - +// insertChain is the internal implementation of insertChain, which assumes that +// 1) chains are contiguous, and 2) The chain mutex is held. +// +// This method is split out so that import batches that require re-injecting +// historical blocks can do so without releasing the lock, which could lead to +// racey behaviour. If a sidechain import is in progress, and the historic state +// is imported, but then new canon-head is added before the actual sidechain +// completes, then the historic state could be pruned again +func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, []interface{}, []*types.Log, error) { + // If the chain is terminating, don't even bother starting u + if atomic.LoadInt32(&bc.procInterrupt) == 1 { + return 0, nil, nil, nil + } // Start a parallel signature recovery (signer will fluke on fork transition, minimal perf loss) senderCacher.recoverFromBlocks(types.MakeSigner(bc.chainConfig, chain[0].Number()), chain) @@ -1171,8 +1105,6 @@ func (bc *BlockChain) insertChainInternal(chain types.Blocks, verifySeals bool) events = make([]interface{}, 0, len(chain)) lastCanon *types.Block coalescedLogs []*types.Log - block *types.Block - err error ) // Start the parallel header verifier headers := make([]*types.Header, len(chain)) @@ -1185,52 +1117,51 @@ func (bc *BlockChain) insertChainInternal(chain types.Blocks, verifySeals bool) abort, results := bc.engine.VerifyHeaders(bc, headers, seals) defer close(abort) - // Peek the error for the first block - batch := newBatch(chain, results, bc.Validator()) - if block, err = batch.next(); err != nil { - if err == consensus.ErrPrunedAncestor { - return bc.insertSidechainInternal(batch, err) - } else if err == consensus.ErrFutureBlock || - (err == consensus.ErrUnknownAncestor && bc.futureBlocks.Contains(batch.first().ParentHash())) { + // Peek the error for the first block to decide the directing import logic + it := newInsertIterator(chain, results, bc.Validator()) - // The first block is a future block - // We can shove that one and any child blocks (that fail because of UnknownAncestor) into the future-queue - for block != nil && (batch.index == 0 || err == consensus.ErrUnknownAncestor) { - block := batch.current() - if futureError := bc.addFutureBlock(block); futureError != nil { - return batch.index, events, coalescedLogs, futureError - } - block, err = batch.next() + block, err := it.next() + switch { + // First block is pruned, insert as sidechain and reorg only if TD grows enough + case err == consensus.ErrPrunedAncestor: + return bc.insertSidechain(it) + + // First block is future, shove it (and all children) to the future queue (unknown ancestor) + case err == consensus.ErrFutureBlock || (err == consensus.ErrUnknownAncestor && bc.futureBlocks.Contains(it.first().ParentHash())): + for block != nil && (it.index == 0 || err == consensus.ErrUnknownAncestor) { + if err := bc.addFutureBlock(block); err != nil { + return it.index, events, coalescedLogs, err } - stats.queued += batch.processed() - stats.ignored += batch.remaining() - - // If there are any still remaining, mark as ignored - return batch.index, events, coalescedLogs, err - } else if err == ErrKnownBlock { - - // Block and state both already known -- there can be two explanations. - // 1. We did a roll-back, and should now do a re-import - // 2. The block is stored as a sidechain, and is lying about it's stateroot, and passes a stateroot - // from the canonical chain, which has not been verified. - - // Skip all known blocks that are blocks behind us - currentNum := bc.CurrentBlock().NumberU64() - for block != nil && err == ErrKnownBlock && currentNum >= block.NumberU64() { - // We ignore these - stats.ignored++ - block, err = batch.next() - } - // Falls through to the block import - } else { - // Some other error - stats.ignored += len(batch.chain) - bc.reportBlock(block, nil, err) - return batch.index, events, coalescedLogs, err + block, err = it.next() } + stats.queued += it.processed() + stats.ignored += it.remaining() + + // If there are any still remaining, mark as ignored + return it.index, events, coalescedLogs, err + + // First block (and state) is known + // 1. We did a roll-back, and should now do a re-import + // 2. The block is stored as a sidechain, and is lying about it's stateroot, and passes a stateroot + // from the canonical chain, which has not been verified. + case err == ErrKnownBlock: + // Skip all known blocks that behind us + current := bc.CurrentBlock().NumberU64() + + for block != nil && err == ErrKnownBlock && current >= block.NumberU64() { + stats.ignored++ + block, err = it.next() + } + // Falls through to the block import + + // Some other error occurred, abort + case err != nil: + stats.ignored += len(it.chain) + bc.reportBlock(block, nil, err) + return it.index, events, coalescedLogs, err } - // No validation errors - for ; block != nil && err == nil; block, err = batch.next() { + // No validation errors for the first block (or chain prefix skipped) + for ; block != nil && err == nil; block, err = it.next() { // If the chain is terminating, stop processing blocks if atomic.LoadInt32(&bc.procInterrupt) == 1 { log.Debug("Premature abort during blocks processing") @@ -1239,43 +1170,43 @@ func (bc *BlockChain) insertChainInternal(chain types.Blocks, verifySeals bool) // If the header is a banned one, straight out abort if BadHashes[block.Hash()] { bc.reportBlock(block, nil, ErrBlacklistedHash) - return batch.index, events, coalescedLogs, ErrBlacklistedHash + return it.index, events, coalescedLogs, ErrBlacklistedHash } - bstart := time.Now() - var parent *types.Block - parent = batch.previous() + // Retrieve the parent block and it's state to execute on top + start := time.Now() + + parent := it.previous() if parent == nil { parent = bc.GetBlock(block.ParentHash(), block.NumberU64()-1) } - state, err := state.New(parent.Root(), bc.stateCache) if err != nil { - return batch.index, events, coalescedLogs, err + return it.index, events, coalescedLogs, err } // Process block using the parent state as reference point. receipts, logs, usedGas, err := bc.processor.Process(block, state, bc.vmConfig) if err != nil { bc.reportBlock(block, receipts, err) - return batch.index, events, coalescedLogs, err + return it.index, events, coalescedLogs, err } // Validate the state using the default validator if err := bc.Validator().ValidateState(block, parent, state, receipts, usedGas); err != nil { bc.reportBlock(block, receipts, err) - return batch.index, events, coalescedLogs, err + return it.index, events, coalescedLogs, err } - proctime := time.Since(bstart) + proctime := time.Since(start) // Write the block to the chain and get the status. status, err := bc.WriteBlockWithState(block, receipts, state) if err != nil { - return batch.index, events, coalescedLogs, err + return it.index, events, coalescedLogs, err } switch status { case CanonStatTy: log.Debug("Inserted new block", "number", block.Number(), "hash", block.Hash(), "uncles", len(block.Uncles()), "txs", len(block.Transactions()), "gas", block.GasUsed(), - "elapsed", common.PrettyDuration(time.Since(bstart)), - "root", block.Root().String()) + "elapsed", common.PrettyDuration(time.Since(start)), + "root", block.Root()) coalescedLogs = append(coalescedLogs, logs...) events = append(events, ChainEvent{block, block.Hash(), logs}) @@ -1286,192 +1217,152 @@ func (bc *BlockChain) insertChainInternal(chain types.Blocks, verifySeals bool) case SideStatTy: log.Debug("Inserted forked block", "number", block.Number(), "hash", block.Hash(), - "diff", block.Difficulty(), "elapsed", common.PrettyDuration(time.Since(bstart)), + "diff", block.Difficulty(), "elapsed", common.PrettyDuration(time.Since(start)), "txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()), - "root", block.Root().String()) + "root", block.Root()) events = append(events, ChainSideEvent{block}) } - blockInsertTimer.UpdateSince(bstart) + blockInsertTimer.UpdateSince(start) stats.processed++ stats.usedGas += usedGas cache, _ := bc.stateCache.TrieDB().Size() - stats.report(chain, batch.index, cache) + stats.report(chain, it.index, cache) } - - // Any blocks remaining here? If so, the only ones we need to care about are - // shoving future blocks into queue + // Any blocks remaining here? The only ones we care about are the future ones if block != nil && err == consensus.ErrFutureBlock { - if futureErr := bc.addFutureBlock(block); futureErr != nil { - return batch.index, events, coalescedLogs, futureErr + if err := bc.addFutureBlock(block); err != nil { + return it.index, events, coalescedLogs, err } - for block, err = batch.next(); block != nil && err == consensus.ErrUnknownAncestor; { - if futureErr := bc.addFutureBlock(block); futureErr != nil { - return batch.index, events, coalescedLogs, futureErr + block, err = it.next() + + for ; block != nil && err == consensus.ErrUnknownAncestor; block, err = it.next() { + if err := bc.addFutureBlock(block); err != nil { + return it.index, events, coalescedLogs, err } stats.queued++ - block, err = batch.next() } } - stats.ignored += batch.remaining() + stats.ignored += it.remaining() + // Append a single chain head event if we've progressed the chain if lastCanon != nil && bc.CurrentBlock().Hash() == lastCanon.Hash() { events = append(events, ChainHeadEvent{lastCanon}) } - return 0, events, coalescedLogs, err + return it.index, events, coalescedLogs, err } -// insertSidechainInternal should be called when an import batch hits upon a pruned ancestor error, which happens when -// an sidechain with a sufficiently old fork-block is found. It writes all (header-and-body-valid) blocks to disk, then -// tries to switch over to the new chain if the TD exceeded the current chain. -// It assumes that relevant locks are held already (hence 'Internal') -func (bc *BlockChain) insertSidechainInternal(batch *importBatch, err error) (int, []interface{}, []*types.Log, error) { - // If we're given a chain of blocks, and the first one is pruned, that means we're getting a - // sidechain imported. On the sidechain, we validate headers, but do not validate body and state - // (and actually import them) until the sidechain reaches a higher TD. - // Until then, we store them in the database (assuming that the header PoW check works out) +// insertSidechain is called when an import batch hits upon a pruned ancestor +// error, which happens when a sidechain with a sufficiently old fork-block is +// found. +// +// The method writes all (header-and-body-valid) blocks to disk, then tries to +// switch over to the new chain if the TD exceeded the current chain. +func (bc *BlockChain) insertSidechain(it *insertIterator) (int, []interface{}, []*types.Log, error) { var ( - externTd *big.Int - canonHeadNumber = bc.CurrentBlock().NumberU64() - events = make([]interface{}, 0) - coalescedLogs []*types.Log + externTd *big.Int + current = bc.CurrentBlock().NumberU64() ) - // The first sidechain block error is already verified to be ErrPrunedAncestor. Since we don't import - // them here, we expect ErrUnknownAncestor for the remaining ones. Any other errors means that - // the block is invalid, and should not be written to disk. - block := batch.current() - for block != nil && (err == consensus.ErrPrunedAncestor) { - // Check the canonical stateroot for that number - if remoteNum := block.NumberU64(); canonHeadNumber >= remoteNum { - canonBlock := bc.GetBlockByNumber(remoteNum) - if canonBlock != nil && canonBlock.Root() == block.Root() { - // This is most likely a shadow-state attack. - // When a fork is imported into the database, and it eventually reaches a block height which is - // not pruned, we just found that the state already exist! This means that the sidechain block + // The first sidechain block error is already verified to be ErrPrunedAncestor. + // Since we don't import them here, we expect ErrUnknownAncestor for the remaining + // ones. Any other errors means that the block is invalid, and should not be written + // to disk. + block, err := it.current(), consensus.ErrPrunedAncestor + for ; block != nil && (err == consensus.ErrPrunedAncestor); block, err = it.next() { + // Check the canonical state root for that number + if number := block.NumberU64(); current >= number { + if canonical := bc.GetBlockByNumber(number); canonical != nil && canonical.Root() == block.Root() { + // This is most likely a shadow-state attack. When a fork is imported into the + // database, and it eventually reaches a block height which is not pruned, we + // just found that the state already exist! This means that the sidechain block // refers to a state which already exists in our canon chain. - // If left unchecked, we would now proceed importing the blocks, without actually having verified - // the state of the previous blocks. - log.Warn("Sidechain ghost-state attack detected", "blocknum", block.NumberU64(), - "sidechain root", block.Root(), "canon root", canonBlock.Root()) + // + // If left unchecked, we would now proceed importing the blocks, without actually + // having verified the state of the previous blocks. + log.Warn("Sidechain ghost-state attack detected", "number", block.NumberU64(), "sideroot", block.Root(), "canonroot", canonical.Root()) + // If someone legitimately side-mines blocks, they would still be imported as usual. However, // we cannot risk writing unverified blocks to disk when they obviously target the pruning // mechanism. - return batch.index, events, coalescedLogs, fmt.Errorf("sidechain ghost-state attack detected") + return it.index, nil, nil, errors.New("sidechain ghost-state attack") } } if externTd == nil { externTd = bc.GetTd(block.ParentHash(), block.NumberU64()-1) } externTd = new(big.Int).Add(externTd, block.Difficulty()) + if !bc.HasBlock(block.Hash(), block.NumberU64()) { + start := time.Now() if err := bc.WriteBlockWithoutState(block, externTd); err != nil { - return batch.index, events, coalescedLogs, err + return it.index, nil, nil, err } + log.Debug("Inserted sidechain block", "number", block.Number(), "hash", block.Hash(), + "diff", block.Difficulty(), "elapsed", common.PrettyDuration(time.Since(start)), + "txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()), + "root", block.Root()) } - block, err = batch.next() } - // At this point, we've written all sidechain blocks to database. Loop ended either on some other error, - // or all were processed. If there was some other error, we can ignore the rest of those blocks. + // At this point, we've written all sidechain blocks to database. Loop ended + // either on some other error or all were processed. If there was some other + // error, we can ignore the rest of those blocks. // // If the externTd was larger than our local TD, we now need to reimport the previous // blocks to regenerate the required state - currentBlock := bc.CurrentBlock() - localTd := bc.GetTd(currentBlock.Hash(), currentBlock.NumberU64()) - // don't process until the competitor TD goes above the canonical TD + localTd := bc.GetTd(bc.CurrentBlock().Hash(), current) if localTd.Cmp(externTd) > 0 { - // If we have hit a sidechain, we may have to reimport pruned blocks - log.Info("Sidechain stored", "start", batch.first().NumberU64(), "end", batch.current().NumberU64(), "sidechain TD", externTd, "local TD", localTd) - return batch.index, events, coalescedLogs, err + log.Info("Sidechain written to disk", "start", it.first().NumberU64(), "end", it.previous().NumberU64(), "sidetd", externTd, "localtd", localTd) + return it.index, nil, nil, err } - // Competitor chain beat canonical. Before we reprocess to get the common ancestor, investigate if - // any blocks in the chain are 'known bad' blocks. - for index, b := range batch.chain { - if bc.badBlocks.Contains(b.Hash()) { - log.Info("Sidechain import aborted, bad block found", "index", index, "hash", b.Hash()) - return index, events, coalescedLogs, fmt.Errorf("known bad block %d 0x%x", b.NumberU64(), b.Hash()) - } - } - // gather all blocks from the common ancestor - var parents []*types.Block - // Import all the pruned blocks to make the state available - parent := bc.GetBlock(batch.first().ParentHash(), batch.first().NumberU64()-1) - for !bc.HasState(parent.Root()) { - if bc.badBlocks.Contains(parent.Hash()) { - log.Info("Sidechain parent processing aborted, bad block found", "number", parent.NumberU64(), "hash", parent.Hash()) - return 0, events, coalescedLogs, fmt.Errorf("known bad block %d 0x%x", parent.NumberU64(), parent.Hash()) - } - parents = append(parents, parent) - parent = bc.GetBlock(parent.ParentHash(), parent.NumberU64()-1) - } - for j := 0; j < len(parents)/2; j++ { - parents[j], parents[len(parents)-1-j] = parents[len(parents)-1-j], parents[j] - } - // Import all the pruned blocks to make the state available - // During re-import, we can disable PoW-verification, since these are already verified - log.Info("Inserting parent blocks for reprocessing", "first", parents[0].NumberU64(), "count", len(parents), "last", parents[len(parents)-1].NumberU64) - _, evs, logs, err := bc.insertChainInternal(parents, false) - events, coalescedLogs = evs, logs - if err != nil { - return 0, events, coalescedLogs, err - } - log.Info("Inserting sidechain blocks for processing") - errindex, events, coalescedLogs, err := bc.insertChainInternal(batch.chain[0:batch.index], false) - return errindex, events, coalescedLogs, err -} - -// insertStats tracks and reports on block insertion. -type insertStats struct { - queued, processed, ignored int - usedGas uint64 - lastIndex int - startTime mclock.AbsTime -} - -// statsReportLimit is the time limit during import and export after which we -// always print out progress. This avoids the user wondering what's going on. -const statsReportLimit = 8 * time.Second - -// report prints statistics if some number of blocks have been processed -// or more than a few seconds have passed since the last message. -func (st *insertStats) report(chain []*types.Block, index int, cache common.StorageSize) { - // Fetch the timings for the batch + // Gather all the sidechain hashes (full blocks may be memory heavy) var ( - now = mclock.Now() - elapsed = time.Duration(now) - time.Duration(st.startTime) + hashes []common.Hash + numbers []uint64 ) - // If we're at the last block of the batch or report period reached, log - if index == len(chain)-1 || elapsed >= statsReportLimit { - var ( - end = chain[index] - txs = countTransactions(chain[st.lastIndex : index+1]) - ) - context := []interface{}{ - "blocks", st.processed, "txs", txs, "mgas", float64(st.usedGas) / 1000000, - "elapsed", common.PrettyDuration(elapsed), "mgasps", float64(st.usedGas) * 1000 / float64(elapsed), - "number", end.Number(), "hash", end.Hash(), - } - if timestamp := time.Unix(end.Time().Int64(), 0); time.Since(timestamp) > time.Minute { - context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...) - } - context = append(context, []interface{}{"cache", cache}...) + parent := bc.GetHeader(it.previous().Hash(), it.previous().NumberU64()) + for parent != nil && !bc.HasState(parent.Root) { + hashes = append(hashes, parent.Hash()) + numbers = append(numbers, parent.Number.Uint64()) - if st.queued > 0 { - context = append(context, []interface{}{"queued", st.queued}...) - } - if st.ignored > 0 { - context = append(context, []interface{}{"ignored", st.ignored}...) - } - log.Info("Imported new chain segment", context...) - - *st = insertStats{startTime: now, lastIndex: index + 1} + parent = bc.GetHeader(parent.ParentHash, parent.Number.Uint64()-1) } -} - -func countTransactions(chain []*types.Block) (c int) { - for _, b := range chain { - c += len(b.Transactions()) + if parent == nil { + return it.index, nil, nil, errors.New("missing parent") } - return c + // Import all the pruned blocks to make the state available + var ( + blocks []*types.Block + memory common.StorageSize + ) + for i := len(hashes) - 1; i >= 0; i-- { + // Append the next block to our batch + block := bc.GetBlock(hashes[i], numbers[i]) + + blocks = append(blocks, block) + memory += block.Size() + + // If memory use grew too large, import and continue. Sadly we need to discard + // all raised events and logs from notifications since we're too heavy on the + // memory here. + if len(blocks) >= 2048 || memory > 64*1024*1024 { + log.Info("Importing heavy sidechain segment", "blocks", len(blocks), "start", blocks[0].NumberU64(), "end", block.NumberU64()) + if _, _, _, err := bc.insertChain(blocks, false); err != nil { + return 0, nil, nil, err + } + blocks, memory = blocks[:0], 0 + + // If the chain is terminating, stop processing blocks + if atomic.LoadInt32(&bc.procInterrupt) == 1 { + log.Debug("Premature abort during blocks processing") + return 0, nil, nil, nil + } + } + } + if len(blocks) > 0 { + log.Info("Importing sidechain segment", "start", blocks[0].NumberU64(), "end", blocks[len(blocks)-1].NumberU64()) + return bc.insertChain(blocks, false) + } + return 0, nil, nil, nil } // reorgs takes two blocks, an old chain and a new chain and will reconstruct the blocks and inserts them diff --git a/core/blockchain_insert.go b/core/blockchain_insert.go new file mode 100644 index 0000000000..70bea35445 --- /dev/null +++ b/core/blockchain_insert.go @@ -0,0 +1,143 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package core + +import ( + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" +) + +// insertStats tracks and reports on block insertion. +type insertStats struct { + queued, processed, ignored int + usedGas uint64 + lastIndex int + startTime mclock.AbsTime +} + +// statsReportLimit is the time limit during import and export after which we +// always print out progress. This avoids the user wondering what's going on. +const statsReportLimit = 8 * time.Second + +// report prints statistics if some number of blocks have been processed +// or more than a few seconds have passed since the last message. +func (st *insertStats) report(chain []*types.Block, index int, cache common.StorageSize) { + // Fetch the timings for the batch + var ( + now = mclock.Now() + elapsed = time.Duration(now) - time.Duration(st.startTime) + ) + // If we're at the last block of the batch or report period reached, log + if index == len(chain)-1 || elapsed >= statsReportLimit { + // Count the number of transactions in this segment + var txs int + for _, block := range chain[st.lastIndex : index+1] { + txs += len(block.Transactions()) + } + end := chain[index] + + // Assemble the log context and send it to the logger + context := []interface{}{ + "blocks", st.processed, "txs", txs, "mgas", float64(st.usedGas) / 1000000, + "elapsed", common.PrettyDuration(elapsed), "mgasps", float64(st.usedGas) * 1000 / float64(elapsed), + "number", end.Number(), "hash", end.Hash(), + } + if timestamp := time.Unix(end.Time().Int64(), 0); time.Since(timestamp) > time.Minute { + context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...) + } + context = append(context, []interface{}{"cache", cache}...) + + if st.queued > 0 { + context = append(context, []interface{}{"queued", st.queued}...) + } + if st.ignored > 0 { + context = append(context, []interface{}{"ignored", st.ignored}...) + } + log.Info("Imported new chain segment", context...) + + // Bump the stats reported to the next section + *st = insertStats{startTime: now, lastIndex: index + 1} + } +} + +// insertIterator is a helper to assist during chain import. +type insertIterator struct { + chain types.Blocks + results <-chan error + index int + validator Validator +} + +// newInsertIterator creates a new iterator based on the given blocks, which are +// assumed to be a contiguous chain. +func newInsertIterator(chain types.Blocks, results <-chan error, validator Validator) *insertIterator { + return &insertIterator{ + chain: chain, + results: results, + index: -1, + validator: validator, + } +} + +// next returns the next block in the iterator, along with any potential validation +// error for that block. When the end is reached, it will return (nil, nil). +func (it *insertIterator) next() (*types.Block, error) { + if it.index+1 >= len(it.chain) { + it.index = len(it.chain) + return nil, nil + } + it.index++ + if err := <-it.results; err != nil { + return it.chain[it.index], err + } + return it.chain[it.index], it.validator.ValidateBody(it.chain[it.index]) +} + +// current returns the current block that's being processed. +func (it *insertIterator) current() *types.Block { + if it.index < 0 || it.index+1 >= len(it.chain) { + return nil + } + return it.chain[it.index] +} + +// previous returns the previous block was being processed, or nil +func (it *insertIterator) previous() *types.Block { + if it.index < 1 { + return nil + } + return it.chain[it.index-1] +} + +// first returns the first block in the it. +func (it *insertIterator) first() *types.Block { + return it.chain[0] +} + +// remaining returns the number of remaining blocks. +func (it *insertIterator) remaining() int { + return len(it.chain) - it.index +} + +// processed returns the number of processed blocks. +func (it *insertIterator) processed() int { + return it.index + 1 +} diff --git a/core/blockchain_test.go b/core/blockchain_test.go index aef810050b..5ab29e205f 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -579,11 +579,11 @@ func testInsertNonceError(t *testing.T, full bool) { blockchain.hc.engine = blockchain.engine failRes, err = blockchain.InsertHeaderChain(headers, 1) } - // Check that the returned error indicates the failure. + // Check that the returned error indicates the failure if failRes != failAt { - t.Errorf("test %d: failure index mismatch: have %d, want %d", i, failRes, failAt) + t.Errorf("test %d: failure (%v) index mismatch: have %d, want %d", i, err, failRes, failAt) } - // Check that all no blocks after the failing block have been inserted. + // Check that all blocks after the failing block have been inserted for j := 0; j < i-failAt; j++ { if full { if block := blockchain.GetBlockByNumber(failNum + uint64(j)); block != nil { @@ -1345,7 +1345,7 @@ func TestLargeReorgTrieGC(t *testing.T) { t.Fatalf("failed to insert shared chain: %v", err) } if _, err := chain.InsertChain(original); err != nil { - t.Fatalf("failed to insert shared chain: %v", err) + t.Fatalf("failed to insert original chain: %v", err) } // Ensure that the state associated with the forking point is pruned away if node, _ := chain.stateCache.TrieDB().Node(shared[len(shared)-1].Root()); node != nil { From 4c181e4fb98bb88503cccd6147026b6c2b7b56f6 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 21 Nov 2018 14:36:56 +0100 Subject: [PATCH 070/159] swarm/state: refactor InmemoryStore (#18143) --- swarm/network/simulations/overlay.go | 4 +- swarm/network/stream/delivery.go | 2 +- swarm/state/dbstore.go | 21 +++++++ swarm/state/inmemorystore.go | 94 ---------------------------- swarm/state/store.go | 26 -------- 5 files changed, 24 insertions(+), 123 deletions(-) delete mode 100644 swarm/state/inmemorystore.go delete mode 100644 swarm/state/store.go diff --git a/swarm/network/simulations/overlay.go b/swarm/network/simulations/overlay.go index caf7ff1f2d..284ae6398a 100644 --- a/swarm/network/simulations/overlay.go +++ b/swarm/network/simulations/overlay.go @@ -64,12 +64,12 @@ func init() { type Simulation struct { mtx sync.Mutex - stores map[enode.ID]*state.InmemoryStore + stores map[enode.ID]state.Store } func NewSimulation() *Simulation { return &Simulation{ - stores: make(map[enode.ID]*state.InmemoryStore), + stores: make(map[enode.ID]state.Store), } } diff --git a/swarm/network/stream/delivery.go b/swarm/network/stream/delivery.go index 0109fbdef2..64d754336f 100644 --- a/swarm/network/stream/delivery.go +++ b/swarm/network/stream/delivery.go @@ -169,7 +169,7 @@ func (d *Delivery) handleRetrieveRequestMsg(ctx context.Context, sp *Peer, req * go func() { chunk, err := d.chunkStore.Get(ctx, req.Addr) if err != nil { - log.Warn("ChunkStore.Get can not retrieve chunk", "err", err) + log.Warn("ChunkStore.Get can not retrieve chunk", "peer", sp.ID().String(), "addr", req.Addr, "hopcount", req.HopCount, "err", err) return } if req.SkipCheck { diff --git a/swarm/state/dbstore.go b/swarm/state/dbstore.go index b0aa92e272..fc5dd8f7ca 100644 --- a/swarm/state/dbstore.go +++ b/swarm/state/dbstore.go @@ -22,6 +22,7 @@ import ( "errors" "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/storage" ) // ErrNotFound is returned when no results are returned from the database @@ -30,6 +31,15 @@ var ErrNotFound = errors.New("ErrorNotFound") // ErrInvalidArgument is returned when the argument type does not match the expected type var ErrInvalidArgument = errors.New("ErrorInvalidArgument") +// Store defines methods required to get, set, delete values for different keys +// and close the underlying resources. +type Store interface { + Get(key string, i interface{}) (err error) + Put(key string, i interface{}) (err error) + Delete(key string) (err error) + Close() error +} + // DBStore uses LevelDB to store values. type DBStore struct { db *leveldb.DB @@ -46,6 +56,17 @@ func NewDBStore(path string) (s *DBStore, err error) { }, nil } +// NewInmemoryStore returns a new instance of DBStore. To be used only in tests and simulations. +func NewInmemoryStore() *DBStore { + db, err := leveldb.Open(storage.NewMemStorage(), nil) + if err != nil { + panic(err) + } + return &DBStore{ + db: db, + } +} + // Get retrieves a persisted value for a specific key. If there is no results // ErrNotFound is returned. The provided parameter should be either a byte slice or // a struct that implements the encoding.BinaryUnmarshaler interface diff --git a/swarm/state/inmemorystore.go b/swarm/state/inmemorystore.go deleted file mode 100644 index 3ba48592bc..0000000000 --- a/swarm/state/inmemorystore.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package state - -import ( - "encoding" - "encoding/json" - "sync" -) - -// InmemoryStore is the reference implementation of Store interface that is supposed -// to be used in tests. -type InmemoryStore struct { - db map[string][]byte - mu sync.RWMutex -} - -// NewInmemoryStore returns a new instance of InmemoryStore. -func NewInmemoryStore() *InmemoryStore { - return &InmemoryStore{ - db: make(map[string][]byte), - } -} - -// Get retrieves a value stored for a specific key. If there is no value found, -// ErrNotFound is returned. -func (s *InmemoryStore) Get(key string, i interface{}) (err error) { - s.mu.RLock() - defer s.mu.RUnlock() - - bytes, ok := s.db[key] - if !ok { - return ErrNotFound - } - - unmarshaler, ok := i.(encoding.BinaryUnmarshaler) - if !ok { - return json.Unmarshal(bytes, i) - } - - return unmarshaler.UnmarshalBinary(bytes) -} - -// Put stores a value for a specific key. -func (s *InmemoryStore) Put(key string, i interface{}) (err error) { - s.mu.Lock() - defer s.mu.Unlock() - var bytes []byte - - marshaler, ok := i.(encoding.BinaryMarshaler) - if !ok { - if bytes, err = json.Marshal(i); err != nil { - return err - } - } else { - if bytes, err = marshaler.MarshalBinary(); err != nil { - return err - } - } - - s.db[key] = bytes - return nil -} - -// Delete removes value stored under a specific key. -func (s *InmemoryStore) Delete(key string) (err error) { - s.mu.Lock() - defer s.mu.Unlock() - - if _, ok := s.db[key]; !ok { - return ErrNotFound - } - delete(s.db, key) - return nil -} - -// Close does not do anything. -func (s *InmemoryStore) Close() error { - return nil -} diff --git a/swarm/state/store.go b/swarm/state/store.go deleted file mode 100644 index fb7fe258f1..0000000000 --- a/swarm/state/store.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package state - -// Store defines methods required to get, set, delete values for different keys -// and close the underlying resources. -type Store interface { - Get(key string, i interface{}) (err error) - Put(key string, i interface{}) (err error) - Delete(key string) (err error) - Close() error -} From 070caec4bd72b1a958ece5f2b8fcc2bb92d44796 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Wed, 21 Nov 2018 20:49:13 +0100 Subject: [PATCH 071/159] swarm/network/stream: use swarm/mock/mem as mock global store (#18157) --- swarm/network/stream/common_test.go | 16 ------------- swarm/network/stream/snapshot_sync_test.go | 27 +++++----------------- swarm/network/stream/syncer_test.go | 21 ++++------------- 3 files changed, 11 insertions(+), 53 deletions(-) diff --git a/swarm/network/stream/common_test.go b/swarm/network/stream/common_test.go index c5f1fa176a..e0a7f7e120 100644 --- a/swarm/network/stream/common_test.go +++ b/swarm/network/stream/common_test.go @@ -38,7 +38,6 @@ import ( "github.com/ethereum/go-ethereum/swarm/pot" "github.com/ethereum/go-ethereum/swarm/state" "github.com/ethereum/go-ethereum/swarm/storage" - mockdb "github.com/ethereum/go-ethereum/swarm/storage/mock/db" "github.com/ethereum/go-ethereum/swarm/testutil" colorable "github.com/mattn/go-colorable" ) @@ -69,21 +68,6 @@ func init() { log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) } -func createGlobalStore() (string, *mockdb.GlobalStore, error) { - var globalStore *mockdb.GlobalStore - globalStoreDir, err := ioutil.TempDir("", "global.store") - if err != nil { - log.Error("Error initiating global store temp directory!", "err", err) - return "", nil, err - } - globalStore, err = mockdb.NewGlobalStore(globalStoreDir) - if err != nil { - log.Error("Error initiating global store!", "err", err) - return "", nil, err - } - return globalStoreDir, globalStore, nil -} - func newStreamerTester(t *testing.T, registryOptions *RegistryOptions) (*p2ptest.ProtocolTester, *Registry, *storage.LocalStore, func(), error) { // setup addr := network.RandomAddr() // tested peers peer address diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go index 6b92c32ae9..4e56f71b54 100644 --- a/swarm/network/stream/snapshot_sync_test.go +++ b/swarm/network/stream/snapshot_sync_test.go @@ -35,7 +35,8 @@ import ( "github.com/ethereum/go-ethereum/swarm/pot" "github.com/ethereum/go-ethereum/swarm/state" "github.com/ethereum/go-ethereum/swarm/storage" - mockdb "github.com/ethereum/go-ethereum/swarm/storage/mock/db" + "github.com/ethereum/go-ethereum/swarm/storage/mock" + mockmem "github.com/ethereum/go-ethereum/swarm/storage/mock/mem" "github.com/ethereum/go-ethereum/swarm/testutil" ) @@ -268,20 +269,9 @@ func runSim(conf *synctestConfig, ctx context.Context, sim *simulation.Simulatio // File retrieval check is repeated until all uploaded files are retrieved from all nodes // or until the timeout is reached. - var gDir string - var globalStore *mockdb.GlobalStore + var globalStore mock.GlobalStorer if *useMockStore { - gDir, globalStore, err = createGlobalStore() - if err != nil { - return fmt.Errorf("Something went wrong; using mockStore enabled but globalStore is nil") - } - defer func() { - os.RemoveAll(gDir) - err := globalStore.Close() - if err != nil { - log.Error("Error closing global store! %v", "err", err) - } - }() + globalStore = mockmem.NewGlobalStore() } REPEAT: for { @@ -476,14 +466,9 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int) return err } - var gDir string - var globalStore *mockdb.GlobalStore + var globalStore mock.GlobalStorer if *useMockStore { - gDir, globalStore, err = createGlobalStore() - if err != nil { - return fmt.Errorf("Something went wrong; using mockStore enabled but globalStore is nil") - } - defer os.RemoveAll(gDir) + globalStore = mockmem.NewGlobalStore() } // File retrieval check is repeated until all uploaded files are retrieved from all nodes // or until the timeout is reached. diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index fe20bab266..5764efc92d 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -35,7 +35,8 @@ import ( "github.com/ethereum/go-ethereum/swarm/network/simulation" "github.com/ethereum/go-ethereum/swarm/state" "github.com/ethereum/go-ethereum/swarm/storage" - mockdb "github.com/ethereum/go-ethereum/swarm/storage/mock/db" + "github.com/ethereum/go-ethereum/swarm/storage/mock" + mockmem "github.com/ethereum/go-ethereum/swarm/storage/mock/mem" "github.com/ethereum/go-ethereum/swarm/testutil" ) @@ -48,7 +49,7 @@ func TestSyncerSimulation(t *testing.T) { testSyncBetweenNodes(t, 16, 1, dataChunkCount, true, 1) } -func createMockStore(globalStore *mockdb.GlobalStore, id enode.ID, addr *network.BzzAddr) (lstore storage.ChunkStore, datadir string, err error) { +func createMockStore(globalStore mock.GlobalStorer, id enode.ID, addr *network.BzzAddr) (lstore storage.ChunkStore, datadir string, err error) { address := common.BytesToAddress(id.Bytes()) mockStore := globalStore.NewNodeStore(address) params := storage.NewDefaultLocalStoreParams() @@ -70,8 +71,7 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck sim := simulation.New(map[string]simulation.ServiceFunc{ "streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) { var store storage.ChunkStore - var globalStore *mockdb.GlobalStore - var gDir, datadir string + var datadir string node := ctx.Config.Node() addr := network.NewAddr(node) @@ -79,11 +79,7 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck addr.OAddr[0] = byte(0) if *useMockStore { - gDir, globalStore, err = createGlobalStore() - if err != nil { - return nil, nil, fmt.Errorf("Something went wrong; using mockStore enabled but globalStore is nil") - } - store, datadir, err = createMockStore(globalStore, node.ID(), addr) + store, datadir, err = createMockStore(mockmem.NewGlobalStore(), node.ID(), addr) } else { store, datadir, err = createTestLocalStorageForID(node.ID(), addr) } @@ -94,13 +90,6 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck cleanup = func() { store.Close() os.RemoveAll(datadir) - if *useMockStore { - err := globalStore.Close() - if err != nil { - log.Error("Error closing global store! %v", "err", err) - } - os.RemoveAll(gDir) - } } localStore := store.(*storage.LocalStore) netStore, err := storage.NewNetStore(localStore, nil) From e0d091e0909fc94958a33608ae4dad0825482813 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 22 Nov 2018 10:00:16 +0100 Subject: [PATCH 072/159] core: better printout of receipts in bad block reports (#18156) * core/blockchain: better printout of receipts in bad block reports * fix splleing --- core/blockchain.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index d173b2de29..4de7f007b5 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1469,8 +1469,10 @@ func (bc *BlockChain) reportBlock(block *types.Block, receipts types.Receipts, e bc.addBadBlock(block) var receiptString string - for _, receipt := range receipts { - receiptString += fmt.Sprintf("\t%v\n", receipt) + for i, receipt := range receipts { + receiptString += fmt.Sprintf("\t %d: cumulative: %v gas: %v contract: %v status: %v tx: %v logs: %v bloom: %x state: %x\n", + i, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.ContractAddress.Hex(), + receipt.Status, receipt.TxHash.Hex(), receipt.Logs, receipt.Bloom, receipt.PostState) } log.Error(fmt.Sprintf(` ########## BAD BLOCK ######### From 9d5e3e06370dace249305d91624cb83cabf3c25f Mon Sep 17 00:00:00 2001 From: Enrique Fynn Date: Thu, 22 Nov 2018 14:03:50 +0100 Subject: [PATCH 073/159] params: add Constantinople block to AllXYZProtocolChanges (#18162) * params: Add Constantinople block to AllCliqueProtocolChanges * params: Add Constantinople block to AllEthashProtocolChanges --- params/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/params/config.go b/params/config.go index 007e4a66d8..7f04e0e675 100644 --- a/params/config.go +++ b/params/config.go @@ -111,14 +111,14 @@ var ( // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil} + AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil} // AllCliqueProtocolChanges contains every protocol change (EIPs) introduced // and accepted by the Ethereum core developers into the Clique consensus. // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}} + AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}} TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil} TestRules = TestChainConfig.Rules(new(big.Int)) From 2843001ac218040f7e773159596625654f4e4148 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 22 Nov 2018 14:14:31 +0100 Subject: [PATCH 074/159] trie: fix overflow in write cache parent tracking (#18165) trie/database: fix overflow in parent tracking --- trie/database.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trie/database.go b/trie/database.go index 71190b3f3f..739a98addb 100644 --- a/trie/database.go +++ b/trie/database.go @@ -141,7 +141,7 @@ type cachedNode struct { node node // Cached collapsed trie node, or raw rlp data size uint16 // Byte size of the useful cached data - parents uint16 // Number of live nodes referencing this one + parents uint32 // Number of live nodes referencing this one children map[common.Hash]uint16 // External children referenced by this node flushPrev common.Hash // Previous node in the flush-list From 6b2cc8950e55b1d93afb3c82bb1081cf931d62cf Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 22 Nov 2018 15:32:50 +0100 Subject: [PATCH 075/159] travis: increase open file limits (#18155) --- .travis.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.travis.yml b/.travis.yml index c1cc7c4aa7..33a4f89497 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,14 @@ matrix: - os: osx go: 1.11.x script: + - echo "Increase the maximum number of open file descriptors on macOS" + - NOFILE=20480 + - sudo sysctl -w kern.maxfiles=$NOFILE + - sudo sysctl -w kern.maxfilesperproc=$NOFILE + - sudo launchctl limit maxfiles $NOFILE $NOFILE + - sudo launchctl limit maxfiles + - ulimit -S -n $NOFILE + - ulimit -n - unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703 - go run build/ci.go install - go run build/ci.go test -coverage $TEST_PACKAGES From 76f5f662ccaf5190eb283ab8b5d607587e1ab8f9 Mon Sep 17 00:00:00 2001 From: Ferenc Szabo Date: Fri, 23 Nov 2018 01:32:34 +0100 Subject: [PATCH 076/159] cmd/swarm: FUSE do not require --ipcpath (#18112) - Have `${DataDir}/bzzd.ipc` as IPC path default. - Respect the `--datadir` flag. - Keep only the global `--ipcpath` flag and drop the local `--ipcpath` flag as flags might overwrite each other. (Note: before global `--ipcpath` was ignored even if it was set) fixes ethersphere#795 --- cmd/swarm/config_test.go | 24 ++++++++++++------------ cmd/swarm/fs.go | 36 +++++++++++++++++------------------- cmd/swarm/fs_test.go | 30 ++++++++++++++++++++++++++---- cmd/utils/flags.go | 24 ++++++++++++++---------- 4 files changed, 69 insertions(+), 45 deletions(-) diff --git a/cmd/swarm/config_test.go b/cmd/swarm/config_test.go index 02198f878e..18be316e5f 100644 --- a/cmd/swarm/config_test.go +++ b/cmd/swarm/config_test.go @@ -26,14 +26,14 @@ import ( "testing" "time" + "github.com/docker/docker/pkg/reexec" + "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm" "github.com/ethereum/go-ethereum/swarm/api" - - "github.com/docker/docker/pkg/reexec" ) -func TestDumpConfig(t *testing.T) { +func TestConfigDump(t *testing.T) { swarm := runSwarm(t, "dumpconfig") defaultConf := api.NewConfig() out, err := tomlSettings.Marshal(&defaultConf) @@ -91,8 +91,8 @@ func TestConfigCmdLineOverrides(t *testing.T) { fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(), fmt.Sprintf("--%s", SwarmDeliverySkipCheckFlag.Name), fmt.Sprintf("--%s", EnsAPIFlag.Name), "", - "--datadir", dir, - "--ipcpath", conf.IPCPath, + fmt.Sprintf("--%s", utils.DataDirFlag.Name), dir, + fmt.Sprintf("--%s", utils.IPCPathFlag.Name), conf.IPCPath, } node.Cmd = runSwarm(t, flags...) node.Cmd.InputLine(testPassphrase) @@ -189,9 +189,9 @@ func TestConfigFileOverrides(t *testing.T) { flags := []string{ fmt.Sprintf("--%s", SwarmTomlConfigPathFlag.Name), f.Name(), fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(), - "--ens-api", "", - "--ipcpath", conf.IPCPath, - "--datadir", dir, + fmt.Sprintf("--%s", EnsAPIFlag.Name), "", + fmt.Sprintf("--%s", utils.DataDirFlag.Name), dir, + fmt.Sprintf("--%s", utils.IPCPathFlag.Name), conf.IPCPath, } node.Cmd = runSwarm(t, flags...) node.Cmd.InputLine(testPassphrase) @@ -407,9 +407,9 @@ func TestConfigCmdLineOverridesFile(t *testing.T) { fmt.Sprintf("--%s", SwarmSyncDisabledFlag.Name), fmt.Sprintf("--%s", SwarmTomlConfigPathFlag.Name), f.Name(), fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(), - "--ens-api", "", - "--datadir", dir, - "--ipcpath", conf.IPCPath, + fmt.Sprintf("--%s", EnsAPIFlag.Name), "", + fmt.Sprintf("--%s", utils.DataDirFlag.Name), dir, + fmt.Sprintf("--%s", utils.IPCPathFlag.Name), conf.IPCPath, } node.Cmd = runSwarm(t, flags...) node.Cmd.InputLine(testPassphrase) @@ -466,7 +466,7 @@ func TestConfigCmdLineOverridesFile(t *testing.T) { node.Shutdown() } -func TestValidateConfig(t *testing.T) { +func TestConfigValidate(t *testing.T) { for _, c := range []struct { cfg *api.Config err string diff --git a/cmd/swarm/fs.go b/cmd/swarm/fs.go index b970b2e8c2..edeeddff84 100644 --- a/cmd/swarm/fs.go +++ b/cmd/swarm/fs.go @@ -24,7 +24,7 @@ import ( "time" "github.com/ethereum/go-ethereum/cmd/utils" - "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm/fuse" "gopkg.in/urfave/cli.v1" @@ -41,27 +41,24 @@ var fsCommand = cli.Command{ Action: mount, CustomHelpTemplate: helpTemplate, Name: "mount", - Flags: []cli.Flag{utils.IPCPathFlag}, Usage: "mount a swarm hash to a mount point", - ArgsUsage: "swarm fs mount --ipcpath ", + ArgsUsage: "swarm fs mount ", Description: "Mounts a Swarm manifest hash to a given mount point. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file", }, { Action: unmount, CustomHelpTemplate: helpTemplate, Name: "unmount", - Flags: []cli.Flag{utils.IPCPathFlag}, Usage: "unmount a swarmfs mount", - ArgsUsage: "swarm fs unmount --ipcpath ", + ArgsUsage: "swarm fs unmount ", Description: "Unmounts a swarmfs mount residing at . This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file", }, { Action: listMounts, CustomHelpTemplate: helpTemplate, Name: "list", - Flags: []cli.Flag{utils.IPCPathFlag}, Usage: "list swarmfs mounts", - ArgsUsage: "swarm fs list --ipcpath ", + ArgsUsage: "swarm fs list", Description: "Lists all mounted swarmfs volumes. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file", }, }, @@ -70,7 +67,7 @@ var fsCommand = cli.Command{ func mount(cliContext *cli.Context) { args := cliContext.Args() if len(args) < 2 { - utils.Fatalf("Usage: swarm fs mount --ipcpath ") + utils.Fatalf("Usage: swarm fs mount ") } client, err := dialRPC(cliContext) @@ -97,7 +94,7 @@ func unmount(cliContext *cli.Context) { args := cliContext.Args() if len(args) < 1 { - utils.Fatalf("Usage: swarm fs unmount --ipcpath ") + utils.Fatalf("Usage: swarm fs unmount ") } client, err := dialRPC(cliContext) if err != nil { @@ -145,20 +142,21 @@ func listMounts(cliContext *cli.Context) { } func dialRPC(ctx *cli.Context) (*rpc.Client, error) { - var endpoint string + endpoint := getIPCEndpoint(ctx) + log.Info("IPC endpoint", "path", endpoint) + return rpc.Dial(endpoint) +} - if ctx.IsSet(utils.IPCPathFlag.Name) { - endpoint = ctx.String(utils.IPCPathFlag.Name) - } else { - utils.Fatalf("swarm ipc endpoint not specified") - } +func getIPCEndpoint(ctx *cli.Context) string { + cfg := defaultNodeConfig + utils.SetNodeConfig(ctx, &cfg) - if endpoint == "" { - endpoint = node.DefaultIPCEndpoint(clientIdentifier) - } else if strings.HasPrefix(endpoint, "rpc:") || strings.HasPrefix(endpoint, "ipc:") { + endpoint := cfg.IPCEndpoint() + + if strings.HasPrefix(endpoint, "rpc:") || strings.HasPrefix(endpoint, "ipc:") { // Backwards compatibility with geth < 1.5 which required // these prefixes. endpoint = endpoint[4:] } - return rpc.Dial(endpoint) + return endpoint } diff --git a/cmd/swarm/fs_test.go b/cmd/swarm/fs_test.go index ac4223b661..5f58d6c0d8 100644 --- a/cmd/swarm/fs_test.go +++ b/cmd/swarm/fs_test.go @@ -20,6 +20,7 @@ package main import ( "bytes" + "fmt" "io" "io/ioutil" "os" @@ -28,6 +29,7 @@ import ( "testing" "time" + "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/log" ) @@ -36,6 +38,26 @@ type testFile struct { content string } +// TestCLISwarmFsDefaultIPCPath tests if the most basic fs command, i.e., list +// can find and correctly connect to a running Swarm node on the default +// IPCPath. +func TestCLISwarmFsDefaultIPCPath(t *testing.T) { + cluster := newTestCluster(t, 1) + defer cluster.Shutdown() + + handlingNode := cluster.Nodes[0] + list := runSwarm(t, []string{ + "--datadir", handlingNode.Dir, + "fs", + "list", + }...) + + list.WaitExit() + if list.Err != nil { + t.Fatal(list.Err) + } +} + // TestCLISwarmFs is a high-level test of swarmfs // // This test fails on travis for macOS as this executable exits with code 1 @@ -59,9 +81,9 @@ func TestCLISwarmFs(t *testing.T) { log.Debug("swarmfs cli test: mounting first run", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath)) mount := runSwarm(t, []string{ + fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath), "fs", "mount", - "--ipcpath", filepath.Join(handlingNode.Dir, handlingNode.IpcPath), mhash, mountPoint, }...) @@ -101,9 +123,9 @@ func TestCLISwarmFs(t *testing.T) { log.Debug("swarmfs cli test: unmounting first run...", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath)) unmount := runSwarm(t, []string{ + fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath), "fs", "unmount", - "--ipcpath", filepath.Join(handlingNode.Dir, handlingNode.IpcPath), mountPoint, }...) _, matches := unmount.ExpectRegexp(hashRegexp) @@ -136,9 +158,9 @@ func TestCLISwarmFs(t *testing.T) { //remount, check files newMount := runSwarm(t, []string{ + fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath), "fs", "mount", - "--ipcpath", filepath.Join(handlingNode.Dir, handlingNode.IpcPath), hash, // the latest hash secondMountPoint, }...) @@ -172,9 +194,9 @@ func TestCLISwarmFs(t *testing.T) { log.Debug("swarmfs cli test: unmounting second run", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath)) unmountSec := runSwarm(t, []string{ + fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath), "fs", "unmount", - "--ipcpath", filepath.Join(handlingNode.Dir, handlingNode.IpcPath), secondMountPoint, }...) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index d7b698c7e2..d0597c2f18 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -978,16 +978,7 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) { setWS(ctx, cfg) setNodeUserIdent(ctx, cfg) - switch { - case ctx.GlobalIsSet(DataDirFlag.Name): - cfg.DataDir = ctx.GlobalString(DataDirFlag.Name) - case ctx.GlobalBool(DeveloperFlag.Name): - cfg.DataDir = "" // unless explicitly requested, use memory databases - case ctx.GlobalBool(TestnetFlag.Name): - cfg.DataDir = filepath.Join(node.DefaultDataDir(), "testnet") - case ctx.GlobalBool(RinkebyFlag.Name): - cfg.DataDir = filepath.Join(node.DefaultDataDir(), "rinkeby") - } + setDataDir(ctx, cfg) if ctx.GlobalIsSet(KeyStoreDirFlag.Name) { cfg.KeyStoreDir = ctx.GlobalString(KeyStoreDirFlag.Name) @@ -1000,6 +991,19 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) { } } +func setDataDir(ctx *cli.Context, cfg *node.Config) { + switch { + case ctx.GlobalIsSet(DataDirFlag.Name): + cfg.DataDir = ctx.GlobalString(DataDirFlag.Name) + case ctx.GlobalBool(DeveloperFlag.Name): + cfg.DataDir = "" // unless explicitly requested, use memory databases + case ctx.GlobalBool(TestnetFlag.Name): + cfg.DataDir = filepath.Join(node.DefaultDataDir(), "testnet") + case ctx.GlobalBool(RinkebyFlag.Name): + cfg.DataDir = filepath.Join(node.DefaultDataDir(), "rinkeby") + } +} + func setGPO(ctx *cli.Context, cfg *gasprice.Config) { if ctx.GlobalIsSet(GpoBlocksFlag.Name) { cfg.Blocks = ctx.GlobalInt(GpoBlocksFlag.Name) From b24ef5e05d158e1533cf339aea866e0584447cef Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 23 Nov 2018 10:14:09 +0100 Subject: [PATCH 077/159] eth: increase timeout in TestBroadcastBlock (#18064) --- eth/handler_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eth/handler_test.go b/eth/handler_test.go index 7811cd4804..44824fd0b7 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -585,7 +585,7 @@ func testBroadcastBlock(t *testing.T, totalPeers, broadcastExpected int) { } }(peer) } - timeoutCh := time.NewTimer(time.Millisecond * 100).C + timeout := time.After(300 * time.Millisecond) var receivedCount int outer: for { @@ -597,7 +597,7 @@ outer: if receivedCount == totalPeers { break outer } - case <-timeoutCh: + case <-timeout: break outer } } From 2a113f6d72d3b612c6686fa2ab8c30f9e7e6e972 Mon Sep 17 00:00:00 2001 From: mr_franklin Date: Fri, 23 Nov 2018 17:16:14 +0800 Subject: [PATCH 078/159] core: return error if repair block failed (#18126) * core: return error if repair block failed * make error a bit shorter --- core/blockchain.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/blockchain.go b/core/blockchain.go index 3a9caf01e0..bcd53d4987 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -445,7 +445,11 @@ func (bc *BlockChain) repair(head **types.Block) error { return nil } // Otherwise rewind one block and recheck state availability there - (*head) = bc.GetBlock((*head).ParentHash(), (*head).NumberU64()-1) + block := bc.GetBlock((*head).ParentHash(), (*head).NumberU64()-1) + if block == nil { + return fmt.Errorf("missing block %d [%x]", (*head).NumberU64()-1, (*head).ParentHash()) + } + (*head) = block } } From a5898ba621c51e420097a48b4a3f4fa75165a8b0 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 26 Nov 2018 09:55:45 +0100 Subject: [PATCH 079/159] config: add constantinople block to testchainconfig --- params/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/params/config.go b/params/config.go index 7f04e0e675..3570cb204f 100644 --- a/params/config.go +++ b/params/config.go @@ -120,7 +120,7 @@ var ( // adding flags to the config to also have to set these fields. AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}} - TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil} + TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil} TestRules = TestChainConfig.Rules(new(big.Int)) ) From 38592a13a3c41f7b6cf29ace473852b68cdb6690 Mon Sep 17 00:00:00 2001 From: Jaynti Kanani Date: Mon, 26 Nov 2018 14:29:04 +0530 Subject: [PATCH 080/159] fix mixHash/nonce for parity compatible network (#18166) --- core/types/block.go | 4 ++-- core/types/gen_header_json.go | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/core/types/block.go b/core/types/block.go index 8a21bba1e7..9d11f60d83 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -81,8 +81,8 @@ type Header struct { GasUsed uint64 `json:"gasUsed" gencodec:"required"` Time *big.Int `json:"timestamp" gencodec:"required"` Extra []byte `json:"extraData" gencodec:"required"` - MixDigest common.Hash `json:"mixHash" gencodec:"required"` - Nonce BlockNonce `json:"nonce" gencodec:"required"` + MixDigest common.Hash `json:"mixHash"` + Nonce BlockNonce `json:"nonce"` } // field type overrides for gencodec diff --git a/core/types/gen_header_json.go b/core/types/gen_header_json.go index 1b92cd9cf4..59a1c9c439 100644 --- a/core/types/gen_header_json.go +++ b/core/types/gen_header_json.go @@ -13,6 +13,7 @@ import ( var _ = (*headerMarshaling)(nil) +// MarshalJSON marshals as JSON. func (h Header) MarshalJSON() ([]byte, error) { type Header struct { ParentHash common.Hash `json:"parentHash" gencodec:"required"` @@ -28,8 +29,8 @@ func (h Header) MarshalJSON() ([]byte, error) { GasUsed hexutil.Uint64 `json:"gasUsed" gencodec:"required"` Time *hexutil.Big `json:"timestamp" gencodec:"required"` Extra hexutil.Bytes `json:"extraData" gencodec:"required"` - MixDigest common.Hash `json:"mixHash" gencodec:"required"` - Nonce BlockNonce `json:"nonce" gencodec:"required"` + MixDigest common.Hash `json:"mixHash"` + Nonce BlockNonce `json:"nonce"` Hash common.Hash `json:"hash"` } var enc Header @@ -52,6 +53,7 @@ func (h Header) MarshalJSON() ([]byte, error) { return json.Marshal(&enc) } +// UnmarshalJSON unmarshals from JSON. func (h *Header) UnmarshalJSON(input []byte) error { type Header struct { ParentHash *common.Hash `json:"parentHash" gencodec:"required"` @@ -67,8 +69,8 @@ func (h *Header) UnmarshalJSON(input []byte) error { GasUsed *hexutil.Uint64 `json:"gasUsed" gencodec:"required"` Time *hexutil.Big `json:"timestamp" gencodec:"required"` Extra *hexutil.Bytes `json:"extraData" gencodec:"required"` - MixDigest *common.Hash `json:"mixHash" gencodec:"required"` - Nonce *BlockNonce `json:"nonce" gencodec:"required"` + MixDigest *common.Hash `json:"mixHash"` + Nonce *BlockNonce `json:"nonce"` } var dec Header if err := json.Unmarshal(input, &dec); err != nil { @@ -126,13 +128,11 @@ func (h *Header) UnmarshalJSON(input []byte) error { return errors.New("missing required field 'extraData' for Header") } h.Extra = *dec.Extra - if dec.MixDigest == nil { - return errors.New("missing required field 'mixHash' for Header") + if dec.MixDigest != nil { + h.MixDigest = *dec.MixDigest } - h.MixDigest = *dec.MixDigest - if dec.Nonce == nil { - return errors.New("missing required field 'nonce' for Header") + if dec.Nonce != nil { + h.Nonce = *dec.Nonce } - h.Nonce = *dec.Nonce return nil } From f0515800e6e92044466b095bbb2374465c53b6f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Mon, 26 Nov 2018 12:34:33 +0100 Subject: [PATCH 081/159] les: fix fetcher syncing logic (#18072) --- les/fetcher.go | 56 ++++++++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/les/fetcher.go b/les/fetcher.go index f0d3b188db..2615f69df9 100644 --- a/les/fetcher.go +++ b/les/fetcher.go @@ -141,36 +141,39 @@ func (f *lightFetcher) syncLoop() { s := requesting requesting = false var ( - rq *distReq - reqID uint64 + rq *distReq + reqID uint64 + syncing bool ) if !f.syncing && !(newAnnounce && s) { - rq, reqID = f.nextRequest() + rq, reqID, syncing = f.nextRequest() } - syncing := f.syncing f.lock.Unlock() if rq != nil { requesting = true - _, ok := <-f.pm.reqDist.queue(rq) - if !ok { + if _, ok := <-f.pm.reqDist.queue(rq); ok { + if syncing { + f.lock.Lock() + f.syncing = true + f.lock.Unlock() + } else { + go func() { + time.Sleep(softRequestTimeout) + f.reqMu.Lock() + req, ok := f.requested[reqID] + if ok { + req.timeout = true + f.requested[reqID] = req + } + f.reqMu.Unlock() + // keep starting new requests while possible + f.requestChn <- false + }() + } + } else { f.requestChn <- false } - - if !syncing { - go func() { - time.Sleep(softRequestTimeout) - f.reqMu.Lock() - req, ok := f.requested[reqID] - if ok { - req.timeout = true - f.requested[reqID] = req - } - f.reqMu.Unlock() - // keep starting new requests while possible - f.requestChn <- false - }() - } } case reqID := <-f.timeoutChn: f.reqMu.Lock() @@ -209,6 +212,7 @@ func (f *lightFetcher) syncLoop() { f.checkSyncedHeaders(p) f.syncing = false f.lock.Unlock() + f.requestChn <- false } } } @@ -405,7 +409,7 @@ func (f *lightFetcher) requestedID(reqID uint64) bool { // nextRequest selects the peer and announced head to be requested next, amount // to be downloaded starting from the head backwards is also returned -func (f *lightFetcher) nextRequest() (*distReq, uint64) { +func (f *lightFetcher) nextRequest() (*distReq, uint64, bool) { var ( bestHash common.Hash bestAmount uint64 @@ -427,14 +431,12 @@ func (f *lightFetcher) nextRequest() (*distReq, uint64) { } } if bestTd == f.maxConfirmedTd { - return nil, 0 + return nil, 0, false } - f.syncing = bestSyncing - var rq *distReq reqID := genReqID() - if f.syncing { + if bestSyncing { rq = &distReq{ getCost: func(dp distPeer) uint64 { return 0 @@ -500,7 +502,7 @@ func (f *lightFetcher) nextRequest() (*distReq, uint64) { }, } } - return rq, reqID + return rq, reqID, bestSyncing } // deliverHeaders delivers header download request responses for processing From 93854bbad4fb819375463d04cf6b7e66b7f272f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Mon, 26 Nov 2018 12:39:38 +0100 Subject: [PATCH 082/159] swarm/network/simulation: fix New function for-loop scope (#18161) --- swarm/network/simulation/node_test.go | 35 +++++++++++++++++++++ swarm/network/simulation/simulation.go | 7 +++++ swarm/network/simulation/simulation_test.go | 13 ++++++++ swarm/network/stream/delivery.go | 2 +- 4 files changed, 56 insertions(+), 1 deletion(-) diff --git a/swarm/network/simulation/node_test.go b/swarm/network/simulation/node_test.go index 086ab606f6..01346ef14d 100644 --- a/swarm/network/simulation/node_test.go +++ b/swarm/network/simulation/node_test.go @@ -160,6 +160,41 @@ func TestAddNodeWithService(t *testing.T) { } } +func TestAddNodeMultipleServices(t *testing.T) { + sim := New(map[string]ServiceFunc{ + "noop1": noopServiceFunc, + "noop2": noopService2Func, + }) + defer sim.Close() + + id, err := sim.AddNode() + if err != nil { + t.Fatal(err) + } + + n := sim.Net.GetNode(id).Node.(*adapters.SimNode) + if n.Service("noop1") == nil { + t.Error("service noop1 not found on node") + } + if n.Service("noop2") == nil { + t.Error("service noop2 not found on node") + } +} + +func TestAddNodeDuplicateServiceError(t *testing.T) { + sim := New(map[string]ServiceFunc{ + "noop1": noopServiceFunc, + "noop2": noopServiceFunc, + }) + defer sim.Close() + + wantErr := "duplicate service: *simulation.noopService" + _, err := sim.AddNode() + if err.Error() != wantErr { + t.Errorf("got error %q, want %q", err, wantErr) + } +} + func TestAddNodes(t *testing.T) { sim := New(noopServiceFuncMap) defer sim.Close() diff --git a/swarm/network/simulation/simulation.go b/swarm/network/simulation/simulation.go index f6d3ce2290..e5435b9f0e 100644 --- a/swarm/network/simulation/simulation.go +++ b/swarm/network/simulation/simulation.go @@ -68,6 +68,10 @@ type ServiceFunc func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Se // New creates a new Simulation instance with new // simulations.Network initialized with provided services. +// Services map must have unique keys as service names and +// every ServiceFunc must return a node.Service of the unique type. +// This restriction is required by node.Node.Start() function +// which is used to start node.Service returned by ServiceFunc. func New(services map[string]ServiceFunc) (s *Simulation) { s = &Simulation{ buckets: make(map[enode.ID]*sync.Map), @@ -76,6 +80,9 @@ func New(services map[string]ServiceFunc) (s *Simulation) { adapterServices := make(map[string]adapters.ServiceFunc, len(services)) for name, serviceFunc := range services { + // Scope this variables correctly + // as they will be in the adapterServices[name] function accessed later. + name, serviceFunc := name, serviceFunc s.serviceNames = append(s.serviceNames, name) adapterServices[name] = func(ctx *adapters.ServiceContext) (node.Service, error) { b := new(sync.Map) diff --git a/swarm/network/simulation/simulation_test.go b/swarm/network/simulation/simulation_test.go index eed09bf508..ca8599d7c0 100644 --- a/swarm/network/simulation/simulation_test.go +++ b/swarm/network/simulation/simulation_test.go @@ -205,3 +205,16 @@ func (t *noopService) Start(server *p2p.Server) error { func (t *noopService) Stop() error { return nil } + +// a helper function for most basic noop service +// of a different type then noopService to test +// multiple services on one node. +func noopService2Func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) { + return new(noopService2), nil, nil +} + +// noopService2 is the service that does not do anything +// but implements node.Service interface. +type noopService2 struct { + noopService +} diff --git a/swarm/network/stream/delivery.go b/swarm/network/stream/delivery.go index 64d754336f..a5c9fa7e86 100644 --- a/swarm/network/stream/delivery.go +++ b/swarm/network/stream/delivery.go @@ -255,7 +255,7 @@ func (d *Delivery) RequestFromPeers(ctx context.Context, req *network.Request) ( } sp = d.getPeer(id) if sp == nil { - log.Warn("Delivery.RequestFromPeers: peer not found", "id", id) + //log.Warn("Delivery.RequestFromPeers: peer not found", "id", id) return true } spID = &id From f5e6634fd2fd2be5520c77edb7a5cd375eb28fd4 Mon Sep 17 00:00:00 2001 From: Elad Date: Mon, 26 Nov 2018 17:23:15 +0530 Subject: [PATCH 083/159] swarm/api: improve not found error msg (#18171) --- swarm/api/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 7bb631967c..099a469396 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -472,7 +472,7 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage // no entry found status = http.StatusNotFound apiGetNotFound.Inc(1) - err = fmt.Errorf("manifest entry for '%s' not found", path) + err = fmt.Errorf("Not found: could not find resource '%s'", path) log.Trace("manifest entry not found", "key", contentAddr, "path", path) } return From ca228569e4aedd5ebbe853ec61761561d86579a6 Mon Sep 17 00:00:00 2001 From: Sheldon <11510383@mail.sustc.edu.cn> Date: Mon, 26 Nov 2018 20:27:49 +0800 Subject: [PATCH 084/159] light: odrTrie tryUpdate should use update (#18107) TryUpdate does not call t.trie.TryUpdate(key, value) and calls t.trie.TryDelete instead. The update operation simply deletes the corresponding entry, though it could retrieve later by odr. However, it adds further network overhead. --- light/trie.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/light/trie.go b/light/trie.go index c07e99461c..ab4e18b43e 100644 --- a/light/trie.go +++ b/light/trie.go @@ -108,7 +108,7 @@ func (t *odrTrie) TryGet(key []byte) ([]byte, error) { func (t *odrTrie) TryUpdate(key, value []byte) error { key = crypto.Keccak256(key) return t.do(key, func() error { - return t.trie.TryDelete(key) + return t.trie.TryUpdate(key, value) }) } From 197d609b9a3a9b5436a9902dcc8db38bb3e7543b Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 26 Nov 2018 13:52:04 +0100 Subject: [PATCH 085/159] swarm/pss: Message handler refactor (#18169) --- swarm/network/kademlia.go | 26 +- swarm/pss/api.go | 12 +- swarm/pss/client/client.go | 2 +- swarm/pss/handshake.go | 2 +- swarm/pss/notify/notify.go | 4 +- swarm/pss/notify/notify_test.go | 4 +- swarm/pss/protocol_test.go | 5 +- swarm/pss/pss.go | 171 ++++++++--- swarm/pss/pss_test.go | 493 +++++++++++++++++++++++++++++--- swarm/pss/types.go | 34 ++- 10 files changed, 644 insertions(+), 109 deletions(-) diff --git a/swarm/network/kademlia.go b/swarm/network/kademlia.go index cd94741be3..5fda51e3eb 100644 --- a/swarm/network/kademlia.go +++ b/swarm/network/kademlia.go @@ -81,14 +81,15 @@ func NewKadParams() *KadParams { // Kademlia is a table of live peers and a db of known peers (node records) type Kademlia struct { lock sync.RWMutex - *KadParams // Kademlia configuration parameters - base []byte // immutable baseaddress of the table - addrs *pot.Pot // pots container for known peer addresses - conns *pot.Pot // pots container for live peer connections - depth uint8 // stores the last current depth of saturation - nDepth int // stores the last neighbourhood depth - nDepthC chan int // returned by DepthC function to signal neighbourhood depth change - addrCountC chan int // returned by AddrCountC function to signal peer count change + *KadParams // Kademlia configuration parameters + base []byte // immutable baseaddress of the table + addrs *pot.Pot // pots container for known peer addresses + conns *pot.Pot // pots container for live peer connections + depth uint8 // stores the last current depth of saturation + nDepth int // stores the last neighbourhood depth + nDepthC chan int // returned by DepthC function to signal neighbourhood depth change + addrCountC chan int // returned by AddrCountC function to signal peer count change + Pof func(pot.Val, pot.Val, int) (int, bool) // function for calculating kademlia routing distance between two addresses } // NewKademlia creates a Kademlia table for base address addr @@ -103,6 +104,7 @@ func NewKademlia(addr []byte, params *KadParams) *Kademlia { KadParams: params, addrs: pot.NewPot(nil, 0), conns: pot.NewPot(nil, 0), + Pof: pof, } } @@ -289,6 +291,7 @@ func (k *Kademlia) On(p *Peer) (uint8, bool) { // neighbourhood depth on each change. // Not receiving from the returned channel will block On function // when the neighbourhood depth is changed. +// TODO: Why is this exported, and if it should be; why can't we have more subscribers than one? func (k *Kademlia) NeighbourhoodDepthC() <-chan int { k.lock.Lock() defer k.lock.Unlock() @@ -429,7 +432,12 @@ func (k *Kademlia) eachAddr(base []byte, o int, f func(*BzzAddr, int, bool) bool // neighbourhoodDepth returns the proximity order that defines the distance of // the nearest neighbour set with cardinality >= MinProxBinSize // if there is altogether less than MinProxBinSize peers it returns 0 -// caller must hold the lock +func (k *Kademlia) NeighbourhoodDepth() (depth int) { + k.lock.RLock() + defer k.lock.RUnlock() + return k.neighbourhoodDepth() +} + func (k *Kademlia) neighbourhoodDepth() (depth int) { if k.conns.Size() < k.MinProxBinSize { return 0 diff --git a/swarm/pss/api.go b/swarm/pss/api.go index eba7bb722c..dd55b2a702 100644 --- a/swarm/pss/api.go +++ b/swarm/pss/api.go @@ -51,7 +51,7 @@ func NewAPI(ps *Pss) *API { // // All incoming messages to the node matching this topic will be encapsulated in the APIMsg // struct and sent to the subscriber -func (pssapi *API) Receive(ctx context.Context, topic Topic) (*rpc.Subscription, error) { +func (pssapi *API) Receive(ctx context.Context, topic Topic, raw bool, prox bool) (*rpc.Subscription, error) { notifier, supported := rpc.NotifierFromContext(ctx) if !supported { return nil, fmt.Errorf("Subscribe not supported") @@ -59,7 +59,7 @@ func (pssapi *API) Receive(ctx context.Context, topic Topic) (*rpc.Subscription, psssub := notifier.CreateSubscription() - handler := func(msg []byte, p *p2p.Peer, asymmetric bool, keyid string) error { + hndlr := NewHandler(func(msg []byte, p *p2p.Peer, asymmetric bool, keyid string) error { apimsg := &APIMsg{ Msg: hexutil.Bytes(msg), Asymmetric: asymmetric, @@ -69,9 +69,15 @@ func (pssapi *API) Receive(ctx context.Context, topic Topic) (*rpc.Subscription, log.Warn(fmt.Sprintf("notification on pss sub topic rpc (sub %v) msg %v failed!", psssub.ID, msg)) } return nil + }) + if raw { + hndlr.caps.raw = true + } + if prox { + hndlr.caps.prox = true } - deregf := pssapi.Register(&topic, handler) + deregf := pssapi.Register(&topic, hndlr) go func() { defer deregf() select { diff --git a/swarm/pss/client/client.go b/swarm/pss/client/client.go index d541081d3e..5ee387aa79 100644 --- a/swarm/pss/client/client.go +++ b/swarm/pss/client/client.go @@ -236,7 +236,7 @@ func (c *Client) RunProtocol(ctx context.Context, proto *p2p.Protocol) error { topichex := topicobj.String() msgC := make(chan pss.APIMsg) c.peerPool[topicobj] = make(map[string]*pssRPCRW) - sub, err := c.rpc.Subscribe(ctx, "pss", msgC, "receive", topichex) + sub, err := c.rpc.Subscribe(ctx, "pss", msgC, "receive", topichex, false, false) if err != nil { return fmt.Errorf("pss event subscription failed: %v", err) } diff --git a/swarm/pss/handshake.go b/swarm/pss/handshake.go index e3ead77d04..5486abafa9 100644 --- a/swarm/pss/handshake.go +++ b/swarm/pss/handshake.go @@ -486,7 +486,7 @@ func (api *HandshakeAPI) Handshake(pubkeyid string, topic Topic, sync bool, flus // Activate handshake functionality on a topic func (api *HandshakeAPI) AddHandshake(topic Topic) error { - api.ctrl.deregisterFuncs[topic] = api.ctrl.pss.Register(&topic, api.ctrl.handler) + api.ctrl.deregisterFuncs[topic] = api.ctrl.pss.Register(&topic, NewHandler(api.ctrl.handler)) return nil } diff --git a/swarm/pss/notify/notify.go b/swarm/pss/notify/notify.go index 3731fb9dbb..d3c89058b2 100644 --- a/swarm/pss/notify/notify.go +++ b/swarm/pss/notify/notify.go @@ -113,7 +113,7 @@ func NewController(ps *pss.Pss) *Controller { notifiers: make(map[string]*notifier), subscriptions: make(map[string]*subscription), } - ctrl.pss.Register(&controlTopic, ctrl.Handler) + ctrl.pss.Register(&controlTopic, pss.NewHandler(ctrl.Handler)) return ctrl } @@ -336,7 +336,7 @@ func (c *Controller) handleNotifyWithKeyMsg(msg *Msg) error { // \TODO keep track of and add actual address updaterAddr := pss.PssAddress([]byte{}) c.pss.SetSymmetricKey(symkey, topic, &updaterAddr, true) - c.pss.Register(&topic, c.Handler) + c.pss.Register(&topic, pss.NewHandler(c.Handler)) return c.subscriptions[msg.namestring].handler(msg.namestring, msg.Payload[:len(msg.Payload)-symKeyLength]) } diff --git a/swarm/pss/notify/notify_test.go b/swarm/pss/notify/notify_test.go index d4d383a6ba..6100195b09 100644 --- a/swarm/pss/notify/notify_test.go +++ b/swarm/pss/notify/notify_test.go @@ -121,7 +121,7 @@ func TestStart(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*2) defer cancel() rmsgC := make(chan *pss.APIMsg) - rightSub, err := rightRpc.Subscribe(ctx, "pss", rmsgC, "receive", controlTopic) + rightSub, err := rightRpc.Subscribe(ctx, "pss", rmsgC, "receive", controlTopic, false, false) if err != nil { t.Fatal(err) } @@ -174,7 +174,7 @@ func TestStart(t *testing.T) { t.Fatalf("expected payload length %d, have %d", len(updateMsg)+symKeyLength, len(dMsg.Payload)) } - rightSubUpdate, err := rightRpc.Subscribe(ctx, "pss", rmsgC, "receive", rsrcTopic) + rightSubUpdate, err := rightRpc.Subscribe(ctx, "pss", rmsgC, "receive", rsrcTopic, false, false) if err != nil { t.Fatal(err) } diff --git a/swarm/pss/protocol_test.go b/swarm/pss/protocol_test.go index 4ef3e90a04..520c48a202 100644 --- a/swarm/pss/protocol_test.go +++ b/swarm/pss/protocol_test.go @@ -92,7 +92,7 @@ func testProtocol(t *testing.T) { lmsgC := make(chan APIMsg) lctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() - lsub, err := clients[0].Subscribe(lctx, "pss", lmsgC, "receive", topic) + lsub, err := clients[0].Subscribe(lctx, "pss", lmsgC, "receive", topic, false, false) if err != nil { t.Fatal(err) } @@ -100,7 +100,7 @@ func testProtocol(t *testing.T) { rmsgC := make(chan APIMsg) rctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() - rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic) + rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic, false, false) if err != nil { t.Fatal(err) } @@ -130,6 +130,7 @@ func testProtocol(t *testing.T) { log.Debug("lnode ok") case cerr := <-lctx.Done(): t.Fatalf("test message timed out: %v", cerr) + return } select { case <-rmsgC: diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index e1e24e1f54..d0986d280b 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -23,11 +23,13 @@ import ( "crypto/rand" "errors" "fmt" + "hash" "sync" "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" @@ -136,10 +138,10 @@ type Pss struct { symKeyDecryptCacheCapacity int // max amount of symkeys to keep. // message handling - handlers map[Topic]map[*Handler]bool // topic and version based pss payload handlers. See pss.Handle() - handlersMu sync.RWMutex - allowRaw bool - hashPool sync.Pool + handlers map[Topic]map[*handler]bool // topic and version based pss payload handlers. See pss.Handle() + handlersMu sync.RWMutex + hashPool sync.Pool + topicHandlerCaps map[Topic]*handlerCaps // caches capabilities of each topic's handlers (see handlerCap* consts in types.go) // process quitC chan struct{} @@ -180,11 +182,12 @@ func NewPss(k *network.Kademlia, params *PssParams) (*Pss, error) { symKeyDecryptCache: make([]*string, params.SymKeyCacheCapacity), symKeyDecryptCacheCapacity: params.SymKeyCacheCapacity, - handlers: make(map[Topic]map[*Handler]bool), - allowRaw: params.AllowRaw, + handlers: make(map[Topic]map[*handler]bool), + topicHandlerCaps: make(map[Topic]*handlerCaps), + hashPool: sync.Pool{ New: func() interface{} { - return storage.MakeHashFunc(storage.DefaultHash)() + return sha3.NewKeccak256() }, }, } @@ -313,30 +316,54 @@ func (p *Pss) PublicKey() *ecdsa.PublicKey { // // Returns a deregister function which needs to be called to // deregister the handler, -func (p *Pss) Register(topic *Topic, handler Handler) func() { +func (p *Pss) Register(topic *Topic, hndlr *handler) func() { p.handlersMu.Lock() defer p.handlersMu.Unlock() handlers := p.handlers[*topic] if handlers == nil { - handlers = make(map[*Handler]bool) + handlers = make(map[*handler]bool) p.handlers[*topic] = handlers + log.Debug("registered handler", "caps", hndlr.caps) } - handlers[&handler] = true - return func() { p.deregister(topic, &handler) } + if hndlr.caps == nil { + hndlr.caps = &handlerCaps{} + } + handlers[hndlr] = true + if _, ok := p.topicHandlerCaps[*topic]; !ok { + p.topicHandlerCaps[*topic] = &handlerCaps{} + } + if hndlr.caps.raw { + p.topicHandlerCaps[*topic].raw = true + } + if hndlr.caps.prox { + p.topicHandlerCaps[*topic].prox = true + } + return func() { p.deregister(topic, hndlr) } } -func (p *Pss) deregister(topic *Topic, h *Handler) { +func (p *Pss) deregister(topic *Topic, hndlr *handler) { p.handlersMu.Lock() defer p.handlersMu.Unlock() handlers := p.handlers[*topic] - if len(handlers) == 1 { + if len(handlers) > 1 { delete(p.handlers, *topic) + // topic caps might have changed now that a handler is gone + caps := &handlerCaps{} + for h := range handlers { + if h.caps.raw { + caps.raw = true + } + if h.caps.prox { + caps.prox = true + } + } + p.topicHandlerCaps[*topic] = caps return } - delete(handlers, h) + delete(handlers, hndlr) } // get all registered handlers for respective topics -func (p *Pss) getHandlers(topic Topic) map[*Handler]bool { +func (p *Pss) getHandlers(topic Topic) map[*handler]bool { p.handlersMu.RLock() defer p.handlersMu.RUnlock() return p.handlers[topic] @@ -348,12 +375,11 @@ func (p *Pss) getHandlers(topic Topic) map[*Handler]bool { // Only passes error to pss protocol handler if payload is not valid pssmsg func (p *Pss) handlePssMsg(ctx context.Context, msg interface{}) error { metrics.GetOrRegisterCounter("pss.handlepssmsg", nil).Inc(1) - pssmsg, ok := msg.(*PssMsg) - if !ok { return fmt.Errorf("invalid message type. Expected *PssMsg, got %T ", msg) } + log.Trace("handler", "self", label(p.Kademlia.BaseAddr()), "topic", label(pssmsg.Payload.Topic[:])) if int64(pssmsg.Expire) < time.Now().Unix() { metrics.GetOrRegisterCounter("pss.expire", nil).Inc(1) log.Warn("pss filtered expired message", "from", common.ToHex(p.Kademlia.BaseAddr()), "to", common.ToHex(pssmsg.To)) @@ -365,13 +391,34 @@ func (p *Pss) handlePssMsg(ctx context.Context, msg interface{}) error { } p.addFwdCache(pssmsg) - if !p.isSelfPossibleRecipient(pssmsg) { - log.Trace("pss was for someone else :'( ... forwarding", "pss", common.ToHex(p.BaseAddr())) + psstopic := Topic(pssmsg.Payload.Topic) + + // raw is simplest handler contingency to check, so check that first + var isRaw bool + if pssmsg.isRaw() { + if !p.topicHandlerCaps[psstopic].raw { + log.Debug("No handler for raw message", "topic", psstopic) + return nil + } + isRaw = true + } + + // check if we can be recipient: + // - no prox handler on message and partial address matches + // - prox handler on message and we are in prox regardless of partial address match + // store this result so we don't calculate again on every handler + var isProx bool + if _, ok := p.topicHandlerCaps[psstopic]; ok { + isProx = p.topicHandlerCaps[psstopic].prox + } + isRecipient := p.isSelfPossibleRecipient(pssmsg, isProx) + if !isRecipient { + log.Trace("pss was for someone else :'( ... forwarding", "pss", common.ToHex(p.BaseAddr()), "prox", isProx) return p.enqueue(pssmsg) } - log.Trace("pss for us, yay! ... let's process!", "pss", common.ToHex(p.BaseAddr())) - if err := p.process(pssmsg); err != nil { + log.Trace("pss for us, yay! ... let's process!", "pss", common.ToHex(p.BaseAddr()), "prox", isProx, "raw", isRaw, "topic", label(pssmsg.Payload.Topic[:])) + if err := p.process(pssmsg, isRaw, isProx); err != nil { qerr := p.enqueue(pssmsg) if qerr != nil { return fmt.Errorf("process fail: processerr %v, queueerr: %v", err, qerr) @@ -384,7 +431,7 @@ func (p *Pss) handlePssMsg(ctx context.Context, msg interface{}) error { // Entry point to processing a message for which the current node can be the intended recipient. // Attempts symmetric and asymmetric decryption with stored keys. // Dispatches message to all handlers matching the message topic -func (p *Pss) process(pssmsg *PssMsg) error { +func (p *Pss) process(pssmsg *PssMsg, raw bool, prox bool) error { metrics.GetOrRegisterCounter("pss.process", nil).Inc(1) var err error @@ -397,10 +444,8 @@ func (p *Pss) process(pssmsg *PssMsg) error { envelope := pssmsg.Payload psstopic := Topic(envelope.Topic) - if pssmsg.isRaw() { - if !p.allowRaw { - return errors.New("raw message support disabled") - } + + if raw { payload = pssmsg.Payload.Data } else { if pssmsg.isSym() { @@ -422,19 +467,27 @@ func (p *Pss) process(pssmsg *PssMsg) error { return err } } - p.executeHandlers(psstopic, payload, from, asymmetric, keyid) + p.executeHandlers(psstopic, payload, from, raw, prox, asymmetric, keyid) return nil } -func (p *Pss) executeHandlers(topic Topic, payload []byte, from *PssAddress, asymmetric bool, keyid string) { +func (p *Pss) executeHandlers(topic Topic, payload []byte, from *PssAddress, raw bool, prox bool, asymmetric bool, keyid string) { handlers := p.getHandlers(topic) peer := p2p.NewPeer(enode.ID{}, fmt.Sprintf("%x", from), []p2p.Cap{}) - for f := range handlers { - err := (*f)(payload, peer, asymmetric, keyid) + for h := range handlers { + if !h.caps.raw && raw { + log.Warn("norawhandler") + continue + } + if !h.caps.prox && prox { + log.Warn("noproxhandler") + continue + } + err := (h.f)(payload, peer, asymmetric, keyid) if err != nil { - log.Warn("Pss handler %p failed: %v", f, err) + log.Warn("Pss handler failed", "err", err) } } } @@ -445,9 +498,23 @@ func (p *Pss) isSelfRecipient(msg *PssMsg) bool { } // test match of leftmost bytes in given message to node's Kademlia address -func (p *Pss) isSelfPossibleRecipient(msg *PssMsg) bool { +func (p *Pss) isSelfPossibleRecipient(msg *PssMsg, prox bool) bool { local := p.Kademlia.BaseAddr() - return bytes.Equal(msg.To, local[:len(msg.To)]) + + // if a partial address matches we are possible recipient regardless of prox + // if not and prox is not set, we are surely not + if bytes.Equal(msg.To, local[:len(msg.To)]) { + + return true + } else if !prox { + return false + } + + depth := p.Kademlia.NeighbourhoodDepth() + po, _ := p.Kademlia.Pof(p.Kademlia.BaseAddr(), msg.To, 0) + log.Trace("selfpossible", "po", po, "depth", depth) + + return depth <= po } ///////////////////////////////////////////////////////////////////// @@ -684,9 +751,6 @@ func (p *Pss) enqueue(msg *PssMsg) error { // // Will fail if raw messages are disallowed func (p *Pss) SendRaw(address PssAddress, topic Topic, msg []byte) error { - if !p.allowRaw { - return errors.New("Raw messages not enabled") - } pssMsgParams := &msgParams{ raw: true, } @@ -699,7 +763,17 @@ func (p *Pss) SendRaw(address PssAddress, topic Topic, msg []byte) error { pssMsg.Expire = uint32(time.Now().Add(p.msgTTL).Unix()) pssMsg.Payload = payload p.addFwdCache(pssMsg) - return p.enqueue(pssMsg) + err := p.enqueue(pssMsg) + if err != nil { + return err + } + + // if we have a proxhandler on this topic + // also deliver message to ourselves + if p.isSelfPossibleRecipient(pssMsg, true) && p.topicHandlerCaps[topic].prox { + return p.process(pssMsg, true, true) + } + return nil } // Send a message using symmetric encryption @@ -800,7 +874,16 @@ func (p *Pss) send(to []byte, topic Topic, msg []byte, asymmetric bool, key []by pssMsg.To = to pssMsg.Expire = uint32(time.Now().Add(p.msgTTL).Unix()) pssMsg.Payload = envelope - return p.enqueue(pssMsg) + err = p.enqueue(pssMsg) + if err != nil { + return err + } + if _, ok := p.topicHandlerCaps[topic]; ok { + if p.isSelfPossibleRecipient(pssMsg, true) && p.topicHandlerCaps[topic].prox { + return p.process(pssMsg, true, true) + } + } + return nil } // Forwards a pss message to the peer(s) closest to the to recipient address in the PssMsg struct @@ -895,6 +978,10 @@ func (p *Pss) cleanFwdCache() { } } +func label(b []byte) string { + return fmt.Sprintf("%04x", b[:2]) +} + // add a message to the cache func (p *Pss) addFwdCache(msg *PssMsg) error { metrics.GetOrRegisterCounter("pss.addfwdcache", nil).Inc(1) @@ -934,10 +1021,14 @@ func (p *Pss) checkFwdCache(msg *PssMsg) bool { // Digest of message func (p *Pss) digest(msg *PssMsg) pssDigest { - hasher := p.hashPool.Get().(storage.SwarmHash) + return p.digestBytes(msg.serialize()) +} + +func (p *Pss) digestBytes(msg []byte) pssDigest { + hasher := p.hashPool.Get().(hash.Hash) defer p.hashPool.Put(hasher) hasher.Reset() - hasher.Write(msg.serialize()) + hasher.Write(msg) digest := pssDigest{} key := hasher.Sum(nil) copy(digest[:], key[:digestLength]) diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index 66a90be620..32404aaaf9 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -48,20 +48,23 @@ import ( "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm/network" + "github.com/ethereum/go-ethereum/swarm/pot" "github.com/ethereum/go-ethereum/swarm/state" whisper "github.com/ethereum/go-ethereum/whisper/whisperv5" ) var ( - initOnce = sync.Once{} - debugdebugflag = flag.Bool("vv", false, "veryverbose") - debugflag = flag.Bool("v", false, "verbose") - longrunning = flag.Bool("longrunning", false, "do run long-running tests") - w *whisper.Whisper - wapi *whisper.PublicWhisperAPI - psslogmain log.Logger - pssprotocols map[string]*protoCtrl - useHandshake bool + initOnce = sync.Once{} + loglevel = flag.Int("loglevel", 2, "logging verbosity") + longrunning = flag.Bool("longrunning", false, "do run long-running tests") + w *whisper.Whisper + wapi *whisper.PublicWhisperAPI + psslogmain log.Logger + pssprotocols map[string]*protoCtrl + useHandshake bool + noopHandlerFunc = func(msg []byte, p *p2p.Peer, asymmetric bool, keyid string) error { + return nil + } ) func init() { @@ -75,16 +78,9 @@ func init() { func initTest() { initOnce.Do( func() { - loglevel := log.LvlInfo - if *debugflag { - loglevel = log.LvlDebug - } else if *debugdebugflag { - loglevel = log.LvlTrace - } - psslogmain = log.New("psslog", "*") hs := log.StreamHandler(os.Stderr, log.TerminalFormat(true)) - hf := log.LvlFilterHandler(loglevel, hs) + hf := log.LvlFilterHandler(log.Lvl(*loglevel), hs) h := log.CallerFileHandler(hf) log.Root().SetHandler(h) @@ -280,15 +276,14 @@ func TestAddressMatch(t *testing.T) { } pssmsg := &PssMsg{ - To: remoteaddr, - Payload: &whisper.Envelope{}, + To: remoteaddr, } // differ from first byte if ps.isSelfRecipient(pssmsg) { t.Fatalf("isSelfRecipient true but %x != %x", remoteaddr, localaddr) } - if ps.isSelfPossibleRecipient(pssmsg) { + if ps.isSelfPossibleRecipient(pssmsg, false) { t.Fatalf("isSelfPossibleRecipient true but %x != %x", remoteaddr[:8], localaddr[:8]) } @@ -297,7 +292,7 @@ func TestAddressMatch(t *testing.T) { if ps.isSelfRecipient(pssmsg) { t.Fatalf("isSelfRecipient true but %x != %x", remoteaddr, localaddr) } - if !ps.isSelfPossibleRecipient(pssmsg) { + if !ps.isSelfPossibleRecipient(pssmsg, false) { t.Fatalf("isSelfPossibleRecipient false but %x == %x", remoteaddr[:8], localaddr[:8]) } @@ -306,13 +301,342 @@ func TestAddressMatch(t *testing.T) { if !ps.isSelfRecipient(pssmsg) { t.Fatalf("isSelfRecipient false but %x == %x", remoteaddr, localaddr) } - if !ps.isSelfPossibleRecipient(pssmsg) { + if !ps.isSelfPossibleRecipient(pssmsg, false) { t.Fatalf("isSelfPossibleRecipient false but %x == %x", remoteaddr[:8], localaddr[:8]) } + +} + +// test that message is handled by sender if a prox handler exists and sender is in prox of message +func TestProxShortCircuit(t *testing.T) { + + // sender node address + localAddr := network.RandomAddr().Over() + localPotAddr := pot.NewAddressFromBytes(localAddr) + + // set up kademlia + kadParams := network.NewKadParams() + kad := network.NewKademlia(localAddr, kadParams) + peerCount := kad.MinBinSize + 1 + + // set up pss + privKey, err := crypto.GenerateKey() + pssp := NewPssParams().WithPrivateKey(privKey) + ps, err := NewPss(kad, pssp) + if err != nil { + t.Fatal(err.Error()) + } + + // create kademlia peers, so we have peers both inside and outside minproxlimit + var peers []*network.Peer + proxMessageAddress := pot.RandomAddressAt(localPotAddr, peerCount).Bytes() + distantMessageAddress := pot.RandomAddressAt(localPotAddr, 0).Bytes() + + for i := 0; i < peerCount; i++ { + rw := &p2p.MsgPipeRW{} + ptpPeer := p2p.NewPeer(enode.ID{}, "wanna be with me? [ ] yes [ ] no", []p2p.Cap{}) + protoPeer := protocols.NewPeer(ptpPeer, rw, &protocols.Spec{}) + peerAddr := pot.RandomAddressAt(localPotAddr, i) + bzzPeer := &network.BzzPeer{ + Peer: protoPeer, + BzzAddr: &network.BzzAddr{ + OAddr: peerAddr.Bytes(), + UAddr: []byte(fmt.Sprintf("%x", peerAddr[:])), + }, + } + peer := network.NewPeer(bzzPeer, kad) + kad.On(peer) + peers = append(peers, peer) + } + + // register it marking prox capability + delivered := make(chan struct{}) + rawHandlerFunc := func(msg []byte, p *p2p.Peer, asymmetric bool, keyid string) error { + log.Trace("in allowraw handler") + delivered <- struct{}{} + return nil + } + topic := BytesToTopic([]byte{0x2a}) + hndlrProxDereg := ps.Register(&topic, &handler{ + f: rawHandlerFunc, + caps: &handlerCaps{ + raw: true, + prox: true, + }, + }) + defer hndlrProxDereg() + + // send message too far away for sender to be in prox + // reception of this message should time out + errC := make(chan error) + go func() { + err := ps.SendRaw(distantMessageAddress, topic, []byte("foo")) + if err != nil { + errC <- err + } + }() + + ctx, cancel := context.WithTimeout(context.TODO(), time.Second) + defer cancel() + select { + case <-delivered: + t.Fatal("raw distant message delivered") + case err := <-errC: + t.Fatal(err) + case <-ctx.Done(): + } + + // send message that should be within sender prox + // this message should be delivered + go func() { + err := ps.SendRaw(proxMessageAddress, topic, []byte("bar")) + if err != nil { + errC <- err + } + }() + + ctx, cancel = context.WithTimeout(context.TODO(), time.Second) + defer cancel() + select { + case <-delivered: + case err := <-errC: + t.Fatal(err) + case <-ctx.Done(): + t.Fatal("raw timeout") + } + + // try the same prox message with sym and asym send + proxAddrPss := PssAddress(proxMessageAddress) + symKeyId, err := ps.GenerateSymmetricKey(topic, &proxAddrPss, true) + go func() { + err := ps.SendSym(symKeyId, topic, []byte("baz")) + if err != nil { + errC <- err + } + }() + ctx, cancel = context.WithTimeout(context.TODO(), time.Second) + defer cancel() + select { + case <-delivered: + case err := <-errC: + t.Fatal(err) + case <-ctx.Done(): + t.Fatal("sym timeout") + } + + err = ps.SetPeerPublicKey(&privKey.PublicKey, topic, &proxAddrPss) + if err != nil { + t.Fatal(err) + } + pubKeyId := hexutil.Encode(crypto.FromECDSAPub(&privKey.PublicKey)) + go func() { + err := ps.SendAsym(pubKeyId, topic, []byte("xyzzy")) + if err != nil { + errC <- err + } + }() + ctx, cancel = context.WithTimeout(context.TODO(), time.Second) + defer cancel() + select { + case <-delivered: + case err := <-errC: + t.Fatal(err) + case <-ctx.Done(): + t.Fatal("asym timeout") + } } -// -func TestHandlerConditions(t *testing.T) { +// verify that node can be set as recipient regardless of explicit message address match if minimum one handler of a topic is explicitly set to allow it +// note that in these tests we use the raw capability on handlers for convenience +func TestAddressMatchProx(t *testing.T) { + + // recipient node address + localAddr := network.RandomAddr().Over() + localPotAddr := pot.NewAddressFromBytes(localAddr) + + // set up kademlia + kadparams := network.NewKadParams() + kad := network.NewKademlia(localAddr, kadparams) + nnPeerCount := kad.MinBinSize + peerCount := nnPeerCount + 2 + + // set up pss + privKey, err := crypto.GenerateKey() + pssp := NewPssParams().WithPrivateKey(privKey) + ps, err := NewPss(kad, pssp) + if err != nil { + t.Fatal(err.Error()) + } + + // create kademlia peers, so we have peers both inside and outside minproxlimit + var peers []*network.Peer + for i := 0; i < peerCount; i++ { + rw := &p2p.MsgPipeRW{} + ptpPeer := p2p.NewPeer(enode.ID{}, "362436 call me anytime", []p2p.Cap{}) + protoPeer := protocols.NewPeer(ptpPeer, rw, &protocols.Spec{}) + peerAddr := pot.RandomAddressAt(localPotAddr, i) + bzzPeer := &network.BzzPeer{ + Peer: protoPeer, + BzzAddr: &network.BzzAddr{ + OAddr: peerAddr.Bytes(), + UAddr: []byte(fmt.Sprintf("%x", peerAddr[:])), + }, + } + peer := network.NewPeer(bzzPeer, kad) + kad.On(peer) + peers = append(peers, peer) + } + + // TODO: create a test in the network package to make a table with n peers where n-m are proxpeers + // meanwhile test regression for kademlia since we are compiling the test parameters from different packages + var proxes int + var conns int + kad.EachConn(nil, peerCount, func(p *network.Peer, po int, prox bool) bool { + conns++ + if prox { + proxes++ + } + log.Trace("kadconn", "po", po, "peer", p, "prox", prox) + return true + }) + if proxes != nnPeerCount { + t.Fatalf("expected %d proxpeers, have %d", nnPeerCount, proxes) + } else if conns != peerCount { + t.Fatalf("expected %d peers total, have %d", peerCount, proxes) + } + + // remote address distances from localAddr to try and the expected outcomes if we use prox handler + remoteDistances := []int{ + 255, + nnPeerCount + 1, + nnPeerCount, + nnPeerCount - 1, + 0, + } + expects := []bool{ + true, + true, + true, + false, + false, + } + + // first the unit test on the method that calculates possible receipient using prox + for i, distance := range remoteDistances { + pssMsg := newPssMsg(&msgParams{}) + pssMsg.To = make([]byte, len(localAddr)) + copy(pssMsg.To, localAddr) + var byteIdx = distance / 8 + pssMsg.To[byteIdx] ^= 1 << uint(7-(distance%8)) + log.Trace(fmt.Sprintf("addrmatch %v", bytes.Equal(pssMsg.To, localAddr))) + if ps.isSelfPossibleRecipient(pssMsg, true) != expects[i] { + t.Fatalf("expected distance %d to be %v", distance, expects[i]) + } + } + + // we move up to higher level and test the actual message handler + // for each distance check if we are possible recipient when prox variant is used is set + + // this handler will increment a counter for every message that gets passed to the handler + var receives int + rawHandlerFunc := func(msg []byte, p *p2p.Peer, asymmetric bool, keyid string) error { + log.Trace("in allowraw handler") + receives++ + return nil + } + + // register it marking prox capability + topic := BytesToTopic([]byte{0x2a}) + hndlrProxDereg := ps.Register(&topic, &handler{ + f: rawHandlerFunc, + caps: &handlerCaps{ + raw: true, + prox: true, + }, + }) + + // test the distances + var prevReceive int + for i, distance := range remoteDistances { + remotePotAddr := pot.RandomAddressAt(localPotAddr, distance) + remoteAddr := remotePotAddr.Bytes() + + var data [32]byte + rand.Read(data[:]) + pssMsg := newPssMsg(&msgParams{raw: true}) + pssMsg.To = remoteAddr + pssMsg.Expire = uint32(time.Now().Unix() + 4200) + pssMsg.Payload = &whisper.Envelope{ + Topic: whisper.TopicType(topic), + Data: data[:], + } + + log.Trace("withprox addrs", "local", localAddr, "remote", remoteAddr) + ps.handlePssMsg(context.TODO(), pssMsg) + if (!expects[i] && prevReceive != receives) || (expects[i] && prevReceive == receives) { + t.Fatalf("expected distance %d recipient %v when prox is set for handler", distance, expects[i]) + } + prevReceive = receives + } + + // now add a non prox-capable handler and test + ps.Register(&topic, &handler{ + f: rawHandlerFunc, + caps: &handlerCaps{ + raw: true, + }, + }) + receives = 0 + prevReceive = 0 + for i, distance := range remoteDistances { + remotePotAddr := pot.RandomAddressAt(localPotAddr, distance) + remoteAddr := remotePotAddr.Bytes() + + var data [32]byte + rand.Read(data[:]) + pssMsg := newPssMsg(&msgParams{raw: true}) + pssMsg.To = remoteAddr + pssMsg.Expire = uint32(time.Now().Unix() + 4200) + pssMsg.Payload = &whisper.Envelope{ + Topic: whisper.TopicType(topic), + Data: data[:], + } + + log.Trace("withprox addrs", "local", localAddr, "remote", remoteAddr) + ps.handlePssMsg(context.TODO(), pssMsg) + if (!expects[i] && prevReceive != receives) || (expects[i] && prevReceive == receives) { + t.Fatalf("expected distance %d recipient %v when prox is set for handler", distance, expects[i]) + } + prevReceive = receives + } + + // now deregister the prox capable handler, now none of the messages will be handled + hndlrProxDereg() + receives = 0 + + for _, distance := range remoteDistances { + remotePotAddr := pot.RandomAddressAt(localPotAddr, distance) + remoteAddr := remotePotAddr.Bytes() + + pssMsg := newPssMsg(&msgParams{raw: true}) + pssMsg.To = remoteAddr + pssMsg.Expire = uint32(time.Now().Unix() + 4200) + pssMsg.Payload = &whisper.Envelope{ + Topic: whisper.TopicType(topic), + Data: []byte(remotePotAddr.String()), + } + + log.Trace("noprox addrs", "local", localAddr, "remote", remoteAddr) + ps.handlePssMsg(context.TODO(), pssMsg) + if receives != 0 { + t.Fatalf("expected distance %d to not be recipient when prox is not set for handler", distance) + } + + } +} + +// verify that message queueing happens when it should, and that expired and corrupt messages are dropped +func TestMessageProcessing(t *testing.T) { t.Skip("Disabled due to probable faulty logic for outbox expectations") // setup @@ -326,13 +650,12 @@ func TestHandlerConditions(t *testing.T) { ps := newTestPss(privkey, network.NewKademlia(addr, network.NewKadParams()), NewPssParams()) // message should pass - msg := &PssMsg{ - To: addr, - Expire: uint32(time.Now().Add(time.Second * 60).Unix()), - Payload: &whisper.Envelope{ - Topic: [4]byte{}, - Data: []byte{0x66, 0x6f, 0x6f}, - }, + msg := newPssMsg(&msgParams{}) + msg.To = addr + msg.Expire = uint32(time.Now().Add(time.Second * 60).Unix()) + msg.Payload = &whisper.Envelope{ + Topic: [4]byte{}, + Data: []byte{0x66, 0x6f, 0x6f}, } if err := ps.handlePssMsg(context.TODO(), msg); err != nil { t.Fatal(err.Error()) @@ -498,6 +821,7 @@ func TestKeys(t *testing.T) { } } +// check that we can retrieve previously added public key entires per topic and peer func TestGetPublickeyEntries(t *testing.T) { privkey, err := crypto.GenerateKey() @@ -557,7 +881,7 @@ OUTER: } // forwarding should skip peers that do not have matching pss capabilities -func TestMismatch(t *testing.T) { +func TestPeerCapabilityMismatch(t *testing.T) { // create privkey for forwarder node privkey, err := crypto.GenerateKey() @@ -615,6 +939,76 @@ func TestMismatch(t *testing.T) { } +// verifies that message handlers for raw messages only are invoked when minimum one handler for the topic exists in which raw messages are explicitly allowed +func TestRawAllow(t *testing.T) { + + // set up pss like so many times before + privKey, err := crypto.GenerateKey() + if err != nil { + t.Fatal(err) + } + baseAddr := network.RandomAddr() + kad := network.NewKademlia((baseAddr).Over(), network.NewKadParams()) + ps := newTestPss(privKey, kad, nil) + topic := BytesToTopic([]byte{0x2a}) + + // create handler innards that increments every time a message hits it + var receives int + rawHandlerFunc := func(msg []byte, p *p2p.Peer, asymmetric bool, keyid string) error { + log.Trace("in allowraw handler") + receives++ + return nil + } + + // wrap this handler function with a handler without raw capability and register it + hndlrNoRaw := &handler{ + f: rawHandlerFunc, + } + ps.Register(&topic, hndlrNoRaw) + + // test it with a raw message, should be poo-poo + pssMsg := newPssMsg(&msgParams{ + raw: true, + }) + pssMsg.To = baseAddr.OAddr + pssMsg.Expire = uint32(time.Now().Unix() + 4200) + pssMsg.Payload = &whisper.Envelope{ + Topic: whisper.TopicType(topic), + } + ps.handlePssMsg(context.TODO(), pssMsg) + if receives > 0 { + t.Fatalf("Expected handler not to be executed with raw cap off") + } + + // now wrap the same handler function with raw capabilities and register it + hndlrRaw := &handler{ + f: rawHandlerFunc, + caps: &handlerCaps{ + raw: true, + }, + } + deregRawHandler := ps.Register(&topic, hndlrRaw) + + // should work now + pssMsg.Payload.Data = []byte("Raw Deal") + ps.handlePssMsg(context.TODO(), pssMsg) + if receives == 0 { + t.Fatalf("Expected handler to be executed with raw cap on") + } + + // now deregister the raw capable handler + prevReceives := receives + deregRawHandler() + + // check that raw messages fail again + pssMsg.Payload.Data = []byte("Raw Trump") + ps.handlePssMsg(context.TODO(), pssMsg) + if receives != prevReceives { + t.Fatalf("Expected handler not to be executed when raw handler is retracted") + } +} + +// verifies that nodes can send and receive raw (verbatim) messages func TestSendRaw(t *testing.T) { t.Run("32", testSendRaw) t.Run("8", testSendRaw) @@ -658,13 +1052,13 @@ func testSendRaw(t *testing.T) { lmsgC := make(chan APIMsg) lctx, lcancel := context.WithTimeout(context.Background(), time.Second*10) defer lcancel() - lsub, err := clients[0].Subscribe(lctx, "pss", lmsgC, "receive", topic) + lsub, err := clients[0].Subscribe(lctx, "pss", lmsgC, "receive", topic, true, false) log.Trace("lsub", "id", lsub) defer lsub.Unsubscribe() rmsgC := make(chan APIMsg) rctx, rcancel := context.WithTimeout(context.Background(), time.Second*10) defer rcancel() - rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic) + rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic, true, false) log.Trace("rsub", "id", rsub) defer rsub.Unsubscribe() @@ -757,13 +1151,13 @@ func testSendSym(t *testing.T) { lmsgC := make(chan APIMsg) lctx, lcancel := context.WithTimeout(context.Background(), time.Second*10) defer lcancel() - lsub, err := clients[0].Subscribe(lctx, "pss", lmsgC, "receive", topic) + lsub, err := clients[0].Subscribe(lctx, "pss", lmsgC, "receive", topic, false, false) log.Trace("lsub", "id", lsub) defer lsub.Unsubscribe() rmsgC := make(chan APIMsg) rctx, rcancel := context.WithTimeout(context.Background(), time.Second*10) defer rcancel() - rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic) + rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic, false, false) log.Trace("rsub", "id", rsub) defer rsub.Unsubscribe() @@ -872,13 +1266,13 @@ func testSendAsym(t *testing.T) { lmsgC := make(chan APIMsg) lctx, lcancel := context.WithTimeout(context.Background(), time.Second*10) defer lcancel() - lsub, err := clients[0].Subscribe(lctx, "pss", lmsgC, "receive", topic) + lsub, err := clients[0].Subscribe(lctx, "pss", lmsgC, "receive", topic, false, false) log.Trace("lsub", "id", lsub) defer lsub.Unsubscribe() rmsgC := make(chan APIMsg) rctx, rcancel := context.WithTimeout(context.Background(), time.Second*10) defer rcancel() - rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic) + rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic, false, false) log.Trace("rsub", "id", rsub) defer rsub.Unsubscribe() @@ -1037,7 +1431,7 @@ func testNetwork(t *testing.T) { msgC := make(chan APIMsg) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - sub, err := rpcclient.Subscribe(ctx, "pss", msgC, "receive", topic) + sub, err := rpcclient.Subscribe(ctx, "pss", msgC, "receive", topic, false, false) if err != nil { t.Fatal(err) } @@ -1209,7 +1603,7 @@ func TestDeduplication(t *testing.T) { rmsgC := make(chan APIMsg) rctx, cancel := context.WithTimeout(context.Background(), time.Second*1) defer cancel() - rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic) + rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic, false, false) log.Trace("rsub", "id", rsub) defer rsub.Unsubscribe() @@ -1392,8 +1786,8 @@ func benchmarkSymkeyBruteforceChangeaddr(b *testing.B) { if err != nil { b.Fatalf("could not generate whisper envelope: %v", err) } - ps.Register(&topic, func(msg []byte, p *p2p.Peer, asymmetric bool, keyid string) error { - return nil + ps.Register(&topic, &handler{ + f: noopHandlerFunc, }) pssmsgs = append(pssmsgs, &PssMsg{ To: to, @@ -1402,7 +1796,7 @@ func benchmarkSymkeyBruteforceChangeaddr(b *testing.B) { } b.ResetTimer() for i := 0; i < b.N; i++ { - if err := ps.process(pssmsgs[len(pssmsgs)-(i%len(pssmsgs))-1]); err != nil { + if err := ps.process(pssmsgs[len(pssmsgs)-(i%len(pssmsgs))-1], false, false); err != nil { b.Fatalf("pss processing failed: %v", err) } } @@ -1476,15 +1870,15 @@ func benchmarkSymkeyBruteforceSameaddr(b *testing.B) { if err != nil { b.Fatalf("could not generate whisper envelope: %v", err) } - ps.Register(&topic, func(msg []byte, p *p2p.Peer, asymmetric bool, keyid string) error { - return nil + ps.Register(&topic, &handler{ + f: noopHandlerFunc, }) pssmsg := &PssMsg{ To: addr[len(addr)-1][:], Payload: env, } for i := 0; i < b.N; i++ { - if err := ps.process(pssmsg); err != nil { + if err := ps.process(pssmsg, false, false); err != nil { b.Fatalf("pss processing failed: %v", err) } } @@ -1581,7 +1975,12 @@ func newServices(allowRaw bool) adapters.Services { if useHandshake { SetHandshakeController(ps, NewHandshakeParams()) } - ps.Register(&PingTopic, pp.Handle) + ps.Register(&PingTopic, &handler{ + f: pp.Handle, + caps: &handlerCaps{ + raw: true, + }, + }) ps.addAPI(rpc.API{ Namespace: "psstest", Version: "0.3", diff --git a/swarm/pss/types.go b/swarm/pss/types.go index 56c2c51dc0..ba963067cb 100644 --- a/swarm/pss/types.go +++ b/swarm/pss/types.go @@ -159,9 +159,39 @@ func (msg *PssMsg) String() string { } // Signature for a message handler function for a PssMsg -// // Implementations of this type are passed to Pss.Register together with a topic, -type Handler func(msg []byte, p *p2p.Peer, asymmetric bool, keyid string) error +type HandlerFunc func(msg []byte, p *p2p.Peer, asymmetric bool, keyid string) error + +type handlerCaps struct { + raw bool + prox bool +} + +// Handler defines code to be executed upon reception of content. +type handler struct { + f HandlerFunc + caps *handlerCaps +} + +// NewHandler returns a new message handler +func NewHandler(f HandlerFunc) *handler { + return &handler{ + f: f, + caps: &handlerCaps{}, + } +} + +// WithRaw is a chainable method that allows raw messages to be handled. +func (h *handler) WithRaw() *handler { + h.caps.raw = true + return h +} + +// WithProxBin is a chainable method that allows sending messages with full addresses to neighbourhoods using the kademlia depth as reference +func (h *handler) WithProxBin() *handler { + h.caps.prox = true + return h +} // the stateStore handles saving and loading PSS peers and their corresponding keys // it is currently unimplemented From 0699287440527dedbc7d6f881fc466057b7822bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Mon, 26 Nov 2018 16:09:32 +0100 Subject: [PATCH 086/159] tests: Add flag to use EVMC for state tests (#18084) --- tests/state_test.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/state_test.go b/tests/state_test.go index ad77e4f339..964405382c 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -18,10 +18,12 @@ package tests import ( "bytes" + "flag" "fmt" "reflect" "testing" + "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/core/vm" ) @@ -65,8 +67,17 @@ func TestState(t *testing.T) { // Transactions with gasLimit above this value will not get a VM trace on failure. const traceErrorLimit = 400000 +// The VM config for state tests that accepts --vm.* command line arguments. +var testVMConfig = func() vm.Config { + vmconfig := vm.Config{} + flag.StringVar(&vmconfig.EVMInterpreter, utils.EVMInterpreterFlag.Name, utils.EVMInterpreterFlag.Value, utils.EVMInterpreterFlag.Usage) + flag.StringVar(&vmconfig.EWASMInterpreter, utils.EWASMInterpreterFlag.Name, utils.EWASMInterpreterFlag.Value, utils.EWASMInterpreterFlag.Usage) + flag.Parse() + return vmconfig +}() + func withTrace(t *testing.T, gasLimit uint64, test func(vm.Config) error) { - err := test(vm.Config{}) + err := test(testVMConfig) if err == nil { return } From 2714e8f091117b4f110198008348bfc19233ed60 Mon Sep 17 00:00:00 2001 From: Javier Peletier Date: Mon, 26 Nov 2018 16:10:22 +0100 Subject: [PATCH 087/159] Remove multihash from Swarm bzz:// for Feeds (#18175) --- cmd/swarm/swarm-smoke/feed_upload_and_sync.go | 6 +- swarm/OWNERS | 1 - swarm/api/api.go | 25 ++-- swarm/api/client/client_test.go | 112 +++++++++++++----- swarm/api/http/server_test.go | 104 ++++++++++------ swarm/multihash/multihash.go | 92 -------------- swarm/multihash/multihash_test.go | 53 --------- 7 files changed, 167 insertions(+), 226 deletions(-) delete mode 100644 swarm/multihash/multihash.go delete mode 100644 swarm/multihash/multihash_test.go diff --git a/cmd/swarm/swarm-smoke/feed_upload_and_sync.go b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go index 1371d66548..0328d656f3 100644 --- a/cmd/swarm/swarm-smoke/feed_upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go @@ -16,7 +16,6 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/swarm/multihash" "github.com/ethereum/go-ethereum/swarm/storage/feed" colorable "github.com/mattn/go-colorable" "github.com/pborman/uuid" @@ -36,7 +35,7 @@ func cliFeedUploadAndSync(c *cli.Context) error { generateEndpoints(scheme, cluster, from, to) - log.Info("generating and uploading MRUs to " + endpoints[0] + " and syncing") + log.Info("generating and uploading feeds to " + endpoints[0] + " and syncing") // create a random private key to sign updates with and derive the address pkFile, err := ioutil.TempFile("", "swarm-feed-smoke-test") @@ -218,8 +217,7 @@ func cliFeedUploadAndSync(c *cli.Context) error { if err != nil { return err } - multihashHex := hexutil.Encode(multihash.ToMultihash(hashBytes)) - + multihashHex := hexutil.Encode(hashBytes) fileHash, err := digest(f) if err != nil { return err diff --git a/swarm/OWNERS b/swarm/OWNERS index d4204e08cf..4b9ca96ebe 100644 --- a/swarm/OWNERS +++ b/swarm/OWNERS @@ -7,7 +7,6 @@ swarm ├── fuse ────────────────── @jmozah, @holisticode ├── grafana_dashboards ──── @nonsense ├── metrics ─────────────── @nonsense, @holisticode -├── multihash ───────────── @nolash ├── network ─────────────── ethersphere │ ├── bitvector ───────── @zelig, @janos, @gbalint │ ├── priorityqueue ───── @zelig, @janos, @gbalint diff --git a/swarm/api/api.go b/swarm/api/api.go index 099a469396..33a8e3539e 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -42,7 +42,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/log" - "github.com/ethereum/go-ethereum/swarm/multihash" "github.com/ethereum/go-ethereum/swarm/spancontext" "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/storage/feed" @@ -417,7 +416,7 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage return reader, mimeType, status, nil, err } // get the data of the update - _, rsrcData, err := a.feed.GetContent(entry.Feed) + _, contentAddr, err := a.feed.GetContent(entry.Feed) if err != nil { apiGetNotFound.Inc(1) status = http.StatusNotFound @@ -425,23 +424,23 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage return reader, mimeType, status, nil, err } - // extract multihash - decodedMultihash, err := multihash.FromMultihash(rsrcData) - if err != nil { + // extract content hash + if len(contentAddr) != storage.AddressLength { apiGetInvalid.Inc(1) status = http.StatusUnprocessableEntity - log.Warn("invalid multihash in feed update", "err", err) - return reader, mimeType, status, nil, err + errorMessage := fmt.Sprintf("invalid swarm hash in feed update. Expected %d bytes. Got %d", storage.AddressLength, len(contentAddr)) + log.Warn(errorMessage) + return reader, mimeType, status, nil, errors.New(errorMessage) } - manifestAddr = storage.Address(decodedMultihash) - log.Trace("feed update contains multihash", "key", manifestAddr) + manifestAddr = storage.Address(contentAddr) + log.Trace("feed update contains swarm hash", "key", manifestAddr) - // get the manifest the multihash digest points to + // get the manifest the swarm hash points to trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil, NOOPDecrypt) if err != nil { apiGetNotFound.Inc(1) status = http.StatusNotFound - log.Warn(fmt.Sprintf("loadManifestTrie (feed update multihash) error: %v", err)) + log.Warn(fmt.Sprintf("loadManifestTrie (feed update) error: %v", err)) return reader, mimeType, status, nil, err } @@ -451,8 +450,8 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage if entry == nil { status = http.StatusNotFound apiGetNotFound.Inc(1) - err = fmt.Errorf("manifest (feed update multihash) entry for '%s' not found", path) - log.Trace("manifest (feed update multihash) entry not found", "key", manifestAddr, "path", path) + err = fmt.Errorf("manifest (feed update) entry for '%s' not found", path) + log.Trace("manifest (feed update) entry not found", "key", manifestAddr, "path", path) return reader, mimeType, status, nil, err } } diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go index 76b3493978..39f6e4797d 100644 --- a/swarm/api/client/client_test.go +++ b/swarm/api/client/client_test.go @@ -25,13 +25,13 @@ import ( "sort" "testing" + "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/swarm/api" swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" - "github.com/ethereum/go-ethereum/swarm/multihash" "github.com/ethereum/go-ethereum/swarm/storage/feed" ) @@ -368,58 +368,99 @@ func newTestSigner() (*feed.GenericSigner, error) { return feed.NewGenericSigner(privKey), nil } -// test the transparent resolving of multihash feed updates with bzz:// scheme +// Test the transparent resolving of feed updates with bzz:// scheme // -// first upload data, and store the multihash to the resulting manifest in a feed update -// retrieving the update with the multihash should return the manifest pointing directly to the data +// First upload data to bzz:, and store the Swarm hash to the resulting manifest in a feed update. +// This effectively uses a feed to store a pointer to content rather than the content itself +// Retrieving the update with the Swarm hash should return the manifest pointing directly to the data // and raw retrieve of that hash should return the data -func TestClientCreateFeedMultihash(t *testing.T) { +func TestClientBzzWithFeed(t *testing.T) { signer, _ := newTestSigner() + // Initialize a Swarm test server srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) - client := NewClient(srv.URL) + swarmClient := NewClient(srv.URL) defer srv.Close() - // add the data our multihash aliased manifest will point to - databytes := []byte("bar") + // put together some data for our test: + dataBytes := []byte(` + // + // Create some data our manifest will point to. Data that could be very big and wouldn't fit in a feed update. + // So what we are going to do is upload it to Swarm bzz:// and obtain a **manifest hash** pointing to it: + // + // MANIFEST HASH --> DATA + // + // Then, we store that **manifest hash** into a Swarm Feed update. Once we have done this, + // we can use the **feed manifest hash** in bzz:// instead, this way: bzz://feed-manifest-hash. + // + // FEED MANIFEST HASH --> MANIFEST HASH --> DATA + // + // Given that we can update the feed at any time with a new **manifest hash** but the **feed manifest hash** + // stays constant, we have effectively created a fixed address to changing content. (Applause) + // + // FEED MANIFEST HASH (the same) --> MANIFEST HASH(2) --> DATA(2) + // + `) - swarmHash, err := client.UploadRaw(bytes.NewReader(databytes), int64(len(databytes)), false) - if err != nil { - t.Fatalf("Error uploading raw test data: %s", err) + // Create a virtual File out of memory containing the above data + f := &File{ + ReadCloser: ioutil.NopCloser(bytes.NewReader(dataBytes)), + ManifestEntry: api.ManifestEntry{ + ContentType: "text/plain", + Mode: 0660, + Size: int64(len(dataBytes)), + }, } - s := common.FromHex(swarmHash) - mh := multihash.ToMultihash(s) + // upload data to bzz:// and retrieve the content-addressed manifest hash, hex-encoded. + manifestAddressHex, err := swarmClient.Upload(f, "", false) + if err != nil { + t.Fatalf("Error creating manifest: %s", err) + } - // our feed topic - topic, _ := feed.NewTopic("foo.eth", nil) + // convert the hex-encoded manifest hash to a 32-byte slice + manifestAddress := common.FromHex(manifestAddressHex) - createRequest := feed.NewFirstRequest(topic) + if len(manifestAddress) != storage.AddressLength { + t.Fatalf("Something went wrong. Got a hash of an unexpected length. Expected %d bytes. Got %d", storage.AddressLength, len(manifestAddress)) + } - createRequest.SetData(mh) - if err := createRequest.Sign(signer); err != nil { + // Now create a **feed manifest**. For that, we need a topic: + topic, _ := feed.NewTopic("interesting topic indeed", nil) + + // Build a feed request to update data + request := feed.NewFirstRequest(topic) + + // Put the 32-byte address of the manifest into the feed update + request.SetData(manifestAddress) + + // Sign the update + if err := request.Sign(signer); err != nil { t.Fatalf("Error signing update: %s", err) } - feedManifestHash, err := client.CreateFeedWithManifest(createRequest) - + // Publish the update and at the same time request a **feed manifest** to be created + feedManifestAddressHex, err := swarmClient.CreateFeedWithManifest(request) if err != nil { t.Fatalf("Error creating feed manifest: %s", err) } - correctManifestAddrHex := "bb056a5264c295c2b0f613c8409b9c87ce9d71576ace02458160df4cc894210b" - if feedManifestHash != correctManifestAddrHex { - t.Fatalf("Response feed manifest mismatch, expected '%s', got '%s'", correctManifestAddrHex, feedManifestHash) + // Check we have received the exact **feed manifest** to be expected + // given the topic and user signing the updates: + correctFeedManifestAddrHex := "747c402e5b9dc715a25a4393147512167bab018a007fad7cdcd9adc7fce1ced2" + if feedManifestAddressHex != correctFeedManifestAddrHex { + t.Fatalf("Response feed manifest mismatch, expected '%s', got '%s'", correctFeedManifestAddrHex, feedManifestAddressHex) } // Check we get a not found error when trying to get feed updates with a made-up manifest - _, err = client.QueryFeed(nil, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb") + _, err = swarmClient.QueryFeed(nil, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb") if err != ErrNoFeedUpdatesFound { t.Fatalf("Expected to receive ErrNoFeedUpdatesFound error. Got: %s", err) } - reader, err := client.QueryFeed(nil, correctManifestAddrHex) + // If we query the feed directly we should get **manifest hash** back: + reader, err := swarmClient.QueryFeed(nil, correctFeedManifestAddrHex) if err != nil { t.Fatalf("Error retrieving feed updates: %s", err) } @@ -428,10 +469,27 @@ func TestClientCreateFeedMultihash(t *testing.T) { if err != nil { t.Fatal(err) } - if !bytes.Equal(mh, gotData) { - t.Fatalf("Expected: %v, got %v", mh, gotData) + + //Check that indeed the **manifest hash** is retrieved + if !bytes.Equal(manifestAddress, gotData) { + t.Fatalf("Expected: %v, got %v", manifestAddress, gotData) } + // Now the final test we were looking for: Use bzz:// and that should resolve all manifests + // and return the original data directly: + f, err = swarmClient.Download(feedManifestAddressHex, "") + if err != nil { + t.Fatal(err) + } + gotData, err = ioutil.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + // Check that we get back the original data: + if !bytes.Equal(dataBytes, gotData) { + t.Fatalf("Expected: %v, got %v", manifestAddress, gotData) + } } // TestClientCreateUpdateFeed will check that feeds can be created and updated via the HTTP client. diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 1ef3deece2..e82762ce05 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -45,7 +45,6 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" swarm "github.com/ethereum/go-ethereum/swarm/api/client" - "github.com/ethereum/go-ethereum/swarm/multihash" "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/storage/feed" "github.com/ethereum/go-ethereum/swarm/testutil" @@ -69,60 +68,91 @@ func newTestSigner() (*feed.GenericSigner, error) { return feed.NewGenericSigner(privKey), nil } -// test the transparent resolving of multihash-containing feed updates with bzz:// scheme +// Test the transparent resolving of feed updates with bzz:// scheme // -// first upload data, and store the multihash to the resulting manifest in a feed update -// retrieving the update with the multihash should return the manifest pointing directly to the data +// First upload data to bzz:, and store the Swarm hash to the resulting manifest in a feed update. +// This effectively uses a feed to store a pointer to content rather than the content itself +// Retrieving the update with the Swarm hash should return the manifest pointing directly to the data // and raw retrieve of that hash should return the data -func TestBzzFeedMultihash(t *testing.T) { +func TestBzzWithFeed(t *testing.T) { signer, _ := newTestSigner() + // Initialize Swarm test server srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() - // add the data our multihash aliased manifest will point to - databytes := "bar" - testBzzUrl := fmt.Sprintf("%s/bzz:/", srv.URL) - resp, err := http.Post(testBzzUrl, "text/plain", bytes.NewReader([]byte(databytes))) + // put together some data for our test: + dataBytes := []byte(` + // + // Create some data our manifest will point to. Data that could be very big and wouldn't fit in a feed update. + // So what we are going to do is upload it to Swarm bzz:// and obtain a **manifest hash** pointing to it: + // + // MANIFEST HASH --> DATA + // + // Then, we store that **manifest hash** into a Swarm Feed update. Once we have done this, + // we can use the **feed manifest hash** in bzz:// instead, this way: bzz://feed-manifest-hash. + // + // FEED MANIFEST HASH --> MANIFEST HASH --> DATA + // + // Given that we can update the feed at any time with a new **manifest hash** but the **feed manifest hash** + // stays constant, we have effectively created a fixed address to changing content. (Applause) + // + // FEED MANIFEST HASH (the same) --> MANIFEST HASH(2) --> DATA(2) ... + // + `) + + // POST data to bzz and get back a content-addressed **manifest hash** pointing to it. + resp, err := http.Post(fmt.Sprintf("%s/bzz:/", srv.URL), "text/plain", bytes.NewReader([]byte(dataBytes))) if err != nil { t.Fatal(err) } + defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Fatalf("err %s", resp.Status) } - b, err := ioutil.ReadAll(resp.Body) + manifestAddressHex, err := ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) } - s := common.FromHex(string(b)) - mh := multihash.ToMultihash(s) - log.Info("added data", "manifest", string(b), "data", common.ToHex(mh)) + manifestAddress := common.FromHex(string(manifestAddressHex)) - topic, _ := feed.NewTopic("foo.eth", nil) + log.Info("added data", "manifest", string(manifestAddressHex)) + + // At this point we have uploaded the data and have a manifest pointing to it + // Now store that manifest address in a feed update. + // We also want a feed manifest, so we can use it to refer to the feed. + + // First, create a topic for our feed: + topic, _ := feed.NewTopic("interesting topic indeed", nil) + + // Create a feed update request: updateRequest := feed.NewFirstRequest(topic) - updateRequest.SetData(mh) + // Store the **manifest address** as data into the feed update. + updateRequest.SetData(manifestAddress) + // Sign the update if err := updateRequest.Sign(signer); err != nil { t.Fatal(err) } - log.Info("added data", "manifest", string(b), "data", common.ToHex(mh)) + log.Info("added data", "data", common.ToHex(manifestAddress)) - testUrl, err := url.Parse(fmt.Sprintf("%s/bzz-feed:/", srv.URL)) + // Build the feed update http request: + feedUpdateURL, err := url.Parse(fmt.Sprintf("%s/bzz-feed:/", srv.URL)) if err != nil { t.Fatal(err) } - query := testUrl.Query() + query := feedUpdateURL.Query() body := updateRequest.AppendValues(query) // this adds all query parameters and returns the data to be posted - query.Set("manifest", "1") // indicate we want a manifest back - testUrl.RawQuery = query.Encode() + query.Set("manifest", "1") // indicate we want a feed manifest back + feedUpdateURL.RawQuery = query.Encode() - // create the multihash update - resp, err = http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body)) + // submit the feed update request to Swarm + resp, err = http.Post(feedUpdateURL.String(), "application/octet-stream", bytes.NewReader(body)) if err != nil { t.Fatal(err) } @@ -130,24 +160,25 @@ func TestBzzFeedMultihash(t *testing.T) { if resp.StatusCode != http.StatusOK { t.Fatalf("err %s", resp.Status) } - b, err = ioutil.ReadAll(resp.Body) + + feedManifestAddressHex, err := ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) } - rsrcResp := &storage.Address{} - err = json.Unmarshal(b, rsrcResp) + feedManifestAddress := &storage.Address{} + err = json.Unmarshal(feedManifestAddressHex, feedManifestAddress) if err != nil { - t.Fatalf("data %s could not be unmarshaled: %v", b, err) + t.Fatalf("data %s could not be unmarshaled: %v", feedManifestAddressHex, err) } - correctManifestAddrHex := "bb056a5264c295c2b0f613c8409b9c87ce9d71576ace02458160df4cc894210b" - if rsrcResp.Hex() != correctManifestAddrHex { - t.Fatalf("Response feed manifest address mismatch, expected '%s', got '%s'", correctManifestAddrHex, rsrcResp.Hex()) + correctManifestAddrHex := "747c402e5b9dc715a25a4393147512167bab018a007fad7cdcd9adc7fce1ced2" + if feedManifestAddress.Hex() != correctManifestAddrHex { + t.Fatalf("Response feed manifest address mismatch, expected '%s', got '%s'", correctManifestAddrHex, feedManifestAddress.Hex()) } // get bzz manifest transparent feed update resolve - testBzzUrl = fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp) - resp, err = http.Get(testBzzUrl) + getBzzURL := fmt.Sprintf("%s/bzz:/%s", srv.URL, feedManifestAddress) + resp, err = http.Get(getBzzURL) if err != nil { t.Fatal(err) } @@ -155,12 +186,12 @@ func TestBzzFeedMultihash(t *testing.T) { if resp.StatusCode != http.StatusOK { t.Fatalf("err %s", resp.Status) } - b, err = ioutil.ReadAll(resp.Body) + retrievedData, err := ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) } - if !bytes.Equal(b, []byte(databytes)) { - t.Fatalf("retrieved data mismatch, expected %x, got %x", databytes, b) + if !bytes.Equal(retrievedData, []byte(dataBytes)) { + t.Fatalf("retrieved data mismatch, expected %x, got %x", dataBytes, retrievedData) } } @@ -245,7 +276,8 @@ func TestBzzFeed(t *testing.T) { t.Fatalf("Expected manifest Feed '%s', got '%s'", correctFeedHex, manifest.Entries[0].Feed.Hex()) } - // get bzz manifest transparent feed update resolve + // take the chance to have bzz: crash on resolving a feed update that does not contain + // a swarm hash: testBzzUrl := fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp) resp, err = http.Get(testBzzUrl) if err != nil { @@ -253,7 +285,7 @@ func TestBzzFeed(t *testing.T) { } defer resp.Body.Close() if resp.StatusCode == http.StatusOK { - t.Fatal("Expected error status since feed update does not contain multihash. Received 200 OK") + t.Fatal("Expected error status since feed update does not contain a Swarm hash. Received 200 OK") } _, err = ioutil.ReadAll(resp.Body) if err != nil { diff --git a/swarm/multihash/multihash.go b/swarm/multihash/multihash.go deleted file mode 100644 index 3306e3a6dc..0000000000 --- a/swarm/multihash/multihash.go +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package multihash - -import ( - "bytes" - "encoding/binary" - "errors" - "fmt" -) - -const ( - defaultMultihashLength = 32 - defaultMultihashTypeCode = 0x1b -) - -var ( - multihashTypeCode uint8 - MultihashLength = defaultMultihashLength -) - -func init() { - multihashTypeCode = defaultMultihashTypeCode - MultihashLength = defaultMultihashLength -} - -// check if valid swarm multihash -func isSwarmMultihashType(code uint8) bool { - return code == multihashTypeCode -} - -// GetMultihashLength returns the digest length of the provided multihash -// It will fail if the multihash is not a valid swarm mulithash -func GetMultihashLength(data []byte) (int, int, error) { - cursor := 0 - typ, c := binary.Uvarint(data) - if c <= 0 { - return 0, 0, errors.New("unreadable hashtype field") - } - if !isSwarmMultihashType(uint8(typ)) { - return 0, 0, fmt.Errorf("hash code %x is not a swarm hashtype", typ) - } - cursor += c - hashlength, c := binary.Uvarint(data[cursor:]) - if c <= 0 { - return 0, 0, errors.New("unreadable length field") - } - cursor += c - - // we cheekily assume hashlength < maxint - inthashlength := int(hashlength) - if len(data[c:]) < inthashlength { - return 0, 0, errors.New("length mismatch") - } - return inthashlength, cursor, nil -} - -// FromMulithash returns the digest portion of the multihash -// It will fail if the multihash is not a valid swarm multihash -func FromMultihash(data []byte) ([]byte, error) { - hashLength, _, err := GetMultihashLength(data) - if err != nil { - return nil, err - } - return data[len(data)-hashLength:], nil -} - -// ToMulithash wraps the provided digest data with a swarm mulithash header -func ToMultihash(hashData []byte) []byte { - buf := bytes.NewBuffer(nil) - b := make([]byte, 8) - c := binary.PutUvarint(b, uint64(multihashTypeCode)) - buf.Write(b[:c]) - c = binary.PutUvarint(b, uint64(len(hashData))) - buf.Write(b[:c]) - buf.Write(hashData) - return buf.Bytes() -} diff --git a/swarm/multihash/multihash_test.go b/swarm/multihash/multihash_test.go deleted file mode 100644 index 85df741dda..0000000000 --- a/swarm/multihash/multihash_test.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package multihash - -import ( - "bytes" - "math/rand" - "testing" -) - -// parse multihash, and check that invalid multihashes fail -func TestCheckMultihash(t *testing.T) { - hashbytes := make([]byte, 32) - c, err := rand.Read(hashbytes) - if err != nil { - t.Fatal(err) - } else if c < 32 { - t.Fatal("short read") - } - - expected := ToMultihash(hashbytes) - - l, hl, _ := GetMultihashLength(expected) - if l != 32 { - t.Fatalf("expected length %d, got %d", 32, l) - } else if hl != 2 { - t.Fatalf("expected header length %d, got %d", 2, hl) - } - if _, _, err := GetMultihashLength(expected[1:]); err == nil { - t.Fatal("expected failure on corrupt header") - } - if _, _, err := GetMultihashLength(expected[:len(expected)-2]); err == nil { - t.Fatal("expected failure on short content") - } - dh, _ := FromMultihash(expected) - if !bytes.Equal(dh, hashbytes) { - t.Fatalf("expected content hash %x, got %x", hashbytes, dh) - } -} From bba5fd81921ee700388b4db72db3240d32576b6c Mon Sep 17 00:00:00 2001 From: holisticode Date: Mon, 26 Nov 2018 11:05:18 -0500 Subject: [PATCH 088/159] Accounting metrics reporter (#18136) --- p2p/protocols/accounting.go | 43 ++++-- p2p/protocols/accounting_simulation_test.go | 10 ++ p2p/protocols/reporter.go | 147 ++++++++++++++++++++ p2p/protocols/reporter_test.go | 77 ++++++++++ swarm/swap/swap.go | 5 + swarm/swarm.go | 53 ++++--- 6 files changed, 305 insertions(+), 30 deletions(-) create mode 100644 p2p/protocols/reporter.go create mode 100644 p2p/protocols/reporter_test.go diff --git a/p2p/protocols/accounting.go b/p2p/protocols/accounting.go index 06a1a58454..770406a27e 100644 --- a/p2p/protocols/accounting.go +++ b/p2p/protocols/accounting.go @@ -16,29 +16,32 @@ package protocols -import "github.com/ethereum/go-ethereum/metrics" +import ( + "time" + + "github.com/ethereum/go-ethereum/metrics" +) //define some metrics var ( - //NOTE: these metrics just define the interfaces and are currently *NOT persisted* over sessions //All metrics are cumulative //total amount of units credited - mBalanceCredit = metrics.NewRegisteredCounterForced("account.balance.credit", nil) + mBalanceCredit metrics.Counter //total amount of units debited - mBalanceDebit = metrics.NewRegisteredCounterForced("account.balance.debit", nil) + mBalanceDebit metrics.Counter //total amount of bytes credited - mBytesCredit = metrics.NewRegisteredCounterForced("account.bytes.credit", nil) + mBytesCredit metrics.Counter //total amount of bytes debited - mBytesDebit = metrics.NewRegisteredCounterForced("account.bytes.debit", nil) + mBytesDebit metrics.Counter //total amount of credited messages - mMsgCredit = metrics.NewRegisteredCounterForced("account.msg.credit", nil) + mMsgCredit metrics.Counter //total amount of debited messages - mMsgDebit = metrics.NewRegisteredCounterForced("account.msg.debit", nil) + mMsgDebit metrics.Counter //how many times local node had to drop remote peers - mPeerDrops = metrics.NewRegisteredCounterForced("account.peerdrops", nil) + mPeerDrops metrics.Counter //how many times local node overdrafted and dropped - mSelfDrops = metrics.NewRegisteredCounterForced("account.selfdrops", nil) + mSelfDrops metrics.Counter ) //Prices defines how prices are being passed on to the accounting instance @@ -105,6 +108,26 @@ func NewAccounting(balance Balance, po Prices) *Accounting { return ah } +//SetupAccountingMetrics creates a separate registry for p2p accounting metrics; +//this registry should be independent of any other metrics as it persists at different endpoints. +//It also instantiates the given metrics and starts the persisting go-routine which +//at the passed interval writes the metrics to a LevelDB +func SetupAccountingMetrics(reportInterval time.Duration, path string) *AccountingMetrics { + //create an empty registry + registry := metrics.NewRegistry() + //instantiate the metrics + mBalanceCredit = metrics.NewRegisteredCounterForced("account.balance.credit", registry) + mBalanceDebit = metrics.NewRegisteredCounterForced("account.balance.debit", registry) + mBytesCredit = metrics.NewRegisteredCounterForced("account.bytes.credit", registry) + mBytesDebit = metrics.NewRegisteredCounterForced("account.bytes.debit", registry) + mMsgCredit = metrics.NewRegisteredCounterForced("account.msg.credit", registry) + mMsgDebit = metrics.NewRegisteredCounterForced("account.msg.debit", registry) + mPeerDrops = metrics.NewRegisteredCounterForced("account.peerdrops", registry) + mSelfDrops = metrics.NewRegisteredCounterForced("account.selfdrops", registry) + //create the DB and start persisting + return NewAccountingMetrics(registry, reportInterval, path) +} + //Implement Hook.Send // Send takes a peer, a size and a msg and // - calculates the cost for the local node sending a msg of size to peer using the Prices interface diff --git a/p2p/protocols/accounting_simulation_test.go b/p2p/protocols/accounting_simulation_test.go index 65b737abed..e90a1d81d2 100644 --- a/p2p/protocols/accounting_simulation_test.go +++ b/p2p/protocols/accounting_simulation_test.go @@ -20,7 +20,10 @@ import ( "context" "flag" "fmt" + "io/ioutil" "math/rand" + "os" + "path/filepath" "reflect" "sync" "testing" @@ -66,6 +69,13 @@ func init() { func TestAccountingSimulation(t *testing.T) { //setup the balances objects for every node bal := newBalances(*nodes) + //setup the metrics system or tests will fail trying to write metrics + dir, err := ioutil.TempDir("", "account-sim") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + SetupAccountingMetrics(1*time.Second, filepath.Join(dir, "metrics.db")) //define the node.Service for this test services := adapters.Services{ "accounting": func(ctx *adapters.ServiceContext) (node.Service, error) { diff --git a/p2p/protocols/reporter.go b/p2p/protocols/reporter.go new file mode 100644 index 0000000000..215d4fe31b --- /dev/null +++ b/p2p/protocols/reporter.go @@ -0,0 +1,147 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package protocols + +import ( + "encoding/binary" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + + "github.com/syndtr/goleveldb/leveldb" +) + +//AccountMetrics abstracts away the metrics DB and +//the reporter to persist metrics +type AccountingMetrics struct { + reporter *reporter +} + +//Close will be called when the node is being shutdown +//for a graceful cleanup +func (am *AccountingMetrics) Close() { + close(am.reporter.quit) + am.reporter.db.Close() +} + +//reporter is an internal structure used to write p2p accounting related +//metrics to a LevelDB. It will periodically write the accrued metrics to the DB. +type reporter struct { + reg metrics.Registry //the registry for these metrics (independent of other metrics) + interval time.Duration //duration at which the reporter will persist metrics + db *leveldb.DB //the actual DB + quit chan struct{} //quit the reporter loop +} + +//NewMetricsDB creates a new LevelDB instance used to persist metrics defined +//inside p2p/protocols/accounting.go +func NewAccountingMetrics(r metrics.Registry, d time.Duration, path string) *AccountingMetrics { + var val = make([]byte, 8) + var err error + + //Create the LevelDB + db, err := leveldb.OpenFile(path, nil) + if err != nil { + log.Error(err.Error()) + return nil + } + + //Check for all defined metrics that there is a value in the DB + //If there is, assign it to the metric. This means that the node + //has been running before and that metrics have been persisted. + metricsMap := map[string]metrics.Counter{ + "account.balance.credit": mBalanceCredit, + "account.balance.debit": mBalanceDebit, + "account.bytes.credit": mBytesCredit, + "account.bytes.debit": mBytesDebit, + "account.msg.credit": mMsgCredit, + "account.msg.debit": mMsgDebit, + "account.peerdrops": mPeerDrops, + "account.selfdrops": mSelfDrops, + } + //iterate the map and get the values + for key, metric := range metricsMap { + val, err = db.Get([]byte(key), nil) + //until the first time a value is being written, + //this will return an error. + //it could be beneficial though to log errors later, + //but that would require a different logic + if err == nil { + metric.Inc(int64(binary.BigEndian.Uint64(val))) + } + } + + //create the reporter + rep := &reporter{ + reg: r, + interval: d, + db: db, + quit: make(chan struct{}), + } + + //run the go routine + go rep.run() + + m := &AccountingMetrics{ + reporter: rep, + } + + return m +} + +//run is the goroutine which periodically sends the metrics to the configured LevelDB +func (r *reporter) run() { + intervalTicker := time.NewTicker(r.interval) + + for { + select { + case <-intervalTicker.C: + //at each tick send the metrics + if err := r.save(); err != nil { + log.Error("unable to send metrics to LevelDB", "err", err) + //If there is an error in writing, exit the routine; we assume here that the error is + //severe and don't attempt to write again. + //Also, this should prevent leaking when the node is stopped + return + } + case <-r.quit: + //graceful shutdown + return + } + } +} + +//send the metrics to the DB +func (r *reporter) save() error { + //create a LevelDB Batch + batch := leveldb.Batch{} + //for each metric in the registry (which is independent)... + r.reg.Each(func(name string, i interface{}) { + metric, ok := i.(metrics.Counter) + if ok { + //assuming every metric here to be a Counter (separate registry) + //...create a snapshot... + ms := metric.Snapshot() + byteVal := make([]byte, 8) + binary.BigEndian.PutUint64(byteVal, uint64(ms.Count())) + //...and save the value to the DB + batch.Put([]byte(name), byteVal) + } + }) + return r.db.Write(&batch, nil) +} diff --git a/p2p/protocols/reporter_test.go b/p2p/protocols/reporter_test.go new file mode 100644 index 0000000000..b9f06e6744 --- /dev/null +++ b/p2p/protocols/reporter_test.go @@ -0,0 +1,77 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package protocols + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + "time" + + "github.com/ethereum/go-ethereum/log" +) + +//TestReporter tests that the metrics being collected for p2p accounting +//are being persisted and available after restart of a node. +//It simulates restarting by just recreating the DB as if the node had restarted. +func TestReporter(t *testing.T) { + //create a test directory + dir, err := ioutil.TempDir("", "reporter-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + //setup the metrics + log.Debug("Setting up metrics first time") + reportInterval := 5 * time.Millisecond + metrics := SetupAccountingMetrics(reportInterval, filepath.Join(dir, "test.db")) + log.Debug("Done.") + + //do some metrics + mBalanceCredit.Inc(12) + mBytesCredit.Inc(34) + mMsgDebit.Inc(9) + + //give the reporter time to write the metrics to DB + time.Sleep(20 * time.Millisecond) + + //set the metrics to nil - this effectively simulates the node having shut down... + mBalanceCredit = nil + mBytesCredit = nil + mMsgDebit = nil + //close the DB also, or we can't create a new one + metrics.Close() + + //setup the metrics again + log.Debug("Setting up metrics second time") + metrics = SetupAccountingMetrics(reportInterval, filepath.Join(dir, "test.db")) + defer metrics.Close() + log.Debug("Done.") + + //now check the metrics, they should have the same value as before "shutdown" + if mBalanceCredit.Count() != 12 { + t.Fatalf("Expected counter to be %d, but is %d", 12, mBalanceCredit.Count()) + } + if mBytesCredit.Count() != 34 { + t.Fatalf("Expected counter to be %d, but is %d", 23, mBytesCredit.Count()) + } + if mMsgDebit.Count() != 9 { + t.Fatalf("Expected counter to be %d, but is %d", 9, mMsgDebit.Count()) + } +} diff --git a/swarm/swap/swap.go b/swarm/swap/swap.go index 137eb141d9..5d636dc205 100644 --- a/swarm/swap/swap.go +++ b/swarm/swap/swap.go @@ -91,3 +91,8 @@ func (s *Swap) loadState(peer *protocols.Peer) (err error) { } return } + +//Clean up Swap +func (swap *Swap) Close() { + swap.stateStore.Close() +} diff --git a/swarm/swarm.go b/swarm/swarm.go index dc3756d3af..a4ff94051d 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -1,4 +1,4 @@ -// Copyright 2016 The go-ethereum Authors +// Copyright 2018 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -66,20 +66,22 @@ var ( // the swarm stack type Swarm struct { - config *api.Config // swarm configuration - api *api.API // high level api layer (fs/manifest) - dns api.Resolver // DNS registrar - fileStore *storage.FileStore // distributed preimage archive, the local API to the storage with document level storage/retrieval support - streamer *stream.Registry - bzz *network.Bzz // the logistic manager - backend chequebook.Backend // simple blockchain Backend - privateKey *ecdsa.PrivateKey - corsString string - swapEnabled bool - netStore *storage.NetStore - sfs *fuse.SwarmFS // need this to cleanup all the active mounts on node exit - ps *pss.Pss - swap *swap.Swap + config *api.Config // swarm configuration + api *api.API // high level api layer (fs/manifest) + dns api.Resolver // DNS registrar + fileStore *storage.FileStore // distributed preimage archive, the local API to the storage with document level storage/retrieval support + streamer *stream.Registry + bzz *network.Bzz // the logistic manager + backend chequebook.Backend // simple blockchain Backend + privateKey *ecdsa.PrivateKey + corsString string + swapEnabled bool + netStore *storage.NetStore + sfs *fuse.SwarmFS // need this to cleanup all the active mounts on node exit + ps *pss.Pss + swap *swap.Swap + stateStore *state.DBStore + accountingMetrics *protocols.AccountingMetrics tracerClose io.Closer } @@ -134,7 +136,7 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e LightNode: config.LightNodeEnabled, } - stateStore, err := state.NewDBStore(filepath.Join(config.Path, "state-store.db")) + self.stateStore, err = state.NewDBStore(filepath.Join(config.Path, "state-store.db")) if err != nil { return } @@ -179,6 +181,7 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e return nil, err } self.swap = swap.New(balancesStore) + self.accountingMetrics = protocols.SetupAccountingMetrics(10*time.Second, filepath.Join(config.Path, "metrics.db")) } var nodeID enode.ID @@ -203,7 +206,7 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e SyncUpdateDelay: config.SyncUpdateDelay, MaxPeerServers: config.MaxStreamPeerServers, } - self.streamer = stream.NewRegistry(nodeID, delivery, self.netStore, stateStore, registryOptions, self.swap) + self.streamer = stream.NewRegistry(nodeID, delivery, self.netStore, self.stateStore, registryOptions, self.swap) // Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage self.fileStore = storage.NewFileStore(self.netStore, self.config.FileStoreParams) @@ -226,7 +229,7 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e log.Debug("Setup local storage") - self.bzz = network.NewBzz(bzzconfig, to, stateStore, self.streamer.GetSpec(), self.streamer.Run) + self.bzz = network.NewBzz(bzzconfig, to, self.stateStore, self.streamer.GetSpec(), self.streamer.Run) // Pss = postal service over swarm (devp2p over bzz) self.ps, err = pss.NewPss(to, config.Pss) @@ -446,14 +449,24 @@ func (self *Swarm) Stop() error { ch.Stop() ch.Save() } - + if self.swap != nil { + self.swap.Close() + } + if self.accountingMetrics != nil { + self.accountingMetrics.Close() + } if self.netStore != nil { self.netStore.Close() } self.sfs.Stop() stopCounter.Inc(1) self.streamer.Stop() - return self.bzz.Stop() + + err := self.bzz.Stop() + if self.stateStore != nil { + self.stateStore.Close() + } + return err } // implements the node.Service interface From 1cd007ecae437f5cc54a026e592edfad642a0b69 Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 26 Nov 2018 17:13:59 +0100 Subject: [PATCH 089/159] swarm/network: Correct neighborhood depth (#18066) --- swarm/network/kademlia.go | 135 +++++++++++++----- swarm/network/kademlia_test.go | 89 +++++++++++- swarm/network/simulation/example_test.go | 4 + swarm/network/simulation/kademlia.go | 1 + swarm/network/simulation/kademlia_test.go | 1 + swarm/network/stream/delivery_test.go | 2 + swarm/network/stream/intervals_test.go | 2 + .../network/stream/snapshot_retrieval_test.go | 1 + swarm/network/stream/snapshot_sync_test.go | 4 + swarm/network/stream/syncer_test.go | 2 + .../visualized_snapshot_sync_sim_test.go | 3 + swarm/network_test.go | 2 + 12 files changed, 209 insertions(+), 37 deletions(-) diff --git a/swarm/network/kademlia.go b/swarm/network/kademlia.go index 5fda51e3eb..a8ecaa4be4 100644 --- a/swarm/network/kademlia.go +++ b/swarm/network/kademlia.go @@ -177,7 +177,7 @@ func (k *Kademlia) SuggestPeer() (a *BzzAddr, o int, want bool) { k.lock.Lock() defer k.lock.Unlock() minsize := k.MinBinSize - depth := k.neighbourhoodDepth() + depth := depthForPot(k.conns, k.MinProxBinSize, k.base) // if there is a callable neighbour within the current proxBin, connect // this makes sure nearest neighbour set is fully connected var ppo int @@ -308,7 +308,7 @@ func (k *Kademlia) sendNeighbourhoodDepthChange() { // It provides signaling of neighbourhood depth change. // This part of the code is sending new neighbourhood depth to nDepthC if that condition is met. if k.nDepthC != nil { - nDepth := k.neighbourhoodDepth() + nDepth := depthForPot(k.conns, k.MinProxBinSize, k.base) if nDepth != k.nDepth { k.nDepth = nDepth k.nDepthC <- nDepth @@ -364,7 +364,7 @@ func (k *Kademlia) EachBin(base []byte, pof pot.Pof, o int, eachBinFunc func(con var startPo int var endPo int - kadDepth := k.neighbourhoodDepth() + kadDepth := depthForPot(k.conns, k.MinProxBinSize, k.base) k.conns.EachBin(base, pof, o, func(po, size int, f func(func(val pot.Val, i int) bool) bool) bool { if startPo > 0 && endPo != k.MaxProxDisplay { @@ -398,7 +398,7 @@ func (k *Kademlia) eachConn(base []byte, o int, f func(*Peer, int, bool) bool) { if len(base) == 0 { base = k.base } - depth := k.neighbourhoodDepth() + depth := depthForPot(k.conns, k.MinProxBinSize, k.base) k.conns.EachNeighbour(base, pof, func(val pot.Val, po int) bool { if po > o { return true @@ -420,7 +420,7 @@ func (k *Kademlia) eachAddr(base []byte, o int, f func(*BzzAddr, int, bool) bool if len(base) == 0 { base = k.base } - depth := k.neighbourhoodDepth() + depth := depthForPot(k.conns, k.MinProxBinSize, k.base) k.addrs.EachNeighbour(base, pof, func(val pot.Val, po int) bool { if po > o { return true @@ -429,26 +429,72 @@ func (k *Kademlia) eachAddr(base []byte, o int, f func(*BzzAddr, int, bool) bool }) } -// neighbourhoodDepth returns the proximity order that defines the distance of -// the nearest neighbour set with cardinality >= MinProxBinSize -// if there is altogether less than MinProxBinSize peers it returns 0 func (k *Kademlia) NeighbourhoodDepth() (depth int) { k.lock.RLock() defer k.lock.RUnlock() - return k.neighbourhoodDepth() + return depthForPot(k.conns, k.MinProxBinSize, k.base) } -func (k *Kademlia) neighbourhoodDepth() (depth int) { - if k.conns.Size() < k.MinProxBinSize { +// depthForPot returns the proximity order that defines the distance of +// the nearest neighbour set with cardinality >= MinProxBinSize +// if there is altogether less than MinProxBinSize peers it returns 0 +// caller must hold the lock +func depthForPot(p *pot.Pot, minProxBinSize int, pivotAddr []byte) (depth int) { + if p.Size() <= minProxBinSize { return 0 } + + // total number of peers in iteration var size int + + // true if iteration has all prox peers + var b bool + + // last po recorded in iteration + var lastPo int + f := func(v pot.Val, i int) bool { + // po == 256 means that addr is the pivot address(self) + if i == 256 { + return true + } size++ - depth = i - return size < k.MinProxBinSize + + // this means we have all nn-peers. + // depth is by default set to the bin of the farthest nn-peer + if size == minProxBinSize { + b = true + depth = i + return true + } + + // if there are empty bins between farthest nn and current node, + // the depth should recalculated to be + // the farthest of those empty bins + // + // 0 abac ccde + // 1 2a2a + // 2 589f <--- nearest non-nn + // ============ DEPTH 3 =========== + // 3 <--- don't count as empty bins + // 4 <--- don't count as empty bins + // 5 cbcb cdcd <---- furthest nn + // 6 a1a2 b3c4 + if b && i < depth { + depth = i + 1 + lastPo = i + return false + } + lastPo = i + return true + } + p.EachNeighbour(pivotAddr, pof, f) + + // cover edge case where more than one farthest nn + // AND we only have nn-peers + if lastPo == depth { + depth = 0 } - k.conns.EachNeighbour(k.base, pof, f) return depth } @@ -508,7 +554,7 @@ func (k *Kademlia) string() string { liverows := make([]string, k.MaxProxDisplay) peersrows := make([]string, k.MaxProxDisplay) - depth := k.neighbourhoodDepth() + depth := depthForPot(k.conns, k.MinProxBinSize, k.base) rest := k.conns.Size() k.conns.EachBin(k.base, pof, 0, func(po, size int, f func(func(val pot.Val, i int) bool) bool) bool { var rowlen int @@ -578,6 +624,7 @@ type PeerPot struct { // as hexadecimal representations of the address. // used for testing only func NewPeerPotMap(kadMinProxSize int, addrs [][]byte) map[string]*PeerPot { + // create a table of all nodes for health check np := pot.NewPot(nil, 0) for _, addr := range addrs { @@ -586,34 +633,47 @@ func NewPeerPotMap(kadMinProxSize int, addrs [][]byte) map[string]*PeerPot { ppmap := make(map[string]*PeerPot) for i, a := range addrs { - pl := 256 - prev := 256 + + // actual kademlia depth + depth := depthForPot(np, kadMinProxSize, a) + + // upon entering a new iteration + // this will hold the value the po should be + // if it's one higher than the po in the last iteration + prevPo := 256 + + // all empty bins which are outside neighbourhood depth var emptyBins []int + + // all nn-peers var nns [][]byte - np.EachNeighbour(addrs[i], pof, func(val pot.Val, po int) bool { - a := val.([]byte) + + np.EachNeighbour(a, pof, func(val pot.Val, po int) bool { + addr := val.([]byte) + // po == 256 means that addr is the pivot address(self) if po == 256 { return true } - if pl == 256 || pl == po { - nns = append(nns, a) + + // iterate through the neighbours, going from the closest to the farthest + // we calculate the nearest neighbours that should be in the set + // depth in this case equates to: + // 1. Within all bins that are higher or equal than depth there are + // at least minProxBinSize peers connected + // 2. depth-1 bin is not empty + if po >= depth { + nns = append(nns, addr) + prevPo = depth - 1 + return true } - if pl == 256 && len(nns) >= kadMinProxSize { - pl = po - prev = po + for j := prevPo; j > po; j-- { + emptyBins = append(emptyBins, j) } - if prev < pl { - for j := prev; j > po; j-- { - emptyBins = append(emptyBins, j) - } - } - prev = po - 1 + prevPo = po - 1 return true }) - for j := prev; j >= 0; j-- { - emptyBins = append(emptyBins, j) - } - log.Trace(fmt.Sprintf("%x NNS: %s", addrs[i][:4], LogAddrs(nns))) + + log.Trace(fmt.Sprintf("%x NNS: %s, emptyBins: %s", addrs[i][:4], LogAddrs(nns), logEmptyBins(emptyBins))) ppmap[common.Bytes2Hex(a)] = &PeerPot{nns, emptyBins} } return ppmap @@ -628,7 +688,7 @@ func (k *Kademlia) saturation(n int) int { prev++ return prev == po && size >= n }) - depth := k.neighbourhoodDepth() + depth := depthForPot(k.conns, k.MinProxBinSize, k.base) if depth < prev { return depth } @@ -641,8 +701,11 @@ func (k *Kademlia) full(emptyBins []int) (full bool) { prev := 0 e := len(emptyBins) ok := true - depth := k.neighbourhoodDepth() + depth := depthForPot(k.conns, k.MinProxBinSize, k.base) k.conns.EachBin(k.base, pof, 0, func(po, _ int, _ func(func(val pot.Val, i int) bool) bool) bool { + if po >= depth { + return false + } if prev == depth+1 { return true } diff --git a/swarm/network/kademlia_test.go b/swarm/network/kademlia_test.go index d2e051f457..184a2d9421 100644 --- a/swarm/network/kademlia_test.go +++ b/swarm/network/kademlia_test.go @@ -25,6 +25,9 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/protocols" "github.com/ethereum/go-ethereum/swarm/pot" ) @@ -73,6 +76,76 @@ func Register(k *Kademlia, regs ...string) { } } +// tests the validity of neighborhood depth calculations +// +// in particular, it tests that if there are one or more consecutive +// empty bins above the farthest "nearest neighbor-peer" then +// the depth should be set at the farthest of those empty bins +// +// TODO: Make test adapt to change in MinProxBinSize +func TestNeighbourhoodDepth(t *testing.T) { + baseAddressBytes := RandomAddr().OAddr + kad := NewKademlia(baseAddressBytes, NewKadParams()) + + baseAddress := pot.NewAddressFromBytes(baseAddressBytes) + + closerAddress := pot.RandomAddressAt(baseAddress, 7) + closerPeer := newTestDiscoveryPeer(closerAddress, kad) + kad.On(closerPeer) + depth := kad.NeighbourhoodDepth() + if depth != 0 { + t.Fatalf("expected depth 0, was %d", depth) + } + + sameAddress := pot.RandomAddressAt(baseAddress, 7) + samePeer := newTestDiscoveryPeer(sameAddress, kad) + kad.On(samePeer) + depth = kad.NeighbourhoodDepth() + if depth != 0 { + t.Fatalf("expected depth 0, was %d", depth) + } + + midAddress := pot.RandomAddressAt(baseAddress, 4) + midPeer := newTestDiscoveryPeer(midAddress, kad) + kad.On(midPeer) + depth = kad.NeighbourhoodDepth() + if depth != 5 { + t.Fatalf("expected depth 5, was %d", depth) + } + + kad.Off(midPeer) + depth = kad.NeighbourhoodDepth() + if depth != 0 { + t.Fatalf("expected depth 0, was %d", depth) + } + + fartherAddress := pot.RandomAddressAt(baseAddress, 1) + fartherPeer := newTestDiscoveryPeer(fartherAddress, kad) + kad.On(fartherPeer) + depth = kad.NeighbourhoodDepth() + if depth != 2 { + t.Fatalf("expected depth 2, was %d", depth) + } + + midSameAddress := pot.RandomAddressAt(baseAddress, 4) + midSamePeer := newTestDiscoveryPeer(midSameAddress, kad) + kad.Off(closerPeer) + kad.On(midPeer) + kad.On(midSamePeer) + depth = kad.NeighbourhoodDepth() + if depth != 2 { + t.Fatalf("expected depth 2, was %d", depth) + } + + kad.Off(fartherPeer) + log.Trace(kad.string()) + time.Sleep(time.Millisecond) + depth = kad.NeighbourhoodDepth() + if depth != 0 { + t.Fatalf("expected depth 0, was %d", depth) + } +} + func testSuggestPeer(k *Kademlia, expAddr string, expPo int, expWant bool) error { addr, o, want := k.SuggestPeer() if binStr(addr) != expAddr { @@ -376,7 +449,7 @@ func TestKademliaHiveString(t *testing.T) { Register(k, "10000000", "10000001") k.MaxProxDisplay = 8 h := k.String() - expH := "\n=========================================================================\nMon Feb 27 12:10:28 UTC 2017 KΛÐΞMLIΛ hive: queen's address: 000000\npopulation: 2 (4), MinProxBinSize: 2, MinBinSize: 1, MaxBinSize: 4\n000 0 | 2 8100 (0) 8000 (0)\n============ DEPTH: 1 ==========================================\n001 1 4000 | 1 4000 (0)\n002 1 2000 | 1 2000 (0)\n003 0 | 0\n004 0 | 0\n005 0 | 0\n006 0 | 0\n007 0 | 0\n=========================================================================" + expH := "\n=========================================================================\nMon Feb 27 12:10:28 UTC 2017 KΛÐΞMLIΛ hive: queen's address: 000000\npopulation: 2 (4), MinProxBinSize: 2, MinBinSize: 1, MaxBinSize: 4\n============ DEPTH: 0 ==========================================\n000 0 | 2 8100 (0) 8000 (0)\n001 1 4000 | 1 4000 (0)\n002 1 2000 | 1 2000 (0)\n003 0 | 0\n004 0 | 0\n005 0 | 0\n006 0 | 0\n007 0 | 0\n=========================================================================" if expH[104:] != h[104:] { t.Fatalf("incorrect hive output. expected %v, got %v", expH, h) } @@ -644,3 +717,17 @@ func TestKademliaCase5(t *testing.T) { "78fafa0809929a1279ece089a51d12457c2d8416dff859aeb2ccc24bb50df5ec", "1dd39b1257e745f147cbbc3cadd609ccd6207c41056dbc4254bba5d2527d3ee5", "5f61dd66d4d94aec8fcc3ce0e7885c7edf30c43143fa730e2841c5d28e3cd081", "8aa8b0472cb351d967e575ad05c4b9f393e76c4b01ef4b3a54aac5283b78abc9", "4502f385152a915b438a6726ce3ea9342e7a6db91a23c2f6bee83a885ed7eb82", "718677a504249db47525e959ef1784bed167e1c46f1e0275b9c7b588e28a3758", "7c54c6ed1f8376323896ed3a4e048866410de189e9599dd89bf312ca4adb96b5", "18e03bd3378126c09e799a497150da5c24c895aedc84b6f0dbae41fc4bac081a", "23db76ac9e6e58d9f5395ca78252513a7b4118b4155f8462d3d5eec62486cadc", "40ae0e8f065e96c7adb7fa39505136401f01780481e678d718b7f6dbb2c906ec", "c1539998b8bae19d339d6bbb691f4e9daeb0e86847545229e80fe0dffe716e92", "ed139d73a2699e205574c08722ca9f030ad2d866c662f1112a276b91421c3cb9", "5bdb19584b7a36d09ca689422ef7e6bb681b8f2558a6b2177a8f7c812f631022", "636c9de7fe234ffc15d67a504c69702c719f626c17461d3f2918e924cd9d69e2", "de4455413ff9335c440d52458c6544191bd58a16d85f700c1de53b62773064ea", "de1963310849527acabc7885b6e345a56406a8f23e35e436b6d9725e69a79a83", "a80a50a467f561210a114cba6c7fb1489ed43a14d61a9edd70e2eb15c31f074d", "7804f12b8d8e6e4b375b242058242068a3809385e05df0e64973cde805cf729c", "60f9aa320c02c6f2e6370aa740cf7cea38083fa95fca8c99552cda52935c1520", "d8da963602390f6c002c00ce62a84b514edfce9ebde035b277a957264bb54d21", "8463d93256e026fe436abad44697152b9a56ac8e06a0583d318e9571b83d073c", "9a3f78fcefb9a05e40a23de55f6153d7a8b9d973ede43a380bf46bb3b3847de1", "e3bb576f4b3760b9ca6bff59326f4ebfc4a669d263fb7d67ab9797adea54ed13", "4d5cdbd6dcca5bdf819a0fe8d175dc55cc96f088d37462acd5ea14bc6296bdbe", "5a0ed28de7b5258c727cb85447071c74c00a5fbba9e6bc0393bc51944d04ab2a", "61e4ddb479c283c638f4edec24353b6cc7a3a13b930824aad016b0996ca93c47", "7e3610868acf714836cafaaa7b8c009a9ac6e3a6d443e5586cf661530a204ee2", "d74b244d4345d2c86e30a097105e4fb133d53c578320285132a952cdaa64416e", "cfeed57d0f935bfab89e3f630a7c97e0b1605f0724d85a008bbfb92cb47863a8", "580837af95055670e20d494978f60c7f1458dc4b9e389fc7aa4982b2aca3bce3", "df55c0c49e6c8a83d82dfa1c307d3bf6a20e18721c80d8ec4f1f68dc0a137ced", "5f149c51ce581ba32a285439a806c063ced01ccd4211cd024e6a615b8f216f95", "1eb76b00aeb127b10dd1b7cd4c3edeb4d812b5a658f0feb13e85c4d2b7c6fe06", "7a56ba7c3fb7cbfb5561a46a75d95d7722096b45771ec16e6fa7bbfab0b35dfe", "4bae85ad88c28470f0015246d530adc0cd1778bdd5145c3c6b538ee50c4e04bd", "afd1892e2a7145c99ec0ebe9ded0d3fec21089b277a68d47f45961ec5e39e7e0", "953138885d7b36b0ef79e46030f8e61fd7037fbe5ce9e0a94d728e8c8d7eab86", "de761613ef305e4f628cb6bf97d7b7dc69a9d513dc233630792de97bcda777a6", "3f3087280063d09504c084bbf7fdf984347a72b50d097fd5b086ffabb5b3fb4c", "7d18a94bb1ebfdef4d3e454d2db8cb772f30ca57920dd1e402184a9e598581a0", "a7d6fbdc9126d9f10d10617f49fb9f5474ffe1b229f76b7dd27cebba30eccb5d", "fad0246303618353d1387ec10c09ee991eb6180697ed3470ed9a6b377695203d", "1cf66e09ea51ee5c23df26615a9e7420be2ac8063f28f60a3bc86020e94fe6f3", "8269cdaa153da7c358b0b940791af74d7c651cd4d3f5ed13acfe6d0f2c539e7f", "90d52eaaa60e74bf1c79106113f2599471a902d7b1c39ac1f55b20604f453c09", "9788fd0c09190a3f3d0541f68073a2f44c2fcc45bb97558a7c319f36c25a75b3", "10b68fc44157ecfdae238ee6c1ce0333f906ad04d1a4cb1505c8e35c3c87fbb0", "e5284117fdf3757920475c786e0004cb00ba0932163659a89b36651a01e57394", "403ad51d911e113dcd5f9ff58c94f6d278886a2a4da64c3ceca2083282c92de3", ) } + +func newTestDiscoveryPeer(addr pot.Address, kad *Kademlia) *Peer { + rw := &p2p.MsgPipeRW{} + p := p2p.NewPeer(enode.ID{}, "foo", []p2p.Cap{}) + pp := protocols.NewPeer(p, rw, &protocols.Spec{}) + bp := &BzzPeer{ + Peer: pp, + BzzAddr: &BzzAddr{ + OAddr: addr.Bytes(), + UAddr: []byte(fmt.Sprintf("%x", addr[:])), + }, + } + return NewPeer(bp, kad) +} diff --git a/swarm/network/simulation/example_test.go b/swarm/network/simulation/example_test.go index bacc64d53a..7b62046174 100644 --- a/swarm/network/simulation/example_test.go +++ b/swarm/network/simulation/example_test.go @@ -33,6 +33,10 @@ import ( // BucketKeyKademlia key. This allows to use WaitTillHealthy to block until // all nodes have the their Kadmlias healthy. func ExampleSimulation_WaitTillHealthy() { + + log.Error("temporarily disabled as simulations.WaitTillHealthy cannot be trusted") + return + sim := simulation.New(map[string]simulation.ServiceFunc{ "bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) { addr := network.NewAddr(ctx.Config.Node()) diff --git a/swarm/network/simulation/kademlia.go b/swarm/network/simulation/kademlia.go index f895181d9e..7982810ca3 100644 --- a/swarm/network/simulation/kademlia.go +++ b/swarm/network/simulation/kademlia.go @@ -33,6 +33,7 @@ var BucketKeyKademlia BucketKey = "kademlia" // WaitTillHealthy is blocking until the health of all kademlias is true. // If error is not nil, a map of kademlia that was found not healthy is returned. +// TODO: Check correctness since change in kademlia depth calculation logic func (s *Simulation) WaitTillHealthy(ctx context.Context, kadMinProxSize int) (ill map[enode.ID]*network.Kademlia, err error) { // Prepare PeerPot map for checking Kademlia health var ppmap map[string]*network.PeerPot diff --git a/swarm/network/simulation/kademlia_test.go b/swarm/network/simulation/kademlia_test.go index 285644a0f4..024830315a 100644 --- a/swarm/network/simulation/kademlia_test.go +++ b/swarm/network/simulation/kademlia_test.go @@ -28,6 +28,7 @@ import ( ) func TestWaitTillHealthy(t *testing.T) { + sim := New(map[string]ServiceFunc{ "bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) { addr := network.NewAddr(ctx.Config.Node()) diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index a6173a3892..f69f80499b 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -453,6 +453,8 @@ func TestDeliveryFromNodes(t *testing.T) { } func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck bool) { + + t.Skip("temporarily disabled as simulations.WaitTillHealthy cannot be trusted") sim := simulation.New(map[string]simulation.ServiceFunc{ "streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) { node := ctx.Config.Node() diff --git a/swarm/network/stream/intervals_test.go b/swarm/network/stream/intervals_test.go index defb6df50a..668cf586c0 100644 --- a/swarm/network/stream/intervals_test.go +++ b/swarm/network/stream/intervals_test.go @@ -52,6 +52,8 @@ func TestIntervalsLiveAndHistory(t *testing.T) { } func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { + + t.Skip("temporarily disabled as simulations.WaitTillHealthy cannot be trusted") nodes := 2 chunkCount := dataChunkCount externalStreamName := "externalStream" diff --git a/swarm/network/stream/snapshot_retrieval_test.go b/swarm/network/stream/snapshot_retrieval_test.go index 5ea0b1511b..932e28b32f 100644 --- a/swarm/network/stream/snapshot_retrieval_test.go +++ b/swarm/network/stream/snapshot_retrieval_test.go @@ -246,6 +246,7 @@ simulation's `action` function. The snapshot should have 'streamer' in its service list. */ func runRetrievalTest(chunkCount int, nodeCount int) error { + sim := simulation.New(retrievalSimServiceMap) defer sim.Close() diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go index 4e56f71b54..4a632c8c90 100644 --- a/swarm/network/stream/snapshot_sync_test.go +++ b/swarm/network/stream/snapshot_sync_test.go @@ -182,6 +182,8 @@ func streamerFunc(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Servic } func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) { + + t.Skip("temporarily disabled as simulations.WaitTillHealthy cannot be trusted") sim := simulation.New(simServiceMap) defer sim.Close() @@ -329,6 +331,8 @@ assuming that the snapshot file identifies a healthy kademlia network. The snapshot should have 'streamer' in its service list. */ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int) error { + + t.Skip("temporarily disabled as simulations.WaitTillHealthy cannot be trusted") sim := simulation.New(map[string]simulation.ServiceFunc{ "streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) { n := ctx.Config.Node() diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index 5764efc92d..df1a98c92e 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -68,6 +68,8 @@ func createMockStore(globalStore mock.GlobalStorer, id enode.ID, addr *network.B } func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck bool, po uint8) { + + t.Skip("temporarily disabled as simulations.WaitTillHealthy cannot be trusted") sim := simulation.New(map[string]simulation.ServiceFunc{ "streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) { var store storage.ChunkStore diff --git a/swarm/network/stream/visualized_snapshot_sync_sim_test.go b/swarm/network/stream/visualized_snapshot_sync_sim_test.go index 437c17e5e2..f6d618020d 100644 --- a/swarm/network/stream/visualized_snapshot_sync_sim_test.go +++ b/swarm/network/stream/visualized_snapshot_sync_sim_test.go @@ -84,6 +84,8 @@ func watchSim(sim *simulation.Simulation) (context.Context, context.CancelFunc) //This test requests bogus hashes into the network func TestNonExistingHashesWithServer(t *testing.T) { + + t.Skip("temporarily disabled as simulations.WaitTillHealthy cannot be trusted") nodeCount, _, sim := setupSim(retrievalSimServiceMap) defer sim.Close() @@ -143,6 +145,7 @@ func sendSimTerminatedEvent(sim *simulation.Simulation) { //can visualize messages like SendOfferedMsg, WantedHashesMsg, DeliveryMsg func TestSnapshotSyncWithServer(t *testing.T) { + t.Skip("temporarily disabled as simulations.WaitTillHealthy cannot be trusted") nodeCount, chunkCount, sim := setupSim(simServiceMap) defer sim.Close() diff --git a/swarm/network_test.go b/swarm/network_test.go index d84f28147f..41993dfc6b 100644 --- a/swarm/network_test.go +++ b/swarm/network_test.go @@ -259,6 +259,8 @@ type testSwarmNetworkOptions struct { // - May wait for Kademlia on every node to be healthy. // - Checking if a file is retrievable from all nodes. func testSwarmNetwork(t *testing.T, o *testSwarmNetworkOptions, steps ...testSwarmNetworkStep) { + + t.Skip("temporarily disabled as simulations.WaitTillHealthy cannot be trusted") if o == nil { o = new(testSwarmNetworkOptions) } From 4f0d978eaaebdd118af3bfe623782c70601daaed Mon Sep 17 00:00:00 2001 From: Javier Peletier Date: Mon, 26 Nov 2018 17:37:59 +0100 Subject: [PATCH 090/159] cmd/swarm: update should error on manifest mismatch (#18047) * cmd/swarm: fix ethersphere/go-ethereum#979: update should error on manifest mistmatch * cmd/swarm: fixed comments and remove sprintf from log.Info * cmd/swarm: remove unnecessary comment --- cmd/swarm/feeds.go | 6 +++++- cmd/swarm/feeds_test.go | 41 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/cmd/swarm/feeds.go b/cmd/swarm/feeds.go index f26a8cc7dc..6cd971a92c 100644 --- a/cmd/swarm/feeds.go +++ b/cmd/swarm/feeds.go @@ -169,7 +169,6 @@ func feedUpdate(ctx *cli.Context) { query = new(feed.Query) query.User = signer.Address() query.Topic = getTopic(ctx) - } // Retrieve a feed update request @@ -178,6 +177,11 @@ func feedUpdate(ctx *cli.Context) { utils.Fatalf("Error retrieving feed status: %s", err.Error()) } + // Check that the provided signer matches the request to sign + if updateRequest.User != signer.Address() { + utils.Fatalf("Signer address does not match the update request") + } + // set the new data updateRequest.SetData(data) diff --git a/cmd/swarm/feeds_test.go b/cmd/swarm/feeds_test.go index a0cedf0d36..4c40f62a82 100644 --- a/cmd/swarm/feeds_test.go +++ b/cmd/swarm/feeds_test.go @@ -19,7 +19,6 @@ package main import ( "bytes" "encoding/json" - "fmt" "io/ioutil" "os" "testing" @@ -69,7 +68,7 @@ func TestCLIFeedUpdate(t *testing.T) { hexData} // create an update and expect an exit without errors - log.Info(fmt.Sprintf("updating a feed with 'swarm feed update'")) + log.Info("updating a feed with 'swarm feed update'") cmd := runSwarm(t, flags...) cmd.ExpectExit() @@ -116,7 +115,7 @@ func TestCLIFeedUpdate(t *testing.T) { "--user", address.Hex(), } - log.Info(fmt.Sprintf("getting feed info with 'swarm feed info'")) + log.Info("getting feed info with 'swarm feed info'") cmd = runSwarm(t, flags...) _, matches := cmd.ExpectRegexp(`.*`) // regex hack to extract stdout cmd.ExpectExit() @@ -141,9 +140,9 @@ func TestCLIFeedUpdate(t *testing.T) { "--topic", topic.Hex(), } - log.Info(fmt.Sprintf("Publishing manifest with 'swarm feed create'")) + log.Info("Publishing manifest with 'swarm feed create'") cmd = runSwarm(t, flags...) - _, matches = cmd.ExpectRegexp(`[a-f\d]{64}`) // regex hack to extract stdout + _, matches = cmd.ExpectRegexp(`[a-f\d]{64}`) cmd.ExpectExit() manifestAddress := matches[0] // read the received feed manifest @@ -162,4 +161,36 @@ func TestCLIFeedUpdate(t *testing.T) { if !bytes.Equal(data, retrieved) { t.Fatalf("Received %s, expected %s", retrieved, data) } + + // test publishing a manifest for a different user + flags = []string{ + "--bzzapi", srv.URL, + "feed", "create", + "--topic", topic.Hex(), + "--user", "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", // different user + } + + log.Info("Publishing manifest with 'swarm feed create' for a different user") + cmd = runSwarm(t, flags...) + _, matches = cmd.ExpectRegexp(`[a-f\d]{64}`) + cmd.ExpectExit() + + manifestAddress = matches[0] // read the received feed manifest + + // now let's try to update that user's manifest which we don't have the private key for + flags = []string{ + "--bzzapi", srv.URL, + "--bzzaccount", pkFileName, + "feed", "update", + "--manifest", manifestAddress, + hexData} + + // create an update and expect an error given there is a user mismatch + log.Info("updating a feed with 'swarm feed update'") + cmd = runSwarm(t, flags...) + cmd.ExpectRegexp("Fatal:.*") // best way so far to detect a failure. + cmd.ExpectExit() + if cmd.ExitStatus() == 0 { + t.Fatal("Expected nonzero exit code when updating a manifest with the wrong user. Got 0.") + } } From c207edf2a3a6f48b4fc78cc55982d648eedab198 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Mon, 26 Nov 2018 18:49:01 +0100 Subject: [PATCH 091/159] swarm: add database abstractions (shed package) (#18183) --- swarm/shed/db.go | 130 ++++++++++ swarm/shed/db_test.go | 110 ++++++++ swarm/shed/example_store_test.go | 332 ++++++++++++++++++++++++ swarm/shed/field_string.go | 66 +++++ swarm/shed/field_string_test.go | 110 ++++++++ swarm/shed/field_struct.go | 71 ++++++ swarm/shed/field_struct_test.go | 127 +++++++++ swarm/shed/field_uint64.go | 108 ++++++++ swarm/shed/field_uint64_test.go | 194 ++++++++++++++ swarm/shed/index.go | 264 +++++++++++++++++++ swarm/shed/index_test.go | 426 +++++++++++++++++++++++++++++++ swarm/shed/schema.go | 134 ++++++++++ swarm/shed/schema_test.go | 126 +++++++++ swarm/storage/mock/db/db.go | 7 + swarm/storage/mock/mem/mem.go | 16 ++ swarm/storage/mock/mock.go | 7 + swarm/storage/mock/rpc/rpc.go | 6 + swarm/storage/mock/test/test.go | 53 ++++ 18 files changed, 2287 insertions(+) create mode 100644 swarm/shed/db.go create mode 100644 swarm/shed/db_test.go create mode 100644 swarm/shed/example_store_test.go create mode 100644 swarm/shed/field_string.go create mode 100644 swarm/shed/field_string_test.go create mode 100644 swarm/shed/field_struct.go create mode 100644 swarm/shed/field_struct_test.go create mode 100644 swarm/shed/field_uint64.go create mode 100644 swarm/shed/field_uint64_test.go create mode 100644 swarm/shed/index.go create mode 100644 swarm/shed/index_test.go create mode 100644 swarm/shed/schema.go create mode 100644 swarm/shed/schema_test.go diff --git a/swarm/shed/db.go b/swarm/shed/db.go new file mode 100644 index 0000000000..e128b8cbc8 --- /dev/null +++ b/swarm/shed/db.go @@ -0,0 +1,130 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package shed provides a simple abstraction components to compose +// more complex operations on storage data organized in fields and indexes. +// +// Only type which holds logical information about swarm storage chunks data +// and metadata is IndexItem. This part is not generalized mostly for +// performance reasons. +package shed + +import ( + "github.com/ethereum/go-ethereum/metrics" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/iterator" + "github.com/syndtr/goleveldb/leveldb/opt" +) + +// The limit for LevelDB OpenFilesCacheCapacity. +const openFileLimit = 128 + +// DB provides abstractions over LevelDB in order to +// implement complex structures using fields and ordered indexes. +// It provides a schema functionality to store fields and indexes +// information about naming and types. +type DB struct { + ldb *leveldb.DB +} + +// NewDB constructs a new DB and validates the schema +// if it exists in database on the given path. +func NewDB(path string) (db *DB, err error) { + ldb, err := leveldb.OpenFile(path, &opt.Options{ + OpenFilesCacheCapacity: openFileLimit, + }) + if err != nil { + return nil, err + } + db = &DB{ + ldb: ldb, + } + + if _, err = db.getSchema(); err != nil { + if err == leveldb.ErrNotFound { + // save schema with initialized default fields + if err = db.putSchema(schema{ + Fields: make(map[string]fieldSpec), + Indexes: make(map[byte]indexSpec), + }); err != nil { + return nil, err + } + } else { + return nil, err + } + } + return db, nil +} + +// Put wraps LevelDB Put method to increment metrics counter. +func (db *DB) Put(key []byte, value []byte) (err error) { + err = db.ldb.Put(key, value, nil) + if err != nil { + metrics.GetOrRegisterCounter("DB.putFail", nil).Inc(1) + return err + } + metrics.GetOrRegisterCounter("DB.put", nil).Inc(1) + return nil +} + +// Get wraps LevelDB Get method to increment metrics counter. +func (db *DB) Get(key []byte) (value []byte, err error) { + value, err = db.ldb.Get(key, nil) + if err != nil { + if err == leveldb.ErrNotFound { + metrics.GetOrRegisterCounter("DB.getNotFound", nil).Inc(1) + } else { + metrics.GetOrRegisterCounter("DB.getFail", nil).Inc(1) + } + return nil, err + } + metrics.GetOrRegisterCounter("DB.get", nil).Inc(1) + return value, nil +} + +// Delete wraps LevelDB Delete method to increment metrics counter. +func (db *DB) Delete(key []byte) (err error) { + err = db.ldb.Delete(key, nil) + if err != nil { + metrics.GetOrRegisterCounter("DB.deleteFail", nil).Inc(1) + return err + } + metrics.GetOrRegisterCounter("DB.delete", nil).Inc(1) + return nil +} + +// NewIterator wraps LevelDB NewIterator method to increment metrics counter. +func (db *DB) NewIterator() iterator.Iterator { + metrics.GetOrRegisterCounter("DB.newiterator", nil).Inc(1) + + return db.ldb.NewIterator(nil, nil) +} + +// WriteBatch wraps LevelDB Write method to increment metrics counter. +func (db *DB) WriteBatch(batch *leveldb.Batch) (err error) { + err = db.ldb.Write(batch, nil) + if err != nil { + metrics.GetOrRegisterCounter("DB.writebatchFail", nil).Inc(1) + return err + } + metrics.GetOrRegisterCounter("DB.writebatch", nil).Inc(1) + return nil +} + +// Close closes LevelDB database. +func (db *DB) Close() (err error) { + return db.ldb.Close() +} diff --git a/swarm/shed/db_test.go b/swarm/shed/db_test.go new file mode 100644 index 0000000000..45325beeb8 --- /dev/null +++ b/swarm/shed/db_test.go @@ -0,0 +1,110 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package shed + +import ( + "io/ioutil" + "os" + "testing" +) + +// TestNewDB constructs a new DB +// and validates if the schema is initialized properly. +func TestNewDB(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + s, err := db.getSchema() + if err != nil { + t.Fatal(err) + } + if s.Fields == nil { + t.Error("schema fields are empty") + } + if len(s.Fields) != 0 { + t.Errorf("got schema fields length %v, want %v", len(s.Fields), 0) + } + if s.Indexes == nil { + t.Error("schema indexes are empty") + } + if len(s.Indexes) != 0 { + t.Errorf("got schema indexes length %v, want %v", len(s.Indexes), 0) + } +} + +// TestDB_persistence creates one DB, saves a field and closes that DB. +// Then, it constructs another DB and trues to retrieve the saved value. +func TestDB_persistence(t *testing.T) { + dir, err := ioutil.TempDir("", "shed-test-persistence") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + db, err := NewDB(dir) + if err != nil { + t.Fatal(err) + } + stringField, err := db.NewStringField("preserve-me") + if err != nil { + t.Fatal(err) + } + want := "persistent value" + err = stringField.Put(want) + if err != nil { + t.Fatal(err) + } + err = db.Close() + if err != nil { + t.Fatal(err) + } + + db2, err := NewDB(dir) + if err != nil { + t.Fatal(err) + } + stringField2, err := db2.NewStringField("preserve-me") + if err != nil { + t.Fatal(err) + } + got, err := stringField2.Get() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got string %q, want %q", got, want) + } +} + +// newTestDB is a helper function that constructs a +// temporary database and returns a cleanup function that must +// be called to remove the data. +func newTestDB(t *testing.T) (db *DB, cleanupFunc func()) { + t.Helper() + + dir, err := ioutil.TempDir("", "shed-test") + if err != nil { + t.Fatal(err) + } + cleanupFunc = func() { os.RemoveAll(dir) } + db, err = NewDB(dir) + if err != nil { + cleanupFunc() + t.Fatal(err) + } + return db, cleanupFunc +} diff --git a/swarm/shed/example_store_test.go b/swarm/shed/example_store_test.go new file mode 100644 index 0000000000..2ed0be1413 --- /dev/null +++ b/swarm/shed/example_store_test.go @@ -0,0 +1,332 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package shed_test + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + "io/ioutil" + "log" + "os" + "time" + + "github.com/ethereum/go-ethereum/swarm/shed" + "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/syndtr/goleveldb/leveldb" +) + +// Store holds fields and indexes (including their encoding functions) +// and defines operations on them by composing data from them. +// It implements storage.ChunkStore interface. +// It is just an example without any support for parallel operations +// or real world implementation. +type Store struct { + db *shed.DB + + // fields and indexes + schemaName shed.StringField + sizeCounter shed.Uint64Field + accessCounter shed.Uint64Field + retrievalIndex shed.Index + accessIndex shed.Index + gcIndex shed.Index +} + +// New returns new Store. All fields and indexes are initialized +// and possible conflicts with schema from existing database is checked +// automatically. +func New(path string) (s *Store, err error) { + db, err := shed.NewDB(path) + if err != nil { + return nil, err + } + s = &Store{ + db: db, + } + // Identify current storage schema by arbitrary name. + s.schemaName, err = db.NewStringField("schema-name") + if err != nil { + return nil, err + } + // Global ever incrementing index of chunk accesses. + s.accessCounter, err = db.NewUint64Field("access-counter") + if err != nil { + return nil, err + } + // Index storing actual chunk address, data and store timestamp. + s.retrievalIndex, err = db.NewIndex("Address->StoreTimestamp|Data", shed.IndexFuncs{ + EncodeKey: func(fields shed.IndexItem) (key []byte, err error) { + return fields.Address, nil + }, + DecodeKey: func(key []byte) (e shed.IndexItem, err error) { + e.Address = key + return e, nil + }, + EncodeValue: func(fields shed.IndexItem) (value []byte, err error) { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(fields.StoreTimestamp)) + value = append(b, fields.Data...) + return value, nil + }, + DecodeValue: func(value []byte) (e shed.IndexItem, err error) { + e.StoreTimestamp = int64(binary.BigEndian.Uint64(value[:8])) + e.Data = value[8:] + return e, nil + }, + }) + if err != nil { + return nil, err + } + // Index storing access timestamp for a particular address. + // It is needed in order to update gc index keys for iteration order. + s.accessIndex, err = db.NewIndex("Address->AccessTimestamp", shed.IndexFuncs{ + EncodeKey: func(fields shed.IndexItem) (key []byte, err error) { + return fields.Address, nil + }, + DecodeKey: func(key []byte) (e shed.IndexItem, err error) { + e.Address = key + return e, nil + }, + EncodeValue: func(fields shed.IndexItem) (value []byte, err error) { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(fields.AccessTimestamp)) + return b, nil + }, + DecodeValue: func(value []byte) (e shed.IndexItem, err error) { + e.AccessTimestamp = int64(binary.BigEndian.Uint64(value)) + return e, nil + }, + }) + if err != nil { + return nil, err + } + // Index with keys ordered by access timestamp for garbage collection prioritization. + s.gcIndex, err = db.NewIndex("AccessTimestamp|StoredTimestamp|Address->nil", shed.IndexFuncs{ + EncodeKey: func(fields shed.IndexItem) (key []byte, err error) { + b := make([]byte, 16, 16+len(fields.Address)) + binary.BigEndian.PutUint64(b[:8], uint64(fields.AccessTimestamp)) + binary.BigEndian.PutUint64(b[8:16], uint64(fields.StoreTimestamp)) + key = append(b, fields.Address...) + return key, nil + }, + DecodeKey: func(key []byte) (e shed.IndexItem, err error) { + e.AccessTimestamp = int64(binary.BigEndian.Uint64(key[:8])) + e.StoreTimestamp = int64(binary.BigEndian.Uint64(key[8:16])) + e.Address = key[16:] + return e, nil + }, + EncodeValue: func(fields shed.IndexItem) (value []byte, err error) { + return nil, nil + }, + DecodeValue: func(value []byte) (e shed.IndexItem, err error) { + return e, nil + }, + }) + if err != nil { + return nil, err + } + return s, nil +} + +// Put stores the chunk and sets it store timestamp. +func (s *Store) Put(_ context.Context, ch storage.Chunk) (err error) { + return s.retrievalIndex.Put(shed.IndexItem{ + Address: ch.Address(), + Data: ch.Data(), + StoreTimestamp: time.Now().UTC().UnixNano(), + }) +} + +// Get retrieves a chunk with the provided address. +// It updates access and gc indexes by removing the previous +// items from them and adding new items as keys of index entries +// are changed. +func (s *Store) Get(_ context.Context, addr storage.Address) (c storage.Chunk, err error) { + batch := new(leveldb.Batch) + + // Get the chunk data and storage timestamp. + item, err := s.retrievalIndex.Get(shed.IndexItem{ + Address: addr, + }) + if err != nil { + if err == leveldb.ErrNotFound { + return nil, storage.ErrChunkNotFound + } + return nil, err + } + + // Get the chunk access timestamp. + accessItem, err := s.accessIndex.Get(shed.IndexItem{ + Address: addr, + }) + switch err { + case nil: + // Remove gc index entry if access timestamp is found. + err = s.gcIndex.DeleteInBatch(batch, shed.IndexItem{ + Address: item.Address, + StoreTimestamp: accessItem.AccessTimestamp, + AccessTimestamp: item.StoreTimestamp, + }) + if err != nil { + return nil, err + } + case leveldb.ErrNotFound: + // Access timestamp is not found. Do not do anything. + // This is the firs get request. + default: + return nil, err + } + + // Specify new access timestamp + accessTimestamp := time.Now().UTC().UnixNano() + + // Put new access timestamp in access index. + err = s.accessIndex.PutInBatch(batch, shed.IndexItem{ + Address: addr, + AccessTimestamp: accessTimestamp, + }) + if err != nil { + return nil, err + } + + // Put new access timestamp in gc index. + err = s.gcIndex.PutInBatch(batch, shed.IndexItem{ + Address: item.Address, + AccessTimestamp: accessTimestamp, + StoreTimestamp: item.StoreTimestamp, + }) + if err != nil { + return nil, err + } + + // Increment access counter. + // Currently this information is not used anywhere. + _, err = s.accessCounter.IncInBatch(batch) + if err != nil { + return nil, err + } + + // Write the batch. + err = s.db.WriteBatch(batch) + if err != nil { + return nil, err + } + + // Return the chunk. + return storage.NewChunk(item.Address, item.Data), nil +} + +// CollectGarbage is an example of index iteration. +// It provides no reliable garbage collection functionality. +func (s *Store) CollectGarbage() (err error) { + const maxTrashSize = 100 + maxRounds := 10 // arbitrary number, needs to be calculated + + // Run a few gc rounds. + for roundCount := 0; roundCount < maxRounds; roundCount++ { + var garbageCount int + // New batch for a new cg round. + trash := new(leveldb.Batch) + // Iterate through all index items and break when needed. + err = s.gcIndex.IterateAll(func(item shed.IndexItem) (stop bool, err error) { + // Remove the chunk. + err = s.retrievalIndex.DeleteInBatch(trash, item) + if err != nil { + return false, err + } + // Remove the element in gc index. + err = s.gcIndex.DeleteInBatch(trash, item) + if err != nil { + return false, err + } + // Remove the relation in access index. + err = s.accessIndex.DeleteInBatch(trash, item) + if err != nil { + return false, err + } + garbageCount++ + if garbageCount >= maxTrashSize { + return true, nil + } + return false, nil + }) + if err != nil { + return err + } + if garbageCount == 0 { + return nil + } + err = s.db.WriteBatch(trash) + if err != nil { + return err + } + } + return nil +} + +// GetSchema is an example of retrieveing the most simple +// string from a database field. +func (s *Store) GetSchema() (name string, err error) { + name, err = s.schemaName.Get() + if err == leveldb.ErrNotFound { + return "", nil + } + return name, err +} + +// GetSchema is an example of storing the most simple +// string in a database field. +func (s *Store) PutSchema(name string) (err error) { + return s.schemaName.Put(name) +} + +// Close closes the underlying database. +func (s *Store) Close() error { + return s.db.Close() +} + +// Example_store constructs a simple storage implementation using shed package. +func Example_store() { + dir, err := ioutil.TempDir("", "ephemeral") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(dir) + + s, err := New(dir) + if err != nil { + log.Fatal(err) + } + defer s.Close() + + ch := storage.GenerateRandomChunk(1024) + err = s.Put(context.Background(), ch) + if err != nil { + log.Fatal(err) + } + + got, err := s.Get(context.Background(), ch.Address()) + if err != nil { + log.Fatal(err) + } + + fmt.Println(bytes.Equal(got.Data(), ch.Data())) + + //Output: true +} diff --git a/swarm/shed/field_string.go b/swarm/shed/field_string.go new file mode 100644 index 0000000000..a7e8f0c754 --- /dev/null +++ b/swarm/shed/field_string.go @@ -0,0 +1,66 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package shed + +import ( + "github.com/syndtr/goleveldb/leveldb" +) + +// StringField is the most simple field implementation +// that stores an arbitrary string under a specific LevelDB key. +type StringField struct { + db *DB + key []byte +} + +// NewStringField retruns a new Instance of StringField. +// It validates its name and type against the database schema. +func (db *DB) NewStringField(name string) (f StringField, err error) { + key, err := db.schemaFieldKey(name, "string") + if err != nil { + return f, err + } + return StringField{ + db: db, + key: key, + }, nil +} + +// Get returns a string value from database. +// If the value is not found, an empty string is returned +// an no error. +func (f StringField) Get() (val string, err error) { + b, err := f.db.Get(f.key) + if err != nil { + if err == leveldb.ErrNotFound { + return "", nil + } + return "", err + } + return string(b), nil +} + +// Put stores a string in the database. +func (f StringField) Put(val string) (err error) { + return f.db.Put(f.key, []byte(val)) +} + +// PutInBatch stores a string in a batch that can be +// saved later in database. +func (f StringField) PutInBatch(batch *leveldb.Batch, val string) { + batch.Put(f.key, []byte(val)) +} diff --git a/swarm/shed/field_string_test.go b/swarm/shed/field_string_test.go new file mode 100644 index 0000000000..4215075bca --- /dev/null +++ b/swarm/shed/field_string_test.go @@ -0,0 +1,110 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package shed + +import ( + "testing" + + "github.com/syndtr/goleveldb/leveldb" +) + +// TestStringField validates put and get operations +// of the StringField. +func TestStringField(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + simpleString, err := db.NewStringField("simple-string") + if err != nil { + t.Fatal(err) + } + + t.Run("get empty", func(t *testing.T) { + got, err := simpleString.Get() + if err != nil { + t.Fatal(err) + } + want := "" + if got != want { + t.Errorf("got string %q, want %q", got, want) + } + }) + + t.Run("put", func(t *testing.T) { + want := "simple string value" + err = simpleString.Put(want) + if err != nil { + t.Fatal(err) + } + got, err := simpleString.Get() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got string %q, want %q", got, want) + } + + t.Run("overwrite", func(t *testing.T) { + want := "overwritten string value" + err = simpleString.Put(want) + if err != nil { + t.Fatal(err) + } + got, err := simpleString.Get() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got string %q, want %q", got, want) + } + }) + }) + + t.Run("put in batch", func(t *testing.T) { + batch := new(leveldb.Batch) + want := "simple string batch value" + simpleString.PutInBatch(batch, want) + err = db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + got, err := simpleString.Get() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got string %q, want %q", got, want) + } + + t.Run("overwrite", func(t *testing.T) { + batch := new(leveldb.Batch) + want := "overwritten string batch value" + simpleString.PutInBatch(batch, want) + err = db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + got, err := simpleString.Get() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got string %q, want %q", got, want) + } + }) + }) +} diff --git a/swarm/shed/field_struct.go b/swarm/shed/field_struct.go new file mode 100644 index 0000000000..90daee7fc4 --- /dev/null +++ b/swarm/shed/field_struct.go @@ -0,0 +1,71 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package shed + +import ( + "github.com/ethereum/go-ethereum/rlp" + "github.com/syndtr/goleveldb/leveldb" +) + +// StructField is a helper to store complex structure by +// encoding it in RLP format. +type StructField struct { + db *DB + key []byte +} + +// NewStructField returns a new StructField. +// It validates its name and type against the database schema. +func (db *DB) NewStructField(name string) (f StructField, err error) { + key, err := db.schemaFieldKey(name, "struct-rlp") + if err != nil { + return f, err + } + return StructField{ + db: db, + key: key, + }, nil +} + +// Get unmarshals data from the database to a provided val. +// If the data is not found leveldb.ErrNotFound is returned. +func (f StructField) Get(val interface{}) (err error) { + b, err := f.db.Get(f.key) + if err != nil { + return err + } + return rlp.DecodeBytes(b, val) +} + +// Put marshals provided val and saves it to the database. +func (f StructField) Put(val interface{}) (err error) { + b, err := rlp.EncodeToBytes(val) + if err != nil { + return err + } + return f.db.Put(f.key, b) +} + +// PutInBatch marshals provided val and puts it into the batch. +func (f StructField) PutInBatch(batch *leveldb.Batch, val interface{}) (err error) { + b, err := rlp.EncodeToBytes(val) + if err != nil { + return err + } + batch.Put(f.key, b) + return nil +} diff --git a/swarm/shed/field_struct_test.go b/swarm/shed/field_struct_test.go new file mode 100644 index 0000000000..cc0be01863 --- /dev/null +++ b/swarm/shed/field_struct_test.go @@ -0,0 +1,127 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package shed + +import ( + "testing" + + "github.com/syndtr/goleveldb/leveldb" +) + +// TestStructField validates put and get operations +// of the StructField. +func TestStructField(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + complexField, err := db.NewStructField("complex-field") + if err != nil { + t.Fatal(err) + } + + type complexStructure struct { + A string + } + + t.Run("get empty", func(t *testing.T) { + var s complexStructure + err := complexField.Get(&s) + if err != leveldb.ErrNotFound { + t.Fatalf("got error %v, want %v", err, leveldb.ErrNotFound) + } + want := "" + if s.A != want { + t.Errorf("got string %q, want %q", s.A, want) + } + }) + + t.Run("put", func(t *testing.T) { + want := complexStructure{ + A: "simple string value", + } + err = complexField.Put(want) + if err != nil { + t.Fatal(err) + } + var got complexStructure + err = complexField.Get(&got) + if err != nil { + t.Fatal(err) + } + if got.A != want.A { + t.Errorf("got string %q, want %q", got.A, want.A) + } + + t.Run("overwrite", func(t *testing.T) { + want := complexStructure{ + A: "overwritten string value", + } + err = complexField.Put(want) + if err != nil { + t.Fatal(err) + } + var got complexStructure + err = complexField.Get(&got) + if err != nil { + t.Fatal(err) + } + if got.A != want.A { + t.Errorf("got string %q, want %q", got.A, want.A) + } + }) + }) + + t.Run("put in batch", func(t *testing.T) { + batch := new(leveldb.Batch) + want := complexStructure{ + A: "simple string batch value", + } + complexField.PutInBatch(batch, want) + err = db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + var got complexStructure + err := complexField.Get(&got) + if err != nil { + t.Fatal(err) + } + if got.A != want.A { + t.Errorf("got string %q, want %q", got, want) + } + + t.Run("overwrite", func(t *testing.T) { + batch := new(leveldb.Batch) + want := complexStructure{ + A: "overwritten string batch value", + } + complexField.PutInBatch(batch, want) + err = db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + var got complexStructure + err := complexField.Get(&got) + if err != nil { + t.Fatal(err) + } + if got.A != want.A { + t.Errorf("got string %q, want %q", got, want) + } + }) + }) +} diff --git a/swarm/shed/field_uint64.go b/swarm/shed/field_uint64.go new file mode 100644 index 0000000000..80e0069ae4 --- /dev/null +++ b/swarm/shed/field_uint64.go @@ -0,0 +1,108 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package shed + +import ( + "encoding/binary" + + "github.com/syndtr/goleveldb/leveldb" +) + +// Uint64Field provides a way to have a simple counter in the database. +// It transparently encodes uint64 type value to bytes. +type Uint64Field struct { + db *DB + key []byte +} + +// NewUint64Field returns a new Uint64Field. +// It validates its name and type against the database schema. +func (db *DB) NewUint64Field(name string) (f Uint64Field, err error) { + key, err := db.schemaFieldKey(name, "uint64") + if err != nil { + return f, err + } + return Uint64Field{ + db: db, + key: key, + }, nil +} + +// Get retrieves a uint64 value from the database. +// If the value is not found in the database a 0 value +// is returned and no error. +func (f Uint64Field) Get() (val uint64, err error) { + b, err := f.db.Get(f.key) + if err != nil { + if err == leveldb.ErrNotFound { + return 0, nil + } + return 0, err + } + return binary.BigEndian.Uint64(b), nil +} + +// Put encodes uin64 value and stores it in the database. +func (f Uint64Field) Put(val uint64) (err error) { + return f.db.Put(f.key, encodeUint64(val)) +} + +// PutInBatch stores a uint64 value in a batch +// that can be saved later in the database. +func (f Uint64Field) PutInBatch(batch *leveldb.Batch, val uint64) { + batch.Put(f.key, encodeUint64(val)) +} + +// Inc increments a uint64 value in the database. +// This operation is not goroutine save. +func (f Uint64Field) Inc() (val uint64, err error) { + val, err = f.Get() + if err != nil { + if err == leveldb.ErrNotFound { + val = 0 + } else { + return 0, err + } + } + val++ + return val, f.Put(val) +} + +// IncInBatch increments a uint64 value in the batch +// by retreiving a value from the database, not the same batch. +// This operation is not goroutine save. +func (f Uint64Field) IncInBatch(batch *leveldb.Batch) (val uint64, err error) { + val, err = f.Get() + if err != nil { + if err == leveldb.ErrNotFound { + val = 0 + } else { + return 0, err + } + } + val++ + f.PutInBatch(batch, val) + return val, nil +} + +// encode transforms uint64 to 8 byte long +// slice in big endian encoding. +func encodeUint64(val uint64) (b []byte) { + b = make([]byte, 8) + binary.BigEndian.PutUint64(b, val) + return b +} diff --git a/swarm/shed/field_uint64_test.go b/swarm/shed/field_uint64_test.go new file mode 100644 index 0000000000..69ade71ba3 --- /dev/null +++ b/swarm/shed/field_uint64_test.go @@ -0,0 +1,194 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package shed + +import ( + "testing" + + "github.com/syndtr/goleveldb/leveldb" +) + +// TestUint64Field validates put and get operations +// of the Uint64Field. +func TestUint64Field(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + counter, err := db.NewUint64Field("counter") + if err != nil { + t.Fatal(err) + } + + t.Run("get empty", func(t *testing.T) { + got, err := counter.Get() + if err != nil { + t.Fatal(err) + } + var want uint64 + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } + }) + + t.Run("put", func(t *testing.T) { + var want uint64 = 42 + err = counter.Put(want) + if err != nil { + t.Fatal(err) + } + got, err := counter.Get() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } + + t.Run("overwrite", func(t *testing.T) { + var want uint64 = 84 + err = counter.Put(want) + if err != nil { + t.Fatal(err) + } + got, err := counter.Get() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } + }) + }) + + t.Run("put in batch", func(t *testing.T) { + batch := new(leveldb.Batch) + var want uint64 = 42 + counter.PutInBatch(batch, want) + err = db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + got, err := counter.Get() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } + + t.Run("overwrite", func(t *testing.T) { + batch := new(leveldb.Batch) + var want uint64 = 84 + counter.PutInBatch(batch, want) + err = db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + got, err := counter.Get() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } + }) + }) +} + +// TestUint64Field_Inc validates Inc operation +// of the Uint64Field. +func TestUint64Field_Inc(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + counter, err := db.NewUint64Field("counter") + if err != nil { + t.Fatal(err) + } + + var want uint64 = 1 + got, err := counter.Inc() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } + + want = 2 + got, err = counter.Inc() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } +} + +// TestUint64Field_IncInBatch validates IncInBatch operation +// of the Uint64Field. +func TestUint64Field_IncInBatch(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + counter, err := db.NewUint64Field("counter") + if err != nil { + t.Fatal(err) + } + + batch := new(leveldb.Batch) + var want uint64 = 1 + got, err := counter.IncInBatch(batch) + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } + err = db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + got, err = counter.Get() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } + + batch2 := new(leveldb.Batch) + want = 2 + got, err = counter.IncInBatch(batch2) + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } + err = db.WriteBatch(batch2) + if err != nil { + t.Fatal(err) + } + got, err = counter.Get() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } +} diff --git a/swarm/shed/index.go b/swarm/shed/index.go new file mode 100644 index 0000000000..ba803e3c23 --- /dev/null +++ b/swarm/shed/index.go @@ -0,0 +1,264 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package shed + +import ( + "github.com/syndtr/goleveldb/leveldb" +) + +// IndexItem holds fields relevant to Swarm Chunk data and metadata. +// All information required for swarm storage and operations +// on that storage must be defined here. +// This structure is logically connected to swarm storage, +// the only part of this package that is not generalized, +// mostly for performance reasons. +// +// IndexItem is a type that is used for retrieving, storing and encoding +// chunk data and metadata. It is passed as an argument to Index encoding +// functions, get function and put function. +// But it is also returned with additional data from get function call +// and as the argument in iterator function definition. +type IndexItem struct { + Address []byte + Data []byte + AccessTimestamp int64 + StoreTimestamp int64 + // UseMockStore is a pointer to identify + // an unset state of the field in Join function. + UseMockStore *bool +} + +// Merge is a helper method to construct a new +// IndexItem by filling up fields with default values +// of a particular IndexItem with values from another one. +func (i IndexItem) Merge(i2 IndexItem) (new IndexItem) { + if i.Address == nil { + i.Address = i2.Address + } + if i.Data == nil { + i.Data = i2.Data + } + if i.AccessTimestamp == 0 { + i.AccessTimestamp = i2.AccessTimestamp + } + if i.StoreTimestamp == 0 { + i.StoreTimestamp = i2.StoreTimestamp + } + if i.UseMockStore == nil { + i.UseMockStore = i2.UseMockStore + } + return i +} + +// Index represents a set of LevelDB key value pairs that have common +// prefix. It holds functions for encoding and decoding keys and values +// to provide transparent actions on saved data which inclide: +// - getting a particular IndexItem +// - saving a particular IndexItem +// - iterating over a sorted LevelDB keys +// It implements IndexIteratorInterface interface. +type Index struct { + db *DB + prefix []byte + encodeKeyFunc func(fields IndexItem) (key []byte, err error) + decodeKeyFunc func(key []byte) (e IndexItem, err error) + encodeValueFunc func(fields IndexItem) (value []byte, err error) + decodeValueFunc func(value []byte) (e IndexItem, err error) +} + +// IndexFuncs structure defines functions for encoding and decoding +// LevelDB keys and values for a specific index. +type IndexFuncs struct { + EncodeKey func(fields IndexItem) (key []byte, err error) + DecodeKey func(key []byte) (e IndexItem, err error) + EncodeValue func(fields IndexItem) (value []byte, err error) + DecodeValue func(value []byte) (e IndexItem, err error) +} + +// NewIndex returns a new Index instance with defined name and +// encoding functions. The name must be unique and will be validated +// on database schema for a key prefix byte. +func (db *DB) NewIndex(name string, funcs IndexFuncs) (f Index, err error) { + id, err := db.schemaIndexPrefix(name) + if err != nil { + return f, err + } + prefix := []byte{id} + return Index{ + db: db, + prefix: prefix, + // This function adjusts Index LevelDB key + // by appending the provided index id byte. + // This is needed to avoid collisions between keys of different + // indexes as all index ids are unique. + encodeKeyFunc: func(e IndexItem) (key []byte, err error) { + key, err = funcs.EncodeKey(e) + if err != nil { + return nil, err + } + return append(append(make([]byte, 0, len(key)+1), prefix...), key...), nil + }, + // This function reverses the encodeKeyFunc constructed key + // to transparently work with index keys without their index ids. + // It assumes that index keys are prefixed with only one byte. + decodeKeyFunc: func(key []byte) (e IndexItem, err error) { + return funcs.DecodeKey(key[1:]) + }, + encodeValueFunc: funcs.EncodeValue, + decodeValueFunc: funcs.DecodeValue, + }, nil +} + +// Get accepts key fields represented as IndexItem to retrieve a +// value from the index and return maximum available information +// from the index represented as another IndexItem. +func (f Index) Get(keyFields IndexItem) (out IndexItem, err error) { + key, err := f.encodeKeyFunc(keyFields) + if err != nil { + return out, err + } + value, err := f.db.Get(key) + if err != nil { + return out, err + } + out, err = f.decodeValueFunc(value) + if err != nil { + return out, err + } + return out.Merge(keyFields), nil +} + +// Put accepts IndexItem to encode information from it +// and save it to the database. +func (f Index) Put(i IndexItem) (err error) { + key, err := f.encodeKeyFunc(i) + if err != nil { + return err + } + value, err := f.encodeValueFunc(i) + if err != nil { + return err + } + return f.db.Put(key, value) +} + +// PutInBatch is the same as Put method, but it just +// saves the key/value pair to the batch instead +// directly to the database. +func (f Index) PutInBatch(batch *leveldb.Batch, i IndexItem) (err error) { + key, err := f.encodeKeyFunc(i) + if err != nil { + return err + } + value, err := f.encodeValueFunc(i) + if err != nil { + return err + } + batch.Put(key, value) + return nil +} + +// Delete accepts IndexItem to remove a key/value pair +// from the database based on its fields. +func (f Index) Delete(keyFields IndexItem) (err error) { + key, err := f.encodeKeyFunc(keyFields) + if err != nil { + return err + } + return f.db.Delete(key) +} + +// DeleteInBatch is the same as Delete just the operation +// is performed on the batch instead on the database. +func (f Index) DeleteInBatch(batch *leveldb.Batch, keyFields IndexItem) (err error) { + key, err := f.encodeKeyFunc(keyFields) + if err != nil { + return err + } + batch.Delete(key) + return nil +} + +// IndexIterFunc is a callback on every IndexItem that is decoded +// by iterating on an Index keys. +// By returning a true for stop variable, iteration will +// stop, and by returning the error, that error will be +// propagated to the called iterator method on Index. +type IndexIterFunc func(item IndexItem) (stop bool, err error) + +// IterateAll iterates over all keys of the Index. +func (f Index) IterateAll(fn IndexIterFunc) (err error) { + it := f.db.NewIterator() + defer it.Release() + + for ok := it.Seek(f.prefix); ok; ok = it.Next() { + key := it.Key() + if key[0] != f.prefix[0] { + break + } + keyIndexItem, err := f.decodeKeyFunc(key) + if err != nil { + return err + } + valueIndexItem, err := f.decodeValueFunc(it.Value()) + if err != nil { + return err + } + stop, err := fn(keyIndexItem.Merge(valueIndexItem)) + if err != nil { + return err + } + if stop { + break + } + } + return it.Error() +} + +// IterateFrom iterates over Index keys starting from the key +// encoded from the provided IndexItem. +func (f Index) IterateFrom(start IndexItem, fn IndexIterFunc) (err error) { + startKey, err := f.encodeKeyFunc(start) + if err != nil { + return err + } + it := f.db.NewIterator() + defer it.Release() + + for ok := it.Seek(startKey); ok; ok = it.Next() { + key := it.Key() + if key[0] != f.prefix[0] { + break + } + keyIndexItem, err := f.decodeKeyFunc(key) + if err != nil { + return err + } + valueIndexItem, err := f.decodeValueFunc(it.Value()) + if err != nil { + return err + } + stop, err := fn(keyIndexItem.Merge(valueIndexItem)) + if err != nil { + return err + } + if stop { + break + } + } + return it.Error() +} diff --git a/swarm/shed/index_test.go b/swarm/shed/index_test.go new file mode 100644 index 0000000000..ba82216dfe --- /dev/null +++ b/swarm/shed/index_test.go @@ -0,0 +1,426 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package shed + +import ( + "bytes" + "encoding/binary" + "fmt" + "sort" + "testing" + "time" + + "github.com/syndtr/goleveldb/leveldb" +) + +// Index functions for the index that is used in tests in this file. +var retrievalIndexFuncs = IndexFuncs{ + EncodeKey: func(fields IndexItem) (key []byte, err error) { + return fields.Address, nil + }, + DecodeKey: func(key []byte) (e IndexItem, err error) { + e.Address = key + return e, nil + }, + EncodeValue: func(fields IndexItem) (value []byte, err error) { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(fields.StoreTimestamp)) + value = append(b, fields.Data...) + return value, nil + }, + DecodeValue: func(value []byte) (e IndexItem, err error) { + e.StoreTimestamp = int64(binary.BigEndian.Uint64(value[:8])) + e.Data = value[8:] + return e, nil + }, +} + +// TestIndex validates put, get and delete functions of the Index implementation. +func TestIndex(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + index, err := db.NewIndex("retrieval", retrievalIndexFuncs) + if err != nil { + t.Fatal(err) + } + + t.Run("put", func(t *testing.T) { + want := IndexItem{ + Address: []byte("put-hash"), + Data: []byte("DATA"), + StoreTimestamp: time.Now().UTC().UnixNano(), + } + + err := index.Put(want) + if err != nil { + t.Fatal(err) + } + got, err := index.Get(IndexItem{ + Address: want.Address, + }) + if err != nil { + t.Fatal(err) + } + checkIndexItem(t, got, want) + + t.Run("overwrite", func(t *testing.T) { + want := IndexItem{ + Address: []byte("put-hash"), + Data: []byte("New DATA"), + StoreTimestamp: time.Now().UTC().UnixNano(), + } + + err = index.Put(want) + if err != nil { + t.Fatal(err) + } + got, err := index.Get(IndexItem{ + Address: want.Address, + }) + if err != nil { + t.Fatal(err) + } + checkIndexItem(t, got, want) + }) + }) + + t.Run("put in batch", func(t *testing.T) { + want := IndexItem{ + Address: []byte("put-in-batch-hash"), + Data: []byte("DATA"), + StoreTimestamp: time.Now().UTC().UnixNano(), + } + + batch := new(leveldb.Batch) + index.PutInBatch(batch, want) + err := db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + got, err := index.Get(IndexItem{ + Address: want.Address, + }) + if err != nil { + t.Fatal(err) + } + checkIndexItem(t, got, want) + + t.Run("overwrite", func(t *testing.T) { + want := IndexItem{ + Address: []byte("put-in-batch-hash"), + Data: []byte("New DATA"), + StoreTimestamp: time.Now().UTC().UnixNano(), + } + + batch := new(leveldb.Batch) + index.PutInBatch(batch, want) + db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + got, err := index.Get(IndexItem{ + Address: want.Address, + }) + if err != nil { + t.Fatal(err) + } + checkIndexItem(t, got, want) + }) + }) + + t.Run("put in batch twice", func(t *testing.T) { + // ensure that the last item of items with the same db keys + // is actually saved + batch := new(leveldb.Batch) + address := []byte("put-in-batch-twice-hash") + + // put the first item + index.PutInBatch(batch, IndexItem{ + Address: address, + Data: []byte("DATA"), + StoreTimestamp: time.Now().UTC().UnixNano(), + }) + + want := IndexItem{ + Address: address, + Data: []byte("New DATA"), + StoreTimestamp: time.Now().UTC().UnixNano(), + } + // then put the item that will produce the same key + // but different value in the database + index.PutInBatch(batch, want) + db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + got, err := index.Get(IndexItem{ + Address: address, + }) + if err != nil { + t.Fatal(err) + } + checkIndexItem(t, got, want) + }) + + t.Run("delete", func(t *testing.T) { + want := IndexItem{ + Address: []byte("delete-hash"), + Data: []byte("DATA"), + StoreTimestamp: time.Now().UTC().UnixNano(), + } + + err := index.Put(want) + if err != nil { + t.Fatal(err) + } + got, err := index.Get(IndexItem{ + Address: want.Address, + }) + if err != nil { + t.Fatal(err) + } + checkIndexItem(t, got, want) + + err = index.Delete(IndexItem{ + Address: want.Address, + }) + if err != nil { + t.Fatal(err) + } + + wantErr := leveldb.ErrNotFound + got, err = index.Get(IndexItem{ + Address: want.Address, + }) + if err != wantErr { + t.Fatalf("got error %v, want %v", err, wantErr) + } + }) + + t.Run("delete in batch", func(t *testing.T) { + want := IndexItem{ + Address: []byte("delete-in-batch-hash"), + Data: []byte("DATA"), + StoreTimestamp: time.Now().UTC().UnixNano(), + } + + err := index.Put(want) + if err != nil { + t.Fatal(err) + } + got, err := index.Get(IndexItem{ + Address: want.Address, + }) + if err != nil { + t.Fatal(err) + } + checkIndexItem(t, got, want) + + batch := new(leveldb.Batch) + index.DeleteInBatch(batch, IndexItem{ + Address: want.Address, + }) + err = db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + + wantErr := leveldb.ErrNotFound + got, err = index.Get(IndexItem{ + Address: want.Address, + }) + if err != wantErr { + t.Fatalf("got error %v, want %v", err, wantErr) + } + }) +} + +// TestIndex_iterate validates index iterator functions for correctness. +func TestIndex_iterate(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + index, err := db.NewIndex("retrieval", retrievalIndexFuncs) + if err != nil { + t.Fatal(err) + } + + items := []IndexItem{ + { + Address: []byte("iterate-hash-01"), + Data: []byte("data80"), + }, + { + Address: []byte("iterate-hash-03"), + Data: []byte("data22"), + }, + { + Address: []byte("iterate-hash-05"), + Data: []byte("data41"), + }, + { + Address: []byte("iterate-hash-02"), + Data: []byte("data84"), + }, + { + Address: []byte("iterate-hash-06"), + Data: []byte("data1"), + }, + } + batch := new(leveldb.Batch) + for _, i := range items { + index.PutInBatch(batch, i) + } + err = db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + item04 := IndexItem{ + Address: []byte("iterate-hash-04"), + Data: []byte("data0"), + } + err = index.Put(item04) + if err != nil { + t.Fatal(err) + } + items = append(items, item04) + + sort.SliceStable(items, func(i, j int) bool { + return bytes.Compare(items[i].Address, items[j].Address) < 0 + }) + + t.Run("all", func(t *testing.T) { + var i int + err := index.IterateAll(func(item IndexItem) (stop bool, err error) { + if i > len(items)-1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + want := items[i] + checkIndexItem(t, item, want) + i++ + return false, nil + }) + if err != nil { + t.Fatal(err) + } + }) + + t.Run("from", func(t *testing.T) { + startIndex := 2 + i := startIndex + err := index.IterateFrom(items[startIndex], func(item IndexItem) (stop bool, err error) { + if i > len(items)-1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + want := items[i] + checkIndexItem(t, item, want) + i++ + return false, nil + }) + if err != nil { + t.Fatal(err) + } + }) + + t.Run("stop", func(t *testing.T) { + var i int + stopIndex := 3 + var count int + err := index.IterateAll(func(item IndexItem) (stop bool, err error) { + if i > len(items)-1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + want := items[i] + checkIndexItem(t, item, want) + count++ + if i == stopIndex { + return true, nil + } + i++ + return false, nil + }) + if err != nil { + t.Fatal(err) + } + wantItemsCount := stopIndex + 1 + if count != wantItemsCount { + t.Errorf("got %v items, expected %v", count, wantItemsCount) + } + }) + + t.Run("no overflow", func(t *testing.T) { + secondIndex, err := db.NewIndex("second-index", retrievalIndexFuncs) + if err != nil { + t.Fatal(err) + } + + secondIndexItem := IndexItem{ + Address: []byte("iterate-hash-10"), + Data: []byte("data-second"), + } + err = secondIndex.Put(secondIndexItem) + if err != nil { + t.Fatal(err) + } + + var i int + err = index.IterateAll(func(item IndexItem) (stop bool, err error) { + if i > len(items)-1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + want := items[i] + checkIndexItem(t, item, want) + i++ + return false, nil + }) + if err != nil { + t.Fatal(err) + } + + i = 0 + err = secondIndex.IterateAll(func(item IndexItem) (stop bool, err error) { + if i > 1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + checkIndexItem(t, item, secondIndexItem) + i++ + return false, nil + }) + if err != nil { + t.Fatal(err) + } + }) +} + +// checkIndexItem is a test helper function that compares if two Index items are the same. +func checkIndexItem(t *testing.T, got, want IndexItem) { + t.Helper() + + if !bytes.Equal(got.Address, want.Address) { + t.Errorf("got hash %q, expected %q", string(got.Address), string(want.Address)) + } + if !bytes.Equal(got.Data, want.Data) { + t.Errorf("got data %q, expected %q", string(got.Data), string(want.Data)) + } + if got.StoreTimestamp != want.StoreTimestamp { + t.Errorf("got store timestamp %v, expected %v", got.StoreTimestamp, want.StoreTimestamp) + } + if got.AccessTimestamp != want.AccessTimestamp { + t.Errorf("got access timestamp %v, expected %v", got.AccessTimestamp, want.AccessTimestamp) + } +} diff --git a/swarm/shed/schema.go b/swarm/shed/schema.go new file mode 100644 index 0000000000..cfb7c6d642 --- /dev/null +++ b/swarm/shed/schema.go @@ -0,0 +1,134 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package shed + +import ( + "encoding/json" + "errors" + "fmt" +) + +var ( + // LevelDB key value for storing the schema. + keySchema = []byte{0} + // LevelDB key prefix for all field type. + // LevelDB keys will be constructed by appending name values to this prefix. + keyPrefixFields byte = 1 + // LevelDB key prefix from which indexing keys start. + // Every index has its own key prefix and this value defines the first one. + keyPrefixIndexStart byte = 2 // Q: or maybe a higher number like 7, to have more space for potential specific perfixes +) + +// schema is used to serialize known database structure information. +type schema struct { + Fields map[string]fieldSpec `json:"fields"` // keys are field names + Indexes map[byte]indexSpec `json:"indexes"` // keys are index prefix bytes +} + +// fieldSpec holds information about a particular field. +// It does not need Name field as it is contained in the +// schema.Field map key. +type fieldSpec struct { + Type string `json:"type"` +} + +// indxSpec holds information about a particular index. +// It does not contain index type, as indexes do not have type. +type indexSpec struct { + Name string `json:"name"` +} + +// schemaFieldKey retrives the complete LevelDB key for +// a particular field form the schema definition. +func (db *DB) schemaFieldKey(name, fieldType string) (key []byte, err error) { + if name == "" { + return nil, errors.New("field name can not be blank") + } + if fieldType == "" { + return nil, errors.New("field type can not be blank") + } + s, err := db.getSchema() + if err != nil { + return nil, err + } + var found bool + for n, f := range s.Fields { + if n == name { + if f.Type != fieldType { + return nil, fmt.Errorf("field %q of type %q stored as %q in db", name, fieldType, f.Type) + } + break + } + } + if !found { + s.Fields[name] = fieldSpec{ + Type: fieldType, + } + err := db.putSchema(s) + if err != nil { + return nil, err + } + } + return append([]byte{keyPrefixFields}, []byte(name)...), nil +} + +// schemaIndexID retrieves the complete LevelDB prefix for +// a particular index. +func (db *DB) schemaIndexPrefix(name string) (id byte, err error) { + if name == "" { + return 0, errors.New("index name can not be blank") + } + s, err := db.getSchema() + if err != nil { + return 0, err + } + nextID := keyPrefixIndexStart + for i, f := range s.Indexes { + if i >= nextID { + nextID = i + 1 + } + if f.Name == name { + return i, nil + } + } + id = nextID + s.Indexes[id] = indexSpec{ + Name: name, + } + return id, db.putSchema(s) +} + +// getSchema retrieves the complete schema from +// the database. +func (db *DB) getSchema() (s schema, err error) { + b, err := db.Get(keySchema) + if err != nil { + return s, err + } + err = json.Unmarshal(b, &s) + return s, err +} + +// putSchema stores the complete schema to +// the database. +func (db *DB) putSchema(s schema) (err error) { + b, err := json.Marshal(s) + if err != nil { + return err + } + return db.Put(keySchema, b) +} diff --git a/swarm/shed/schema_test.go b/swarm/shed/schema_test.go new file mode 100644 index 0000000000..a0c1838c8b --- /dev/null +++ b/swarm/shed/schema_test.go @@ -0,0 +1,126 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package shed + +import ( + "bytes" + "testing" +) + +// TestDB_schemaFieldKey validates correctness of schemaFieldKey. +func TestDB_schemaFieldKey(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + t.Run("empty name or type", func(t *testing.T) { + _, err := db.schemaFieldKey("", "") + if err == nil { + t.Errorf("error not returned, but expected") + } + _, err = db.schemaFieldKey("", "type") + if err == nil { + t.Errorf("error not returned, but expected") + } + + _, err = db.schemaFieldKey("test", "") + if err == nil { + t.Errorf("error not returned, but expected") + } + }) + + t.Run("same field", func(t *testing.T) { + key1, err := db.schemaFieldKey("test", "undefined") + if err != nil { + t.Fatal(err) + } + + key2, err := db.schemaFieldKey("test", "undefined") + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(key1, key2) { + t.Errorf("schema keys for the same field name are not the same: %q, %q", string(key1), string(key2)) + } + }) + + t.Run("different fields", func(t *testing.T) { + key1, err := db.schemaFieldKey("test1", "undefined") + if err != nil { + t.Fatal(err) + } + + key2, err := db.schemaFieldKey("test2", "undefined") + if err != nil { + t.Fatal(err) + } + + if bytes.Equal(key1, key2) { + t.Error("schema keys for the same field name are the same, but must not be") + } + }) + + t.Run("same field name different types", func(t *testing.T) { + _, err := db.schemaFieldKey("the-field", "one-type") + if err != nil { + t.Fatal(err) + } + + _, err = db.schemaFieldKey("the-field", "another-type") + if err == nil { + t.Errorf("error not returned, but expected") + } + }) +} + +// TestDB_schemaIndexPrefix validates correctness of schemaIndexPrefix. +func TestDB_schemaIndexPrefix(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + t.Run("same name", func(t *testing.T) { + id1, err := db.schemaIndexPrefix("test") + if err != nil { + t.Fatal(err) + } + + id2, err := db.schemaIndexPrefix("test") + if err != nil { + t.Fatal(err) + } + + if id1 != id2 { + t.Errorf("schema keys for the same field name are not the same: %v, %v", id1, id2) + } + }) + + t.Run("different names", func(t *testing.T) { + id1, err := db.schemaIndexPrefix("test1") + if err != nil { + t.Fatal(err) + } + + id2, err := db.schemaIndexPrefix("test2") + if err != nil { + t.Fatal(err) + } + + if id1 == id2 { + t.Error("schema ids for the same index name are the same, but must not be") + } + }) +} diff --git a/swarm/storage/mock/db/db.go b/swarm/storage/mock/db/db.go index 43bfa24f05..73ae199e8b 100644 --- a/swarm/storage/mock/db/db.go +++ b/swarm/storage/mock/db/db.go @@ -86,6 +86,13 @@ func (s *GlobalStore) Put(addr common.Address, key []byte, data []byte) error { return s.db.Write(batch, nil) } +// Delete removes the chunk reference to node with address addr. +func (s *GlobalStore) Delete(addr common.Address, key []byte) error { + batch := new(leveldb.Batch) + batch.Delete(nodeDBKey(addr, key)) + return s.db.Write(batch, nil) +} + // HasKey returns whether a node with addr contains the key. func (s *GlobalStore) HasKey(addr common.Address, key []byte) bool { has, err := s.db.Has(nodeDBKey(addr, key), nil) diff --git a/swarm/storage/mock/mem/mem.go b/swarm/storage/mock/mem/mem.go index 8878309d0e..3a0a2beb8d 100644 --- a/swarm/storage/mock/mem/mem.go +++ b/swarm/storage/mock/mem/mem.go @@ -83,6 +83,22 @@ func (s *GlobalStore) Put(addr common.Address, key []byte, data []byte) error { return nil } +// Delete removes the chunk data for node with address addr. +func (s *GlobalStore) Delete(addr common.Address, key []byte) error { + s.mu.Lock() + defer s.mu.Unlock() + + var count int + if _, ok := s.nodes[string(key)]; ok { + delete(s.nodes[string(key)], addr) + count = len(s.nodes[string(key)]) + } + if count == 0 { + delete(s.data, string(key)) + } + return nil +} + // HasKey returns whether a node with addr contains the key. func (s *GlobalStore) HasKey(addr common.Address, key []byte) bool { s.mu.Lock() diff --git a/swarm/storage/mock/mock.go b/swarm/storage/mock/mock.go index 81340f9274..1fb71b70a2 100644 --- a/swarm/storage/mock/mock.go +++ b/swarm/storage/mock/mock.go @@ -70,6 +70,12 @@ func (n *NodeStore) Put(key []byte, data []byte) error { return n.store.Put(n.addr, key, data) } +// Delete removes chunk data for a key for a node that has the address +// provided on NodeStore initialization. +func (n *NodeStore) Delete(key []byte) error { + return n.store.Delete(n.addr, key) +} + // GlobalStorer defines methods for mock db store // that stores chunk data for all swarm nodes. // It is used in tests to construct mock NodeStores @@ -77,6 +83,7 @@ func (n *NodeStore) Put(key []byte, data []byte) error { type GlobalStorer interface { Get(addr common.Address, key []byte) (data []byte, err error) Put(addr common.Address, key []byte, data []byte) error + Delete(addr common.Address, key []byte) error HasKey(addr common.Address, key []byte) bool // NewNodeStore creates an instance of NodeStore // to be used by a single swarm node with diff --git a/swarm/storage/mock/rpc/rpc.go b/swarm/storage/mock/rpc/rpc.go index 6e735f6988..8cd6c83a7a 100644 --- a/swarm/storage/mock/rpc/rpc.go +++ b/swarm/storage/mock/rpc/rpc.go @@ -73,6 +73,12 @@ func (s *GlobalStore) Put(addr common.Address, key []byte, data []byte) error { return err } +// Delete calls a Delete method to RPC server. +func (s *GlobalStore) Delete(addr common.Address, key []byte) error { + err := s.client.Call(nil, "mockStore_delete", addr, key) + return err +} + // HasKey calls a HasKey method to RPC server. func (s *GlobalStore) HasKey(addr common.Address, key []byte) bool { var has bool diff --git a/swarm/storage/mock/test/test.go b/swarm/storage/mock/test/test.go index 02da3af553..10180985f3 100644 --- a/swarm/storage/mock/test/test.go +++ b/swarm/storage/mock/test/test.go @@ -72,6 +72,31 @@ func MockStore(t *testing.T, globalStore mock.GlobalStorer, n int) { } } } + t.Run("delete", func(t *testing.T) { + chunkAddr := storage.Address([]byte("1234567890abcd")) + for _, addr := range addrs { + err := globalStore.Put(addr, chunkAddr, []byte("data")) + if err != nil { + t.Fatalf("put data to store %s key %s: %v", addr.Hex(), chunkAddr.Hex(), err) + } + } + firstNodeAddr := addrs[0] + if err := globalStore.Delete(firstNodeAddr, chunkAddr); err != nil { + t.Fatalf("delete from store %s key %s: %v", firstNodeAddr.Hex(), chunkAddr.Hex(), err) + } + for i, addr := range addrs { + _, err := globalStore.Get(addr, chunkAddr) + if i == 0 { + if err != mock.ErrNotFound { + t.Errorf("get data from store %s key %s: expected mock.ErrNotFound error, got %v", addr.Hex(), chunkAddr.Hex(), err) + } + } else { + if err != nil { + t.Errorf("get data from store %s key %s: %v", addr.Hex(), chunkAddr.Hex(), err) + } + } + } + }) }) t.Run("NodeStore", func(t *testing.T) { @@ -114,6 +139,34 @@ func MockStore(t *testing.T, globalStore mock.GlobalStorer, n int) { } } } + t.Run("delete", func(t *testing.T) { + chunkAddr := storage.Address([]byte("1234567890abcd")) + var chosenStore *mock.NodeStore + for addr, store := range nodes { + if chosenStore == nil { + chosenStore = store + } + err := store.Put(chunkAddr, []byte("data")) + if err != nil { + t.Fatalf("put data to store %s key %s: %v", addr.Hex(), chunkAddr.Hex(), err) + } + } + if err := chosenStore.Delete(chunkAddr); err != nil { + t.Fatalf("delete key %s: %v", chunkAddr.Hex(), err) + } + for addr, store := range nodes { + _, err := store.Get(chunkAddr) + if store == chosenStore { + if err != mock.ErrNotFound { + t.Errorf("get data from store %s key %s: expected mock.ErrNotFound error, got %v", addr.Hex(), chunkAddr.Hex(), err) + } + } else { + if err != nil { + t.Errorf("get data from store %s key %s: %v", addr.Hex(), chunkAddr.Hex(), err) + } + } + } + }) }) } From 695a5cce1e4c0db173dd2a798ed0dc14ee9ca2a1 Mon Sep 17 00:00:00 2001 From: holisticode Date: Mon, 26 Nov 2018 13:34:40 -0500 Subject: [PATCH 092/159] Increase bzz version (#18184) * swarm/network/stream/: added stream protocol version match tests * Increase BZZ version due to streamer version change; version tests * swarm/network: increased hive and test protocol version --- swarm/network/protocol.go | 4 +- swarm/network/protocol_test.go | 2 +- swarm/network/stream/syncer_test.go | 167 ++++++++++++++++++++++++++++ 3 files changed, 170 insertions(+), 3 deletions(-) diff --git a/swarm/network/protocol.go b/swarm/network/protocol.go index 66ae94a88b..4b9b28cdc3 100644 --- a/swarm/network/protocol.go +++ b/swarm/network/protocol.go @@ -44,7 +44,7 @@ const ( // BzzSpec is the spec of the generic swarm handshake var BzzSpec = &protocols.Spec{ Name: "bzz", - Version: 7, + Version: 8, MaxMsgSize: 10 * 1024 * 1024, Messages: []interface{}{ HandshakeMsg{}, @@ -54,7 +54,7 @@ var BzzSpec = &protocols.Spec{ // DiscoverySpec is the spec for the bzz discovery subprotocols var DiscoverySpec = &protocols.Spec{ Name: "hive", - Version: 6, + Version: 8, MaxMsgSize: 10 * 1024 * 1024, Messages: []interface{}{ peersMsg{}, diff --git a/swarm/network/protocol_test.go b/swarm/network/protocol_test.go index f0d2666288..53ceda7448 100644 --- a/swarm/network/protocol_test.go +++ b/swarm/network/protocol_test.go @@ -31,7 +31,7 @@ import ( ) const ( - TestProtocolVersion = 7 + TestProtocolVersion = 8 TestProtocolNetworkID = 3 ) diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index df1a98c92e..3e3cee18d6 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -234,3 +234,170 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck t.Fatal(result.Error) } } + +//TestSameVersionID just checks that if the version is not changed, +//then streamer peers see each other +func TestSameVersionID(t *testing.T) { + //test version ID + v := uint(1) + sim := simulation.New(map[string]simulation.ServiceFunc{ + "streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) { + var store storage.ChunkStore + var datadir string + + node := ctx.Config.Node() + addr := network.NewAddr(node) + + store, datadir, err = createTestLocalStorageForID(node.ID(), addr) + if err != nil { + return nil, nil, err + } + bucket.Store(bucketKeyStore, store) + cleanup = func() { + store.Close() + os.RemoveAll(datadir) + } + localStore := store.(*storage.LocalStore) + netStore, err := storage.NewNetStore(localStore, nil) + if err != nil { + return nil, nil, err + } + bucket.Store(bucketKeyDB, netStore) + kad := network.NewKademlia(addr.Over(), network.NewKadParams()) + delivery := NewDelivery(kad, netStore) + netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New + + bucket.Store(bucketKeyDelivery, delivery) + + r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ + Retrieval: RetrievalDisabled, + Syncing: SyncingAutoSubscribe, + }, nil) + //assign to each node the same version ID + r.spec.Version = v + + bucket.Store(bucketKeyRegistry, r) + + return r, cleanup, nil + + }, + }) + defer sim.Close() + + //connect just two nodes + log.Info("Adding nodes to simulation") + _, err := sim.AddNodesAndConnectChain(2) + if err != nil { + t.Fatal(err) + } + + log.Info("Starting simulation") + ctx := context.Background() + //make sure they have time to connect + time.Sleep(200 * time.Millisecond) + result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error { + //get the pivot node's filestore + nodes := sim.UpNodeIDs() + + item, ok := sim.NodeItem(nodes[0], bucketKeyRegistry) + if !ok { + return fmt.Errorf("No filestore") + } + registry := item.(*Registry) + + //the peers should connect, thus getting the peer should not return nil + if registry.getPeer(nodes[1]) == nil { + t.Fatal("Expected the peer to not be nil, but it is") + } + return nil + }) + if result.Error != nil { + t.Fatal(result.Error) + } + log.Info("Simulation ended") +} + +//TestDifferentVersionID proves that if the streamer protocol version doesn't match, +//then the peers are not connected at streamer level +func TestDifferentVersionID(t *testing.T) { + //create a variable to hold the version ID + v := uint(0) + sim := simulation.New(map[string]simulation.ServiceFunc{ + "streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) { + var store storage.ChunkStore + var datadir string + + node := ctx.Config.Node() + addr := network.NewAddr(node) + + store, datadir, err = createTestLocalStorageForID(node.ID(), addr) + if err != nil { + return nil, nil, err + } + bucket.Store(bucketKeyStore, store) + cleanup = func() { + store.Close() + os.RemoveAll(datadir) + } + localStore := store.(*storage.LocalStore) + netStore, err := storage.NewNetStore(localStore, nil) + if err != nil { + return nil, nil, err + } + bucket.Store(bucketKeyDB, netStore) + kad := network.NewKademlia(addr.Over(), network.NewKadParams()) + delivery := NewDelivery(kad, netStore) + netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New + + bucket.Store(bucketKeyDelivery, delivery) + + r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ + Retrieval: RetrievalDisabled, + Syncing: SyncingAutoSubscribe, + }, nil) + + //increase the version ID for each node + v++ + r.spec.Version = v + + bucket.Store(bucketKeyRegistry, r) + + return r, cleanup, nil + + }, + }) + defer sim.Close() + + //connect the nodes + log.Info("Adding nodes to simulation") + _, err := sim.AddNodesAndConnectChain(2) + if err != nil { + t.Fatal(err) + } + + log.Info("Starting simulation") + ctx := context.Background() + //make sure they have time to connect + time.Sleep(200 * time.Millisecond) + result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error { + //get the pivot node's filestore + nodes := sim.UpNodeIDs() + + item, ok := sim.NodeItem(nodes[0], bucketKeyRegistry) + if !ok { + return fmt.Errorf("No filestore") + } + registry := item.(*Registry) + + //getting the other peer should fail due to the different version numbers + if registry.getPeer(nodes[1]) != nil { + t.Fatal("Expected the peer to be nil, but it is not") + } + return nil + }) + if result.Error != nil { + t.Fatal(result.Error) + } + log.Info("Simulation ended") + +} From 8b9f46941924a04ce1091532a74def03e70c7559 Mon Sep 17 00:00:00 2001 From: Liang Ma Date: Tue, 27 Nov 2018 10:52:30 +0000 Subject: [PATCH 093/159] p2p/protocols: fix minor comments typo (#18185) --- p2p/protocols/protocol.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/protocols/protocol.go b/p2p/protocols/protocol.go index 7dddd852fa..b16720dd39 100644 --- a/p2p/protocols/protocol.go +++ b/p2p/protocols/protocol.go @@ -381,7 +381,7 @@ func (p *Peer) handleIncoming(handle func(ctx context.Context, msg interface{}) // * arguments // * context // * the local handshake to be sent to the remote peer -// * funcion to be called on the remote handshake (can be nil) +// * function to be called on the remote handshake (can be nil) // * expects a remote handshake back of the same type // * the dialing peer needs to send the handshake first and then waits for remote // * the listening peer waits for the remote handshake and then sends it From 89fe24bbcc2395b9fbd449f5aeb9da8df073991d Mon Sep 17 00:00:00 2001 From: ANOTHEL Date: Tue, 27 Nov 2018 21:00:57 +0900 Subject: [PATCH 094/159] p2p/discv5: minor code simplification (#18188) * Update net.go more simple * Update net.go --- p2p/discv5/net.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/p2p/discv5/net.go b/p2p/discv5/net.go index a6cabf0803..976eae7df3 100644 --- a/p2p/discv5/net.go +++ b/p2p/discv5/net.go @@ -567,12 +567,11 @@ loop: net.ticketStore.searchLookupDone(res.target, res.nodes, func(n *Node, topic Topic) []byte { if n.state != nil && n.state.canQuery { return net.conn.send(n, topicQueryPacket, topicQuery{Topic: topic}) // TODO: set expiration - } else { - if n.state == unknown { - net.ping(n, n.addr()) - } - return nil } + if n.state == unknown { + net.ping(n, n.addr()) + } + return nil }) case <-statsDump.C: From edc39aaedf526bfb7757a445c1f9dd42f45dc8d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 27 Nov 2018 14:50:47 +0200 Subject: [PATCH 095/159] p2p/discv5: gofmt --- p2p/discv5/net.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/discv5/net.go b/p2p/discv5/net.go index 976eae7df3..cdeb28dd5b 100644 --- a/p2p/discv5/net.go +++ b/p2p/discv5/net.go @@ -571,7 +571,7 @@ loop: if n.state == unknown { net.ping(n, n.addr()) } - return nil + return nil }) case <-statsDump.C: From bfed28a42187c7bf7b8f50401fe239278a4ed530 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 28 Nov 2018 09:29:05 +0100 Subject: [PATCH 096/159] core: more detailed metrics for block processing (#18119) --- core/blockchain.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/core/blockchain.go b/core/blockchain.go index bcd53d4987..a672c0ab64 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -47,7 +47,10 @@ import ( ) var ( - blockInsertTimer = metrics.NewRegisteredTimer("chain/inserts", nil) + blockInsertTimer = metrics.NewRegisteredTimer("chain/inserts", nil) + blockValidationTimer = metrics.NewRegisteredTimer("chain/validation", nil) + blockExecutionTimer = metrics.NewRegisteredTimer("chain/execution", nil) + blockWriteTimer = metrics.NewRegisteredTimer("chain/write", nil) ErrNoGenesis = errors.New("Genesis not found in chain") ) @@ -1188,7 +1191,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, [] return it.index, events, coalescedLogs, err } // Process block using the parent state as reference point. + t0 := time.Now() receipts, logs, usedGas, err := bc.processor.Process(block, state, bc.vmConfig) + t1 := time.Now() if err != nil { bc.reportBlock(block, receipts, err) return it.index, events, coalescedLogs, err @@ -1198,13 +1203,19 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, [] bc.reportBlock(block, receipts, err) return it.index, events, coalescedLogs, err } + t2 := time.Now() proctime := time.Since(start) // Write the block to the chain and get the status. status, err := bc.WriteBlockWithState(block, receipts, state) + t3 := time.Now() if err != nil { return it.index, events, coalescedLogs, err } + blockInsertTimer.UpdateSince(start) + blockExecutionTimer.Update(t1.Sub(t0)) + blockValidationTimer.Update(t2.Sub(t1)) + blockWriteTimer.Update(t3.Sub(t2)) switch status { case CanonStatTy: log.Debug("Inserted new block", "number", block.Number(), "hash", block.Hash(), From 174083c3ae0b6217dff0ced98ec7f73e53e24b04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 28 Nov 2018 13:31:42 +0200 Subject: [PATCH 097/159] eth/downloader: fix light client cht binary search issue --- eth/downloader/downloader.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index f81a5cbac2..3a177ab9de 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -99,6 +99,7 @@ type Downloader struct { mode SyncMode // Synchronisation mode defining the strategy used (per sync cycle) mux *event.TypeMux // Event multiplexer to announce sync operation events + genesis uint64 // Genesis block number to limit sync to (e.g. light client CHT) queue *queue // Scheduler for selecting the hashes to download peers *peerSet // Set of active peers from which download can proceed stateDB ethdb.Database @@ -664,7 +665,28 @@ func (d *Downloader) findAncestor(p *peerConnection, remoteHeader *types.Header) } p.log.Debug("Looking for common ancestor", "local", localHeight, "remote", remoteHeight) if localHeight >= MaxForkAncestry { + // We're above the max reorg threshold, find the earliest fork point floor = int64(localHeight - MaxForkAncestry) + + // If we're doing a light sync, ensure the floor doesn't go below the CHT, as + // all headers before that point will be missing. + if d.mode == LightSync { + // If we dont know the current CHT position, find it + if d.genesis == 0 { + header := d.lightchain.CurrentHeader() + for header != nil { + d.genesis = header.Number.Uint64() + if floor >= int64(d.genesis)-1 { + break + } + header = d.lightchain.GetHeaderByHash(header.ParentHash) + } + } + // We already know the "genesis" block number, cap floor to that + if floor < int64(d.genesis)-1 { + floor = int64(d.genesis) - 1 + } + } } from, count, skip, max := calculateRequestSpan(remoteHeight, localHeight) From d606a7a46a62483f7fd8110de97ce91f70c4766e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 28 Nov 2018 13:53:33 +0200 Subject: [PATCH 098/159] params: update CHTs for the v1.8.19 release --- params/config.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/params/config.go b/params/config.go index 3570cb204f..ba9fac1c4a 100644 --- a/params/config.go +++ b/params/config.go @@ -49,10 +49,10 @@ var ( // MainnetTrustedCheckpoint contains the light client trusted checkpoint for the main network. MainnetTrustedCheckpoint = &TrustedCheckpoint{ Name: "mainnet", - SectionIndex: 203, - SectionHead: common.HexToHash("0xc9e05fc67c6a9815adc8072eb18805b53da53a9a6a273e05541e1b7542cf937a"), - CHTRoot: common.HexToHash("0xb85f42447d59f7c3e6679b9a37ed983593fd52efd6251b883592662e95769d5b"), - BloomRoot: common.HexToHash("0xf93d50cb4c49b403c6fd33cd60896d3b36184275be0a51bae4df5e8844ac624c"), + SectionIndex: 206, + SectionHead: common.HexToHash("0x9fa677c7c0580136f5a86d9b2fd29b112e531f0284396298b8809bcb6787b538"), + CHTRoot: common.HexToHash("0x7f32dfb29e341b4c8c10ea2e06a812bcea470366f635b7a8b3d0856684cd76f4"), + BloomRoot: common.HexToHash("0x0169e174f0a8172aec217556d8a25c7ba7ca52aacff170325230a75740ff1eaf"), } // TestnetChainConfig contains the chain parameters to run a node on the Ropsten test network. @@ -73,10 +73,10 @@ var ( // TestnetTrustedCheckpoint contains the light client trusted checkpoint for the Ropsten test network. TestnetTrustedCheckpoint = &TrustedCheckpoint{ Name: "testnet", - SectionIndex: 134, - SectionHead: common.HexToHash("0x17053ecbe045bebefaa01e7716cc85a4e22647e181416cc1098ccbb73a088931"), - CHTRoot: common.HexToHash("0x4d2b86422e46ed76f0e3f50f06632c409f809c8375e53c8bc0f782bcb93dd49a"), - BloomRoot: common.HexToHash("0xccba62232ee56c2967afc58f136a47ba7dc545ae586e6be666430d94516306c7"), + SectionIndex: 136, + SectionHead: common.HexToHash("0xe5d80bb08d92bbc12dfe510c64cba01eafcbb4ba585e7c7ab7f8a93c6f295ab3"), + CHTRoot: common.HexToHash("0xe3ca77ab0cb51eec74f4f7458e36aee207c68768387b39cb0bcff0940a6264d8"), + BloomRoot: common.HexToHash("0x30c8eeadac5539d3dcd6e88915d1a07cb2f3a1d6ebe7e553e3ee783c04c68c2d"), } // RinkebyChainConfig contains the chain parameters to run a node on the Rinkeby test network. @@ -100,10 +100,10 @@ var ( // RinkebyTrustedCheckpoint contains the light client trusted checkpoint for the Rinkeby test network. RinkebyTrustedCheckpoint = &TrustedCheckpoint{ Name: "rinkeby", - SectionIndex: 100, - SectionHead: common.HexToHash("0xf18f9b43e16f37b12e68818536ffe455ff18d676274ffdd856a8520ed61bb514"), - CHTRoot: common.HexToHash("0x473f5d603b1fedad75d97fd58692130b9ac9ade1aca01eb9363d79bd1c43c791"), - BloomRoot: common.HexToHash("0xa39ced3ddbb87e909c7531df2afb6414bea9c9a60ab94da9c6b467535f05326e"), + SectionIndex: 103, + SectionHead: common.HexToHash("0x9f38b903852831bf4fa7992f7fd43d8b26da2deb82b421fb845cf6faee54e056"), + CHTRoot: common.HexToHash("0x2d710c2cea468d2e604838000d658ee213e4abb07f90c4f71f5cd7f8510aa708"), + BloomRoot: common.HexToHash("0xcc401060280c2cc82697ea5ecef8cac61e52063c37533a2e9609332419704d5f"), } // AllEthashProtocolChanges contains every protocol change (EIPs) introduced From dae82f098570e15d44584f0d7f350713f4774727 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 28 Nov 2018 14:42:37 +0200 Subject: [PATCH 099/159] params, swarm: release Geth v1.8.19 and Swarm v0.3.7 --- params/version.go | 8 ++++---- swarm/version/version.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/params/version.go b/params/version.go index b9dcc2a842..cbb23e16dd 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 19 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 8 // Minor version component of the current release + VersionPatch = 19 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. diff --git a/swarm/version/version.go b/swarm/version/version.go index 17ef34f5f4..d75763fe95 100644 --- a/swarm/version/version.go +++ b/swarm/version/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 0 // Major version component of the current release - VersionMinor = 3 // Minor version component of the current release - VersionPatch = 7 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 0 // Major version component of the current release + VersionMinor = 3 // Minor version component of the current release + VersionPatch = 7 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From a7501d0c419f8c1dc8b4004c115dd2d4e3b524b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 28 Nov 2018 14:44:00 +0200 Subject: [PATCH 100/159] params, swarm: start Geth v1.8.20 and Swarm v0.3.8 release cycle --- params/version.go | 8 ++++---- swarm/version/version.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/params/version.go b/params/version.go index cbb23e16dd..2291d5f3c5 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 19 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 8 // Minor version component of the current release + VersionPatch = 20 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. diff --git a/swarm/version/version.go b/swarm/version/version.go index d75763fe95..57ac05a864 100644 --- a/swarm/version/version.go +++ b/swarm/version/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 0 // Major version component of the current release - VersionMinor = 3 // Minor version component of the current release - VersionPatch = 7 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 0 // Major version component of the current release + VersionMinor = 3 // Minor version component of the current release + VersionPatch = 8 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From 3dba6a6d27cc98814a23555d84cf6f016fe4635d Mon Sep 17 00:00:00 2001 From: Franklin Date: Wed, 28 Nov 2018 22:42:07 +0800 Subject: [PATCH 101/159] remove unrelated code --- core/blockchain.go | 6 +----- rpc/ipc.go | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index ee1c0702b5..d173b2de29 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -445,11 +445,7 @@ func (bc *BlockChain) repair(head **types.Block) error { return nil } // Otherwise rewind one block and recheck state availability there - block := bc.GetBlock((*head).ParentHash(), (*head).NumberU64()-1) - if block == nil { - return fmt.Errorf("failed to repair block, can not get block at height %d", (*head).NumberU64()) - } - (*head) = block + (*head) = bc.GetBlock((*head).ParentHash(), (*head).NumberU64()-1) } } diff --git a/rpc/ipc.go b/rpc/ipc.go index 1467f7a0c9..4cce1cf74b 100644 --- a/rpc/ipc.go +++ b/rpc/ipc.go @@ -24,7 +24,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/netutil" ) -// ServeListener accepts connections on l, serving IPC-RPC on them. +// ServeListener accepts connections on l, serving JSON-RPC on them. func (srv *Server) ServeListener(l net.Listener) error { for { conn, err := l.Accept() From 3d21d455dca6e75a635ab5742c66c80ee175c3e7 Mon Sep 17 00:00:00 2001 From: Matthew Di Ferrante Date: Thu, 29 Nov 2018 09:29:12 +0100 Subject: [PATCH 102/159] cmd/evm: commit statedb if dump is requested (#18208) Add a call `statedb.Commit(true)` if the `Dump` flag is on, as otherwise the `storage` output in the dump is always empty. --- cmd/evm/runner.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index 962fc021d7..a9c8a38cae 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -206,6 +206,7 @@ func runCmd(ctx *cli.Context) error { execTime := time.Since(tstart) if ctx.GlobalBool(DumpFlag.Name) { + statedb.Commit(true) statedb.IntermediateRoot(true) fmt.Println(string(statedb.Dump())) } From 7c657fc78957d7ed302eba68bb6122b515364ac4 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 29 Nov 2018 09:51:57 +0100 Subject: [PATCH 103/159] tests, core: update tests and make STATICCALL cause touch-delete (#18187) --- core/vm/evm.go | 6 ++++++ tests/init.go | 9 +++++++++ tests/testdata | 2 +- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/core/vm/evm.go b/core/vm/evm.go index 968d2219ea..ba4d1e9eb8 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -339,6 +339,12 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte contract := NewContract(caller, to, new(big.Int), gas) contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr)) + // We do an AddBalance of zero here, just in order to trigger a touch. + // This doesn't matter on Mainnet, where all empties are gone at the time of Byzantium, + // but is the correct thing to do and matters on other networks, in tests, and potential + // future scenarios + evm.StateDB.AddBalance(addr, bigZero) + // When an error was returned by the EVM or when setting the creation code // above we revert to the snapshot and consume any gas remaining. Additionally // when we're in Homestead this also counts for code storage gas errors. diff --git a/tests/init.go b/tests/init.go index f0a4943c13..db0457b6d7 100644 --- a/tests/init.go +++ b/tests/init.go @@ -86,6 +86,15 @@ var Forks = map[string]*params.ChainConfig{ EIP158Block: big.NewInt(0), ByzantiumBlock: big.NewInt(5), }, + "ByzantiumToConstantinopleAt5": { + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(5), + }, } // UnsupportedForkError is returned when a test requests a fork that isn't implemented. diff --git a/tests/testdata b/tests/testdata index 95a3092038..c02a2a17c0 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit 95a309203890e6244c6d4353ca411671973c13b5 +Subproject commit c02a2a17c0288a255572b37dc7ec1fcb838b9dbf From 8380a1303ce6a2fa4760a8641e47e7196a941b4c Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 28 Nov 2018 20:53:02 +0100 Subject: [PATCH 104/159] vendor: update leveldb --- .../syndtr/goleveldb/leveldb/cache/cache.go | 1 - .../leveldb/comparer/bytes_comparer.go | 4 +- .../goleveldb/leveldb/comparer/comparer.go | 2 +- .../github.com/syndtr/goleveldb/leveldb/db.go | 2 +- .../syndtr/goleveldb/leveldb/db_compaction.go | 24 +++----- .../syndtr/goleveldb/leveldb/db_util.go | 2 +- .../syndtr/goleveldb/leveldb/iterator/iter.go | 4 +- .../syndtr/goleveldb/leveldb/opt/options.go | 13 +++++ .../syndtr/goleveldb/leveldb/session_util.go | 4 +- .../syndtr/goleveldb/leveldb/table.go | 30 +++++----- vendor/vendor.json | 58 +++++++++---------- 11 files changed, 76 insertions(+), 68 deletions(-) diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/cache/cache.go b/vendor/github.com/syndtr/goleveldb/leveldb/cache/cache.go index c5940b232c..c36ad32359 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/cache/cache.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/cache/cache.go @@ -331,7 +331,6 @@ func (r *Cache) delete(n *Node) bool { return deleted } } - return false } // Nodes returns number of 'cache node' in the map. diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/comparer/bytes_comparer.go b/vendor/github.com/syndtr/goleveldb/leveldb/comparer/bytes_comparer.go index 14dddf88dd..abf9fb65c7 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/comparer/bytes_comparer.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/comparer/bytes_comparer.go @@ -29,7 +29,7 @@ func (bytesComparer) Separator(dst, a, b []byte) []byte { // Do not shorten if one string is a prefix of the other } else if c := a[i]; c < 0xff && c+1 < b[i] { dst = append(dst, a[:i+1]...) - dst[i]++ + dst[len(dst)-1]++ return dst } return nil @@ -39,7 +39,7 @@ func (bytesComparer) Successor(dst, b []byte) []byte { for i, c := range b { if c != 0xff { dst = append(dst, b[:i+1]...) - dst[i]++ + dst[len(dst)-1]++ return dst } } diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/comparer/comparer.go b/vendor/github.com/syndtr/goleveldb/leveldb/comparer/comparer.go index 14a28f16fc..2c522db23b 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/comparer/comparer.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/comparer/comparer.go @@ -36,7 +36,7 @@ type Comparer interface { // by any users of this package. Name() string - // Bellow are advanced functions used used to reduce the space requirements + // Bellow are advanced functions used to reduce the space requirements // for internal data structures such as index blocks. // Separator appends a sequence of bytes x to dst such that a <= x && x < b, diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db.go b/vendor/github.com/syndtr/goleveldb/leveldb/db.go index e7ac065418..b27c38d37e 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/db.go @@ -182,7 +182,7 @@ func Open(stor storage.Storage, o *opt.Options) (db *DB, err error) { err = s.recover() if err != nil { - if !os.IsNotExist(err) || s.o.GetErrorIfMissing() { + if !os.IsNotExist(err) || s.o.GetErrorIfMissing() || s.o.GetReadOnly() { return } err = s.create() diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go b/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go index 28e50906ad..0c1b9a53b8 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go @@ -663,7 +663,7 @@ type cCmd interface { } type cAuto struct { - // Note for table compaction, an empty ackC represents it's a compaction waiting command. + // Note for table compaction, an non-empty ackC represents it's a compaction waiting command. ackC chan<- error } @@ -777,8 +777,8 @@ func (db *DB) mCompaction() { func (db *DB) tCompaction() { var ( - x cCmd - ackQ, waitQ []cCmd + x cCmd + waitQ []cCmd ) defer func() { @@ -787,10 +787,6 @@ func (db *DB) tCompaction() { panic(x) } } - for i := range ackQ { - ackQ[i].ack(ErrClosed) - ackQ[i] = nil - } for i := range waitQ { waitQ[i].ack(ErrClosed) waitQ[i] = nil @@ -821,11 +817,6 @@ func (db *DB) tCompaction() { waitQ = waitQ[:0] } } else { - for i := range ackQ { - ackQ[i].ack(nil) - ackQ[i] = nil - } - ackQ = ackQ[:0] for i := range waitQ { waitQ[i].ack(nil) waitQ[i] = nil @@ -844,9 +835,12 @@ func (db *DB) tCompaction() { switch cmd := x.(type) { case cAuto: if cmd.ackC != nil { - waitQ = append(waitQ, x) - } else { - ackQ = append(ackQ, x) + // Check the write pause state before caching it. + if db.resumeWrite() { + x.ack(nil) + } else { + waitQ = append(waitQ, x) + } } case cRange: x.ack(db.tableRangeCompaction(cmd.level, cmd.min, cmd.max)) diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db_util.go b/vendor/github.com/syndtr/goleveldb/leveldb/db_util.go index 7ecd960d2c..3f0654894b 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db_util.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/db_util.go @@ -84,7 +84,7 @@ func (db *DB) checkAndCleanFiles() error { var mfds []storage.FileDesc for num, present := range tmap { if !present { - mfds = append(mfds, storage.FileDesc{storage.TypeTable, num}) + mfds = append(mfds, storage.FileDesc{Type: storage.TypeTable, Num: num}) db.logf("db@janitor table missing @%d", num) } } diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/iterator/iter.go b/vendor/github.com/syndtr/goleveldb/leveldb/iterator/iter.go index b16e3a7045..96fb0f6859 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/iterator/iter.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/iterator/iter.go @@ -40,11 +40,11 @@ type IteratorSeeker interface { Seek(key []byte) bool // Next moves the iterator to the next key/value pair. - // It returns whether the iterator is exhausted. + // It returns false if the iterator is exhausted. Next() bool // Prev moves the iterator to the previous key/value pair. - // It returns whether the iterator is exhausted. + // It returns false if the iterator is exhausted. Prev() bool } diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/opt/options.go b/vendor/github.com/syndtr/goleveldb/leveldb/opt/options.go index 44e7d9adce..528b164233 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/opt/options.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/opt/options.go @@ -158,6 +158,12 @@ type Options struct { // The default value is 8MiB. BlockCacheCapacity int + // BlockCacheEvictRemoved allows enable forced-eviction on cached block belonging + // to removed 'sorted table'. + // + // The default if false. + BlockCacheEvictRemoved bool + // BlockRestartInterval is the number of keys between restart points for // delta encoding of keys. // @@ -384,6 +390,13 @@ func (o *Options) GetBlockCacheCapacity() int { return o.BlockCacheCapacity } +func (o *Options) GetBlockCacheEvictRemoved() bool { + if o == nil { + return false + } + return o.BlockCacheEvictRemoved +} + func (o *Options) GetBlockRestartInterval() int { if o == nil || o.BlockRestartInterval <= 0 { return DefaultBlockRestartInterval diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/session_util.go b/vendor/github.com/syndtr/goleveldb/leveldb/session_util.go index 92328933cc..40cb2cf957 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/session_util.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/session_util.go @@ -36,7 +36,7 @@ func (s *session) logf(format string, v ...interface{}) { s.stor.Log(fmt.Sprintf func (s *session) newTemp() storage.FileDesc { num := atomic.AddInt64(&s.stTempFileNum, 1) - 1 - return storage.FileDesc{storage.TypeTemp, num} + return storage.FileDesc{Type: storage.TypeTemp, Num: num} } func (s *session) addFileRef(fd storage.FileDesc, ref int) int { @@ -190,7 +190,7 @@ func (s *session) recordCommited(rec *sessionRecord) { // Create a new manifest file; need external synchronization. func (s *session) newManifest(rec *sessionRecord, v *version) (err error) { - fd := storage.FileDesc{storage.TypeManifest, s.allocFileNum()} + fd := storage.FileDesc{Type: storage.TypeManifest, Num: s.allocFileNum()} writer, err := s.stor.Create(fd) if err != nil { return diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/table.go b/vendor/github.com/syndtr/goleveldb/leveldb/table.go index 81d18a531e..1fac60d050 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/table.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/table.go @@ -78,7 +78,7 @@ func newTableFile(fd storage.FileDesc, size int64, imin, imax internalKey) *tFil } func tableFileFromRecord(r atRecord) *tFile { - return newTableFile(storage.FileDesc{storage.TypeTable, r.num}, r.size, r.imin, r.imax) + return newTableFile(storage.FileDesc{Type: storage.TypeTable, Num: r.num}, r.size, r.imin, r.imax) } // tFiles hold multiple tFile. @@ -290,16 +290,17 @@ func (x *tFilesSortByNum) Less(i, j int) bool { // Table operations. type tOps struct { - s *session - noSync bool - cache *cache.Cache - bcache *cache.Cache - bpool *util.BufferPool + s *session + noSync bool + evictRemoved bool + cache *cache.Cache + bcache *cache.Cache + bpool *util.BufferPool } // Creates an empty table and returns table writer. func (t *tOps) create() (*tWriter, error) { - fd := storage.FileDesc{storage.TypeTable, t.s.allocFileNum()} + fd := storage.FileDesc{Type: storage.TypeTable, Num: t.s.allocFileNum()} fw, err := t.s.stor.Create(fd) if err != nil { return nil, err @@ -422,7 +423,7 @@ func (t *tOps) remove(f *tFile) { } else { t.s.logf("table@remove removed @%d", f.fd.Num) } - if t.bcache != nil { + if t.evictRemoved && t.bcache != nil { t.bcache.EvictNS(uint64(f.fd.Num)) } }) @@ -451,7 +452,7 @@ func newTableOps(s *session) *tOps { if !s.o.GetDisableBlockCache() { var bcacher cache.Cacher if s.o.GetBlockCacheCapacity() > 0 { - bcacher = cache.NewLRU(s.o.GetBlockCacheCapacity()) + bcacher = s.o.GetBlockCacher().New(s.o.GetBlockCacheCapacity()) } bcache = cache.NewCache(bcacher) } @@ -459,11 +460,12 @@ func newTableOps(s *session) *tOps { bpool = util.NewBufferPool(s.o.GetBlockSize() + 5) } return &tOps{ - s: s, - noSync: s.o.GetNoSync(), - cache: cache.NewCache(cacher), - bcache: bcache, - bpool: bpool, + s: s, + noSync: s.o.GetNoSync(), + evictRemoved: s.o.GetBlockCacheEvictRemoved(), + cache: cache.NewCache(cacher), + bcache: bcache, + bpool: bpool, } } diff --git a/vendor/vendor.json b/vendor/vendor.json index 1bfe09da72..a4fee11aa5 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -455,76 +455,76 @@ "revisionTime": "2017-07-05T02:17:15Z" }, { - "checksumSHA1": "k6zbR5hiI10hkWtiK91rIY5s5/E=", + "checksumSHA1": "LV0VMVON7xY1ttV+s2ph83ntmDQ=", "path": "github.com/syndtr/goleveldb/leveldb", - "revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445", - "revisionTime": "2018-07-08T03:05:51Z" + "revision": "b001fa50d6b27f3f0bb175a87d0cb55426d0a0ae", + "revisionTime": "2018-11-28T10:09:59Z" }, { - "checksumSHA1": "EKIow7XkgNdWvR/982ffIZxKG8Y=", + "checksumSHA1": "mPNraL2edpk/2FYq26rSXfMHbJg=", "path": "github.com/syndtr/goleveldb/leveldb/cache", - "revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445", - "revisionTime": "2018-07-08T03:05:51Z" + "revision": "b001fa50d6b27f3f0bb175a87d0cb55426d0a0ae", + "revisionTime": "2018-11-28T10:09:59Z" }, { - "checksumSHA1": "5KPgnvCPlR0ysDAqo6jApzRQ3tw=", + "checksumSHA1": "UA+PKDKWlDnE2OZblh23W6wZwbY=", "path": "github.com/syndtr/goleveldb/leveldb/comparer", - "revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445", - "revisionTime": "2018-07-08T03:05:51Z" + "revision": "b001fa50d6b27f3f0bb175a87d0cb55426d0a0ae", + "revisionTime": "2018-11-28T10:09:59Z" }, { "checksumSHA1": "1DRAxdlWzS4U0xKN/yQ/fdNN7f0=", "path": "github.com/syndtr/goleveldb/leveldb/errors", - "revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445", - "revisionTime": "2018-07-08T03:05:51Z" + "revision": "b001fa50d6b27f3f0bb175a87d0cb55426d0a0ae", + "revisionTime": "2018-11-28T10:09:59Z" }, { "checksumSHA1": "eqKeD6DS7eNCtxVYZEHHRKkyZrw=", "path": "github.com/syndtr/goleveldb/leveldb/filter", - "revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445", - "revisionTime": "2018-07-08T03:05:51Z" + "revision": "b001fa50d6b27f3f0bb175a87d0cb55426d0a0ae", + "revisionTime": "2018-11-28T10:09:59Z" }, { - "checksumSHA1": "weSsccMav4BCerDpSLzh3mMxAYo=", + "checksumSHA1": "hPyFsMiqZ1OB7MX+6wIAA6nsdtc=", "path": "github.com/syndtr/goleveldb/leveldb/iterator", - "revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445", - "revisionTime": "2018-07-08T03:05:51Z" + "revision": "b001fa50d6b27f3f0bb175a87d0cb55426d0a0ae", + "revisionTime": "2018-11-28T10:09:59Z" }, { "checksumSHA1": "gJY7bRpELtO0PJpZXgPQ2BYFJ88=", "path": "github.com/syndtr/goleveldb/leveldb/journal", - "revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445", - "revisionTime": "2018-07-08T03:05:51Z" + "revision": "b001fa50d6b27f3f0bb175a87d0cb55426d0a0ae", + "revisionTime": "2018-11-28T10:09:59Z" }, { "checksumSHA1": "MtYY1b2234y/MlS+djL8tXVAcQs=", "path": "github.com/syndtr/goleveldb/leveldb/memdb", - "revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445", - "revisionTime": "2018-07-08T03:05:51Z" + "revision": "b001fa50d6b27f3f0bb175a87d0cb55426d0a0ae", + "revisionTime": "2018-11-28T10:09:59Z" }, { - "checksumSHA1": "UmQeotV+m8/FduKEfLOhjdp18rs=", + "checksumSHA1": "o2TorI3z+vc+EBMJ8XeFoUmXBtU=", "path": "github.com/syndtr/goleveldb/leveldb/opt", - "revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445", - "revisionTime": "2018-07-08T03:05:51Z" + "revision": "b001fa50d6b27f3f0bb175a87d0cb55426d0a0ae", + "revisionTime": "2018-11-28T10:09:59Z" }, { "checksumSHA1": "ZnyuciM+R19NG8L5YS3TIJdo1e8=", "path": "github.com/syndtr/goleveldb/leveldb/storage", - "revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445", - "revisionTime": "2018-07-08T03:05:51Z" + "revision": "b001fa50d6b27f3f0bb175a87d0cb55426d0a0ae", + "revisionTime": "2018-11-28T10:09:59Z" }, { "checksumSHA1": "gWFPMz8OQeul0t54RM66yMTX49g=", "path": "github.com/syndtr/goleveldb/leveldb/table", - "revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445", - "revisionTime": "2018-07-08T03:05:51Z" + "revision": "b001fa50d6b27f3f0bb175a87d0cb55426d0a0ae", + "revisionTime": "2018-11-28T10:09:59Z" }, { "checksumSHA1": "V/Dh7NV0/fy/5jX1KaAjmGcNbzI=", "path": "github.com/syndtr/goleveldb/leveldb/util", - "revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445", - "revisionTime": "2018-07-08T03:05:51Z" + "revision": "b001fa50d6b27f3f0bb175a87d0cb55426d0a0ae", + "revisionTime": "2018-11-28T10:09:59Z" }, { "checksumSHA1": "nD6S4KB0S+YHxVMDDE+w3PyXaMk=", From 55a4ff806fc1c8018055c56abcc6857502447902 Mon Sep 17 00:00:00 2001 From: zah Date: Thu, 29 Nov 2018 11:57:00 +0200 Subject: [PATCH 105/159] remove a no-op line in the code (#17760) --- les/flowcontrol/control.go | 1 - 1 file changed, 1 deletion(-) diff --git a/les/flowcontrol/control.go b/les/flowcontrol/control.go index d50eb809cc..8ef4ba511f 100644 --- a/les/flowcontrol/control.go +++ b/les/flowcontrol/control.go @@ -82,7 +82,6 @@ func (peer *ClientNode) RequestProcessed(cost uint64) (bv, realCost uint64) { time := mclock.Now() peer.recalcBV(time) peer.bufValue -= cost - peer.recalcBV(time) rcValue, rcost := peer.cm.processed(peer.cmNode, time) if rcValue < peer.params.BufLimit { bv := peer.params.BufLimit - rcValue From a4428c505e84aaa1982d4cc522e0c0ffe608890a Mon Sep 17 00:00:00 2001 From: Eugene Valeyev Date: Thu, 29 Nov 2018 13:02:31 +0300 Subject: [PATCH 106/159] mobile: added constructor for BigInts (#17828) --- mobile/big.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mobile/big.go b/mobile/big.go index dd7b158786..86ea93245a 100644 --- a/mobile/big.go +++ b/mobile/big.go @@ -84,6 +84,13 @@ func (bi *BigInt) SetString(x string, base int) { // BigInts represents a slice of big ints. type BigInts struct{ bigints []*big.Int } +// NewBigInts creates a slice of uninitialized big numbers. +func NewBigInts(size int) *BigInts { + return &BigInts{ + bigints: make([]*big.Int, size), + } +} + // Size returns the number of big ints in the slice. func (bi *BigInts) Size() int { return len(bi.bigints) From 32d35c9c088463efac49aeb0f3e6d48cfb373a40 Mon Sep 17 00:00:00 2001 From: Wenbiao Zheng Date: Thu, 29 Nov 2018 18:04:56 +0800 Subject: [PATCH 107/159] accounts/keystore: delete the redundant keystore in filename (#17930) * accounts/keystore: reduce file name length * accounts/keystore: reduce code line width --- accounts/keystore/account_cache.go | 5 ++++- accounts/keystore/key.go | 8 ++++++-- .../keystore/{keystore_passphrase.go => passphrase.go} | 1 + .../{keystore_passphrase_test.go => passphrase_test.go} | 0 accounts/keystore/{keystore_plain.go => plain.go} | 0 .../keystore/{keystore_plain_test.go => plain_test.go} | 0 accounts/keystore/presale.go | 8 +++++++- accounts/keystore/{keystore_wallet.go => wallet.go} | 0 8 files changed, 18 insertions(+), 4 deletions(-) rename accounts/keystore/{keystore_passphrase.go => passphrase.go} (99%) rename accounts/keystore/{keystore_passphrase_test.go => passphrase_test.go} (100%) rename accounts/keystore/{keystore_plain.go => plain.go} (100%) rename accounts/keystore/{keystore_plain_test.go => plain_test.go} (100%) rename accounts/keystore/{keystore_wallet.go => wallet.go} (100%) diff --git a/accounts/keystore/account_cache.go b/accounts/keystore/account_cache.go index da3a46eb80..8f660e282f 100644 --- a/accounts/keystore/account_cache.go +++ b/accounts/keystore/account_cache.go @@ -265,7 +265,10 @@ func (ac *accountCache) scanAccounts() error { case (addr == common.Address{}): log.Debug("Failed to decode keystore key", "path", path, "err", "missing or zero address") default: - return &accounts.Account{Address: addr, URL: accounts.URL{Scheme: KeyStoreScheme, Path: path}} + return &accounts.Account{ + Address: addr, + URL: accounts.URL{Scheme: KeyStoreScheme, Path: path}, + } } return nil } diff --git a/accounts/keystore/key.go b/accounts/keystore/key.go index 0564751c43..84d8df0c5a 100644 --- a/accounts/keystore/key.go +++ b/accounts/keystore/key.go @@ -171,7 +171,10 @@ func storeNewKey(ks keyStore, rand io.Reader, auth string) (*Key, accounts.Accou if err != nil { return nil, accounts.Account{}, err } - a := accounts.Account{Address: key.Address, URL: accounts.URL{Scheme: KeyStoreScheme, Path: ks.JoinPath(keyFileName(key.Address))}} + a := accounts.Account{ + Address: key.Address, + URL: accounts.URL{Scheme: KeyStoreScheme, Path: ks.JoinPath(keyFileName(key.Address))}, + } if err := ks.StoreKey(a.URL.Path, key, auth); err != nil { zeroKey(key.PrivateKey) return nil, a, err @@ -224,5 +227,6 @@ func toISO8601(t time.Time) string { } else { tz = fmt.Sprintf("%03d00", offset/3600) } - return fmt.Sprintf("%04d-%02d-%02dT%02d-%02d-%02d.%09d%s", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), tz) + return fmt.Sprintf("%04d-%02d-%02dT%02d-%02d-%02d.%09d%s", + t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), tz) } diff --git a/accounts/keystore/keystore_passphrase.go b/accounts/keystore/passphrase.go similarity index 99% rename from accounts/keystore/keystore_passphrase.go rename to accounts/keystore/passphrase.go index 9794f32fe1..a0b6cf5385 100644 --- a/accounts/keystore/keystore_passphrase.go +++ b/accounts/keystore/passphrase.go @@ -233,6 +233,7 @@ func DecryptKey(keyjson []byte, auth string) (*Key, error) { PrivateKey: key, }, nil } + func DecryptDataV3(cryptoJson CryptoJSON, auth string) ([]byte, error) { if cryptoJson.Cipher != "aes-128-ctr" { return nil, fmt.Errorf("Cipher not supported: %v", cryptoJson.Cipher) diff --git a/accounts/keystore/keystore_passphrase_test.go b/accounts/keystore/passphrase_test.go similarity index 100% rename from accounts/keystore/keystore_passphrase_test.go rename to accounts/keystore/passphrase_test.go diff --git a/accounts/keystore/keystore_plain.go b/accounts/keystore/plain.go similarity index 100% rename from accounts/keystore/keystore_plain.go rename to accounts/keystore/plain.go diff --git a/accounts/keystore/keystore_plain_test.go b/accounts/keystore/plain_test.go similarity index 100% rename from accounts/keystore/keystore_plain_test.go rename to accounts/keystore/plain_test.go diff --git a/accounts/keystore/presale.go b/accounts/keystore/presale.go index 1554294e14..03055245f5 100644 --- a/accounts/keystore/presale.go +++ b/accounts/keystore/presale.go @@ -38,7 +38,13 @@ func importPreSaleKey(keyStore keyStore, keyJSON []byte, password string) (accou return accounts.Account{}, nil, err } key.Id = uuid.NewRandom() - a := accounts.Account{Address: key.Address, URL: accounts.URL{Scheme: KeyStoreScheme, Path: keyStore.JoinPath(keyFileName(key.Address))}} + a := accounts.Account{ + Address: key.Address, + URL: accounts.URL{ + Scheme: KeyStoreScheme, + Path: keyStore.JoinPath(keyFileName(key.Address)), + }, + } err = keyStore.StoreKey(a.URL.Path, key, password) return a, key, err } diff --git a/accounts/keystore/keystore_wallet.go b/accounts/keystore/wallet.go similarity index 100% rename from accounts/keystore/keystore_wallet.go rename to accounts/keystore/wallet.go From cf62bd2e8878d880aa9d5b20c05bb348111f6298 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 29 Nov 2018 12:47:29 +0200 Subject: [PATCH 108/159] cmd/utils: max out the OS file allowance, don't cap to 2K --- cmd/utils/flags.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index d0597c2f18..6a285fcb39 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -824,17 +824,12 @@ func setIPC(ctx *cli.Context, cfg *node.Config) { // makeDatabaseHandles raises out the number of allowed file handles per process // for Geth and returns half of the allowance to assign to the database. func makeDatabaseHandles() int { - limit, err := fdlimit.Current() + limit, err := fdlimit.Maximum() if err != nil { Fatalf("Failed to retrieve file descriptor allowance: %v", err) } - if limit < 2048 { - if err := fdlimit.Raise(2048); err != nil { - Fatalf("Failed to raise file descriptor allowance: %v", err) - } - } - if limit > 2048 { // cap database file descriptors even if more is available - limit = 2048 + if err := fdlimit.Raise(uint64(limit)); err != nil { + Fatalf("Failed to raise file descriptor allowance: %v", err) } return limit / 2 // Leave half for networking and other stuff } From 7e7781ffaa5c5486c5e7b66dea4986e063524f63 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 29 Nov 2018 17:43:15 +0100 Subject: [PATCH 109/159] cmd/swarm: add flag for application name (swarm or swarm-private) (#18189) * cmd/swarm: add flag for application name (swarm or swarm-private) * cmd/swarm/swarm-smoke: return correct exit code * cmd/swarm/swarm-smoke: remove colorable * remove swarm/grafana_dashboards --- cmd/swarm/swarm-smoke/feed_upload_and_sync.go | 2 +- cmd/swarm/swarm-smoke/main.go | 8 + cmd/swarm/swarm-smoke/upload_and_sync.go | 11 +- swarm/grafana_dashboards/ldbstore.json | 2278 ------------ swarm/grafana_dashboards/swarm.json | 3198 ----------------- 5 files changed, 14 insertions(+), 5483 deletions(-) delete mode 100644 swarm/grafana_dashboards/ldbstore.json delete mode 100644 swarm/grafana_dashboards/swarm.json diff --git a/cmd/swarm/swarm-smoke/feed_upload_and_sync.go b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go index 0328d656f3..7ec1528263 100644 --- a/cmd/swarm/swarm-smoke/feed_upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go @@ -33,7 +33,7 @@ func cliFeedUploadAndSync(c *cli.Context) error { defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now()) - generateEndpoints(scheme, cluster, from, to) + generateEndpoints(scheme, cluster, appName, from, to) log.Info("generating and uploading feeds to " + endpoints[0] + " and syncing") diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index 4ff17fd5b7..845998dc12 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -29,6 +29,7 @@ var ( endpoints []string includeLocalhost bool cluster string + appName string scheme string filesize int from int @@ -49,6 +50,12 @@ func main() { Usage: "cluster to point to (prod or a given namespace)", Destination: &cluster, }, + cli.StringFlag{ + Name: "app", + Value: "swarm", + Usage: "application to point to (swarm or swarm-private)", + Destination: &appName, + }, cli.IntFlag{ Name: "cluster-from", Value: 8501, @@ -107,5 +114,6 @@ func main() { err := app.Run(os.Args) if err != nil { log.Error(err.Error()) + os.Exit(1) } } diff --git a/cmd/swarm/swarm-smoke/upload_and_sync.go b/cmd/swarm/swarm-smoke/upload_and_sync.go index 7872421d3d..3843457dc4 100644 --- a/cmd/swarm/swarm-smoke/upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/upload_and_sync.go @@ -33,20 +33,19 @@ import ( "time" "github.com/ethereum/go-ethereum/log" - colorable "github.com/mattn/go-colorable" "github.com/pborman/uuid" cli "gopkg.in/urfave/cli.v1" ) -func generateEndpoints(scheme string, cluster string, from int, to int) { +func generateEndpoints(scheme string, cluster string, app string, from int, to int) { if cluster == "prod" { for port := from; port <= to; port++ { endpoints = append(endpoints, fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, port)) } } else { for port := from; port <= to; port++ { - endpoints = append(endpoints, fmt.Sprintf("%s://swarm-%v-%s.stg.swarm-gateways.net", scheme, port, cluster)) + endpoints = append(endpoints, fmt.Sprintf("%s://%s-%v-%s.stg.swarm-gateways.net", scheme, app, port, cluster)) } } @@ -57,11 +56,11 @@ func generateEndpoints(scheme string, cluster string, from int, to int) { func cliUploadAndSync(c *cli.Context) error { log.PrintOrigins(true) - log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) - defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now()) + defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "kb", filesize) }(time.Now()) - generateEndpoints(scheme, cluster, from, to) + generateEndpoints(scheme, cluster, appName, from, to) log.Info("uploading to " + endpoints[0] + " and syncing") diff --git a/swarm/grafana_dashboards/ldbstore.json b/swarm/grafana_dashboards/ldbstore.json deleted file mode 100644 index 2d64380ba8..0000000000 --- a/swarm/grafana_dashboards/ldbstore.json +++ /dev/null @@ -1,2278 +0,0 @@ -{ - "annotations": { - "list": [ - { - "$$hashKey": "object:325", - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 1, - "id": 5, - "iteration": 1527598894689, - "links": [], - "panels": [ - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 40, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 42, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.localstore.get.cachehit.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LocalStore get cachehit", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 43, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.localstore.get.cachemiss.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LocalStore get cachemiss", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 7 - }, - "id": 44, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.localstore.getorcreaterequest.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Total LocalStore.GetOrCreateRequest", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 7 - }, - "id": 47, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.localstore.getorcreaterequest.errfetching.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LocalStore GetOrCreateRequest ErrFetching", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 13 - }, - "id": 45, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.localstore.getorcreaterequest.hit.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LocalStore.GetOrCreateRequest hit", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 13 - }, - "id": 49, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.localstore.getorcreaterequest.miss.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LocalStore GetOrCreateRequest miss", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 19 - }, - "id": 48, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.localstore.get.error.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LocalStore get error", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 19 - }, - "id": 46, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.localstore.get.errfetching.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LocalStore get ErrFetching", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "LocalStore", - "type": "row" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 27, - "panels": [], - "title": "LDBStore", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 2 - }, - "id": 29, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.ldbstore.get.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LDBStore get", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 2 - }, - "id": 30, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.ldbstore.put.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LDBStore put", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 31, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.ldbstore.synciterator.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LDBStore SyncIterator", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 32, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.ldbstore.synciterator.seek.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LDBStore SyncIterator Seek/Next", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 14 - }, - "id": 50, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.ldbstore.collectgarbage.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LDBStore Collect Garbage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 14 - }, - "id": 51, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.ldbstore.collectgarbage.delete.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LDBStore Collect Garbage - Actual Deletes", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 20 - }, - "id": 34, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 39 - }, - "id": 36, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.ldbdatabase.get.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LDBDatabase get", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 39 - }, - "id": 37, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.ldbdatabase.write.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LDBDatabase write", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 45 - }, - "id": 38, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.ldbdatabase.newiterator.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LDBDatabase NewIterator", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "LDBDatabase", - "type": "row" - } - ], - "refresh": "10s", - "schemaVersion": 16, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "auto": false, - "auto_count": 30, - "auto_min": "10s", - "current": { - "text": "10s", - "value": "10s" - }, - "hide": 0, - "label": "resolution", - "name": "myinterval", - "options": [ - { - "selected": false, - "text": "5s", - "value": "5s" - }, - { - "selected": true, - "text": "10s", - "value": "10s" - }, - { - "selected": false, - "text": "30s", - "value": "30s" - }, - { - "selected": false, - "text": "100s", - "value": "100s" - } - ], - "query": "5s,10s,30s,100s", - "refresh": 2, - "type": "interval" - }, - { - "allValue": null, - "current": { - "text": "swarm_30399 + swarm_30400 + swarm_30401", - "value": [ - "swarm_30399", - "swarm_30400", - "swarm_30401" - ] - }, - "datasource": "metrics", - "hide": 0, - "includeAll": true, - "label": null, - "multi": true, - "name": "host", - "options": [], - "query": "SHOW TAG VALUES WITH KEY = \"host\"", - "refresh": 1, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "swarm.http.request.GET.time.span", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "LDBStore and LDBDatabase", - "uid": "zS6beG7iz", - "version": 28 -} diff --git a/swarm/grafana_dashboards/swarm.json b/swarm/grafana_dashboards/swarm.json deleted file mode 100644 index 3ee244d154..0000000000 --- a/swarm/grafana_dashboards/swarm.json +++ /dev/null @@ -1,3198 +0,0 @@ -{ - "annotations": { - "list": [ - { - "$$hashKey": "object:147", - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 1, - "id": 2, - "iteration": 1527598859072, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 34, - "panels": [], - "title": "P2P", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 36, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "none" - ], - "type": "fill" - } - ], - "measurement": "swarm.peer.send.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "P2P Send() - messages sent", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 37, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "p95($tag_host)", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "none" - ], - "type": "fill" - } - ], - "measurement": "swarm.peer.send_t.span", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "p95" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "P2P Send() timer - 95%ile", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ns", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 10 - }, - "id": 38, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "1 $tag_host", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "none" - ], - "type": "fill" - } - ], - "measurement": "swarm.peer.sendpriority.1.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [] - }, - { - "alias": "2 $tag_host", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "none" - ], - "type": "fill" - } - ], - "measurement": "swarm.peer.sendpriority.2.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [] - }, - { - "alias": "3 $tag_host", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "none" - ], - "type": "fill" - } - ], - "measurement": "swarm.peer.sendpriority.3.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "C", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "P2P SendPriority() - messages sent", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 10 - }, - "id": 39, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "1 $tag_host", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "none" - ], - "type": "fill" - } - ], - "measurement": "swarm.peer.sendpriority_t.1.span", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "p95" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [] - }, - { - "alias": "2 $tag_host", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "none" - ], - "type": "fill" - } - ], - "measurement": "swarm.peer.sendpriority_t.2.span", - "orderByTime": "ASC", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "p95" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "P2P SendPriority() timer - 95%ile", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ns", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 19 - }, - "id": 40, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "none" - ], - "type": "fill" - } - ], - "measurement": "swarm.registry.peers.gauge", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "last" - } - ] - ], - "tags": [] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Registry Peers", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 32, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 2 - }, - "id": 14, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.stack.uptime.gauge", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Uptime", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ns", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "Uptime", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 29 - }, - "id": 28, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 7 - }, - "id": 2, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "GET", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "swarm.http.request.GET.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - }, - { - "alias": "POST", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "swarm.http.request.POST.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Total HTTP Requests", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 7 - }, - "id": 26, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.http.request.GET.time.span", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "p95" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "HTTP GET requests 95% timer", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ns", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 13 - }, - "id": 15, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.http.request.GET.time.span", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "p50" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "HTTP GET requests 50% timer", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ns", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 13 - }, - "id": 8, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "POST", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.http.request.POST.time.span", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "p95" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "HTTP POST requests 95% timer", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ns", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "HTTP", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 30 - }, - "id": 30, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 16, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.lazychunkreader.read.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LazyChunkReader read() calls", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 18, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.lazychunkreader.read.err.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LazyChunkReader read errors", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 13 - }, - "id": 17, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.lazychunkreader.read.bytes.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LazyChunkReader bytes read", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "LazyChunkReader", - "type": "row" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 31 - }, - "id": 25, - "panels": [], - "title": "All measurements", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 3, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.api.get.count.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "API Get (BZZ)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 13, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.network.stream.request_from_peers.count.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Request from peers", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 38 - }, - "id": 11, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.network.stream.received_chunks.count.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Received chunks", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 38 - }, - "id": 12, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.storage.cache.requests.size.gauge", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "max" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Requests cache entries", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 44 - }, - "id": 9, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.network.stream.handle_retrieve_request_msg.count.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Handle retrieve request msg", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 44 - }, - "id": 20, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.syncer.setnextbatch.iterator.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "syncer setnextbatch iterator calls", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 50 - }, - "id": 21, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.peer.handlewantedhashesmsg.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "peer HandleWantedHashesMsg", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 50 - }, - "id": 22, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.peer.handlesubscribemsg.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "peer HandleSubscribeMsg", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 56 - }, - "id": 23, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.peer.handlewantedhashesmsg.actualget.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "peer HandleWantedHashesMsg actual get", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 56 - }, - "id": 19, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.peer.handleofferedhashes.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "peer OfferedHashesMsg", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "refresh": "30s", - "schemaVersion": 16, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "auto": false, - "auto_count": 30, - "auto_min": "10s", - "current": { - "text": "10s", - "value": "10s" - }, - "hide": 0, - "label": "resolution", - "name": "myinterval", - "options": [ - { - "selected": false, - "text": "5s", - "value": "5s" - }, - { - "selected": true, - "text": "10s", - "value": "10s" - }, - { - "selected": false, - "text": "30s", - "value": "30s" - }, - { - "selected": false, - "text": "100s", - "value": "100s" - } - ], - "query": "5s,10s,30s,100s", - "refresh": 2, - "type": "interval" - }, - { - "allValue": null, - "current": { - "text": "swarm_30399 + swarm_30400 + swarm_30401 + swarm_30402", - "value": [ - "swarm_30399", - "swarm_30400", - "swarm_30401", - "swarm_30402" - ] - }, - "datasource": "metrics", - "hide": 0, - "includeAll": true, - "label": null, - "multi": true, - "name": "host", - "options": [], - "query": "SHOW TAG VALUES WITH KEY = \"host\"", - "refresh": 1, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "swarm.http.request.GET.time.span", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Swarm", - "uid": "vmEtxxgmz", - "version": 138 -} From ef8ced4151006c84e77ab8db88cf6fc1bb09a716 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 30 Nov 2018 10:22:53 +0100 Subject: [PATCH 110/159] vendor: update github.com/karalabe/hid (#18213) Fixes #15101 because hidapi is no longer being called from an init function. --- vendor/github.com/karalabe/hid/appveyor.yml | 4 ++-- vendor/github.com/karalabe/hid/hid_disabled.go | 2 +- vendor/github.com/karalabe/hid/hid_enabled.go | 12 ++++++------ vendor/vendor.json | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/vendor/github.com/karalabe/hid/appveyor.yml b/vendor/github.com/karalabe/hid/appveyor.yml index f43958747c..84b3c95ff7 100644 --- a/vendor/github.com/karalabe/hid/appveyor.yml +++ b/vendor/github.com/karalabe/hid/appveyor.yml @@ -22,8 +22,8 @@ environment: install: - rmdir C:\go /s /q - - appveyor DownloadFile https://storage.googleapis.com/golang/go1.8.windows-%GOARCH%.zip - - 7z x go1.8.windows-%GOARCH%.zip -y -oC:\ > NUL + - appveyor DownloadFile https://storage.googleapis.com/golang/go1.10.1.windows-%GOARCH%.zip + - 7z x go1.10.1.windows-%GOARCH%.zip -y -oC:\ > NUL - go version - gcc --version diff --git a/vendor/github.com/karalabe/hid/hid_disabled.go b/vendor/github.com/karalabe/hid/hid_disabled.go index 1f40263791..0f266ba58a 100644 --- a/vendor/github.com/karalabe/hid/hid_disabled.go +++ b/vendor/github.com/karalabe/hid/hid_disabled.go @@ -36,7 +36,7 @@ func (info DeviceInfo) Open() (*Device, error) { // Close releases the HID USB device handle. On platforms that this file implements // the method is just a noop. -func (dev *Device) Close() {} +func (dev *Device) Close() error { return nil } // Write sends an output report to a HID device. On platforms that this file // implements the method just returns an error. diff --git a/vendor/github.com/karalabe/hid/hid_enabled.go b/vendor/github.com/karalabe/hid/hid_enabled.go index 419273be6a..e95e5792d9 100644 --- a/vendor/github.com/karalabe/hid/hid_enabled.go +++ b/vendor/github.com/karalabe/hid/hid_enabled.go @@ -41,6 +41,7 @@ package hid #endif */ import "C" + import ( "errors" "runtime" @@ -57,11 +58,6 @@ import ( // > "subsequent calls will cause the hid manager to release previously enumerated devices" var enumerateLock sync.Mutex -func init() { - // Initialize the HIDAPI library - C.hid_init() -} - // Supported returns whether this platform is supported by the HID library or not. // The goal of this method is to allow programatically handling platforms that do // not support USB HID and not having to fall back to build constraints. @@ -113,6 +109,9 @@ func Enumerate(vendorID uint16, productID uint16) []DeviceInfo { // Open connects to an HID device by its path name. func (info DeviceInfo) Open() (*Device, error) { + enumerateLock.Lock() + defer enumerateLock.Unlock() + path := C.CString(info.Path) defer C.free(unsafe.Pointer(path)) @@ -135,7 +134,7 @@ type Device struct { } // Close releases the HID USB device handle. -func (dev *Device) Close() { +func (dev *Device) Close() error { dev.lock.Lock() defer dev.lock.Unlock() @@ -143,6 +142,7 @@ func (dev *Device) Close() { C.hid_close(dev.device) dev.device = nil } + return nil } // Write sends an output report to a HID device. diff --git a/vendor/vendor.json b/vendor/vendor.json index a4fee11aa5..883d02097d 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -267,10 +267,10 @@ "revisionTime": "2017-04-30T22:20:11Z" }, { - "checksumSHA1": "UpjhOUZ1+0zNt+iIvdtECSHXmTs=", + "checksumSHA1": "6XsjAARQFvlW6dS15al0ibTFPOQ=", "path": "github.com/karalabe/hid", - "revision": "f00545f9f3748e591590be3732d913c77525b10f", - "revisionTime": "2017-08-21T10:38:37Z", + "revision": "d815e0c1a2e2082a287a2806bc90bc8fc7b276a9", + "revisionTime": "2018-11-28T19:21:57Z", "tree": true }, { From 54abb97e3be4fb2d6f557758aaa3a8291b0979fe Mon Sep 17 00:00:00 2001 From: needkane <604476380@qq.com> Date: Sat, 1 Dec 2018 05:38:37 +0800 Subject: [PATCH 111/159] p2p: use errors.New instead of fmt.Errorf (#18193) --- p2p/server.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/p2p/server.go b/p2p/server.go index 6678608634..566f01ffc5 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -22,7 +22,6 @@ import ( "crypto/ecdsa" "encoding/hex" "errors" - "fmt" "net" "sort" "sync" @@ -391,7 +390,7 @@ type sharedUDPConn struct { func (s *sharedUDPConn) ReadFromUDP(b []byte) (n int, addr *net.UDPAddr, err error) { packet, ok := <-s.unhandled if !ok { - return 0, nil, fmt.Errorf("Connection was closed") + return 0, nil, errors.New("Connection was closed") } l := len(packet.Data) if l > len(b) { @@ -425,7 +424,7 @@ func (srv *Server) Start() (err error) { // static fields if srv.PrivateKey == nil { - return fmt.Errorf("Server.PrivateKey must be set to a non-nil key") + return errors.New("Server.PrivateKey must be set to a non-nil key") } if srv.newTransport == nil { srv.newTransport = newRLPX @@ -903,7 +902,7 @@ func (srv *Server) setupConn(c *conn, flags connFlag, dialDest *enode.Node) erro if dialDest != nil { dialPubkey = new(ecdsa.PublicKey) if err := dialDest.Load((*enode.Secp256k1)(dialPubkey)); err != nil { - return fmt.Errorf("dial destination doesn't have a secp256k1 public key") + return errors.New("dial destination doesn't have a secp256k1 public key") } } // Run the encryption handshake. @@ -937,7 +936,7 @@ func (srv *Server) setupConn(c *conn, flags connFlag, dialDest *enode.Node) erro return err } if id := c.node.ID(); !bytes.Equal(crypto.Keccak256(phs.ID), id[:]) { - clog.Trace("Wrong devp2p handshake identity", "phsid", fmt.Sprintf("%x", phs.ID)) + clog.Trace("Wrong devp2p handshake identity", "phsid", hex.EncodeToString(phs.ID)) return DiscUnexpectedIdentity } c.caps, c.name = phs.Caps, phs.Name From 085f89172f0af9d9685d228ad163d639041f1217 Mon Sep 17 00:00:00 2001 From: lash Date: Sat, 1 Dec 2018 07:07:18 +0100 Subject: [PATCH 112/159] swarm/pss: Add same api interface for all Send* methods (#18218) --- swarm/pss/api.go | 4 ++++ swarm/pss/pss_test.go | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/swarm/pss/api.go b/swarm/pss/api.go index dd55b2a702..587382d729 100644 --- a/swarm/pss/api.go +++ b/swarm/pss/api.go @@ -164,6 +164,10 @@ func (pssapi *API) SendSym(symkeyhex string, topic Topic, msg hexutil.Bytes) err return pssapi.Pss.SendSym(symkeyhex, topic, msg[:]) } +func (pssapi *API) SendRaw(addr hexutil.Bytes, topic Topic, msg hexutil.Bytes) error { + return pssapi.Pss.SendRaw(PssAddress(addr), topic, msg[:]) +} + func (pssapi *API) GetPeerTopics(pubkeyhex string) ([]Topic, error) { topics, _, err := pssapi.Pss.GetPublickeyPeers(pubkeyhex) return topics, err diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index 32404aaaf9..72f62acd94 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -1064,7 +1064,7 @@ func testSendRaw(t *testing.T) { // send and verify delivery lmsg := []byte("plugh") - err = clients[1].Call(nil, "pss_sendRaw", loaddrhex, topic, lmsg) + err = clients[1].Call(nil, "pss_sendRaw", loaddrhex, topic, hexutil.Encode(lmsg)) if err != nil { t.Fatal(err) } @@ -1077,7 +1077,7 @@ func testSendRaw(t *testing.T) { t.Fatalf("test message (left) timed out: %v", cerr) } rmsg := []byte("xyzzy") - err = clients[0].Call(nil, "pss_sendRaw", roaddrhex, topic, rmsg) + err = clients[0].Call(nil, "pss_sendRaw", roaddrhex, topic, hexutil.Encode(rmsg)) if err != nil { t.Fatal(err) } From efe5886877d3f171fae5adfd1e5f1c585e16946b Mon Sep 17 00:00:00 2001 From: vim88 Date: Sun, 2 Dec 2018 14:00:42 +0200 Subject: [PATCH 113/159] signer/core: Fixes typo of method name in comment. (#18222) --- signer/core/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/signer/core/api.go b/signer/core/api.go index 2b96cdb5f3..e9a3357855 100644 --- a/signer/core/api.go +++ b/signer/core/api.go @@ -82,7 +82,7 @@ type SignerUI interface { // OnSignerStartup is invoked when the signer boots, and tells the UI info about external API location and version // information OnSignerStartup(info StartupInfo) - // OnInputRequried is invoked when clef requires user input, for example master password or + // OnInputRequired is invoked when clef requires user input, for example master password or // pin-code for unlocking hardware wallets OnInputRequired(info UserInputRequest) (UserInputResponse, error) } From f850123ec7500af732b73c2501f1ef36d80ee01d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Kj=C3=A6rstad?= Date: Sun, 2 Dec 2018 13:03:31 +0100 Subject: [PATCH 114/159] Changed http:// to https:// on JSON-RPC link (#18224) Changed http:// to https:// on JSON-RPC link in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4b62bfde03..7593dd0905 100644 --- a/README.md +++ b/README.md @@ -168,7 +168,7 @@ HTTP based JSON-RPC API options: * `--ipcpath` Filename for IPC socket/pipe within the datadir (explicit paths escape it) You'll need to use your own programming environments' capabilities (libraries, tools, etc) to connect -via HTTP, WS or IPC to a Geth node configured with the above flags and you'll need to speak [JSON-RPC](http://www.jsonrpc.org/specification) +via HTTP, WS or IPC to a Geth node configured with the above flags and you'll need to speak [JSON-RPC](https://www.jsonrpc.org/specification) on all transports. You can reuse the same connection for multiple requests! **Note: Please understand the security implications of opening up an HTTP/WS based transport before From 4825d9c3dd7d07eadfda8a6d5c18efb67df67088 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 3 Dec 2018 12:17:08 +0200 Subject: [PATCH 115/159] cmd/puppeth: enforce lowercase network names --- cmd/puppeth/puppeth.go | 4 ++-- cmd/puppeth/wizard_intro.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/puppeth/puppeth.go b/cmd/puppeth/puppeth.go index f9b8fe4811..df3fff51b0 100644 --- a/cmd/puppeth/puppeth.go +++ b/cmd/puppeth/puppeth.go @@ -49,8 +49,8 @@ func main() { rand.Seed(time.Now().UnixNano()) network := c.String("network") - if strings.Contains(network, " ") || strings.Contains(network, "-") { - log.Crit("No spaces or hyphens allowed in network name") + if strings.Contains(network, " ") || strings.Contains(network, "-") || strings.ToLower(network) != network { + log.Crit("No spaces, hyphens or capital letters allowed in network name") } // Start the wizard and relinquish control makeWizard(c.String("network")).run() diff --git a/cmd/puppeth/wizard_intro.go b/cmd/puppeth/wizard_intro.go index 60aa0f7ffb..3db9a1087e 100644 --- a/cmd/puppeth/wizard_intro.go +++ b/cmd/puppeth/wizard_intro.go @@ -61,14 +61,14 @@ func (w *wizard) run() { // Make sure we have a good network name to work with fmt.Println() // Docker accepts hyphens in image names, but doesn't like it for container names if w.network == "" { - fmt.Println("Please specify a network name to administer (no spaces or hyphens, please)") + fmt.Println("Please specify a network name to administer (no spaces, hyphens or capital letters please)") for { w.network = w.readString() - if !strings.Contains(w.network, " ") && !strings.Contains(w.network, "-") { + if !strings.Contains(w.network, " ") && !strings.Contains(w.network, "-") && strings.ToLower(w.network) == w.network { fmt.Printf("\nSweet, you can set this via --network=%s next time!\n\n", w.network) break } - log.Error("I also like to live dangerously, still no spaces or hyphens") + log.Error("I also like to live dangerously, still no spaces, hyphens or capital letters") } } log.Info("Administering Ethereum network", "name", w.network) From 8698fbabf6c7811e8eec6e84512510e9c9a4eb45 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Sat, 24 Nov 2018 23:22:25 +0100 Subject: [PATCH 116/159] cmd/puppeth: implement chainspec converters --- cmd/puppeth/genesis.go | 388 ++++++++++++++--------- cmd/puppeth/module_dashboard.go | 2 +- cmd/puppeth/puppeth.go | 32 ++ cmd/puppeth/puppeth_test.go | 91 ++++++ cmd/puppeth/testdata/stureby_aleth.json | 112 +++++++ cmd/puppeth/testdata/stureby_geth.json | 47 +++ cmd/puppeth/testdata/stureby_parity.json | 181 +++++++++++ cmd/puppeth/wizard.go | 22 ++ cmd/puppeth/wizard_genesis.go | 67 +++- 9 files changed, 780 insertions(+), 162 deletions(-) create mode 100644 cmd/puppeth/puppeth_test.go create mode 100644 cmd/puppeth/testdata/stureby_aleth.json create mode 100644 cmd/puppeth/testdata/stureby_geth.json create mode 100644 cmd/puppeth/testdata/stureby_parity.json diff --git a/cmd/puppeth/genesis.go b/cmd/puppeth/genesis.go index 5f39a889d1..83212a126b 100644 --- a/cmd/puppeth/genesis.go +++ b/cmd/puppeth/genesis.go @@ -20,35 +20,41 @@ import ( "encoding/binary" "errors" "math" + "math/big" + "strings" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + math2 "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/params" ) -// cppEthereumGenesisSpec represents the genesis specification format used by the +// alethGenesisSpec represents the genesis specification format used by the // C++ Ethereum implementation. -type cppEthereumGenesisSpec struct { +type alethGenesisSpec struct { SealEngine string `json:"sealEngine"` Params struct { - AccountStartNonce hexutil.Uint64 `json:"accountStartNonce"` - HomesteadForkBlock hexutil.Uint64 `json:"homesteadForkBlock"` - EIP150ForkBlock hexutil.Uint64 `json:"EIP150ForkBlock"` - EIP158ForkBlock hexutil.Uint64 `json:"EIP158ForkBlock"` - ByzantiumForkBlock hexutil.Uint64 `json:"byzantiumForkBlock"` - ConstantinopleForkBlock hexutil.Uint64 `json:"constantinopleForkBlock"` - NetworkID hexutil.Uint64 `json:"networkID"` - ChainID hexutil.Uint64 `json:"chainID"` - MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"` - MinGasLimit hexutil.Uint64 `json:"minGasLimit"` - MaxGasLimit hexutil.Uint64 `json:"maxGasLimit"` - GasLimitBoundDivisor hexutil.Uint64 `json:"gasLimitBoundDivisor"` - MinimumDifficulty *hexutil.Big `json:"minimumDifficulty"` - DifficultyBoundDivisor *hexutil.Big `json:"difficultyBoundDivisor"` - DurationLimit *hexutil.Big `json:"durationLimit"` - BlockReward *hexutil.Big `json:"blockReward"` + AccountStartNonce math2.HexOrDecimal64 `json:"accountStartNonce"` + MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"` + HomesteadForkBlock hexutil.Uint64 `json:"homesteadForkBlock"` + DaoHardforkBlock math2.HexOrDecimal64 `json:"daoHardforkBlock"` + EIP150ForkBlock hexutil.Uint64 `json:"EIP150ForkBlock"` + EIP158ForkBlock hexutil.Uint64 `json:"EIP158ForkBlock"` + ByzantiumForkBlock hexutil.Uint64 `json:"byzantiumForkBlock"` + ConstantinopleForkBlock hexutil.Uint64 `json:"constantinopleForkBlock"` + MinGasLimit hexutil.Uint64 `json:"minGasLimit"` + MaxGasLimit hexutil.Uint64 `json:"maxGasLimit"` + TieBreakingGas bool `json:"tieBreakingGas"` + GasLimitBoundDivisor math2.HexOrDecimal64 `json:"gasLimitBoundDivisor"` + MinimumDifficulty *hexutil.Big `json:"minimumDifficulty"` + DifficultyBoundDivisor *math2.HexOrDecimal256 `json:"difficultyBoundDivisor"` + DurationLimit *math2.HexOrDecimal256 `json:"durationLimit"` + BlockReward *hexutil.Big `json:"blockReward"` + NetworkID hexutil.Uint64 `json:"networkID"` + ChainID hexutil.Uint64 `json:"chainID"` + AllowFutureBlocks bool `json:"allowFutureBlocks""` } `json:"params"` Genesis struct { @@ -62,57 +68,68 @@ type cppEthereumGenesisSpec struct { GasLimit hexutil.Uint64 `json:"gasLimit"` } `json:"genesis"` - Accounts map[common.Address]*cppEthereumGenesisSpecAccount `json:"accounts"` + Accounts map[common.UnprefixedAddress]*alethGenesisSpecAccount `json:"accounts"` } -// cppEthereumGenesisSpecAccount is the prefunded genesis account and/or precompiled +// alethGenesisSpecAccount is the prefunded genesis account and/or precompiled // contract definition. -type cppEthereumGenesisSpecAccount struct { - Balance *hexutil.Big `json:"balance"` - Nonce uint64 `json:"nonce,omitempty"` - Precompiled *cppEthereumGenesisSpecBuiltin `json:"precompiled,omitempty"` +type alethGenesisSpecAccount struct { + Balance *math2.HexOrDecimal256 `json:"balance"` + Nonce uint64 `json:"nonce,omitempty"` + Precompiled *alethGenesisSpecBuiltin `json:"precompiled,omitempty"` } -// cppEthereumGenesisSpecBuiltin is the precompiled contract definition. -type cppEthereumGenesisSpecBuiltin struct { - Name string `json:"name,omitempty"` - StartingBlock hexutil.Uint64 `json:"startingBlock,omitempty"` - Linear *cppEthereumGenesisSpecLinearPricing `json:"linear,omitempty"` +// alethGenesisSpecBuiltin is the precompiled contract definition. +type alethGenesisSpecBuiltin struct { + Name string `json:"name,omitempty"` + StartingBlock hexutil.Uint64 `json:"startingBlock,omitempty"` + Linear *alethGenesisSpecLinearPricing `json:"linear,omitempty"` } -type cppEthereumGenesisSpecLinearPricing struct { +type alethGenesisSpecLinearPricing struct { Base uint64 `json:"base"` Word uint64 `json:"word"` } -// newCppEthereumGenesisSpec converts a go-ethereum genesis block into a Parity specific +// newAlethGenesisSpec converts a go-ethereum genesis block into a Aleth-specific // chain specification format. -func newCppEthereumGenesisSpec(network string, genesis *core.Genesis) (*cppEthereumGenesisSpec, error) { - // Only ethash is currently supported between go-ethereum and cpp-ethereum +func newAlethGenesisSpec(network string, genesis *core.Genesis) (*alethGenesisSpec, error) { + // Only ethash is currently supported between go-ethereum and aleth if genesis.Config.Ethash == nil { return nil, errors.New("unsupported consensus engine") } - // Reconstruct the chain spec in Parity's format - spec := &cppEthereumGenesisSpec{ + // Reconstruct the chain spec in Aleth format + spec := &alethGenesisSpec{ SealEngine: "Ethash", } + // Some defaults spec.Params.AccountStartNonce = 0 + spec.Params.TieBreakingGas = false + spec.Params.AllowFutureBlocks = false + spec.Params.DaoHardforkBlock = 0 + spec.Params.HomesteadForkBlock = (hexutil.Uint64)(genesis.Config.HomesteadBlock.Uint64()) spec.Params.EIP150ForkBlock = (hexutil.Uint64)(genesis.Config.EIP150Block.Uint64()) spec.Params.EIP158ForkBlock = (hexutil.Uint64)(genesis.Config.EIP158Block.Uint64()) - spec.Params.ByzantiumForkBlock = (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()) - spec.Params.ConstantinopleForkBlock = (hexutil.Uint64)(math.MaxUint64) + + // Byzantium + if num := genesis.Config.ByzantiumBlock; num != nil { + spec.setByzantium(num) + } + // Constantinople + if num := genesis.Config.ConstantinopleBlock; num != nil { + spec.setConstantinople(num) + } spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64()) spec.Params.ChainID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64()) - spec.Params.MaximumExtraDataSize = (hexutil.Uint64)(params.MaximumExtraDataSize) spec.Params.MinGasLimit = (hexutil.Uint64)(params.MinGasLimit) - spec.Params.MaxGasLimit = (hexutil.Uint64)(math.MaxUint64) + spec.Params.MaxGasLimit = (hexutil.Uint64)(math.MaxInt64) spec.Params.MinimumDifficulty = (*hexutil.Big)(params.MinimumDifficulty) - spec.Params.DifficultyBoundDivisor = (*hexutil.Big)(params.DifficultyBoundDivisor) - spec.Params.GasLimitBoundDivisor = (hexutil.Uint64)(params.GasLimitBoundDivisor) - spec.Params.DurationLimit = (*hexutil.Big)(params.DurationLimit) + spec.Params.DifficultyBoundDivisor = (*math2.HexOrDecimal256)(params.DifficultyBoundDivisor) + spec.Params.GasLimitBoundDivisor = (math2.HexOrDecimal64)(params.GasLimitBoundDivisor) + spec.Params.DurationLimit = (*math2.HexOrDecimal256)(params.DurationLimit) spec.Params.BlockReward = (*hexutil.Big)(ethash.FrontierBlockReward) spec.Genesis.Nonce = (hexutil.Bytes)(make([]byte, 8)) @@ -126,77 +143,104 @@ func newCppEthereumGenesisSpec(network string, genesis *core.Genesis) (*cppEther spec.Genesis.ExtraData = (hexutil.Bytes)(genesis.ExtraData) spec.Genesis.GasLimit = (hexutil.Uint64)(genesis.GasLimit) - spec.Accounts = make(map[common.Address]*cppEthereumGenesisSpecAccount) for address, account := range genesis.Alloc { - spec.Accounts[address] = &cppEthereumGenesisSpecAccount{ - Balance: (*hexutil.Big)(account.Balance), - Nonce: account.Nonce, - } - } - spec.Accounts[common.BytesToAddress([]byte{1})].Precompiled = &cppEthereumGenesisSpecBuiltin{ - Name: "ecrecover", Linear: &cppEthereumGenesisSpecLinearPricing{Base: 3000}, - } - spec.Accounts[common.BytesToAddress([]byte{2})].Precompiled = &cppEthereumGenesisSpecBuiltin{ - Name: "sha256", Linear: &cppEthereumGenesisSpecLinearPricing{Base: 60, Word: 12}, - } - spec.Accounts[common.BytesToAddress([]byte{3})].Precompiled = &cppEthereumGenesisSpecBuiltin{ - Name: "ripemd160", Linear: &cppEthereumGenesisSpecLinearPricing{Base: 600, Word: 120}, - } - spec.Accounts[common.BytesToAddress([]byte{4})].Precompiled = &cppEthereumGenesisSpecBuiltin{ - Name: "identity", Linear: &cppEthereumGenesisSpecLinearPricing{Base: 15, Word: 3}, + spec.setAccount(address, account) } + + spec.setPrecompile(1, &alethGenesisSpecBuiltin{Name: "ecrecover", + Linear: &alethGenesisSpecLinearPricing{Base: 3000}}) + spec.setPrecompile(2, &alethGenesisSpecBuiltin{Name: "sha256", + Linear: &alethGenesisSpecLinearPricing{Base: 60, Word: 12}}) + spec.setPrecompile(3, &alethGenesisSpecBuiltin{Name: "ripemd160", + Linear: &alethGenesisSpecLinearPricing{Base: 600, Word: 120}}) + spec.setPrecompile(4, &alethGenesisSpecBuiltin{Name: "identity", + Linear: &alethGenesisSpecLinearPricing{Base: 15, Word: 3}}) if genesis.Config.ByzantiumBlock != nil { - spec.Accounts[common.BytesToAddress([]byte{5})].Precompiled = &cppEthereumGenesisSpecBuiltin{ - Name: "modexp", StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()), - } - spec.Accounts[common.BytesToAddress([]byte{6})].Precompiled = &cppEthereumGenesisSpecBuiltin{ - Name: "alt_bn128_G1_add", StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()), Linear: &cppEthereumGenesisSpecLinearPricing{Base: 500}, - } - spec.Accounts[common.BytesToAddress([]byte{7})].Precompiled = &cppEthereumGenesisSpecBuiltin{ - Name: "alt_bn128_G1_mul", StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()), Linear: &cppEthereumGenesisSpecLinearPricing{Base: 40000}, - } - spec.Accounts[common.BytesToAddress([]byte{8})].Precompiled = &cppEthereumGenesisSpecBuiltin{ - Name: "alt_bn128_pairing_product", StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()), - } + spec.setPrecompile(5, &alethGenesisSpecBuiltin{Name: "modexp", + StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64())}) + spec.setPrecompile(6, &alethGenesisSpecBuiltin{Name: "alt_bn128_G1_add", + StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()), + Linear: &alethGenesisSpecLinearPricing{Base: 500}}) + spec.setPrecompile(7, &alethGenesisSpecBuiltin{Name: "alt_bn128_G1_mul", + StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()), + Linear: &alethGenesisSpecLinearPricing{Base: 40000}}) + spec.setPrecompile(8, &alethGenesisSpecBuiltin{Name: "alt_bn128_pairing_product", + StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64())}) } return spec, nil } +func (spec *alethGenesisSpec) setPrecompile(address byte, data *alethGenesisSpecBuiltin) { + if spec.Accounts == nil { + spec.Accounts = make(map[common.UnprefixedAddress]*alethGenesisSpecAccount) + } + spec.Accounts[common.UnprefixedAddress(common.BytesToAddress([]byte{address}))].Precompiled = data +} + +func (spec *alethGenesisSpec) setAccount(address common.Address, account core.GenesisAccount) { + if spec.Accounts == nil { + spec.Accounts = make(map[common.UnprefixedAddress]*alethGenesisSpecAccount) + } + + a, exist := spec.Accounts[common.UnprefixedAddress(address)] + if !exist { + a = &alethGenesisSpecAccount{} + spec.Accounts[common.UnprefixedAddress(address)] = a + } + a.Balance = (*math2.HexOrDecimal256)(account.Balance) + a.Nonce = account.Nonce + +} + +func (spec *alethGenesisSpec) setByzantium(num *big.Int) { + spec.Params.ByzantiumForkBlock = hexutil.Uint64(num.Uint64()) +} + +func (spec *alethGenesisSpec) setConstantinople(num *big.Int) { + spec.Params.ConstantinopleForkBlock = hexutil.Uint64(num.Uint64()) +} + // parityChainSpec is the chain specification format used by Parity. type parityChainSpec struct { - Name string `json:"name"` - Engine struct { + Name string `json:"name"` + Datadir string `json:"dataDir"` + Engine struct { Ethash struct { Params struct { - MinimumDifficulty *hexutil.Big `json:"minimumDifficulty"` - DifficultyBoundDivisor *hexutil.Big `json:"difficultyBoundDivisor"` - DurationLimit *hexutil.Big `json:"durationLimit"` - BlockReward *hexutil.Big `json:"blockReward"` - HomesteadTransition uint64 `json:"homesteadTransition"` - EIP150Transition uint64 `json:"eip150Transition"` - EIP160Transition uint64 `json:"eip160Transition"` - EIP161abcTransition uint64 `json:"eip161abcTransition"` - EIP161dTransition uint64 `json:"eip161dTransition"` - EIP649Reward *hexutil.Big `json:"eip649Reward"` - EIP100bTransition uint64 `json:"eip100bTransition"` - EIP649Transition uint64 `json:"eip649Transition"` + MinimumDifficulty *hexutil.Big `json:"minimumDifficulty"` + DifficultyBoundDivisor *hexutil.Big `json:"difficultyBoundDivisor"` + DurationLimit *hexutil.Big `json:"durationLimit"` + BlockReward map[string]string `json:"blockReward"` + DifficultyBombDelays map[string]string `json:"difficultyBombDelays"` + HomesteadTransition hexutil.Uint64 `json:"homesteadTransition"` + EIP100bTransition hexutil.Uint64 `json:"eip100bTransition"` } `json:"params"` } `json:"Ethash"` } `json:"engine"` Params struct { - MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"` - MinGasLimit hexutil.Uint64 `json:"minGasLimit"` - GasLimitBoundDivisor hexutil.Uint64 `json:"gasLimitBoundDivisor"` - NetworkID hexutil.Uint64 `json:"networkID"` - MaxCodeSize uint64 `json:"maxCodeSize"` - EIP155Transition uint64 `json:"eip155Transition"` - EIP98Transition uint64 `json:"eip98Transition"` - EIP86Transition uint64 `json:"eip86Transition"` - EIP140Transition uint64 `json:"eip140Transition"` - EIP211Transition uint64 `json:"eip211Transition"` - EIP214Transition uint64 `json:"eip214Transition"` - EIP658Transition uint64 `json:"eip658Transition"` + AccountStartNonce hexutil.Uint64 `json:"accountStartNonce"` + MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"` + MinGasLimit hexutil.Uint64 `json:"minGasLimit"` + GasLimitBoundDivisor math2.HexOrDecimal64 `json:"gasLimitBoundDivisor"` + NetworkID hexutil.Uint64 `json:"networkID"` + ChainID hexutil.Uint64 `json:"chainID"` + MaxCodeSize hexutil.Uint64 `json:"maxCodeSize"` + MaxCodeSizeTransition hexutil.Uint64 `json:"maxCodeSizeTransition"` + EIP98Transition hexutil.Uint64 `json:"eip98Transition"` + EIP150Transition hexutil.Uint64 `json:"eip150Transition"` + EIP160Transition hexutil.Uint64 `json:"eip160Transition"` + EIP161abcTransition hexutil.Uint64 `json:"eip161abcTransition"` + EIP161dTransition hexutil.Uint64 `json:"eip161dTransition"` + EIP155Transition hexutil.Uint64 `json:"eip155Transition"` + EIP140Transition hexutil.Uint64 `json:"eip140Transition"` + EIP211Transition hexutil.Uint64 `json:"eip211Transition"` + EIP214Transition hexutil.Uint64 `json:"eip214Transition"` + EIP658Transition hexutil.Uint64 `json:"eip658Transition"` + EIP145Transition hexutil.Uint64 `json:"eip145Transition"` + EIP1014Transition hexutil.Uint64 `json:"eip1014Transition"` + EIP1052Transition hexutil.Uint64 `json:"eip1052Transition"` + EIP1283Transition hexutil.Uint64 `json:"eip1283Transition"` } `json:"params"` Genesis struct { @@ -215,22 +259,22 @@ type parityChainSpec struct { GasLimit hexutil.Uint64 `json:"gasLimit"` } `json:"genesis"` - Nodes []string `json:"nodes"` - Accounts map[common.Address]*parityChainSpecAccount `json:"accounts"` + Nodes []string `json:"nodes"` + Accounts map[common.UnprefixedAddress]*parityChainSpecAccount `json:"accounts"` } // parityChainSpecAccount is the prefunded genesis account and/or precompiled // contract definition. type parityChainSpecAccount struct { - Balance *hexutil.Big `json:"balance"` - Nonce uint64 `json:"nonce,omitempty"` + Balance math2.HexOrDecimal256 `json:"balance"` + Nonce math2.HexOrDecimal64 `json:"nonce,omitempty"` Builtin *parityChainSpecBuiltin `json:"builtin,omitempty"` } // parityChainSpecBuiltin is the precompiled contract definition. type parityChainSpecBuiltin struct { Name string `json:"name,omitempty"` - ActivateAt uint64 `json:"activate_at,omitempty"` + ActivateAt math2.HexOrDecimal64 `json:"activate_at,omitempty"` Pricing *parityChainSpecPricing `json:"pricing,omitempty"` } @@ -265,34 +309,51 @@ func newParityChainSpec(network string, genesis *core.Genesis, bootnodes []strin } // Reconstruct the chain spec in Parity's format spec := &parityChainSpec{ - Name: network, - Nodes: bootnodes, + Name: network, + Nodes: bootnodes, + Datadir: strings.ToLower(network), } + spec.Engine.Ethash.Params.BlockReward = make(map[string]string) + spec.Engine.Ethash.Params.DifficultyBombDelays = make(map[string]string) + // Frontier spec.Engine.Ethash.Params.MinimumDifficulty = (*hexutil.Big)(params.MinimumDifficulty) spec.Engine.Ethash.Params.DifficultyBoundDivisor = (*hexutil.Big)(params.DifficultyBoundDivisor) spec.Engine.Ethash.Params.DurationLimit = (*hexutil.Big)(params.DurationLimit) - spec.Engine.Ethash.Params.BlockReward = (*hexutil.Big)(ethash.FrontierBlockReward) - spec.Engine.Ethash.Params.HomesteadTransition = genesis.Config.HomesteadBlock.Uint64() - spec.Engine.Ethash.Params.EIP150Transition = genesis.Config.EIP150Block.Uint64() - spec.Engine.Ethash.Params.EIP160Transition = genesis.Config.EIP155Block.Uint64() - spec.Engine.Ethash.Params.EIP161abcTransition = genesis.Config.EIP158Block.Uint64() - spec.Engine.Ethash.Params.EIP161dTransition = genesis.Config.EIP158Block.Uint64() - spec.Engine.Ethash.Params.EIP649Reward = (*hexutil.Big)(ethash.ByzantiumBlockReward) - spec.Engine.Ethash.Params.EIP100bTransition = genesis.Config.ByzantiumBlock.Uint64() - spec.Engine.Ethash.Params.EIP649Transition = genesis.Config.ByzantiumBlock.Uint64() + spec.Engine.Ethash.Params.BlockReward["0x0"] = hexutil.EncodeBig(ethash.FrontierBlockReward) + // Homestead + spec.Engine.Ethash.Params.HomesteadTransition = hexutil.Uint64(genesis.Config.HomesteadBlock.Uint64()) + + // Tangerine Whistle : 150 + // https://github.com/ethereum/EIPs/blob/master/EIPS/eip-608.md + spec.Params.EIP150Transition = hexutil.Uint64(genesis.Config.EIP150Block.Uint64()) + + // Spurious Dragon: 155, 160, 161, 170 + // https://github.com/ethereum/EIPs/blob/master/EIPS/eip-607.md + spec.Params.EIP155Transition = hexutil.Uint64(genesis.Config.EIP155Block.Uint64()) + spec.Params.EIP160Transition = hexutil.Uint64(genesis.Config.EIP155Block.Uint64()) + spec.Params.EIP161abcTransition = hexutil.Uint64(genesis.Config.EIP158Block.Uint64()) + spec.Params.EIP161dTransition = hexutil.Uint64(genesis.Config.EIP158Block.Uint64()) + + // Byzantium + if num := genesis.Config.ByzantiumBlock; num != nil { + spec.setByzantium(num) + } + // Constantinople + if num := genesis.Config.ConstantinopleBlock; num != nil { + spec.setConstantinople(num) + } spec.Params.MaximumExtraDataSize = (hexutil.Uint64)(params.MaximumExtraDataSize) spec.Params.MinGasLimit = (hexutil.Uint64)(params.MinGasLimit) - spec.Params.GasLimitBoundDivisor = (hexutil.Uint64)(params.GasLimitBoundDivisor) + spec.Params.GasLimitBoundDivisor = (math2.HexOrDecimal64)(params.GasLimitBoundDivisor) spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64()) + spec.Params.ChainID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64()) spec.Params.MaxCodeSize = params.MaxCodeSize - spec.Params.EIP155Transition = genesis.Config.EIP155Block.Uint64() - spec.Params.EIP98Transition = math.MaxUint64 - spec.Params.EIP86Transition = math.MaxUint64 - spec.Params.EIP140Transition = genesis.Config.ByzantiumBlock.Uint64() - spec.Params.EIP211Transition = genesis.Config.ByzantiumBlock.Uint64() - spec.Params.EIP214Transition = genesis.Config.ByzantiumBlock.Uint64() - spec.Params.EIP658Transition = genesis.Config.ByzantiumBlock.Uint64() + // geth has it set from zero + spec.Params.MaxCodeSizeTransition = 0 + + // Disable this one + spec.Params.EIP98Transition = math.MaxInt64 spec.Genesis.Seal.Ethereum.Nonce = (hexutil.Bytes)(make([]byte, 8)) binary.LittleEndian.PutUint64(spec.Genesis.Seal.Ethereum.Nonce[:], genesis.Nonce) @@ -305,42 +366,77 @@ func newParityChainSpec(network string, genesis *core.Genesis, bootnodes []strin spec.Genesis.ExtraData = (hexutil.Bytes)(genesis.ExtraData) spec.Genesis.GasLimit = (hexutil.Uint64)(genesis.GasLimit) - spec.Accounts = make(map[common.Address]*parityChainSpecAccount) + spec.Accounts = make(map[common.UnprefixedAddress]*parityChainSpecAccount) for address, account := range genesis.Alloc { - spec.Accounts[address] = &parityChainSpecAccount{ - Balance: (*hexutil.Big)(account.Balance), - Nonce: account.Nonce, + bal := math2.HexOrDecimal256(*account.Balance) + + spec.Accounts[common.UnprefixedAddress(address)] = &parityChainSpecAccount{ + Balance: bal, + Nonce: math2.HexOrDecimal64(account.Nonce), } } - spec.Accounts[common.BytesToAddress([]byte{1})].Builtin = &parityChainSpecBuiltin{ - Name: "ecrecover", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 3000}}, - } - spec.Accounts[common.BytesToAddress([]byte{2})].Builtin = &parityChainSpecBuiltin{ + spec.setPrecompile(1, &parityChainSpecBuiltin{Name: "ecrecover", + Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 3000}}}) + + spec.setPrecompile(2, &parityChainSpecBuiltin{ Name: "sha256", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 60, Word: 12}}, - } - spec.Accounts[common.BytesToAddress([]byte{3})].Builtin = &parityChainSpecBuiltin{ + }) + spec.setPrecompile(3, &parityChainSpecBuiltin{ Name: "ripemd160", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 600, Word: 120}}, - } - spec.Accounts[common.BytesToAddress([]byte{4})].Builtin = &parityChainSpecBuiltin{ + }) + spec.setPrecompile(4, &parityChainSpecBuiltin{ Name: "identity", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 15, Word: 3}}, - } + }) if genesis.Config.ByzantiumBlock != nil { - spec.Accounts[common.BytesToAddress([]byte{5})].Builtin = &parityChainSpecBuiltin{ - Name: "modexp", ActivateAt: genesis.Config.ByzantiumBlock.Uint64(), Pricing: &parityChainSpecPricing{ModExp: &parityChainSpecModExpPricing{Divisor: 20}}, - } - spec.Accounts[common.BytesToAddress([]byte{6})].Builtin = &parityChainSpecBuiltin{ - Name: "alt_bn128_add", ActivateAt: genesis.Config.ByzantiumBlock.Uint64(), Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 500}}, - } - spec.Accounts[common.BytesToAddress([]byte{7})].Builtin = &parityChainSpecBuiltin{ - Name: "alt_bn128_mul", ActivateAt: genesis.Config.ByzantiumBlock.Uint64(), Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 40000}}, - } - spec.Accounts[common.BytesToAddress([]byte{8})].Builtin = &parityChainSpecBuiltin{ - Name: "alt_bn128_pairing", ActivateAt: genesis.Config.ByzantiumBlock.Uint64(), Pricing: &parityChainSpecPricing{AltBnPairing: &parityChainSpecAltBnPairingPricing{Base: 100000, Pair: 80000}}, - } + blnum := math2.HexOrDecimal64(genesis.Config.ByzantiumBlock.Uint64()) + spec.setPrecompile(5, &parityChainSpecBuiltin{ + Name: "modexp", ActivateAt: blnum, Pricing: &parityChainSpecPricing{ModExp: &parityChainSpecModExpPricing{Divisor: 20}}, + }) + spec.setPrecompile(6, &parityChainSpecBuiltin{ + Name: "alt_bn128_add", ActivateAt: blnum, Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 500}}, + }) + spec.setPrecompile(7, &parityChainSpecBuiltin{ + Name: "alt_bn128_mul", ActivateAt: blnum, Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 40000}}, + }) + spec.setPrecompile(8, &parityChainSpecBuiltin{ + Name: "alt_bn128_pairing", ActivateAt: blnum, Pricing: &parityChainSpecPricing{AltBnPairing: &parityChainSpecAltBnPairingPricing{Base: 100000, Pair: 80000}}, + }) } return spec, nil } +func (spec *parityChainSpec) setPrecompile(address byte, data *parityChainSpecBuiltin) { + if spec.Accounts == nil { + spec.Accounts = make(map[common.UnprefixedAddress]*parityChainSpecAccount) + } + a := common.UnprefixedAddress(common.BytesToAddress([]byte{address})) + if _, exist := spec.Accounts[a]; !exist { + spec.Accounts[a] = &parityChainSpecAccount{} + } + spec.Accounts[a].Builtin = data +} + +func (spec *parityChainSpec) setByzantium(num *big.Int) { + spec.Engine.Ethash.Params.BlockReward[hexutil.EncodeBig(num)] = hexutil.EncodeBig(ethash.ByzantiumBlockReward) + spec.Engine.Ethash.Params.DifficultyBombDelays[hexutil.EncodeBig(num)] = hexutil.EncodeUint64(3000000) + n := hexutil.Uint64(num.Uint64()) + spec.Engine.Ethash.Params.EIP100bTransition = n + spec.Params.EIP140Transition = n + spec.Params.EIP211Transition = n + spec.Params.EIP214Transition = n + spec.Params.EIP658Transition = n +} + +func (spec *parityChainSpec) setConstantinople(num *big.Int) { + spec.Engine.Ethash.Params.BlockReward[hexutil.EncodeBig(num)] = hexutil.EncodeBig(ethash.ConstantinopleBlockReward) + spec.Engine.Ethash.Params.DifficultyBombDelays[hexutil.EncodeBig(num)] = hexutil.EncodeUint64(2000000) + n := hexutil.Uint64(num.Uint64()) + spec.Params.EIP145Transition = n + spec.Params.EIP1014Transition = n + spec.Params.EIP1052Transition = n + spec.Params.EIP1283Transition = n +} + // pyEthereumGenesisSpec represents the genesis specification format used by the // Python Ethereum implementation. type pyEthereumGenesisSpec struct { diff --git a/cmd/puppeth/module_dashboard.go b/cmd/puppeth/module_dashboard.go index d22bd81103..cb3ed6e71c 100644 --- a/cmd/puppeth/module_dashboard.go +++ b/cmd/puppeth/module_dashboard.go @@ -640,7 +640,7 @@ func deployDashboard(client *sshClient, network string, conf *config, config *da files[filepath.Join(workdir, network+".json")] = genesis if conf.Genesis.Config.Ethash != nil { - cppSpec, err := newCppEthereumGenesisSpec(network, conf.Genesis) + cppSpec, err := newAlethGenesisSpec(network, conf.Genesis) if err != nil { return nil, err } diff --git a/cmd/puppeth/puppeth.go b/cmd/puppeth/puppeth.go index df3fff51b0..3af1751eae 100644 --- a/cmd/puppeth/puppeth.go +++ b/cmd/puppeth/puppeth.go @@ -18,11 +18,15 @@ package main import ( + "encoding/json" + "io/ioutil" "math/rand" "os" "strings" "time" + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/log" "gopkg.in/urfave/cli.v1" ) @@ -43,6 +47,14 @@ func main() { Usage: "log level to emit to the screen", }, } + app.Commands = []cli.Command{ + cli.Command{ + Action: utils.MigrateFlags(convert), + Name: "convert", + Usage: "Convert from geth genesis into chainspecs for other nodes.", + ArgsUsage: "", + }, + } app.Action = func(c *cli.Context) error { // Set up the logger to print everything and the random generator log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(c.Int("loglevel")), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) @@ -58,3 +70,23 @@ func main() { } app.Run(os.Args) } + +func convert(ctx *cli.Context) error { + // Ensure we have a source genesis + log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) + if len(ctx.Args()) != 1 { + utils.Fatalf("No geth genesis provided") + } + blob, err := ioutil.ReadFile(ctx.Args().First()) + if err != nil { + utils.Fatalf("Could not read file: %v", err) + } + + var genesis core.Genesis + if err := json.Unmarshal(blob, &genesis); err != nil { + utils.Fatalf("Failed parsing genesis: %v", err) + } + basename := strings.TrimRight(ctx.Args().First(), ".json") + convertGenesis(&genesis, basename, basename, []string{}) + return nil +} diff --git a/cmd/puppeth/puppeth_test.go b/cmd/puppeth/puppeth_test.go new file mode 100644 index 0000000000..ae0752d549 --- /dev/null +++ b/cmd/puppeth/puppeth_test.go @@ -0,0 +1,91 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "reflect" + "strings" + "testing" + + "github.com/davecgh/go-spew/spew" + "github.com/ethereum/go-ethereum/core" +) + +func TestConverter_AlethStureby(t *testing.T) { + blob, err := ioutil.ReadFile("testdata/stureby_geth.json") + if err != nil { + t.Fatalf("could not read file: %v", err) + } + var genesis core.Genesis + if err := json.Unmarshal(blob, &genesis); err != nil { + t.Fatalf("failed parsing genesis: %v", err) + } + spec, err := newAlethGenesisSpec("stureby", &genesis) + if err != nil { + t.Fatalf("failed creating chainspec: %v", err) + } + + expBlob, err := ioutil.ReadFile("testdata/stureby_aleth.json") + if err != nil { + t.Fatalf("could not read file: %v", err) + } + expspec := &alethGenesisSpec{} + if err := json.Unmarshal(expBlob, expspec); err != nil { + t.Fatalf("failed parsing genesis: %v", err) + } + if !reflect.DeepEqual(expspec, spec) { + t.Errorf("chainspec mismatch") + c := spew.ConfigState{ + DisablePointerAddresses: true, + SortKeys: true, + } + exp := strings.Split(c.Sdump(expspec), "\n") + got := strings.Split(c.Sdump(spec), "\n") + for i := 0; i < len(exp) && i < len(got); i++ { + if exp[i] != got[i] { + fmt.Printf("got: %v\nexp: %v\n", exp[i], got[i]) + } + } + } +} + +func TestConverter_ParityStureby(t *testing.T) { + blob, err := ioutil.ReadFile("testdata/stureby_geth.json") + if err != nil { + t.Fatalf("could not read file: %v", err) + } + var genesis core.Genesis + if err := json.Unmarshal(blob, &genesis); err != nil { + t.Fatalf("failed parsing genesis: %v", err) + } + spec, err := newParityChainSpec("Stureby", &genesis, []string{}) + if err != nil { + t.Fatalf("failed creating chainspec: %v", err) + } + + expBlob, err := ioutil.ReadFile("testdata/stureby_parity.json") + if err != nil { + t.Fatalf("could not read file: %v", err) + } + expspec := &parityChainSpec{} + if err := json.Unmarshal(expBlob, expspec); err != nil { + t.Fatalf("failed parsing genesis: %v", err) + } + expspec.Nodes = []string{} + + if !reflect.DeepEqual(expspec, spec) { + t.Errorf("chainspec mismatch") + c := spew.ConfigState{ + DisablePointerAddresses: true, + SortKeys: true, + } + exp := strings.Split(c.Sdump(expspec), "\n") + got := strings.Split(c.Sdump(spec), "\n") + for i := 0; i < len(exp) && i < len(got); i++ { + if exp[i] != got[i] { + fmt.Printf("got: %v\nexp: %v\n", exp[i], got[i]) + } + } + } +} diff --git a/cmd/puppeth/testdata/stureby_aleth.json b/cmd/puppeth/testdata/stureby_aleth.json new file mode 100644 index 0000000000..1ef1d8ae18 --- /dev/null +++ b/cmd/puppeth/testdata/stureby_aleth.json @@ -0,0 +1,112 @@ +{ + "sealEngine":"Ethash", + "params":{ + "accountStartNonce":"0x00", + "maximumExtraDataSize":"0x20", + "homesteadForkBlock":"0x2710", + "daoHardforkBlock":"0x00", + "EIP150ForkBlock":"0x3a98", + "EIP158ForkBlock":"0x59d8", + "byzantiumForkBlock":"0x7530", + "constantinopleForkBlock":"0x9c40", + "minGasLimit":"0x1388", + "maxGasLimit":"0x7fffffffffffffff", + "tieBreakingGas":false, + "gasLimitBoundDivisor":"0x0400", + "minimumDifficulty":"0x20000", + "difficultyBoundDivisor":"0x0800", + "durationLimit":"0x0d", + "blockReward":"0x4563918244F40000", + "networkID":"0x4cb2e", + "chainID":"0x4cb2e", + "allowFutureBlocks":false + }, + "genesis":{ + "nonce":"0x0000000000000000", + "difficulty":"0x20000", + "mixHash":"0x0000000000000000000000000000000000000000000000000000000000000000", + "author":"0x0000000000000000000000000000000000000000", + "timestamp":"0x59a4e76d", + "parentHash":"0x0000000000000000000000000000000000000000000000000000000000000000", + "extraData":"0x0000000000000000000000000000000000000000000000000000000b4dc0ffee", + "gasLimit":"0x47b760" + }, + "accounts":{ + "0000000000000000000000000000000000000001":{ + "balance":"1", + "precompiled":{ + "name":"ecrecover", + "linear":{ + "base":3000, + "word":0 + } + } + }, + "0000000000000000000000000000000000000002":{ + "balance":"1", + "precompiled":{ + "name":"sha256", + "linear":{ + "base":60, + "word":12 + } + } + }, + "0000000000000000000000000000000000000003":{ + "balance":"1", + "precompiled":{ + "name":"ripemd160", + "linear":{ + "base":600, + "word":120 + } + } + }, + "0000000000000000000000000000000000000004":{ + "balance":"1", + "precompiled":{ + "name":"identity", + "linear":{ + "base":15, + "word":3 + } + } + }, + "0000000000000000000000000000000000000005":{ + "balance":"1", + "precompiled":{ + "name":"modexp", + "startingBlock":"0x7530" + } + }, + "0000000000000000000000000000000000000006":{ + "balance":"1", + "precompiled":{ + "name":"alt_bn128_G1_add", + "startingBlock":"0x7530", + "linear":{ + "base":500, + "word":0 + } + } + }, + "0000000000000000000000000000000000000007":{ + "balance":"1", + "precompiled":{ + "name":"alt_bn128_G1_mul", + "startingBlock":"0x7530", + "linear":{ + "base":40000, + "word":0 + } + } + }, + "0000000000000000000000000000000000000008":{ + "balance":"1", + "precompiled":{ + "name":"alt_bn128_pairing_product", + "startingBlock":"0x7530" + } + } + } +} diff --git a/cmd/puppeth/testdata/stureby_geth.json b/cmd/puppeth/testdata/stureby_geth.json new file mode 100644 index 0000000000..c8c3b3c956 --- /dev/null +++ b/cmd/puppeth/testdata/stureby_geth.json @@ -0,0 +1,47 @@ +{ + "config": { + "ethash":{}, + "chainId": 314158, + "homesteadBlock": 10000, + "eip150Block": 15000, + "eip150Hash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "eip155Block": 23000, + "eip158Block": 23000, + "byzantiumBlock": 30000, + "constantinopleBlock": 40000 + }, + "nonce": "0x0", + "timestamp": "0x59a4e76d", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "extraData": "0x0000000000000000000000000000000000000000000000000000000b4dc0ffee", + "gasLimit": "0x47b760", + "difficulty": "0x20000", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "coinbase": "0x0000000000000000000000000000000000000000", + "alloc": { + "0000000000000000000000000000000000000001": { + "balance": "0x01" + }, + "0000000000000000000000000000000000000002": { + "balance": "0x01" + }, + "0000000000000000000000000000000000000003": { + "balance": "0x01" + }, + "0000000000000000000000000000000000000004": { + "balance": "0x01" + }, + "0000000000000000000000000000000000000005": { + "balance": "0x01" + }, + "0000000000000000000000000000000000000006": { + "balance": "0x01" + }, + "0000000000000000000000000000000000000007": { + "balance": "0x01" + }, + "0000000000000000000000000000000000000008": { + "balance": "0x01" + } + } +} diff --git a/cmd/puppeth/testdata/stureby_parity.json b/cmd/puppeth/testdata/stureby_parity.json new file mode 100644 index 0000000000..f3fa8386a0 --- /dev/null +++ b/cmd/puppeth/testdata/stureby_parity.json @@ -0,0 +1,181 @@ +{ + "name":"Stureby", + "dataDir":"stureby", + "engine":{ + "Ethash":{ + "params":{ + "minimumDifficulty":"0x20000", + "difficultyBoundDivisor":"0x800", + "durationLimit":"0xd", + "blockReward":{ + "0x0":"0x4563918244f40000", + "0x7530":"0x29a2241af62c0000", + "0x9c40":"0x1bc16d674ec80000" + }, + "homesteadTransition":"0x2710", + "eip100bTransition":"0x7530", + "difficultyBombDelays":{ + "0x7530":"0x2dc6c0", + "0x9c40":"0x1e8480" + } + } + } + }, + "params":{ + "accountStartNonce":"0x0", + "maximumExtraDataSize":"0x20", + "gasLimitBoundDivisor":"0x400", + "minGasLimit":"0x1388", + "networkID":"0x4cb2e", + "chainID":"0x4cb2e", + "maxCodeSize":"0x6000", + "maxCodeSizeTransition":"0x0", + "eip98Transition": "0x7fffffffffffffff", + "eip150Transition":"0x3a98", + "eip160Transition":"0x59d8", + "eip161abcTransition":"0x59d8", + "eip161dTransition":"0x59d8", + "eip155Transition":"0x59d8", + "eip140Transition":"0x7530", + "eip211Transition":"0x7530", + "eip214Transition":"0x7530", + "eip658Transition":"0x7530", + "eip145Transition":"0x9c40", + "eip1014Transition":"0x9c40", + "eip1052Transition":"0x9c40", + "eip1283Transition":"0x9c40" + }, + "genesis":{ + "seal":{ + "ethereum":{ + "nonce":"0x0000000000000000", + "mixHash":"0x0000000000000000000000000000000000000000000000000000000000000000" + } + }, + "difficulty":"0x20000", + "author":"0x0000000000000000000000000000000000000000", + "timestamp":"0x59a4e76d", + "parentHash":"0x0000000000000000000000000000000000000000000000000000000000000000", + "extraData":"0x0000000000000000000000000000000000000000000000000000000b4dc0ffee", + "gasLimit":"0x47b760" + }, + "nodes":[ + "enode://dfa7aca3f5b635fbfe7d0b20575f25e40d9e27b4bfbb3cf74364a42023ad9f25c1a4383bcc8cced86ee511a7d03415345a4df05be37f1dff040e4c780699f1c0@168.61.153.255:31303", + "enode://ef441b20dd70aeabf0eac35c3b8a2854e5ce04db0e30be9152ea9fd129359dcbb3f803993303ff5781c755dfd7223f3fe43505f583cccb740949407677412ba9@40.74.91.252:31303", + "enode://953b5ea1c8987cf46008232a0160324fd00d41320ecf00e23af86ec8f5396b19eb57ddab37c78141be56f62e9077de4f4dfa0747fa768ed8c8531bbfb1046237@40.70.214.166:31303", + "enode://276e613dd4b277a66591e565711e6c8bb107f0905248a9f8f8228c1a87992e156e5114bb9937c02824a9d9d25f76340442cf86e2028bf5293cae19904fb2b98e@35.178.251.52:30303", + "enode://064c820d41e52ed7d426ac64b60506c2998235bedc7e67cb497c6faf7bb4fc54fe56fc82d0add3180b747c0c4f40a1108a6f84d7d0629ed606d504528e61cc57@3.8.5.3:30303", + "enode://90069fdabcc5e684fa5d59430bebbb12755d9362dfe5006a1485b13d71a78a3812d36e74dd7d88e50b51add01e097ea80f16263aeaa4f0230db6c79e2a97e7ca@217.29.191.142:30303", + "enode://0aac74b7fd28726275e466acb5e03bc88a95927e9951eb66b5efb239b2f798ada0690853b2f2823fe4efa408f0f3d4dd258430bc952a5ff70677b8625b3e3b14@40.115.33.57:40404", + "enode://0b96415a10f835106d83e090a0528eed5e7887e5c802a6d084e9f1993a9d0fc713781e6e4101f6365e9b91259712f291acc0a9e6e667e22023050d602c36fbe2@40.115.33.57:40414" + ], + "accounts":{ + "0000000000000000000000000000000000000001":{ + "balance":"1", + "nonce":"0", + "builtin":{ + "name":"ecrecover", + "pricing":{ + "linear":{ + "base":3000, + "word":0 + } + } + } + }, + "0000000000000000000000000000000000000002":{ + "balance":"1", + "nonce":"0", + "builtin":{ + "name":"sha256", + "pricing":{ + "linear":{ + "base":60, + "word":12 + } + } + } + }, + "0000000000000000000000000000000000000003":{ + "balance":"1", + "nonce":"0", + "builtin":{ + "name":"ripemd160", + "pricing":{ + "linear":{ + "base":600, + "word":120 + } + } + } + }, + "0000000000000000000000000000000000000004":{ + "balance":"1", + "nonce":"0", + "builtin":{ + "name":"identity", + "pricing":{ + "linear":{ + "base":15, + "word":3 + } + } + } + }, + "0000000000000000000000000000000000000005":{ + "balance":"1", + "nonce":"0", + "builtin":{ + "name":"modexp", + "activate_at":"0x7530", + "pricing":{ + "modexp":{ + "divisor":20 + } + } + } + }, + "0000000000000000000000000000000000000006":{ + "balance":"1", + "nonce":"0", + "builtin":{ + "name":"alt_bn128_add", + "activate_at":"0x7530", + "pricing":{ + "linear":{ + "base":500, + "word":0 + } + } + } + }, + "0000000000000000000000000000000000000007":{ + "balance":"1", + "nonce":"0", + "builtin":{ + "name":"alt_bn128_mul", + "activate_at":"0x7530", + "pricing":{ + "linear":{ + "base":40000, + "word":0 + } + } + } + }, + "0000000000000000000000000000000000000008":{ + "balance":"1", + "nonce":"0", + "builtin":{ + "name":"alt_bn128_pairing", + "activate_at":"0x7530", + "pricing":{ + "alt_bn128_pairing":{ + "base":100000, + "pair":80000 + } + } + } + } + } +} diff --git a/cmd/puppeth/wizard.go b/cmd/puppeth/wizard.go index b88a61de7d..42fc125e58 100644 --- a/cmd/puppeth/wizard.go +++ b/cmd/puppeth/wizard.go @@ -104,6 +104,28 @@ func (w *wizard) readString() string { } } +// readYesNo reads a yes or no from stdin, returning boolean true for yes +func (w *wizard) readYesNo(def bool) bool { + for { + fmt.Printf("> ") + text, err := w.in.ReadString('\n') + if err != nil { + log.Crit("Failed to read user input", "err", err) + } + text = strings.ToLower(strings.TrimSpace(text)) + if text == "y" || text == "yes" { + return true + } + if text == "n" || text == "no" { + return false + } + if len(text) == 0 { + return def + } + fmt.Println("Valid answers: y, yes, n, no or leave empty for default") + } +} + // readDefaultString reads a single line from stdin, trimming if from spaces. If // an empty line is entered, the default value is returned. func (w *wizard) readDefaultString(def string) string { diff --git a/cmd/puppeth/wizard_genesis.go b/cmd/puppeth/wizard_genesis.go index 6c4cd571fb..681a387dff 100644 --- a/cmd/puppeth/wizard_genesis.go +++ b/cmd/puppeth/wizard_genesis.go @@ -114,9 +114,13 @@ func (w *wizard) makeGenesis() { } break } - // Add a batch of precompile balances to avoid them getting deleted - for i := int64(0); i < 256; i++ { - genesis.Alloc[common.BigToAddress(big.NewInt(i))] = core.GenesisAccount{Balance: big.NewInt(1)} + fmt.Println() + fmt.Println("Should the precompile-addresses (0x1 .. 0xff) be pre-funded with 1 wei? (advisable yes)") + if w.readYesNo(true) { + // Add a batch of precompile balances to avoid them getting deleted + for i := int64(0); i < 256; i++ { + genesis.Alloc[common.BigToAddress(big.NewInt(i))] = core.GenesisAccount{Balance: big.NewInt(1)} + } } // Query the user for some custom extras fmt.Println() @@ -136,47 +140,57 @@ func (w *wizard) manageGenesis() { // Figure out whether to modify or export the genesis fmt.Println() fmt.Println(" 1. Modify existing fork rules") - fmt.Println(" 2. Export genesis configuration") + fmt.Println(" 2. Export genesis configurations") fmt.Println(" 3. Remove genesis configuration") choice := w.read() - switch { - case choice == "1": + switch choice { + case "1": // Fork rule updating requested, iterate over each fork fmt.Println() fmt.Printf("Which block should Homestead come into effect? (default = %v)\n", w.conf.Genesis.Config.HomesteadBlock) w.conf.Genesis.Config.HomesteadBlock = w.readDefaultBigInt(w.conf.Genesis.Config.HomesteadBlock) fmt.Println() - fmt.Printf("Which block should EIP150 come into effect? (default = %v)\n", w.conf.Genesis.Config.EIP150Block) + fmt.Printf("Which block should EIP150 (Tangerine Whistle) come into effect? (default = %v)\n", w.conf.Genesis.Config.EIP150Block) w.conf.Genesis.Config.EIP150Block = w.readDefaultBigInt(w.conf.Genesis.Config.EIP150Block) fmt.Println() - fmt.Printf("Which block should EIP155 come into effect? (default = %v)\n", w.conf.Genesis.Config.EIP155Block) + fmt.Printf("Which block should EIP155 (Spurious Dragon) come into effect? (default = %v)\n", w.conf.Genesis.Config.EIP155Block) w.conf.Genesis.Config.EIP155Block = w.readDefaultBigInt(w.conf.Genesis.Config.EIP155Block) fmt.Println() - fmt.Printf("Which block should EIP158 come into effect? (default = %v)\n", w.conf.Genesis.Config.EIP158Block) + fmt.Printf("Which block should EIP158/161 (also Spurious Dragon) come into effect? (default = %v)\n", w.conf.Genesis.Config.EIP158Block) w.conf.Genesis.Config.EIP158Block = w.readDefaultBigInt(w.conf.Genesis.Config.EIP158Block) fmt.Println() fmt.Printf("Which block should Byzantium come into effect? (default = %v)\n", w.conf.Genesis.Config.ByzantiumBlock) w.conf.Genesis.Config.ByzantiumBlock = w.readDefaultBigInt(w.conf.Genesis.Config.ByzantiumBlock) + fmt.Println() + fmt.Printf("Which block should Constantinople come into effect? (default = %v)\n", w.conf.Genesis.Config.ByzantiumBlock) + w.conf.Genesis.Config.ConstantinopleBlock = w.readDefaultBigInt(w.conf.Genesis.Config.ConstantinopleBlock) + out, _ := json.MarshalIndent(w.conf.Genesis.Config, "", " ") fmt.Printf("Chain configuration updated:\n\n%s\n", out) - case choice == "2": + case "2": // Save whatever genesis configuration we currently have fmt.Println() - fmt.Printf("Which file to save the genesis into? (default = %s.json)\n", w.network) + fmt.Printf("Which base filename to save the genesis specifications into? (default = %s)\n", w.network) out, _ := json.MarshalIndent(w.conf.Genesis, "", " ") - if err := ioutil.WriteFile(w.readDefaultString(fmt.Sprintf("%s.json", w.network)), out, 0644); err != nil { + basename := w.readDefaultString(fmt.Sprintf("%s.json", w.network)) + + gethJson := fmt.Sprintf("%s.json", basename) + if err := ioutil.WriteFile((gethJson), out, 0644); err != nil { log.Error("Failed to save genesis file", "err", err) } - log.Info("Exported existing genesis block") + log.Info("Saved geth genesis as %v", gethJson) + if err := convertGenesis(w.conf.Genesis, basename, w.network, w.conf.bootnodes); err != nil { + log.Error("Conversion failed", "err", err) + } - case choice == "3": + case "3": // Make sure we don't have any services running if len(w.conf.servers()) > 0 { log.Error("Genesis reset requires all services and servers torn down") @@ -186,8 +200,31 @@ func (w *wizard) manageGenesis() { w.conf.Genesis = nil w.conf.flush() - default: log.Error("That's not something I can do") } } + +func saveGenesis(basename, client string, spec interface{}) { + filename := fmt.Sprintf("%s-%s.json", basename, client) + out, _ := json.Marshal(spec) + if err := ioutil.WriteFile(filename, out, 0644); err != nil { + log.Error("failed to save genesis file", "client", client, "err", err) + } + log.Info("saved chainspec", "client", client, "filename", filename) +} + +func convertGenesis(genesis *core.Genesis, basename string, network string, bootnodes []string) error { + if spec, err := newAlethGenesisSpec(network, genesis); err == nil { + saveGenesis(basename, "aleth", spec) + } else { + log.Error("failed to create chain spec", "client", "aleth", "err", err) + } + if spec, err := newParityChainSpec(network, genesis, []string{}); err == nil { + saveGenesis(basename, "parity", spec) + } else { + log.Error("failed to create chain spec", "client", "parity", "err", err) + } + saveGenesis(basename, "harmony", genesis) + return nil +} From 7a5c1b28ddd4ce1e95f6ed01672b9d44dba047f2 Mon Sep 17 00:00:00 2001 From: Corey Lin <514971757@qq.com> Date: Mon, 3 Dec 2018 21:15:22 +0800 Subject: [PATCH 117/159] whisperv6: remove duplicated code (#18015) --- whisper/whisperv6/api_test.go | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/whisper/whisperv6/api_test.go b/whisper/whisperv6/api_test.go index cdbc7fab5e..6d7157f57f 100644 --- a/whisper/whisperv6/api_test.go +++ b/whisper/whisperv6/api_test.go @@ -18,27 +18,12 @@ package whisperv6 import ( "bytes" - "crypto/ecdsa" "testing" "time" - - mapset "github.com/deckarep/golang-set" - "github.com/ethereum/go-ethereum/common" ) func TestMultipleTopicCopyInNewMessageFilter(t *testing.T) { - w := &Whisper{ - privateKeys: make(map[string]*ecdsa.PrivateKey), - symKeys: make(map[string][]byte), - envelopes: make(map[common.Hash]*Envelope), - expirations: make(map[uint32]mapset.Set), - peers: make(map[*Peer]struct{}), - messageQueue: make(chan *Envelope, messageQueueLimit), - p2pMsgQueue: make(chan *Envelope, messageQueueLimit), - quit: make(chan struct{}), - syncAllowance: DefaultSyncAllowance, - } - w.filters = NewFilters(w) + w := New(nil) keyID, err := w.GenerateSymKey() if err != nil { From d4415f5e40fbd155adb3e85e4eae5c7db7832694 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 3 Dec 2018 16:50:59 +0200 Subject: [PATCH 118/159] cmd/puppeth: chain import/export via wizard, minor polishes --- .../{puppeth_test.go => genesis_test.go} | 22 ++- cmd/puppeth/puppeth.go | 43 ++---- cmd/puppeth/wizard.go | 64 ++++++--- cmd/puppeth/wizard_dashboard.go | 4 +- cmd/puppeth/wizard_ethstats.go | 6 +- cmd/puppeth/wizard_explorer.go | 2 +- cmd/puppeth/wizard_faucet.go | 8 +- cmd/puppeth/wizard_genesis.go | 129 +++++++++++++----- cmd/puppeth/wizard_intro.go | 16 ++- cmd/puppeth/wizard_nginx.go | 4 +- cmd/puppeth/wizard_node.go | 4 +- cmd/puppeth/wizard_wallet.go | 2 +- 12 files changed, 194 insertions(+), 110 deletions(-) rename cmd/puppeth/{puppeth_test.go => genesis_test.go} (70%) diff --git a/cmd/puppeth/puppeth_test.go b/cmd/puppeth/genesis_test.go similarity index 70% rename from cmd/puppeth/puppeth_test.go rename to cmd/puppeth/genesis_test.go index ae0752d549..83e7383605 100644 --- a/cmd/puppeth/puppeth_test.go +++ b/cmd/puppeth/genesis_test.go @@ -1,3 +1,19 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + package main import ( @@ -12,7 +28,8 @@ import ( "github.com/ethereum/go-ethereum/core" ) -func TestConverter_AlethStureby(t *testing.T) { +// Tests the go-ethereum to Aleth chainspec conversion for the Stureby testnet. +func TestAlethSturebyConverter(t *testing.T) { blob, err := ioutil.ReadFile("testdata/stureby_geth.json") if err != nil { t.Fatalf("could not read file: %v", err) @@ -50,7 +67,8 @@ func TestConverter_AlethStureby(t *testing.T) { } } -func TestConverter_ParityStureby(t *testing.T) { +// Tests the go-ethereum to Parity chainspec conversion for the Stureby testnet. +func TestParitySturebyConverter(t *testing.T) { blob, err := ioutil.ReadFile("testdata/stureby_geth.json") if err != nil { t.Fatalf("could not read file: %v", err) diff --git a/cmd/puppeth/puppeth.go b/cmd/puppeth/puppeth.go index 3af1751eae..c3de5f9360 100644 --- a/cmd/puppeth/puppeth.go +++ b/cmd/puppeth/puppeth.go @@ -18,15 +18,11 @@ package main import ( - "encoding/json" - "io/ioutil" "math/rand" "os" "strings" "time" - "github.com/ethereum/go-ethereum/cmd/utils" - "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/log" "gopkg.in/urfave/cli.v1" ) @@ -47,46 +43,23 @@ func main() { Usage: "log level to emit to the screen", }, } - app.Commands = []cli.Command{ - cli.Command{ - Action: utils.MigrateFlags(convert), - Name: "convert", - Usage: "Convert from geth genesis into chainspecs for other nodes.", - ArgsUsage: "", - }, - } - app.Action = func(c *cli.Context) error { + app.Before = func(c *cli.Context) error { // Set up the logger to print everything and the random generator log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(c.Int("loglevel")), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) rand.Seed(time.Now().UnixNano()) - network := c.String("network") - if strings.Contains(network, " ") || strings.Contains(network, "-") || strings.ToLower(network) != network { - log.Crit("No spaces, hyphens or capital letters allowed in network name") - } - // Start the wizard and relinquish control - makeWizard(c.String("network")).run() return nil } + app.Action = runWizard app.Run(os.Args) } -func convert(ctx *cli.Context) error { - // Ensure we have a source genesis - log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) - if len(ctx.Args()) != 1 { - utils.Fatalf("No geth genesis provided") +// runWizard start the wizard and relinquish control to it. +func runWizard(c *cli.Context) error { + network := c.String("network") + if strings.Contains(network, " ") || strings.Contains(network, "-") || strings.ToLower(network) != network { + log.Crit("No spaces, hyphens or capital letters allowed in network name") } - blob, err := ioutil.ReadFile(ctx.Args().First()) - if err != nil { - utils.Fatalf("Could not read file: %v", err) - } - - var genesis core.Genesis - if err := json.Unmarshal(blob, &genesis); err != nil { - utils.Fatalf("Failed parsing genesis: %v", err) - } - basename := strings.TrimRight(ctx.Args().First(), ".json") - convertGenesis(&genesis, basename, basename, []string{}) + makeWizard(c.String("network")).run() return nil } diff --git a/cmd/puppeth/wizard.go b/cmd/puppeth/wizard.go index 42fc125e58..83536506c4 100644 --- a/cmd/puppeth/wizard.go +++ b/cmd/puppeth/wizard.go @@ -23,6 +23,7 @@ import ( "io/ioutil" "math/big" "net" + "net/url" "os" "path/filepath" "sort" @@ -104,28 +105,6 @@ func (w *wizard) readString() string { } } -// readYesNo reads a yes or no from stdin, returning boolean true for yes -func (w *wizard) readYesNo(def bool) bool { - for { - fmt.Printf("> ") - text, err := w.in.ReadString('\n') - if err != nil { - log.Crit("Failed to read user input", "err", err) - } - text = strings.ToLower(strings.TrimSpace(text)) - if text == "y" || text == "yes" { - return true - } - if text == "n" || text == "no" { - return false - } - if len(text) == 0 { - return def - } - fmt.Println("Valid answers: y, yes, n, no or leave empty for default") - } -} - // readDefaultString reads a single line from stdin, trimming if from spaces. If // an empty line is entered, the default value is returned. func (w *wizard) readDefaultString(def string) string { @@ -140,6 +119,47 @@ func (w *wizard) readDefaultString(def string) string { return def } +// readDefaultYesNo reads a single line from stdin, trimming if from spaces and +// interpreting it as a 'yes' or a 'no'. If an empty line is entered, the default +// value is returned. +func (w *wizard) readDefaultYesNo(def bool) bool { + for { + fmt.Printf("> ") + text, err := w.in.ReadString('\n') + if err != nil { + log.Crit("Failed to read user input", "err", err) + } + if text = strings.ToLower(strings.TrimSpace(text)); text == "" { + return def + } + if text == "y" || text == "yes" { + return true + } + if text == "n" || text == "no" { + return false + } + log.Error("Invalid input, expected 'y', 'yes', 'n', 'no' or empty") + } +} + +// readURL reads a single line from stdin, trimming if from spaces and trying to +// interpret it as a URL (http, https or file). +func (w *wizard) readURL() *url.URL { + for { + fmt.Printf("> ") + text, err := w.in.ReadString('\n') + if err != nil { + log.Crit("Failed to read user input", "err", err) + } + uri, err := url.Parse(strings.TrimSpace(text)) + if err != nil { + log.Error("Invalid input, expected URL", "err", err) + continue + } + return uri + } +} + // readInt reads a single line from stdin, trimming if from spaces, enforcing it // to parse into an integer. func (w *wizard) readInt() int { diff --git a/cmd/puppeth/wizard_dashboard.go b/cmd/puppeth/wizard_dashboard.go index 1a01631ff4..8a8370845b 100644 --- a/cmd/puppeth/wizard_dashboard.go +++ b/cmd/puppeth/wizard_dashboard.go @@ -137,14 +137,14 @@ func (w *wizard) deployDashboard() { if w.conf.ethstats != "" { fmt.Println() fmt.Println("Include ethstats secret on dashboard (y/n)? (default = yes)") - infos.trusted = w.readDefaultString("y") == "y" + infos.trusted = w.readDefaultYesNo(true) } // Try to deploy the dashboard container on the host nocache := false if existed { fmt.Println() fmt.Printf("Should the dashboard be built from scratch (y/n)? (default = no)\n") - nocache = w.readDefaultString("n") != "n" + nocache = w.readDefaultYesNo(false) } if out, err := deployDashboard(client, w.network, &w.conf, infos, nocache); err != nil { log.Error("Failed to deploy dashboard container", "err", err) diff --git a/cmd/puppeth/wizard_ethstats.go b/cmd/puppeth/wizard_ethstats.go index fb2529c267..58ff3efbe9 100644 --- a/cmd/puppeth/wizard_ethstats.go +++ b/cmd/puppeth/wizard_ethstats.go @@ -67,11 +67,11 @@ func (w *wizard) deployEthstats() { if existed { fmt.Println() fmt.Printf("Keep existing IP %v blacklist (y/n)? (default = yes)\n", infos.banned) - if w.readDefaultString("y") != "y" { + if !w.readDefaultYesNo(true) { // The user might want to clear the entire list, although generally probably not fmt.Println() fmt.Printf("Clear out blacklist and start over (y/n)? (default = no)\n") - if w.readDefaultString("n") != "n" { + if w.readDefaultYesNo(false) { infos.banned = nil } // Offer the user to explicitly add/remove certain IP addresses @@ -106,7 +106,7 @@ func (w *wizard) deployEthstats() { if existed { fmt.Println() fmt.Printf("Should the ethstats be built from scratch (y/n)? (default = no)\n") - nocache = w.readDefaultString("n") != "n" + nocache = w.readDefaultYesNo(false) } trusted := make([]string, 0, len(w.servers)) for _, client := range w.servers { diff --git a/cmd/puppeth/wizard_explorer.go b/cmd/puppeth/wizard_explorer.go index 413511c1c3..a128fb9fb5 100644 --- a/cmd/puppeth/wizard_explorer.go +++ b/cmd/puppeth/wizard_explorer.go @@ -100,7 +100,7 @@ func (w *wizard) deployExplorer() { if existed { fmt.Println() fmt.Printf("Should the explorer be built from scratch (y/n)? (default = no)\n") - nocache = w.readDefaultString("n") != "n" + nocache = w.readDefaultYesNo(false) } if out, err := deployExplorer(client, w.network, chain, infos, nocache); err != nil { log.Error("Failed to deploy explorer container", "err", err) diff --git a/cmd/puppeth/wizard_faucet.go b/cmd/puppeth/wizard_faucet.go index 6f08408947..9068c1d30b 100644 --- a/cmd/puppeth/wizard_faucet.go +++ b/cmd/puppeth/wizard_faucet.go @@ -81,7 +81,7 @@ func (w *wizard) deployFaucet() { if infos.captchaToken != "" { fmt.Println() fmt.Println("Reuse previous reCaptcha API authorization (y/n)? (default = yes)") - if w.readDefaultString("y") != "y" { + if !w.readDefaultYesNo(true) { infos.captchaToken, infos.captchaSecret = "", "" } } @@ -89,7 +89,7 @@ func (w *wizard) deployFaucet() { // No previous authorization (or old one discarded) fmt.Println() fmt.Println("Enable reCaptcha protection against robots (y/n)? (default = no)") - if w.readDefaultString("n") == "n" { + if !w.readDefaultYesNo(false) { log.Warn("Users will be able to requests funds via automated scripts") } else { // Captcha protection explicitly requested, read the site and secret keys @@ -132,7 +132,7 @@ func (w *wizard) deployFaucet() { } else { fmt.Println() fmt.Printf("Reuse previous (%s) funding account (y/n)? (default = yes)\n", key.Address.Hex()) - if w.readDefaultString("y") != "y" { + if !w.readDefaultYesNo(true) { infos.node.keyJSON, infos.node.keyPass = "", "" } } @@ -166,7 +166,7 @@ func (w *wizard) deployFaucet() { if existed { fmt.Println() fmt.Printf("Should the faucet be built from scratch (y/n)? (default = no)\n") - nocache = w.readDefaultString("n") != "n" + nocache = w.readDefaultYesNo(false) } if out, err := deployFaucet(client, w.network, w.conf.bootnodes, infos, nocache); err != nil { log.Error("Failed to deploy faucet container", "err", err) diff --git a/cmd/puppeth/wizard_genesis.go b/cmd/puppeth/wizard_genesis.go index 681a387dff..95da5bd4f3 100644 --- a/cmd/puppeth/wizard_genesis.go +++ b/cmd/puppeth/wizard_genesis.go @@ -20,9 +20,13 @@ import ( "bytes" "encoding/json" "fmt" + "io" "io/ioutil" "math/big" "math/rand" + "net/http" + "os" + "path/filepath" "time" "github.com/ethereum/go-ethereum/common" @@ -40,11 +44,12 @@ func (w *wizard) makeGenesis() { Difficulty: big.NewInt(524288), Alloc: make(core.GenesisAlloc), Config: ¶ms.ChainConfig{ - HomesteadBlock: big.NewInt(1), - EIP150Block: big.NewInt(2), - EIP155Block: big.NewInt(3), - EIP158Block: big.NewInt(3), - ByzantiumBlock: big.NewInt(4), + HomesteadBlock: big.NewInt(1), + EIP150Block: big.NewInt(2), + EIP155Block: big.NewInt(3), + EIP158Block: big.NewInt(3), + ByzantiumBlock: big.NewInt(4), + ConstantinopleBlock: big.NewInt(5), }, } // Figure out which consensus engine to choose @@ -116,7 +121,7 @@ func (w *wizard) makeGenesis() { } fmt.Println() fmt.Println("Should the precompile-addresses (0x1 .. 0xff) be pre-funded with 1 wei? (advisable yes)") - if w.readYesNo(true) { + if w.readDefaultYesNo(true) { // Add a batch of precompile balances to avoid them getting deleted for i := int64(0); i < 256; i++ { genesis.Alloc[common.BigToAddress(big.NewInt(i))] = core.GenesisAccount{Balance: big.NewInt(1)} @@ -134,6 +139,53 @@ func (w *wizard) makeGenesis() { w.conf.flush() } +// importGenesis imports a Geth genesis spec into puppeth. +func (w *wizard) importGenesis() { + // Request the genesis JSON spec URL from the user + fmt.Println() + fmt.Println("Where's the genesis file? (local file or http/https url)") + url := w.readURL() + + // Convert the various allowed URLs to a reader stream + var reader io.Reader + + switch url.Scheme { + case "http", "https": + // Remote web URL, retrieve it via an HTTP client + res, err := http.Get(url.String()) + if err != nil { + log.Error("Failed to retrieve remote genesis", "err", err) + return + } + defer res.Body.Close() + reader = res.Body + + case "": + // Schemaless URL, interpret as a local file + file, err := os.Open(url.String()) + if err != nil { + log.Error("Failed to open local genesis", "err", err) + return + } + defer file.Close() + reader = file + + default: + log.Error("Unsupported genesis URL scheme", "scheme", url.Scheme) + return + } + // Parse the genesis file and inject it successful + var genesis core.Genesis + if err := json.NewDecoder(reader).Decode(&genesis); err != nil { + log.Error("Invalid genesis spec: %v", err) + return + } + log.Info("Imported genesis block") + + w.conf.Genesis = &genesis + w.conf.flush() +} + // manageGenesis permits the modification of chain configuration parameters in // a genesis config and the export of the entire genesis spec. func (w *wizard) manageGenesis() { @@ -168,7 +220,7 @@ func (w *wizard) manageGenesis() { w.conf.Genesis.Config.ByzantiumBlock = w.readDefaultBigInt(w.conf.Genesis.Config.ByzantiumBlock) fmt.Println() - fmt.Printf("Which block should Constantinople come into effect? (default = %v)\n", w.conf.Genesis.Config.ByzantiumBlock) + fmt.Printf("Which block should Constantinople come into effect? (default = %v)\n", w.conf.Genesis.Config.ConstantinopleBlock) w.conf.Genesis.Config.ConstantinopleBlock = w.readDefaultBigInt(w.conf.Genesis.Config.ConstantinopleBlock) out, _ := json.MarshalIndent(w.conf.Genesis.Config, "", " ") @@ -177,18 +229,38 @@ func (w *wizard) manageGenesis() { case "2": // Save whatever genesis configuration we currently have fmt.Println() - fmt.Printf("Which base filename to save the genesis specifications into? (default = %s)\n", w.network) - out, _ := json.MarshalIndent(w.conf.Genesis, "", " ") - basename := w.readDefaultString(fmt.Sprintf("%s.json", w.network)) + fmt.Printf("Which folder to save the genesis specs into? (default = current)\n") + fmt.Printf(" Will create %s.json, %s-aleth.json, %s-harmony.json, %s-parity.json\n", w.network, w.network, w.network, w.network) - gethJson := fmt.Sprintf("%s.json", basename) + folder := w.readDefaultString(".") + if err := os.MkdirAll(folder, 0755); err != nil { + log.Error("Failed to create spec folder", "folder", folder, "err", err) + return + } + out, _ := json.MarshalIndent(w.conf.Genesis, "", " ") + + // Export the native genesis spec used by puppeth and Geth + gethJson := filepath.Join(folder, fmt.Sprintf("%s.json", w.network)) if err := ioutil.WriteFile((gethJson), out, 0644); err != nil { log.Error("Failed to save genesis file", "err", err) + return } - log.Info("Saved geth genesis as %v", gethJson) - if err := convertGenesis(w.conf.Genesis, basename, w.network, w.conf.bootnodes); err != nil { - log.Error("Conversion failed", "err", err) + log.Info("Saved native genesis chain spec", "path", gethJson) + + // Export the genesis spec used by Aleth (formerly C++ Ethereum) + if spec, err := newAlethGenesisSpec(w.network, w.conf.Genesis); err != nil { + log.Error("Failed to create Aleth chain spec", "err", err) + } else { + saveGenesis(folder, w.network, "aleth", spec) } + // Export the genesis spec used by Parity + if spec, err := newParityChainSpec(w.network, w.conf.Genesis, []string{}); err != nil { + log.Error("Failed to create Parity chain spec", "err", err) + } else { + saveGenesis(folder, w.network, "parity", spec) + } + // Export the genesis spec used by Harmony (formerly EthereumJ + saveGenesis(folder, w.network, "harmony", w.conf.Genesis) case "3": // Make sure we don't have any services running @@ -202,29 +274,18 @@ func (w *wizard) manageGenesis() { w.conf.flush() default: log.Error("That's not something I can do") + return } } -func saveGenesis(basename, client string, spec interface{}) { - filename := fmt.Sprintf("%s-%s.json", basename, client) +// saveGenesis JSON encodes an arbitrary genesis spec into a pre-defined file. +func saveGenesis(folder, network, client string, spec interface{}) { + path := filepath.Join(folder, fmt.Sprintf("%s-%s.json", network, client)) + out, _ := json.Marshal(spec) - if err := ioutil.WriteFile(filename, out, 0644); err != nil { - log.Error("failed to save genesis file", "client", client, "err", err) + if err := ioutil.WriteFile(path, out, 0644); err != nil { + log.Error("Failed to save genesis file", "client", client, "err", err) + return } - log.Info("saved chainspec", "client", client, "filename", filename) -} - -func convertGenesis(genesis *core.Genesis, basename string, network string, bootnodes []string) error { - if spec, err := newAlethGenesisSpec(network, genesis); err == nil { - saveGenesis(basename, "aleth", spec) - } else { - log.Error("failed to create chain spec", "client", "aleth", "err", err) - } - if spec, err := newParityChainSpec(network, genesis, []string{}); err == nil { - saveGenesis(basename, "parity", spec) - } else { - log.Error("failed to create chain spec", "client", "parity", "err", err) - } - saveGenesis(basename, "harmony", genesis) - return nil + log.Info("Saved genesis chain spec", "client", client, "path", path) } diff --git a/cmd/puppeth/wizard_intro.go b/cmd/puppeth/wizard_intro.go index 3db9a1087e..75fb04b76f 100644 --- a/cmd/puppeth/wizard_intro.go +++ b/cmd/puppeth/wizard_intro.go @@ -131,7 +131,20 @@ func (w *wizard) run() { case choice == "2": if w.conf.Genesis == nil { - w.makeGenesis() + fmt.Println() + fmt.Println("What would you like to do? (default = create)") + fmt.Println(" 1. Create new genesis from scratch") + fmt.Println(" 2. Import already existing genesis") + + choice := w.read() + switch { + case choice == "" || choice == "1": + w.makeGenesis() + case choice == "2": + w.importGenesis() + default: + log.Error("That's not something I can do") + } } else { w.manageGenesis() } @@ -149,7 +162,6 @@ func (w *wizard) run() { } else { w.manageComponents() } - default: log.Error("That's not something I can do") } diff --git a/cmd/puppeth/wizard_nginx.go b/cmd/puppeth/wizard_nginx.go index 4eeae93a0b..8397b7fd57 100644 --- a/cmd/puppeth/wizard_nginx.go +++ b/cmd/puppeth/wizard_nginx.go @@ -41,12 +41,12 @@ func (w *wizard) ensureVirtualHost(client *sshClient, port int, def string) (str // Reverse proxy is not running, offer to deploy a new one fmt.Println() fmt.Println("Allow sharing the port with other services (y/n)? (default = yes)") - if w.readDefaultString("y") == "y" { + if w.readDefaultYesNo(true) { nocache := false if proxy != nil { fmt.Println() fmt.Printf("Should the reverse-proxy be rebuilt from scratch (y/n)? (default = no)\n") - nocache = w.readDefaultString("n") != "n" + nocache = w.readDefaultYesNo(false) } if out, err := deployNginx(client, w.network, port, nocache); err != nil { log.Error("Failed to deploy reverse-proxy", "err", err) diff --git a/cmd/puppeth/wizard_node.go b/cmd/puppeth/wizard_node.go index 49b10a023a..e37297f6d6 100644 --- a/cmd/puppeth/wizard_node.go +++ b/cmd/puppeth/wizard_node.go @@ -126,7 +126,7 @@ func (w *wizard) deployNode(boot bool) { } else { fmt.Println() fmt.Printf("Reuse previous (%s) signing account (y/n)? (default = yes)\n", key.Address.Hex()) - if w.readDefaultString("y") != "y" { + if !w.readDefaultYesNo(true) { infos.keyJSON, infos.keyPass = "", "" } } @@ -165,7 +165,7 @@ func (w *wizard) deployNode(boot bool) { if existed { fmt.Println() fmt.Printf("Should the node be built from scratch (y/n)? (default = no)\n") - nocache = w.readDefaultString("n") != "n" + nocache = w.readDefaultYesNo(false) } if out, err := deployNode(client, w.network, w.conf.bootnodes, infos, nocache); err != nil { log.Error("Failed to deploy Ethereum node container", "err", err) diff --git a/cmd/puppeth/wizard_wallet.go b/cmd/puppeth/wizard_wallet.go index 7624d11e20..ca1ea5bd25 100644 --- a/cmd/puppeth/wizard_wallet.go +++ b/cmd/puppeth/wizard_wallet.go @@ -96,7 +96,7 @@ func (w *wizard) deployWallet() { if existed { fmt.Println() fmt.Printf("Should the wallet be built from scratch (y/n)? (default = no)\n") - nocache = w.readDefaultString("n") != "n" + nocache = w.readDefaultYesNo(false) } if out, err := deployWallet(client, w.network, w.conf.bootnodes, infos, nocache); err != nil { log.Error("Failed to deploy wallet container", "err", err) From 92639b676aaa7f262d19081256c43cdb78aa0efd Mon Sep 17 00:00:00 2001 From: Vedhavyas Singareddi Date: Tue, 4 Dec 2018 15:27:55 +0100 Subject: [PATCH 119/159] Add packing for dynamic array and slice types (#18051) * added tests for new abi encoding features (#4) * added tests from bytes32[][] and string[] * added offset to other types * formatting * Abi/dynamic types (#5) * Revert "Abi/dynamic types (#5)" (#6) This reverts commit dabca31d797623d43bd780721fc0ad461d24be53. * Abi/dynamic types (#7) * some cleanup * Apply suggestions from code review apply suggestions Co-Authored-By: vedhavyas * added better formatting (#8) * review chnages * better comments --- accounts/abi/argument.go | 19 +++++------- accounts/abi/pack_test.go | 62 ++++++++++++++++++++++++++++++++++++++- accounts/abi/type.go | 58 ++++++++++++++++++++++++++++++------ 3 files changed, 117 insertions(+), 22 deletions(-) diff --git a/accounts/abi/argument.go b/accounts/abi/argument.go index 93b513c346..f544c80dbb 100644 --- a/accounts/abi/argument.go +++ b/accounts/abi/argument.go @@ -243,11 +243,7 @@ func (arguments Arguments) Pack(args ...interface{}) ([]byte, error) { // input offset is the bytes offset for packed output inputOffset := 0 for _, abiArg := range abiArgs { - if abiArg.Type.T == ArrayTy { - inputOffset += 32 * abiArg.Type.Size - } else { - inputOffset += 32 - } + inputOffset += getDynamicTypeOffset(abiArg.Type) } var ret []byte for i, a := range args { @@ -257,14 +253,13 @@ func (arguments Arguments) Pack(args ...interface{}) ([]byte, error) { if err != nil { return nil, err } - // check for a slice type (string, bytes, slice) - if input.Type.requiresLengthPrefix() { - // calculate the offset - offset := inputOffset + len(variableInput) + // check for dynamic types + if isDynamicType(input.Type) { // set the offset - ret = append(ret, packNum(reflect.ValueOf(offset))...) - // Append the packed output to the variable input. The variable input - // will be appended at the end of the input. + ret = append(ret, packNum(reflect.ValueOf(inputOffset))...) + // calculate next offset + inputOffset += len(packed) + // append to variable input variableInput = append(variableInput, packed...) } else { // append the packed value to the input diff --git a/accounts/abi/pack_test.go b/accounts/abi/pack_test.go index 58a5b7a581..ddd2b7362d 100644 --- a/accounts/abi/pack_test.go +++ b/accounts/abi/pack_test.go @@ -324,6 +324,66 @@ func TestPack(t *testing.T) { "foobar", common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000006666f6f6261720000000000000000000000000000000000000000000000000000"), }, + { + "string[]", + []string{"hello", "foobar"}, + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002" + // len(array) = 2 + "0000000000000000000000000000000000000000000000000000000000000040" + // offset 64 to i = 0 + "0000000000000000000000000000000000000000000000000000000000000080" + // offset 128 to i = 1 + "0000000000000000000000000000000000000000000000000000000000000005" + // len(str[0]) = 5 + "68656c6c6f000000000000000000000000000000000000000000000000000000" + // str[0] + "0000000000000000000000000000000000000000000000000000000000000006" + // len(str[1]) = 6 + "666f6f6261720000000000000000000000000000000000000000000000000000"), // str[1] + }, + { + "string[2]", + []string{"hello", "foobar"}, + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040" + // offset to i = 0 + "0000000000000000000000000000000000000000000000000000000000000080" + // offset to i = 1 + "0000000000000000000000000000000000000000000000000000000000000005" + // len(str[0]) = 5 + "68656c6c6f000000000000000000000000000000000000000000000000000000" + // str[0] + "0000000000000000000000000000000000000000000000000000000000000006" + // len(str[1]) = 6 + "666f6f6261720000000000000000000000000000000000000000000000000000"), // str[1] + }, + { + "bytes32[][]", + [][]common.Hash{{{1}, {2}}, {{3}, {4}, {5}}}, + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002" + // len(array) = 2 + "0000000000000000000000000000000000000000000000000000000000000040" + // offset 64 to i = 0 + "00000000000000000000000000000000000000000000000000000000000000a0" + // offset 160 to i = 1 + "0000000000000000000000000000000000000000000000000000000000000002" + // len(array[0]) = 2 + "0100000000000000000000000000000000000000000000000000000000000000" + // array[0][0] + "0200000000000000000000000000000000000000000000000000000000000000" + // array[0][1] + "0000000000000000000000000000000000000000000000000000000000000003" + // len(array[1]) = 3 + "0300000000000000000000000000000000000000000000000000000000000000" + // array[1][0] + "0400000000000000000000000000000000000000000000000000000000000000" + // array[1][1] + "0500000000000000000000000000000000000000000000000000000000000000"), // array[1][2] + }, + + { + "bytes32[][2]", + [][]common.Hash{{{1}, {2}}, {{3}, {4}, {5}}}, + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040" + // offset 64 to i = 0 + "00000000000000000000000000000000000000000000000000000000000000a0" + // offset 160 to i = 1 + "0000000000000000000000000000000000000000000000000000000000000002" + // len(array[0]) = 2 + "0100000000000000000000000000000000000000000000000000000000000000" + // array[0][0] + "0200000000000000000000000000000000000000000000000000000000000000" + // array[0][1] + "0000000000000000000000000000000000000000000000000000000000000003" + // len(array[1]) = 3 + "0300000000000000000000000000000000000000000000000000000000000000" + // array[1][0] + "0400000000000000000000000000000000000000000000000000000000000000" + // array[1][1] + "0500000000000000000000000000000000000000000000000000000000000000"), // array[1][2] + }, + + { + "bytes32[3][2]", + [][]common.Hash{{{1}, {2}, {3}}, {{3}, {4}, {5}}}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000" + // array[0][0] + "0200000000000000000000000000000000000000000000000000000000000000" + // array[0][1] + "0300000000000000000000000000000000000000000000000000000000000000" + // array[0][2] + "0300000000000000000000000000000000000000000000000000000000000000" + // array[1][0] + "0400000000000000000000000000000000000000000000000000000000000000" + // array[1][1] + "0500000000000000000000000000000000000000000000000000000000000000"), // array[1][2] + }, } { typ, err := NewType(test.typ) if err != nil { @@ -336,7 +396,7 @@ func TestPack(t *testing.T) { } if !bytes.Equal(output, test.output) { - t.Errorf("%d failed. Expected bytes: '%x' Got: '%x'", i, test.output, output) + t.Errorf("input %d for typ: %v failed. Expected bytes: '%x' Got: '%x'", i, typ.String(), test.output, output) } } } diff --git a/accounts/abi/type.go b/accounts/abi/type.go index dce89d2b4e..6bfaabf5a5 100644 --- a/accounts/abi/type.go +++ b/accounts/abi/type.go @@ -183,23 +183,39 @@ func (t Type) pack(v reflect.Value) ([]byte, error) { return nil, err } - if t.T == SliceTy || t.T == ArrayTy { - var packed []byte + switch t.T { + case SliceTy, ArrayTy: + var ret []byte + if t.requiresLengthPrefix() { + // append length + ret = append(ret, packNum(reflect.ValueOf(v.Len()))...) + } + + // calculate offset if any + offset := 0 + offsetReq := isDynamicType(*t.Elem) + if offsetReq { + offset = getDynamicTypeOffset(*t.Elem) * v.Len() + } + var tail []byte for i := 0; i < v.Len(); i++ { val, err := t.Elem.pack(v.Index(i)) if err != nil { return nil, err } - packed = append(packed, val...) - } - if t.T == SliceTy { - return packBytesSlice(packed, v.Len()), nil - } else if t.T == ArrayTy { - return packed, nil + if !offsetReq { + ret = append(ret, val...) + continue + } + ret = append(ret, packNum(reflect.ValueOf(offset))...) + offset += len(val) + tail = append(tail, val...) } + return append(ret, tail...), nil + default: + return packElement(t, v), nil } - return packElement(t, v), nil } // requireLengthPrefix returns whether the type requires any sort of length @@ -207,3 +223,27 @@ func (t Type) pack(v reflect.Value) ([]byte, error) { func (t Type) requiresLengthPrefix() bool { return t.T == StringTy || t.T == BytesTy || t.T == SliceTy } + +// isDynamicType returns true if the type is dynamic. +// StringTy, BytesTy, and SliceTy(irrespective of slice element type) are dynamic types +// ArrayTy is considered dynamic if and only if the Array element is a dynamic type. +// This function recursively checks the type for slice and array elements. +func isDynamicType(t Type) bool { + // dynamic types + // array is also a dynamic type if the array type is dynamic + return t.T == StringTy || t.T == BytesTy || t.T == SliceTy || (t.T == ArrayTy && isDynamicType(*t.Elem)) +} + +// getDynamicTypeOffset returns the offset for the type. +// See `isDynamicType` to know which types are considered dynamic. +// If the type t is an array and element type is not a dynamic type, then we consider it a static type and +// return 32 * size of array since length prefix is not required. +// If t is a dynamic type or element type(for slices and arrays) is dynamic, then we simply return 32 as offset. +func getDynamicTypeOffset(t Type) int { + // if it is an array and there are no dynamic types + // then the array is static type + if t.T == ArrayTy && !isDynamicType(*t.Elem) { + return 32 * t.Size + } + return 32 +} From b98d2e9a1c11f797502a3afa711e470a41cbc91d Mon Sep 17 00:00:00 2001 From: holisticode Date: Tue, 4 Dec 2018 12:29:51 -0500 Subject: [PATCH 120/159] swarm/network/stream: Debug log instead of Warn for retrieval failure (#18246) --- swarm/network/stream/delivery.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/swarm/network/stream/delivery.go b/swarm/network/stream/delivery.go index a5c9fa7e86..c73298d9aa 100644 --- a/swarm/network/stream/delivery.go +++ b/swarm/network/stream/delivery.go @@ -39,6 +39,7 @@ const ( var ( processReceivedChunksCount = metrics.NewRegisteredCounter("network.stream.received_chunks.count", nil) handleRetrieveRequestMsgCount = metrics.NewRegisteredCounter("network.stream.handle_retrieve_request_msg.count", nil) + retrieveChunkFail = metrics.NewRegisteredCounter("network.stream.retrieve_chunks_fail.count", nil) requestFromPeersCount = metrics.NewRegisteredCounter("network.stream.request_from_peers.count", nil) requestFromPeersEachCount = metrics.NewRegisteredCounter("network.stream.request_from_peers_each.count", nil) @@ -169,7 +170,8 @@ func (d *Delivery) handleRetrieveRequestMsg(ctx context.Context, sp *Peer, req * go func() { chunk, err := d.chunkStore.Get(ctx, req.Addr) if err != nil { - log.Warn("ChunkStore.Get can not retrieve chunk", "peer", sp.ID().String(), "addr", req.Addr, "hopcount", req.HopCount, "err", err) + retrieveChunkFail.Inc(1) + log.Debug("ChunkStore.Get can not retrieve chunk", "peer", sp.ID().String(), "addr", req.Addr, "hopcount", req.HopCount, "err", err) return } if req.SkipCheck { From 3ac633ba8423851a85cb909a1f1f0f3c0d2c5351 Mon Sep 17 00:00:00 2001 From: Elad Date: Wed, 5 Dec 2018 15:50:55 +0530 Subject: [PATCH 121/159] swarm/api/http: add resetting timer metrics to requests (#18249) --- swarm/api/http/middleware.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/swarm/api/http/middleware.go b/swarm/api/http/middleware.go index f5f70138b3..115a008566 100644 --- a/swarm/api/http/middleware.go +++ b/swarm/api/http/middleware.go @@ -5,6 +5,7 @@ import ( "net/http" "runtime/debug" "strings" + "time" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/api" @@ -73,9 +74,13 @@ func ParseURI(h http.Handler) http.Handler { func InitLoggingResponseWriter(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + startTime := time.Now() + defer metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.time", r.Method), nil).UpdateSince(startTime) + writer := newLoggingResponseWriter(w) h.ServeHTTP(writer, r) log.Info("request served", "ruid", GetRUID(r.Context()), "code", writer.statusCode) + metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.%d.time", r.Method, writer.statusCode), nil).UpdateSince(startTime) }) } From de39513ced05f120b30b0daa49003982cc6b2426 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Thu, 6 Dec 2018 14:34:49 +0100 Subject: [PATCH 122/159] core, internal, eth, miner, les: Take VM config from BlockChain (#17955) Until this commit, when sending an RPC request that called `NewEVM`, a blank `vm.Config` would be taken so as to set some options, based on the default configuration. If some extra configuration switches were passed to the blockchain, those would be ignored. This PR adds a function to get the config from the blockchain, and this is what is now used for RPC calls. Some subsequent changes need to be made, see https://github.com/ethereum/go-ethereum/pull/17955#pullrequestreview-182237244 for the details of the discussion. --- core/blockchain.go | 5 +++++ eth/api_backend.go | 4 ++-- internal/ethapi/api.go | 8 ++++---- internal/ethapi/backend.go | 2 +- les/api_backend.go | 4 ++-- miner/worker.go | 3 +-- 6 files changed, 15 insertions(+), 11 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index a672c0ab64..c29063a73b 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -210,6 +210,11 @@ func (bc *BlockChain) getProcInterrupt() bool { return atomic.LoadInt32(&bc.procInterrupt) == 1 } +// GetVMConfig returns the block chain VM config. +func (bc *BlockChain) GetVMConfig() *vm.Config { + return &bc.vmConfig +} + // loadLastState loads the last known chain state from the database. This method // assumes that the chain manager mutex is held. func (bc *BlockChain) loadLastState() error { diff --git a/eth/api_backend.go b/eth/api_backend.go index 8748d444fa..a48815e0db 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -125,12 +125,12 @@ func (b *EthAPIBackend) GetTd(blockHash common.Hash) *big.Int { return b.eth.blockchain.GetTdByHash(blockHash) } -func (b *EthAPIBackend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) { +func (b *EthAPIBackend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header) (*vm.EVM, func() error, error) { state.SetBalance(msg.From(), math.MaxBig256) vmError := func() error { return nil } context := core.NewEVMContext(msg, header, b.eth.BlockChain(), nil) - return vm.NewEVM(context, state, b.eth.chainConfig, vmCfg), vmError, nil + return vm.NewEVM(context, state, b.eth.chainConfig, *b.eth.blockchain.GetVMConfig()), vmError, nil } func (b *EthAPIBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 43a33e9921..656555b3ba 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -683,7 +683,7 @@ type CallArgs struct { Data hexutil.Bytes `json:"data"` } -func (s *PublicBlockChainAPI) doCall(ctx context.Context, args CallArgs, blockNr rpc.BlockNumber, vmCfg vm.Config, timeout time.Duration) ([]byte, uint64, bool, error) { +func (s *PublicBlockChainAPI) doCall(ctx context.Context, args CallArgs, blockNr rpc.BlockNumber, timeout time.Duration) ([]byte, uint64, bool, error) { defer func(start time.Time) { log.Debug("Executing EVM call finished", "runtime", time.Since(start)) }(time.Now()) state, header, err := s.b.StateAndHeaderByNumber(ctx, blockNr) @@ -724,7 +724,7 @@ func (s *PublicBlockChainAPI) doCall(ctx context.Context, args CallArgs, blockNr defer cancel() // Get a new instance of the EVM. - evm, vmError, err := s.b.GetEVM(ctx, msg, state, header, vmCfg) + evm, vmError, err := s.b.GetEVM(ctx, msg, state, header) if err != nil { return nil, 0, false, err } @@ -748,7 +748,7 @@ func (s *PublicBlockChainAPI) doCall(ctx context.Context, args CallArgs, blockNr // Call executes the given transaction on the state for the given block number. // It doesn't make and changes in the state/blockchain and is useful to execute and retrieve values. func (s *PublicBlockChainAPI) Call(ctx context.Context, args CallArgs, blockNr rpc.BlockNumber) (hexutil.Bytes, error) { - result, _, _, err := s.doCall(ctx, args, blockNr, vm.Config{}, 5*time.Second) + result, _, _, err := s.doCall(ctx, args, blockNr, 5*time.Second) return (hexutil.Bytes)(result), err } @@ -777,7 +777,7 @@ func (s *PublicBlockChainAPI) EstimateGas(ctx context.Context, args CallArgs) (h executable := func(gas uint64) bool { args.Gas = hexutil.Uint64(gas) - _, _, failed, err := s.doCall(ctx, args, rpc.PendingBlockNumber, vm.Config{}, 0) + _, _, failed, err := s.doCall(ctx, args, rpc.PendingBlockNumber, 0) if err != nil || failed { return false } diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index c9ffe230c6..e23ee03b15 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -53,7 +53,7 @@ type Backend interface { GetBlock(ctx context.Context, blockHash common.Hash) (*types.Block, error) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) GetTd(blockHash common.Hash) *big.Int - GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) + GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header) (*vm.EVM, func() error, error) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription SubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription diff --git a/les/api_backend.go b/les/api_backend.go index aa748a4ea1..7531396235 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -105,10 +105,10 @@ func (b *LesApiBackend) GetTd(hash common.Hash) *big.Int { return b.eth.blockchain.GetTdByHash(hash) } -func (b *LesApiBackend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) { +func (b *LesApiBackend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header) (*vm.EVM, func() error, error) { state.SetBalance(msg.From(), math.MaxBig256) context := core.NewEVMContext(msg, header, b.eth.blockchain, nil) - return vm.NewEVM(context, state, b.eth.chainConfig, vmCfg), state.Error, nil + return vm.NewEVM(context, state, b.eth.chainConfig, vm.Config{}), state.Error, nil } func (b *LesApiBackend) SendTx(ctx context.Context, signedTx *types.Transaction) error { diff --git a/miner/worker.go b/miner/worker.go index 8579c5c84b..48473796bc 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -31,7 +31,6 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" @@ -692,7 +691,7 @@ func (w *worker) updateSnapshot() { func (w *worker) commitTransaction(tx *types.Transaction, coinbase common.Address) ([]*types.Log, error) { snap := w.current.state.Snapshot() - receipt, _, err := core.ApplyTransaction(w.config, w.chain, &coinbase, w.current.gasPool, w.current.state, w.current.header, tx, &w.current.header.GasUsed, vm.Config{}) + receipt, _, err := core.ApplyTransaction(w.config, w.chain, &coinbase, w.current.gasPool, w.current.state, w.current.header, tx, &w.current.header.GasUsed, *w.chain.GetVMConfig()) if err != nil { w.current.state.RevertToSnapshot(snap) return nil, err From 661809714e35f69dea23f713dd1e65cff523344c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Fri, 7 Dec 2018 06:51:40 +0100 Subject: [PATCH 123/159] swarm: snapshot load improvement (#18220) * swarm/network: Hive - do not notify peer if discovery is disabled * p2p/simulations: validate all connections on loading a snapshot * p2p/simulations: track all connections in on snapshot loading * p2p/simulations: add snapshotLoadTimeout variable * p2p/simulations: ignore control events in snapshot load * p2p/simulations: simplify event loop synchronization * p2p/simulations: return already connected error from Load function * p2p/simulations: log warning on snapshot loading disconnection --- p2p/simulations/network.go | 75 +++++++++++++++++++++++ swarm/network/hive.go | 2 +- swarm/network/simulation/kademlia_test.go | 1 - 3 files changed, 76 insertions(+), 2 deletions(-) diff --git a/p2p/simulations/network.go b/p2p/simulations/network.go index 92ccfde817..ab9f582c52 100644 --- a/p2p/simulations/network.go +++ b/p2p/simulations/network.go @@ -20,6 +20,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "sync" "time" @@ -705,8 +706,11 @@ func (net *Network) snapshot(addServices []string, removeServices []string) (*Sn return snap, nil } +var snapshotLoadTimeout = 120 * time.Second + // Load loads a network snapshot func (net *Network) Load(snap *Snapshot) error { + // Start nodes. for _, n := range snap.Nodes { if _, err := net.NewNodeWithConfig(n.Node.Config); err != nil { return err @@ -718,6 +722,69 @@ func (net *Network) Load(snap *Snapshot) error { return err } } + + // Prepare connection events counter. + allConnected := make(chan struct{}) // closed when all connections are established + done := make(chan struct{}) // ensures that the event loop goroutine is terminated + defer close(done) + + // Subscribe to event channel. + // It needs to be done outside of the event loop goroutine (created below) + // to ensure that the event channel is blocking before connect calls are made. + events := make(chan *Event) + sub := net.Events().Subscribe(events) + defer sub.Unsubscribe() + + go func() { + // Expected number of connections. + total := len(snap.Conns) + // Set of all established connections from the snapshot, not other connections. + // Key array element 0 is the connection One field value, and element 1 connection Other field. + connections := make(map[[2]enode.ID]struct{}, total) + + for { + select { + case e := <-events: + // Ignore control events as they do not represent + // connect or disconnect (Up) state change. + if e.Control { + continue + } + // Detect only connection events. + if e.Type != EventTypeConn { + continue + } + connection := [2]enode.ID{e.Conn.One, e.Conn.Other} + // Nodes are still not connected or have been disconnected. + if !e.Conn.Up { + // Delete the connection from the set of established connections. + // This will prevent false positive in case disconnections happen. + delete(connections, connection) + log.Warn("load snapshot: unexpected disconnection", "one", e.Conn.One, "other", e.Conn.Other) + continue + } + // Check that the connection is from the snapshot. + for _, conn := range snap.Conns { + if conn.One == e.Conn.One && conn.Other == e.Conn.Other { + // Add the connection to the set of established connections. + connections[connection] = struct{}{} + if len(connections) == total { + // Signal that all nodes are connected. + close(allConnected) + return + } + + break + } + } + case <-done: + // Load function returned, terminate this goroutine. + return + } + } + }() + + // Start connecting. for _, conn := range snap.Conns { if !net.GetNode(conn.One).Up || !net.GetNode(conn.Other).Up { @@ -729,6 +796,14 @@ func (net *Network) Load(snap *Snapshot) error { return err } } + + select { + // Wait until all connections from the snapshot are established. + case <-allConnected: + // Make sure that we do not wait forever. + case <-time.After(snapshotLoadTimeout): + return errors.New("snapshot connections not established") + } return nil } diff --git a/swarm/network/hive.go b/swarm/network/hive.go index 1aa1ae42a2..ebef545929 100644 --- a/swarm/network/hive.go +++ b/swarm/network/hive.go @@ -165,8 +165,8 @@ func (h *Hive) Run(p *BzzPeer) error { // otherwise just send depth to new peer dp.NotifyDepth(depth) } + NotifyPeer(p.BzzAddr, h.Kademlia) } - NotifyPeer(p.BzzAddr, h.Kademlia) defer h.Off(dp) return dp.Run(dp.HandleMsg) } diff --git a/swarm/network/simulation/kademlia_test.go b/swarm/network/simulation/kademlia_test.go index 024830315a..f02b0e5417 100644 --- a/swarm/network/simulation/kademlia_test.go +++ b/swarm/network/simulation/kademlia_test.go @@ -33,7 +33,6 @@ func TestWaitTillHealthy(t *testing.T) { "bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) { addr := network.NewAddr(ctx.Config.Node()) hp := network.NewHiveParams() - hp.Discovery = false config := &network.BzzConfig{ OverlayAddr: addr.Over(), UnderlayAddr: addr.Under(), From f32790fb0508bb484a100d6e5c81bec2c89bebb1 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 7 Dec 2018 14:43:27 +0100 Subject: [PATCH 124/159] node: warn when using deprecated config/resource files (#18199) --- node/config.go | 44 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/node/config.go b/node/config.go index 8f10f4f613..7b32a59087 100644 --- a/node/config.go +++ b/node/config.go @@ -24,6 +24,7 @@ import ( "path/filepath" "runtime" "strings" + "sync" "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/accounts/keystore" @@ -152,6 +153,10 @@ type Config struct { // Logger is a custom logger to use with the p2p.Server. Logger log.Logger `toml:",omitempty"` + + staticNodesWarning bool + trustedNodesWarning bool + oldGethResourceWarning bool } // IPCEndpoint resolves an IPC endpoint based on a configured value, taking into @@ -263,8 +268,8 @@ var isOldGethResource = map[string]bool{ "chaindata": true, "nodes": true, "nodekey": true, - "static-nodes.json": true, - "trusted-nodes.json": true, + "static-nodes.json": false, // no warning for these because they have their + "trusted-nodes.json": false, // own separate warning. } // ResolvePath resolves path in the instance directory. @@ -277,13 +282,15 @@ func (c *Config) ResolvePath(path string) string { } // Backwards-compatibility: ensure that data directory files created // by geth 1.4 are used if they exist. - if c.name() == "geth" && isOldGethResource[path] { + if warn, isOld := isOldGethResource[path]; isOld { oldpath := "" - if c.Name == "geth" { + if c.name() == "geth" { oldpath = filepath.Join(c.DataDir, path) } if oldpath != "" && common.FileExist(oldpath) { - // TODO: print warning + if warn { + c.warnOnce(&c.oldGethResourceWarning, "Using deprecated resource file %s, please move this file to the 'geth' subdirectory of datadir.", oldpath) + } return oldpath } } @@ -337,17 +344,17 @@ func (c *Config) NodeKey() *ecdsa.PrivateKey { // StaticNodes returns a list of node enode URLs configured as static nodes. func (c *Config) StaticNodes() []*enode.Node { - return c.parsePersistentNodes(c.ResolvePath(datadirStaticNodes)) + return c.parsePersistentNodes(&c.staticNodesWarning, c.ResolvePath(datadirStaticNodes)) } // TrustedNodes returns a list of node enode URLs configured as trusted nodes. func (c *Config) TrustedNodes() []*enode.Node { - return c.parsePersistentNodes(c.ResolvePath(datadirTrustedNodes)) + return c.parsePersistentNodes(&c.trustedNodesWarning, c.ResolvePath(datadirTrustedNodes)) } // parsePersistentNodes parses a list of discovery node URLs loaded from a .json // file from within the data directory. -func (c *Config) parsePersistentNodes(path string) []*enode.Node { +func (c *Config) parsePersistentNodes(w *bool, path string) []*enode.Node { // Short circuit if no node config is present if c.DataDir == "" { return nil @@ -355,10 +362,12 @@ func (c *Config) parsePersistentNodes(path string) []*enode.Node { if _, err := os.Stat(path); err != nil { return nil } + c.warnOnce(w, "Found deprecated node list file %s, please use the TOML config file instead.", path) + // Load the nodes from the config file. var nodelist []string if err := common.LoadJSON(path, &nodelist); err != nil { - log.Error(fmt.Sprintf("Can't load node file %s: %v", path, err)) + log.Error(fmt.Sprintf("Can't load node list file: %v", err)) return nil } // Interpret the list as a discovery node array @@ -440,3 +449,20 @@ func makeAccountManager(conf *Config) (*accounts.Manager, string, error) { } return accounts.NewManager(backends...), ephemeral, nil } + +var warnLock sync.Mutex + +func (c *Config) warnOnce(w *bool, format string, args ...interface{}) { + warnLock.Lock() + defer warnLock.Unlock() + + if *w { + return + } + l := c.Logger + if l == nil { + l = log.Root() + } + l.Warn(fmt.Sprintf(format, args...)) + *w = true +} From c1d462ee5d3a22746e90531e06139b703f0e040f Mon Sep 17 00:00:00 2001 From: Javier Peletier Date: Mon, 10 Dec 2018 10:16:19 +0100 Subject: [PATCH 125/159] cmd/puppeth: fix rogue quote in alethGenesisSpec JSON (#18262) --- cmd/puppeth/genesis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/puppeth/genesis.go b/cmd/puppeth/genesis.go index 83212a126b..1025dfe826 100644 --- a/cmd/puppeth/genesis.go +++ b/cmd/puppeth/genesis.go @@ -54,7 +54,7 @@ type alethGenesisSpec struct { BlockReward *hexutil.Big `json:"blockReward"` NetworkID hexutil.Uint64 `json:"networkID"` ChainID hexutil.Uint64 `json:"chainID"` - AllowFutureBlocks bool `json:"allowFutureBlocks""` + AllowFutureBlocks bool `json:"allowFutureBlocks"` } `json:"params"` Genesis struct { From dfa16a3e4e0e0b5b20bfda7b7e89ebd07ea0a1a5 Mon Sep 17 00:00:00 2001 From: Max Sistemich Date: Mon, 10 Dec 2018 10:17:31 +0100 Subject: [PATCH 126/159] eth/tracers: fixed incorrect storage from prestate_tracer (#18253) * eth: fixed incorrect storage from prestate_tracer * eth/tracers: updated assets.go --- eth/tracers/internal/tracers/assets.go | 26 +++++++++---------- .../internal/tracers/prestate_tracer.js | 5 +--- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/eth/tracers/internal/tracers/assets.go b/eth/tracers/internal/tracers/assets.go index 04dd6fe89f..addd328825 100644 --- a/eth/tracers/internal/tracers/assets.go +++ b/eth/tracers/internal/tracers/assets.go @@ -1,14 +1,14 @@ // Code generated by go-bindata. DO NOT EDIT. // sources: -// 4byte_tracer.js -// bigram_tracer.js -// call_tracer.js -// evmdis_tracer.js -// noop_tracer.js -// opcount_tracer.js -// prestate_tracer.js -// trigram_tracer.js -// unigram_tracer.js +// 4byte_tracer.js (2.933kB) +// bigram_tracer.js (1.712kB) +// call_tracer.js (8.596kB) +// evmdis_tracer.js (4.194kB) +// noop_tracer.js (1.271kB) +// opcount_tracer.js (1.372kB) +// prestate_tracer.js (3.892kB) +// trigram_tracer.js (1.788kB) +// unigram_tracer.js (1.51kB) package tracers @@ -28,7 +28,7 @@ import ( func bindataRead(data []byte, name string) ([]byte, error) { gz, err := gzip.NewReader(bytes.NewBuffer(data)) if err != nil { - return nil, fmt.Errorf("Read %q: %v", name, err) + return nil, fmt.Errorf("read %q: %v", name, err) } var buf bytes.Buffer @@ -36,7 +36,7 @@ func bindataRead(data []byte, name string) ([]byte, error) { clErr := gz.Close() if err != nil { - return nil, fmt.Errorf("Read %q: %v", name, err) + return nil, fmt.Errorf("read %q: %v", name, err) } if clErr != nil { return nil, err @@ -197,7 +197,7 @@ func opcount_tracerJs() (*asset, error) { return a, nil } -var _prestate_tracerJs = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xa4\x57\xdd\x6f\x1b\xb9\x11\x7f\xde\xfd\x2b\xa6\x7e\x91\x84\x53\x56\xce\x15\xb8\x02\x72\x5d\x60\xa3\x28\x89\x00\x9d\x6d\x48\x4a\x5d\xf7\x70\x0f\x5c\x72\x76\xc5\x13\x45\x2e\x48\xae\x3e\x10\xf8\x7f\x2f\x86\xfb\x21\xcb\x67\x27\x6e\xeb\x27\x2f\x39\xfc\xcd\xf7\x6f\x46\xa3\x11\x4c\x4c\x79\xb4\xb2\x58\x7b\xf8\xf9\xf2\xfd\xdf\x60\xb5\x46\x28\xcc\x3b\xf4\x6b\xb4\x58\x6d\x21\xad\xfc\xda\x58\x17\x8f\x46\xb0\x5a\x4b\x07\xb9\x54\x08\xd2\x41\xc9\xac\x07\x93\x83\x7f\x26\xaf\x64\x66\x99\x3d\x26\xf1\x68\x54\xbf\x79\xf1\x9a\x10\x72\x8b\x08\xce\xe4\x7e\xcf\x2c\x8e\xe1\x68\x2a\xe0\x4c\x83\x45\x21\x9d\xb7\x32\xab\x3c\x82\xf4\xc0\xb4\x18\x19\x0b\x5b\x23\x64\x7e\x24\x48\xe9\xa1\xd2\x02\x6d\x50\xed\xd1\x6e\x5d\x6b\xc7\xe7\x9b\xaf\x30\x47\xe7\xd0\xc2\x67\xd4\x68\x99\x82\xbb\x2a\x53\x92\xc3\x5c\x72\xd4\x0e\x81\x39\x28\xe9\xc4\xad\x51\x40\x16\xe0\xe8\xe1\x27\x32\x65\xd9\x98\x02\x9f\x4c\xa5\x05\xf3\xd2\xe8\x21\xa0\x24\xcb\x61\x87\xd6\x49\xa3\xe1\xaf\xad\xaa\x06\x70\x08\xc6\x12\x48\x9f\x79\x72\xc0\x82\x29\xe9\xdd\x00\x98\x3e\x82\x62\xfe\xf4\xf4\x0d\x01\x39\xf9\x2d\x40\xea\xa0\x66\x6d\x4a\x04\xbf\x66\x9e\xbc\xde\x4b\xa5\x20\x43\xa8\x1c\xe6\x95\x1a\x12\x5a\x56\x79\xb8\x9f\xad\xbe\xdc\x7e\x5d\x41\x7a\xf3\x00\xf7\xe9\x62\x91\xde\xac\x1e\xae\x60\x2f\xfd\xda\x54\x1e\x70\x87\x35\x94\xdc\x96\x4a\xa2\x80\x3d\xb3\x96\x69\x7f\x04\x93\x13\xc2\xaf\xd3\xc5\xe4\x4b\x7a\xb3\x4a\x3f\xcc\xe6\xb3\xd5\x03\x18\x0b\x9f\x66\xab\x9b\xe9\x72\x09\x9f\x6e\x17\x90\xc2\x5d\xba\x58\xcd\x26\x5f\xe7\xe9\x02\xee\xbe\x2e\xee\x6e\x97\xd3\x04\x96\x48\x56\x21\xbd\xff\x71\xcc\xf3\x90\x3d\x8b\x20\xd0\x33\xa9\x5c\x1b\x89\x07\x53\x81\x5b\x9b\x4a\x09\x58\xb3\x1d\x82\x45\x8e\x72\x87\x02\x18\x70\x53\x1e\xdf\x9c\x54\xc2\x62\xca\xe8\x22\xf8\xfc\x6a\x41\xc2\x2c\x07\x6d\xfc\x10\x1c\x22\xfc\x7d\xed\x7d\x39\x1e\x8d\xf6\xfb\x7d\x52\xe8\x2a\x31\xb6\x18\xa9\x1a\xce\x8d\xfe\x91\xc4\x84\x59\x5a\x74\x9e\x79\x5c\x59\xc6\xd1\x82\xa9\x7c\x59\x79\x07\xae\xca\x73\xc9\x25\x6a\x0f\x52\xe7\xc6\x6e\x43\xa5\x80\x37\xc0\x2d\x32\x8f\xc0\x40\x19\xce\x14\xe0\x01\x79\x15\xee\xea\x48\x87\x72\xb5\x4c\x3b\xc6\xc3\x69\x6e\xcd\x96\x7c\xad\x9c\xa7\x7f\x9c\xc3\x6d\xa6\x50\x40\x81\x1a\x9d\x74\x90\x29\xc3\x37\x49\xfc\x2d\x8e\x9e\x18\x43\x75\x12\x3c\x6c\x84\x42\x6d\xec\xb1\x67\x11\xb2\x4a\x2a\x21\x75\x91\xc4\x51\x2b\x3d\x06\x5d\x29\x35\x8c\x03\x84\x32\x66\x53\x95\x29\xe7\xa6\x0a\xb6\xff\x81\xdc\xd7\x60\xae\x44\x2e\x73\x2a\x0e\xd6\xdd\x7a\x13\xae\x3a\xbd\x26\x23\xf9\x24\x8e\xce\x60\xc6\x90\x57\x3a\xb8\xd3\x67\x42\xd8\x21\x88\x6c\xf0\x2d\x8e\xa2\x1d\xb3\x84\x05\xd7\xe0\xcd\x17\x3c\x84\xcb\xc1\x55\x1c\x45\x32\x87\xbe\x5f\x4b\x97\xb4\xc0\xbf\x31\xce\x7f\x87\xeb\xeb\xeb\xd0\xd4\xb9\xd4\x28\x06\x40\x10\xd1\x4b\x62\xf5\x4d\x94\x31\xc5\x34\xc7\x31\xf4\x2e\x0f\x3d\xf8\x09\x44\x96\x14\xe8\x3f\xd4\xa7\xb5\xb2\xc4\x9b\xa5\xb7\x52\x17\xfd\xf7\xbf\x0c\x86\xe1\x95\x36\xe1\x0d\x34\xe2\x37\xa6\x13\xae\xef\xb9\x11\xe1\xba\xb1\xb9\x96\x9a\x18\xd1\x08\x35\x52\xce\x1b\xcb\x0a\x1c\xc3\xb7\x47\xfa\x7e\x24\xaf\x1e\xe3\xe8\xf1\x2c\xca\xcb\x5a\xe8\x95\x28\x37\x10\x80\xda\xdb\xae\xce\x0b\x49\x9d\xfa\x34\x01\x01\xef\x7b\x49\x58\xb6\xa6\x3c\x4b\xc2\x06\x8f\x3f\xce\x04\x5d\x48\x71\xe8\x2e\x36\x78\x1c\x5c\xc5\xaf\xa6\x28\x69\x8c\xfe\x4d\x8a\xc3\xcb\xf9\x22\xc0\x1d\x53\x1d\x60\x1d\xbf\x25\x21\x9c\xec\x1a\x04\xdd\x41\x07\xc9\xfe\xe5\x1a\x2e\x2e\x0f\x97\xff\xe7\xdf\x45\x63\xc1\x0b\x25\xf3\xcc\xec\x37\x98\xf6\x78\x9e\x4f\x8b\xae\x52\x9e\xda\x4e\xea\x9d\xd9\x10\x81\xae\x29\x4f\x4a\x85\xd4\x98\x92\xaa\xc6\xd5\x0c\x96\x21\x6a\x90\x1e\x2d\x23\x0a\x37\x3b\xb4\x34\xbd\xc0\xa2\xaf\xac\x76\x5d\x3a\x73\xa9\x99\x6a\x81\x9b\xec\x7b\xcb\x78\xdd\xbb\xf5\xf9\x93\x9c\x72\x7f\x08\xd9\x0c\x3e\x8e\x46\x90\x7a\x20\x3f\xa1\x34\x52\xfb\x21\xec\x11\x34\xa2\x20\x02\x12\x28\x2a\xee\x03\x5e\x6f\xc7\x54\x85\xbd\x9a\x64\x88\xaa\xc3\x53\x53\xd1\x44\x7a\x42\x42\xc3\x60\xe0\xd6\xec\xc2\xa8\xcd\x18\xdf\x40\xd3\xf8\xc6\xca\x42\xea\xb8\x89\xe9\x59\xd3\x93\x45\x09\x01\x07\xb3\x42\xcd\x50\xee\xe9\xe4\x43\xc8\x7f\x26\x8b\x99\xf6\xcf\x8a\xa8\x8e\x7c\xfb\x74\xf0\x7b\xd2\x34\x71\xe2\x88\x78\xfb\x3f\x0f\x86\xf0\xfe\x97\xae\x32\xbd\x21\x28\xf8\x31\x98\x37\xaf\x43\xc5\xcf\x2b\xe2\xe5\x67\x41\x0d\x31\xc9\x4f\x41\x6b\xe2\xaa\x8c\xd2\x51\xfb\x19\xe2\x78\xce\x26\x57\xdf\xc1\x3d\xf7\xad\xc5\x6d\x42\x93\x30\x21\x5e\x07\xad\x53\xf4\x11\xb9\xc5\x2d\x4d\x17\xca\x02\x67\x4a\xa1\xed\x39\x08\xdc\x35\x6c\xca\x29\xe4\x0b\xb7\xa5\x3f\xb6\x33\xc7\x33\x5b\xa0\x77\x3f\x36\x2c\xe0\xbc\x7b\xd7\x52\x71\x08\xc5\xb1\x44\xb8\xbe\x86\xde\x64\x31\x4d\x57\xd3\x5e\xd3\x4c\xa3\x11\xdc\x63\xd8\xc8\x32\x25\x33\xa1\x8e\x20\x50\xa1\xc7\xda\x2e\xa3\x43\x88\x3a\x6a\x1a\xd2\x6a\x45\x4b\x0f\x1e\xa4\xf3\x52\x17\x50\x33\xd6\x9e\xe6\x7b\x03\x17\x7a\x84\xb3\xca\x51\xb5\x3e\x1b\x86\xde\xd0\x66\x63\x91\xf8\x8d\xe6\x50\x68\x37\xa6\x64\xb7\x09\xe5\xd2\x3a\x0f\xa5\x62\x1c\x13\xc2\xeb\x8c\x79\x3d\xbf\x0d\x33\x93\xea\x45\x68\xc1\x00\x74\x1a\xb4\x4c\xd1\xa0\x26\xf5\x0e\xfa\x2d\xc6\x20\x8e\x22\xdb\x4a\x3f\xc1\xbe\x3a\x51\x82\xf3\x58\x3e\x25\x04\x5a\x70\x70\x87\x44\xe5\x81\x0d\xea\xa1\x4c\xba\xfe\xf9\x6b\xb3\x05\xa0\x4b\xe2\x88\xde\x3d\xe9\x6b\x65\x8a\xf3\xbe\x16\x75\x58\x78\x65\x2d\xe5\xbf\x1b\x05\x39\xf5\xf8\x1f\x95\xf3\x14\x53\x4b\xe1\x69\xd8\xe2\x25\xb2\x0e\xd4\x4c\x53\x7f\xf0\xe7\x21\x4a\xf3\x33\xcc\x2b\x52\xd7\x4c\xcb\x7a\xab\x2c\x8d\x47\xed\x25\x53\xea\x48\x79\xd8\x5b\x5a\xa7\x68\x81\x1a\x82\x93\x24\x15\x18\x27\x88\x4a\xcd\x55\x25\xea\x32\x08\x75\xdc\xe0\xb9\x60\xf3\xf9\x1e\xb6\x45\xe7\x58\x81\x09\x55\x52\x2e\x0f\xcd\x26\xab\xa1\x57\x93\x5c\x7f\xd0\x4b\x3a\x23\xcf\x29\x46\x99\x22\x69\x8b\x8c\xb8\x3a\x15\xc2\xa2\x73\xfd\x41\xc3\x39\x5d\x66\xef\xd7\xa8\x29\xf8\xa0\x71\x0f\xdd\x8a\xc4\x38\xa7\x95\x51\x0c\x81\x09\x41\xd4\xf6\x6c\x9d\x89\xa3\xc8\xed\xa5\xe7\x6b\x08\x9a\x4c\x79\xea\xc5\x41\x53\xff\x9c\x39\x84\x8b\xe9\xbf\x56\x93\xdb\x8f\xd3\xc9\xed\xdd\xc3\xc5\x18\xce\xce\x96\xb3\x7f\x4f\xbb\xb3\x0f\xe9\x3c\xbd\x99\x4c\x2f\xc6\xa7\x39\x74\xee\x90\x37\xad\x0b\xa4\xd0\x79\xc6\x37\x49\x89\xb8\xe9\x5f\x9e\xf3\xc0\xc9\xc1\x28\xca\x2c\xb2\xcd\xd5\xc9\x98\xba\x41\x1b\x1d\x2d\xe5\xc2\x35\xbc\x1a\xac\xab\xd7\xad\x99\x34\xf2\xfd\x96\xc8\x4f\x2b\x51\xa0\x8a\xef\xda\x91\xce\xe7\x9d\xe7\xf4\x41\xe1\xe8\x0e\x3e\x4e\xe7\xd3\xcf\xe9\x6a\x7a\x26\xb5\x5c\xa5\xab\xd9\xa4\x3e\xfa\xaf\x43\xf4\xfe\xcd\x21\xea\x2d\x97\xab\xdb\xc5\xb4\x37\x6e\xbe\xe6\xb7\xe9\xc7\xde\x9f\x14\x36\x7b\xd3\xf7\x8a\xcc\x9b\x7b\x63\xc5\xff\x92\xab\x27\xbb\x43\xce\x5e\x5a\x1d\x02\x09\x71\x5f\x3d\xfb\x89\x00\x4c\xb7\xfc\x91\xd7\x3f\x93\xa2\xf0\xfe\x45\xc6\x78\x8c\x1f\xe3\xff\x04\x00\x00\xff\xff\xb5\x44\x89\xaf\xbc\x0f\x00\x00") +var _prestate_tracerJs = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x9c\x57\xdf\x6f\x1a\x49\x12\x7e\x9e\xf9\x2b\x4a\xfb\x02\x68\xc9\x90\xec\x49\x7b\x12\x3e\x9f\x34\x21\x24\x41\x62\x6d\x0b\xc8\xf9\x7c\xab\x7d\xe8\xe9\xae\x19\x7a\x69\xba\x47\xfd\x03\x8c\x22\xff\xef\xa7\xea\x99\x01\xc3\xda\x49\xee\xde\xcc\x74\xf5\x57\x55\x5f\x55\x7d\x5d\x1e\x8d\x60\x62\xea\x83\x95\xd5\xda\xc3\x2f\x6f\xdf\xfd\x1d\x56\x6b\x84\xca\xbc\x41\xbf\x46\x8b\x61\x0b\x79\xf0\x6b\x63\x5d\x3a\x1a\xc1\x6a\x2d\x1d\x94\x52\x21\x48\x07\x35\xb3\x1e\x4c\x09\xfe\xc2\x5e\xc9\xc2\x32\x7b\xc8\xd2\xd1\xa8\xb9\xf3\xe2\x31\x21\x94\x16\x11\x9c\x29\xfd\x9e\x59\x1c\xc3\xc1\x04\xe0\x4c\x83\x45\x21\x9d\xb7\xb2\x08\x1e\x41\x7a\x60\x5a\x8c\x8c\x85\xad\x11\xb2\x3c\x10\xa4\xf4\x10\xb4\x40\x1b\x5d\x7b\xb4\x5b\xd7\xc5\xf1\xe9\xe6\x0b\xcc\xd1\x39\xb4\xf0\x09\x35\x5a\xa6\xe0\x2e\x14\x4a\x72\x98\x4b\x8e\xda\x21\x30\x07\x35\x7d\x71\x6b\x14\x50\x44\x38\xba\xf8\x91\x42\x59\xb6\xa1\xc0\x47\x13\xb4\x60\x5e\x1a\x3d\x04\x94\x14\x39\xec\xd0\x3a\x69\x34\xfc\xad\x73\xd5\x02\x0e\xc1\x58\x02\xe9\x33\x4f\x09\x58\x30\x35\xdd\x1b\x00\xd3\x07\x50\xcc\x9f\xae\xfe\x00\x21\xa7\xbc\x05\x48\x1d\xdd\xac\x4d\x8d\xe0\xd7\xcc\x53\xd6\x7b\xa9\x14\x14\x08\xc1\x61\x19\xd4\x90\xd0\x8a\xe0\xe1\x7e\xb6\xfa\x7c\xfb\x65\x05\xf9\xcd\x03\xdc\xe7\x8b\x45\x7e\xb3\x7a\xb8\x82\xbd\xf4\x6b\x13\x3c\xe0\x0e\x1b\x28\xb9\xad\x95\x44\x01\x7b\x66\x2d\xd3\xfe\x00\xa6\x24\x84\xdf\xa6\x8b\xc9\xe7\xfc\x66\x95\xbf\x9f\xcd\x67\xab\x07\x30\x16\x3e\xce\x56\x37\xd3\xe5\x12\x3e\xde\x2e\x20\x87\xbb\x7c\xb1\x9a\x4d\xbe\xcc\xf3\x05\xdc\x7d\x59\xdc\xdd\x2e\xa7\x19\x2c\x91\xa2\x42\xba\xff\x7d\xce\xcb\x58\x3d\x8b\x20\xd0\x33\xa9\x5c\xc7\xc4\x83\x09\xe0\xd6\x26\x28\x01\x6b\xb6\x43\xb0\xc8\x51\xee\x50\x00\x03\x6e\xea\xc3\x0f\x17\x95\xb0\x98\x32\xba\x8a\x39\xbf\xda\x90\x30\x2b\x41\x1b\x3f\x04\x87\x08\xff\x58\x7b\x5f\x8f\x47\xa3\xfd\x7e\x9f\x55\x3a\x64\xc6\x56\x23\xd5\xc0\xb9\xd1\x3f\xb3\x94\x30\x6b\x8b\xce\x33\x8f\x2b\xcb\x38\x5a\x30\xc1\xd7\xc1\x3b\x70\xa1\x2c\x25\x97\xa8\x3d\x48\x5d\x1a\xbb\x8d\x9d\x02\xde\x00\xb7\xc8\x3c\x02\x03\x65\x38\x53\x80\x8f\xc8\x43\x3c\x6b\x98\x8e\xed\x6a\x99\x76\x8c\xc7\xaf\xa5\x35\x5b\xca\x35\x38\x4f\x7f\x38\x87\xdb\x42\xa1\x80\x0a\x35\x3a\xe9\xa0\x50\x86\x6f\xb2\xf4\x6b\x9a\x3c\x0b\x86\xfa\x24\x66\xd8\x1a\xc5\xde\xd8\x63\xcf\x22\x14\x41\x2a\x21\x75\x95\xa5\x49\x67\x3d\x06\x1d\x94\x1a\xa6\x11\x42\x19\xb3\x09\x75\xce\xb9\x09\x31\xf6\x3f\x91\xfb\x06\xcc\xd5\xc8\x65\x49\xcd\xc1\x8e\xa7\xde\xc4\xa3\xa3\x5f\x53\x90\x7d\x96\x26\x67\x30\x63\x28\x83\x8e\xe9\xf4\x99\x10\x76\x08\xa2\x18\x7c\x4d\x93\x64\xc7\x2c\x61\xc1\x35\x78\xf3\x19\x1f\xe3\xe1\xe0\x2a\x4d\x12\x59\x42\xdf\xaf\xa5\xcb\x3a\xe0\xdf\x19\xe7\x7f\xc0\xf5\xf5\x75\x1c\xea\x52\x6a\x14\x03\x20\x88\xe4\x25\xb3\xe6\x24\x29\x98\x62\x9a\xe3\x18\x7a\x6f\x1f\x7b\xf0\x33\x88\x22\xab\xd0\xbf\x6f\xbe\x36\xce\x32\x6f\x96\xde\x4a\x5d\xf5\xdf\xfd\x3a\x18\xc6\x5b\xda\xc4\x3b\xd0\x9a\xdf\x98\xa3\x71\x73\xce\x8d\x88\xc7\x6d\xcc\x8d\xd5\xc4\x88\xd6\xa8\xb5\x72\xde\x58\x56\xe1\x18\xbe\x3e\xd1\xef\x27\xca\xea\x29\x4d\x9e\xce\x58\x5e\x36\x46\xaf\xb0\xdc\x42\x00\x6a\x6f\x8f\x7d\x5e\x49\x9a\xd4\xe7\x05\x88\x78\xdf\x2a\xc2\xb2\x0b\xe5\xa2\x08\x1b\x3c\x7c\xbf\x12\x74\x20\xc5\xe3\xf1\x60\x83\x87\xc1\x55\xfa\x6a\x89\xb2\x36\xe8\xdf\xa5\x78\xfc\xd1\x7a\x5d\xdc\x39\xe3\x75\x49\x56\xa7\x78\x07\x83\x0b\x1e\x2d\xba\xa0\x3c\xb5\xbb\xd4\x3b\xb3\x21\xe1\x5a\x13\x3f\x4a\x45\x4a\x4c\x4d\xd5\x72\x8d\x72\x14\x88\x1a\xa4\x47\xcb\x48\x3a\xcd\x0e\x2d\xbd\x1a\x60\xd1\x07\xab\xdd\x91\xc6\x52\x6a\xa6\x3a\xe0\x96\x75\x6f\x19\x6f\x66\xa6\xf9\xfe\x8c\x4b\xee\x1f\x23\x8b\x31\xbb\xd1\x08\x72\x0f\x94\x22\xd4\x46\x6a\x3f\x84\x3d\x82\x46\x14\x34\xf8\x02\x45\xe0\x3e\xe2\xf5\x76\x4c\x05\xec\x35\xc3\x4d\x12\x19\xaf\x9a\x40\x2f\xc1\xb3\xe1\x1f\xc6\x00\xb7\x66\x17\x9f\xb8\x82\xf1\x0d\xb4\x03\x67\xac\xac\xa4\x4e\x5b\x3a\xcf\x86\x8d\x22\xca\x08\x38\x86\x15\x6b\x45\x45\xa4\x2f\xef\x99\x82\x6b\x28\x64\x35\xd3\xfe\xa2\x78\x0d\xe9\xdd\xd5\xc1\x1f\x59\x3b\x3c\x99\x23\xc1\xeb\xff\x32\x18\xc2\xbb\x5f\x8f\x1d\xe1\x0d\x41\xc1\xf7\xc1\xbc\x79\x1d\x2a\xbd\x6c\x86\x97\xaf\x45\x37\x34\xc1\x3f\x47\xaf\x99\x0b\x05\x95\xa3\xc9\x33\xf2\x78\x3e\xc5\x57\xdf\xc0\x3d\xcf\xad\xc3\x6d\xa9\xc9\x98\x10\xaf\x83\x36\x25\xfa\x80\xdc\xe2\x96\x54\x9d\xaa\xc0\x99\x52\x68\x7b\x0e\xa2\x66\x0c\xdb\x76\x8a\xf5\xc2\x6d\xed\x0f\x9d\xd6\x7b\x66\x2b\xf4\xee\xfb\x81\x45\x9c\x37\x6f\x3a\x09\x8c\x54\x1c\x6a\x84\xeb\x6b\xe8\x4d\x16\xd3\x7c\x35\xed\xb5\x63\x34\x1a\xc1\x3d\xc6\x4d\xa8\x50\xb2\x10\xea\x00\x02\x15\x7a\x6c\xe2\x32\x3a\x52\x74\x94\x84\x21\xad\x34\xb4\x6c\xe0\xa3\x74\x5e\xea\x0a\x1a\xa5\xd8\xd3\xbb\xda\xc2\xc5\x19\xe1\x2c\x38\xea\xd6\x8b\x47\xc8\x1b\xda\x28\x2c\x92\xae\x90\xfe\xc7\x71\x63\x4a\x1e\x37\x90\x52\x5a\xe7\xa1\x56\x8c\x63\x46\x78\xc7\x60\x5e\xaf\x6f\x3b\xc9\xe4\x7a\x11\x47\x30\x02\x9d\x1e\x38\xa6\xe8\x81\x24\xf7\x0e\xfa\x1d\xc6\x20\x4d\x12\xdb\x59\x3f\xc3\xbe\x3a\x49\x82\xf3\x58\x3f\x17\x04\x5a\x2c\x70\x87\x24\xa1\x51\x0d\x9a\xc7\x90\x7c\xfd\xeb\xb7\xf6\xf5\x45\x97\xa5\x09\xdd\x7b\x36\xd7\xca\x54\xe7\x73\x2d\x1a\x5a\x78\xb0\x96\xea\x7f\x94\xe0\x92\x66\xfc\xcf\xe0\x3c\x71\x6a\x89\x9e\x56\x2d\x5e\x12\xc9\x28\x89\xf4\xda\x0e\xfe\x2a\x86\xf4\x6e\xc5\x77\x82\xdc\xb5\xaf\x54\xb3\xcd\xd5\xc6\xa3\xf6\x92\x29\x75\xa0\x3a\xec\x2d\xad\x31\xb4\xb8\x0c\xc1\x49\xb2\x8a\x8a\x13\x4d\xa5\xe6\x2a\x88\xa6\x0d\x62\x1f\xb7\x78\x2e\xc6\x7c\xbe\xff\x6c\xd1\x39\x56\x61\x46\x9d\x54\xca\xc7\x76\x83\xd4\xd0\x6b\x44\xae\x3f\xe8\x65\xc7\x20\xcf\x25\x46\x99\x2a\xeb\x9a\x8c\x64\x3a\x17\xc2\xa2\x73\xfd\x41\xab\x39\xc7\xca\xde\xaf\x51\x13\xf9\xa0\x71\x0f\xc7\xd5\x84\x71\x4e\xab\x9a\x18\x02\x13\x82\xa4\xed\x62\x8d\x48\x93\xc4\xed\xa5\xe7\x6b\x88\x9e\x4c\x7d\x9a\xc5\x41\xdb\xff\x9c\x39\x84\x9f\xa6\xff\x5e\x4d\x6e\x3f\x4c\x27\xb7\x77\x0f\x3f\x8d\xe1\xec\xdb\x72\xf6\x9f\xe9\xf1\xdb\xfb\x7c\x9e\xdf\x4c\xa6\x3f\x8d\xe3\xdb\xfc\x42\x42\xde\x74\x29\x90\x43\xe7\x19\xdf\x64\x35\xe2\xa6\xff\xf6\x5c\x07\x4e\x09\x26\x49\x61\x91\x6d\xae\x4e\xc1\x34\x03\xda\xfa\xe8\x24\x17\xae\xe1\x55\xb2\xae\x5e\x8f\x66\xd2\xda\xf7\x3b\x21\x3f\xad\x22\x51\x2a\xbe\x19\x47\x3e\x9f\x1f\x33\xa7\x1f\x44\xc7\xf1\xc3\x87\xe9\x7c\xfa\x29\x5f\x4d\xcf\xac\x96\xab\x7c\x35\x9b\x34\x9f\xfe\x67\x8a\xde\xfd\x30\x45\xbd\xe5\x72\x75\xbb\x98\xf6\xc6\xed\xaf\xf9\x6d\xfe\xa1\xf7\x17\x87\xed\xbe\xf2\xad\x26\xf3\xe6\xde\x58\xf1\xff\xd4\xea\xd9\xee\x50\xb2\x97\x56\x87\x28\x42\xdc\x87\x8b\xd5\x1c\x98\xee\xf4\xa3\x6c\xfe\x3d\x49\xe2\xfd\x17\x15\xe3\x29\x7d\x4a\xff\x1b\x00\x00\xff\xff\x7c\xdb\x3f\x79\x34\x0f\x00\x00") func prestate_tracerJsBytes() ([]byte, error) { return bindataRead( @@ -213,7 +213,7 @@ func prestate_tracerJs() (*asset, error) { } info := bindataFileInfo{name: "prestate_tracer.js", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} - a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xd0, 0xd5, 0x5, 0x92, 0xed, 0xf4, 0x69, 0x2e, 0x14, 0x48, 0x35, 0x67, 0xcc, 0xf2, 0x3e, 0xc7, 0xf, 0x18, 0x22, 0x7a, 0x4d, 0x6f, 0x31, 0xad, 0x3c, 0x92, 0x77, 0xb4, 0x1, 0x2a, 0xd3, 0x7c}} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xd, 0xb0, 0x72, 0x28, 0xc7, 0x27, 0x97, 0x4d, 0xe, 0xbf, 0x29, 0xe1, 0xa8, 0xd7, 0x52, 0x13, 0xa1, 0x19, 0xc3, 0xfb, 0x8d, 0x5b, 0xcb, 0xdd, 0xa5, 0xd7, 0x98, 0x34, 0x6a, 0xbf, 0x33, 0x6c}} return a, nil } diff --git a/eth/tracers/internal/tracers/prestate_tracer.js b/eth/tracers/internal/tracers/prestate_tracer.js index 99f71d2c33..56aa2b210f 100644 --- a/eth/tracers/internal/tracers/prestate_tracer.js +++ b/eth/tracers/internal/tracers/prestate_tracer.js @@ -40,10 +40,7 @@ var idx = toHex(key); if (this.prestate[acc].storage[idx] === undefined) { - var val = toHex(db.getState(addr, key)); - if (val != "0x0000000000000000000000000000000000000000000000000000000000000000") { - this.prestate[acc].storage[idx] = toHex(db.getState(addr, key)); - } + this.prestate[acc].storage[idx] = toHex(db.getState(addr, key)); } }, From 6a1a4375c680ba4c9ee49ae7cf1e5ab0ab72bdfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 10 Dec 2018 11:36:36 +0200 Subject: [PATCH 127/159] params: set mainnet and Rinkeby Constantinople fork blocks --- params/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/params/config.go b/params/config.go index ba9fac1c4a..7734c6e9cc 100644 --- a/params/config.go +++ b/params/config.go @@ -42,7 +42,7 @@ var ( EIP155Block: big.NewInt(2675000), EIP158Block: big.NewInt(2675000), ByzantiumBlock: big.NewInt(4370000), - ConstantinopleBlock: nil, + ConstantinopleBlock: big.NewInt(7080000), Ethash: new(EthashConfig), } @@ -90,7 +90,7 @@ var ( EIP155Block: big.NewInt(3), EIP158Block: big.NewInt(3), ByzantiumBlock: big.NewInt(1035301), - ConstantinopleBlock: nil, + ConstantinopleBlock: big.NewInt(3660663), Clique: &CliqueConfig{ Period: 15, Epoch: 30000, From 42a914a84ffcc5d123af400c2cbcd34b05e2b0b3 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 15 Oct 2018 12:28:44 +0200 Subject: [PATCH 128/159] cmd/evm, core/vm, eth: implement api methods to do stdjson dump to local filesystem --- cmd/evm/runner.go | 2 +- cmd/evm/staterunner.go | 2 +- .../json_logger.go => core/vm/logger_json.go | 13 +- eth/api_tracer.go | 141 ++++++++++++++++-- internal/web3ext/web3ext.go | 10 ++ 5 files changed, 148 insertions(+), 20 deletions(-) rename cmd/evm/json_logger.go => core/vm/logger_json.go (81%) diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index a9c8a38cae..54b67ce101 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -89,7 +89,7 @@ func runCmd(ctx *cli.Context) error { genesisConfig *core.Genesis ) if ctx.GlobalBool(MachineFlag.Name) { - tracer = NewJSONLogger(logconfig, os.Stdout) + tracer = vm.NewJSONLogger(logconfig, os.Stdout) } else if ctx.GlobalBool(DebugFlag.Name) { debugLogger = vm.NewStructLogger(logconfig) tracer = debugLogger diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index 06c9be380a..b3c69d9b9d 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -68,7 +68,7 @@ func stateTestCmd(ctx *cli.Context) error { ) switch { case ctx.GlobalBool(MachineFlag.Name): - tracer = NewJSONLogger(config, os.Stderr) + tracer = vm.NewJSONLogger(config, os.Stderr) case ctx.GlobalBool(DebugFlag.Name): debugger = vm.NewStructLogger(config) diff --git a/cmd/evm/json_logger.go b/core/vm/logger_json.go similarity index 81% rename from cmd/evm/json_logger.go rename to core/vm/logger_json.go index 50cb4f0e42..ac3c407597 100644 --- a/cmd/evm/json_logger.go +++ b/core/vm/logger_json.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with go-ethereum. If not, see . -package main +package vm import ( "encoding/json" @@ -24,17 +24,16 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" - "github.com/ethereum/go-ethereum/core/vm" ) type JSONLogger struct { encoder *json.Encoder - cfg *vm.LogConfig + cfg *LogConfig } // NewJSONLogger creates a new EVM tracer that prints execution steps as JSON objects // into the provided stream. -func NewJSONLogger(cfg *vm.LogConfig, writer io.Writer) *JSONLogger { +func NewJSONLogger(cfg *LogConfig, writer io.Writer) *JSONLogger { return &JSONLogger{json.NewEncoder(writer), cfg} } @@ -43,8 +42,8 @@ func (l *JSONLogger) CaptureStart(from common.Address, to common.Address, create } // CaptureState outputs state information on the logger. -func (l *JSONLogger) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, contract *vm.Contract, depth int, err error) error { - log := vm.StructLog{ +func (l *JSONLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, contract *Contract, depth int, err error) error { + log := StructLog{ Pc: pc, Op: op, Gas: gas, @@ -65,7 +64,7 @@ func (l *JSONLogger) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cos } // CaptureFault outputs state information on the logger. -func (l *JSONLogger) CaptureFault(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, contract *vm.Contract, depth int, err error) error { +func (l *JSONLogger) CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, contract *Contract, depth int, err error) error { return nil } diff --git a/eth/api_tracer.go b/eth/api_tracer.go index 2ebbcc5fd9..77d52d9441 100644 --- a/eth/api_tracer.go +++ b/eth/api_tracer.go @@ -17,11 +17,13 @@ package eth import ( + "bufio" "bytes" "context" "errors" "fmt" "io/ioutil" + "os" "runtime" "sync" "time" @@ -60,6 +62,13 @@ type TraceConfig struct { Reexec *uint64 } +// StdTraceConfig holds extra parameters to standard-json trace functions. +type StdTraceConfig struct { + *vm.LogConfig + Reexec *uint64 + TxHash *common.Hash +} + // txTraceResult is the result of a single transaction trace. type txTraceResult struct { Result interface{} `json:"result,omitempty"` // Trace results produced by the tracer @@ -391,13 +400,37 @@ func (api *PrivateDebugAPI) TraceBlockFromFile(ctx context.Context, file string, return api.TraceBlock(ctx, blob, config) } -// TraceBadBlock returns the structured logs created during the execution of a block -// within the blockchain 'badblocks' cache -func (api *PrivateDebugAPI) TraceBadBlock(ctx context.Context, index int, config *TraceConfig) ([]*txTraceResult, error) { - if blocks := api.eth.blockchain.BadBlocks(); index < len(blocks) { - return api.traceBlock(ctx, blocks[index], config) +// TraceBadBlockByHash returns the structured logs created during the execution of a block +func (api *PrivateDebugAPI) TraceBadBlock(ctx context.Context, blockHash common.Hash, config *TraceConfig) ([]*txTraceResult, error) { + blocks := api.eth.blockchain.BadBlocks() + for _, block := range blocks { + if block.Hash() == blockHash { + return api.traceBlock(ctx, block, config) + } } - return nil, fmt.Errorf("index out of range") + return nil, fmt.Errorf("hash not found among bad blocks") +} + +// StandardTraceBadBlockToFile dumps the standard-json logs to files on the local filesystem, +// and returns a list of files to the caller. +func (api *PrivateDebugAPI) StandardTraceBadBlockToFile(ctx context.Context, blockHash common.Hash, stdConfig *StdTraceConfig) ([]string, error) { + blocks := api.eth.blockchain.BadBlocks() + for _, block := range blocks { + if block.Hash() == blockHash { + return api.standardTraceBlockToFile(ctx, block, stdConfig) + } + } + return nil, fmt.Errorf("hash not found among bad blocks") +} + +// StandardTraceBlockToFile dumps the standard-json logs to files on the local filesystem, +// and returns a list of files to the caller. +func (api *PrivateDebugAPI) StandardTraceBlockToFile(ctx context.Context, blockHash common.Hash, stdConfig *StdTraceConfig) ([]string, error) { + block := api.eth.blockchain.GetBlockByHash(blockHash) + if block == nil { + return nil, fmt.Errorf("block #%x not found", blockHash) + } + return api.standardTraceBlockToFile(ctx, block, stdConfig) } // traceBlock configures a new tracer according to the provided configuration, and @@ -481,6 +514,92 @@ func (api *PrivateDebugAPI) traceBlock(ctx context.Context, block *types.Block, return results, nil } +// standardTraceBlockToFile configures a new tracer which uses standard-json output, and +// traces either a full block or an individual transaction. The return value will be one filename +// per transaction traced. +func (api *PrivateDebugAPI) standardTraceBlockToFile(ctx context.Context, block *types.Block, stdConfig *StdTraceConfig) ([]string, error) { + // Create the parent state database + if err := api.eth.engine.VerifyHeader(api.eth.blockchain, block.Header(), true); err != nil { + return nil, err + } + parent := api.eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) + if parent == nil { + return nil, fmt.Errorf("parent %x not found", block.ParentHash()) + } + var ( + signer = types.MakeSigner(api.config, block.Number()) + done = false + blockPrefix = fmt.Sprintf("block_0x%x", block.Hash().Bytes()[:4]) + usedLogConfig = &vm.LogConfig{Debug: true} + files []string + reExec_val = defaultTraceReexec + txHash *common.Hash + ) + if stdConfig != nil { + if stdConfig.Reexec != nil { + reExec_val = *stdConfig.Reexec + } + if stdConfig.LogConfig != nil { + usedLogConfig.DisableMemory = stdConfig.LogConfig.DisableMemory + usedLogConfig.DisableStack = stdConfig.LogConfig.DisableStack + usedLogConfig.DisableStorage = stdConfig.LogConfig.DisableStorage + usedLogConfig.Limit = stdConfig.LogConfig.Limit + } + txHash = stdConfig.TxHash + } + statedb, err := api.computeStateDB(parent, reExec_val) + if err != nil { + return nil, err + } + + for i, tx := range block.Transactions() { + var ( + outfile *os.File + err error + ) + msg, _ := tx.AsMessage(signer) + vmctx := core.NewEVMContext(msg, block.Header(), api.eth.blockchain, nil) + vmConf := vm.Config{} + if txHash == nil || bytes.Equal(txHash.Bytes(), tx.Hash().Bytes()) { + prefix := fmt.Sprintf("%v-%d-0x%x-", blockPrefix, i, tx.Hash().Bytes()[:4]) + // Open a file to dump trace into + outfile, err = ioutil.TempFile(os.TempDir(), prefix) + if err != nil { + return nil, err + } + files = append(files, outfile.Name()) + vmConf = vm.Config{ + Debug: true, + Tracer: vm.NewJSONLogger(usedLogConfig, bufio.NewWriter(outfile)), + EnablePreimageRecording: true, + } + if txHash != nil { // Only one tx to trace + done = true + } + } + vmenv := vm.NewEVM(vmctx, statedb, api.config, vmConf) + _, _, _, err = core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.Gas())) + + if outfile != nil { + outfile.Close() + log.Info("Wrote trace", "file", outfile.Name()) + } + if err != nil { + return files, err + } + // Finalize the state so any modifications are written to the trie + statedb.Finalise(true) + + if done { + break + } + } + if txHash != nil && !done { + return nil, fmt.Errorf("transaction hash not found in block") + } + return files, nil +} + // computeStateDB retrieves the state database associated with a certain block. // If no state is locally available for the given block, a number of blocks are // attempted to be reexecuted to generate the desired state. @@ -506,7 +625,7 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (* if err != nil { switch err.(type) { case *trie.MissingNodeError: - return nil, errors.New("required historical state unavailable") + return nil, fmt.Errorf("required historical state unavailable (reexec=%d)", reexec) default: return nil, err } @@ -520,7 +639,7 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (* for block.NumberU64() < origin { // Print progress logs if long enough time elapsed if time.Since(logged) > 8*time.Second { - log.Info("Regenerating historical state", "block", block.NumberU64()+1, "target", origin, "elapsed", time.Since(start)) + log.Info("Regenerating historical state", "block", block.NumberU64()+1, "target", origin, "remaining", origin-block.NumberU64()-1, "elapsed", time.Since(start)) logged = time.Now() } // Retrieve the next block to regenerate and process it @@ -529,15 +648,15 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (* } _, _, _, err := api.eth.blockchain.Processor().Process(block, statedb, vm.Config{}) if err != nil { - return nil, err + return nil, fmt.Errorf("processing block %d failed: %v", block.NumberU64(), err) } // Finalize the state so any modifications are written to the trie - root, err := statedb.Commit(true) + root, err := statedb.Commit(api.eth.blockchain.Config().IsEIP158(block.Number())) if err != nil { return nil, err } if err := statedb.Reset(root); err != nil { - return nil, err + return nil, fmt.Errorf("state reset after block %d failed: %v", block.NumberU64(), err) } database.TrieDB().Reference(root, common.Hash{}) if proot != (common.Hash{}) { diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index a5f3196535..f36835d8ac 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -384,6 +384,16 @@ web3._extend({ params: 1, inputFormatter: [null] }), + new web3._extend.Method({ + name: 'standardTraceBadBlockToFile', + call: 'debug_standardTraceBadBlockToFile', + params: 2, + }), + new web3._extend.Method({ + name: 'standardTraceBlockToFile', + call: 'debug_standardTraceBlockToFile', + params: 2, + }), new web3._extend.Method({ name: 'traceBlockByNumber', call: 'debug_traceBlockByNumber', From 0983d02aa906394fd2be83ebda104730e32c593b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 10 Dec 2018 12:32:40 +0200 Subject: [PATCH 129/159] eth, internal/web3ext: tiny polishes in tracers --- eth/api_tracer.go | 172 ++++++++++++++++++++---------------- internal/web3ext/web3ext.go | 2 + 2 files changed, 97 insertions(+), 77 deletions(-) diff --git a/eth/api_tracer.go b/eth/api_tracer.go index 77d52d9441..0b8f8aa00b 100644 --- a/eth/api_tracer.go +++ b/eth/api_tracer.go @@ -66,7 +66,7 @@ type TraceConfig struct { type StdTraceConfig struct { *vm.LogConfig Reexec *uint64 - TxHash *common.Hash + TxHash common.Hash } // txTraceResult is the result of a single transaction trace. @@ -375,7 +375,7 @@ func (api *PrivateDebugAPI) TraceBlockByNumber(ctx context.Context, number rpc.B func (api *PrivateDebugAPI) TraceBlockByHash(ctx context.Context, hash common.Hash, config *TraceConfig) ([]*txTraceResult, error) { block := api.eth.blockchain.GetBlockByHash(hash) if block == nil { - return nil, fmt.Errorf("block #%x not found", hash) + return nil, fmt.Errorf("block %#x not found", hash) } return api.traceBlock(ctx, block, config) } @@ -400,37 +400,41 @@ func (api *PrivateDebugAPI) TraceBlockFromFile(ctx context.Context, file string, return api.TraceBlock(ctx, blob, config) } -// TraceBadBlockByHash returns the structured logs created during the execution of a block -func (api *PrivateDebugAPI) TraceBadBlock(ctx context.Context, blockHash common.Hash, config *TraceConfig) ([]*txTraceResult, error) { +// TraceBadBlockByHash returns the structured logs created during the execution of +// EVM against a block pulled from the pool of bad ones and returns them as a JSON +// object. +func (api *PrivateDebugAPI) TraceBadBlock(ctx context.Context, hash common.Hash, config *TraceConfig) ([]*txTraceResult, error) { blocks := api.eth.blockchain.BadBlocks() for _, block := range blocks { - if block.Hash() == blockHash { + if block.Hash() == hash { return api.traceBlock(ctx, block, config) } } - return nil, fmt.Errorf("hash not found among bad blocks") + return nil, fmt.Errorf("bad block %#x not found", hash) } -// StandardTraceBadBlockToFile dumps the standard-json logs to files on the local filesystem, -// and returns a list of files to the caller. -func (api *PrivateDebugAPI) StandardTraceBadBlockToFile(ctx context.Context, blockHash common.Hash, stdConfig *StdTraceConfig) ([]string, error) { +// StandardTraceBlockToFile dumps the structured logs created during the +// execution of EVM to the local file system and returns a list of files +// to the caller. +func (api *PrivateDebugAPI) StandardTraceBlockToFile(ctx context.Context, hash common.Hash, config *StdTraceConfig) ([]string, error) { + block := api.eth.blockchain.GetBlockByHash(hash) + if block == nil { + return nil, fmt.Errorf("block %#x not found", hash) + } + return api.standardTraceBlockToFile(ctx, block, config) +} + +// StandardTraceBadBlockToFile dumps the structured logs created during the +// execution of EVM against a block pulled from the pool of bad ones to the +// local file system and returns a list of files to the caller. +func (api *PrivateDebugAPI) StandardTraceBadBlockToFile(ctx context.Context, hash common.Hash, config *StdTraceConfig) ([]string, error) { blocks := api.eth.blockchain.BadBlocks() for _, block := range blocks { - if block.Hash() == blockHash { - return api.standardTraceBlockToFile(ctx, block, stdConfig) + if block.Hash() == hash { + return api.standardTraceBlockToFile(ctx, block, config) } } - return nil, fmt.Errorf("hash not found among bad blocks") -} - -// StandardTraceBlockToFile dumps the standard-json logs to files on the local filesystem, -// and returns a list of files to the caller. -func (api *PrivateDebugAPI) StandardTraceBlockToFile(ctx context.Context, blockHash common.Hash, stdConfig *StdTraceConfig) ([]string, error) { - block := api.eth.blockchain.GetBlockByHash(blockHash) - if block == nil { - return nil, fmt.Errorf("block #%x not found", blockHash) - } - return api.standardTraceBlockToFile(ctx, block, stdConfig) + return nil, fmt.Errorf("bad block %#x not found", hash) } // traceBlock configures a new tracer according to the provided configuration, and @@ -443,7 +447,7 @@ func (api *PrivateDebugAPI) traceBlock(ctx context.Context, block *types.Block, } parent := api.eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) if parent == nil { - return nil, fmt.Errorf("parent %x not found", block.ParentHash()) + return nil, fmt.Errorf("parent %#x not found", block.ParentHash()) } reexec := defaultTraceReexec if config != nil && config.Reexec != nil { @@ -514,90 +518,104 @@ func (api *PrivateDebugAPI) traceBlock(ctx context.Context, block *types.Block, return results, nil } -// standardTraceBlockToFile configures a new tracer which uses standard-json output, and -// traces either a full block or an individual transaction. The return value will be one filename -// per transaction traced. -func (api *PrivateDebugAPI) standardTraceBlockToFile(ctx context.Context, block *types.Block, stdConfig *StdTraceConfig) ([]string, error) { +// standardTraceBlockToFile configures a new tracer which uses standard JSON output, +// and traces either a full block or an individual transaction. The return value will +// be one filename per transaction traced. +func (api *PrivateDebugAPI) standardTraceBlockToFile(ctx context.Context, block *types.Block, config *StdTraceConfig) ([]string, error) { + // If we're tracing a single transaction, make sure it's present + if config != nil && config.TxHash != (common.Hash{}) { + var exists bool + for _, tx := range block.Transactions() { + if exists = (tx.Hash() == config.TxHash); exists { + break + } + } + if !exists { + return nil, fmt.Errorf("transaction %#x not found in block", config.TxHash) + } + } // Create the parent state database if err := api.eth.engine.VerifyHeader(api.eth.blockchain, block.Header(), true); err != nil { return nil, err } parent := api.eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) if parent == nil { - return nil, fmt.Errorf("parent %x not found", block.ParentHash()) + return nil, fmt.Errorf("parent %#x not found", block.ParentHash()) } - var ( - signer = types.MakeSigner(api.config, block.Number()) - done = false - blockPrefix = fmt.Sprintf("block_0x%x", block.Hash().Bytes()[:4]) - usedLogConfig = &vm.LogConfig{Debug: true} - files []string - reExec_val = defaultTraceReexec - txHash *common.Hash - ) - if stdConfig != nil { - if stdConfig.Reexec != nil { - reExec_val = *stdConfig.Reexec - } - if stdConfig.LogConfig != nil { - usedLogConfig.DisableMemory = stdConfig.LogConfig.DisableMemory - usedLogConfig.DisableStack = stdConfig.LogConfig.DisableStack - usedLogConfig.DisableStorage = stdConfig.LogConfig.DisableStorage - usedLogConfig.Limit = stdConfig.LogConfig.Limit - } - txHash = stdConfig.TxHash + reexec := defaultTraceReexec + if config != nil && config.Reexec != nil { + reexec = *config.Reexec } - statedb, err := api.computeStateDB(parent, reExec_val) + statedb, err := api.computeStateDB(parent, reexec) if err != nil { return nil, err } + // Retrieve the tracing configurations, or use default values + var ( + logConfig vm.LogConfig + txHash common.Hash + ) + if config != nil { + if config.LogConfig != nil { + logConfig = *config.LogConfig + } + txHash = config.TxHash + } + logConfig.Debug = true + // Execute transaction, either tracing all or just the requested one + var ( + signer = types.MakeSigner(api.config, block.Number()) + dumps []string + ) for i, tx := range block.Transactions() { + // Prepare the trasaction for un-traced execution var ( - outfile *os.File - err error + msg, _ = tx.AsMessage(signer) + vmctx = core.NewEVMContext(msg, block.Header(), api.eth.blockchain, nil) + + vmConf vm.Config + dump *os.File + err error ) - msg, _ := tx.AsMessage(signer) - vmctx := core.NewEVMContext(msg, block.Header(), api.eth.blockchain, nil) - vmConf := vm.Config{} - if txHash == nil || bytes.Equal(txHash.Bytes(), tx.Hash().Bytes()) { - prefix := fmt.Sprintf("%v-%d-0x%x-", blockPrefix, i, tx.Hash().Bytes()[:4]) - // Open a file to dump trace into - outfile, err = ioutil.TempFile(os.TempDir(), prefix) + // If the transaction needs tracing, swap out the configs + if tx.Hash() == txHash || txHash == (common.Hash{}) { + // Generate a unique temporary file to dump it into + prefix := fmt.Sprintf("block_%#x-%d-%#x-", block.Hash().Bytes()[:4], i, tx.Hash().Bytes()[:4]) + + dump, err = ioutil.TempFile(os.TempDir(), prefix) if err != nil { return nil, err } - files = append(files, outfile.Name()) + dumps = append(dumps, dump.Name()) + + // Swap out the noop logger to the standard tracer vmConf = vm.Config{ Debug: true, - Tracer: vm.NewJSONLogger(usedLogConfig, bufio.NewWriter(outfile)), + Tracer: vm.NewJSONLogger(&logConfig, bufio.NewWriter(dump)), EnablePreimageRecording: true, } - if txHash != nil { // Only one tx to trace - done = true - } } + // Execute the transaction and flush any traces to disk vmenv := vm.NewEVM(vmctx, statedb, api.config, vmConf) _, _, _, err = core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.Gas())) - if outfile != nil { - outfile.Close() - log.Info("Wrote trace", "file", outfile.Name()) + if dump != nil { + dump.Close() + log.Info("Wrote standard trace", "file", dump.Name()) } if err != nil { - return files, err + return dumps, err } // Finalize the state so any modifications are written to the trie statedb.Finalise(true) - if done { + // If we've traced the transaction we were looking for, abort + if tx.Hash() == txHash { break } } - if txHash != nil && !done { - return nil, fmt.Errorf("transaction hash not found in block") - } - return files, nil + return dumps, nil } // computeStateDB retrieves the state database associated with a certain block. @@ -675,7 +693,7 @@ func (api *PrivateDebugAPI) TraceTransaction(ctx context.Context, hash common.Ha // Retrieve the transaction and assemble its EVM context tx, blockHash, _, index := rawdb.ReadTransaction(api.eth.ChainDb(), hash) if tx == nil { - return nil, fmt.Errorf("transaction %x not found", hash) + return nil, fmt.Errorf("transaction %#x not found", hash) } reexec := defaultTraceReexec if config != nil && config.Reexec != nil { @@ -755,11 +773,11 @@ func (api *PrivateDebugAPI) computeTxEnv(blockHash common.Hash, txIndex int, ree // Create the parent state database block := api.eth.blockchain.GetBlockByHash(blockHash) if block == nil { - return nil, vm.Context{}, nil, fmt.Errorf("block %x not found", blockHash) + return nil, vm.Context{}, nil, fmt.Errorf("block %#x not found", blockHash) } parent := api.eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) if parent == nil { - return nil, vm.Context{}, nil, fmt.Errorf("parent %x not found", block.ParentHash()) + return nil, vm.Context{}, nil, fmt.Errorf("parent %#x not found", block.ParentHash()) } statedb, err := api.computeStateDB(parent, reexec) if err != nil { @@ -778,10 +796,10 @@ func (api *PrivateDebugAPI) computeTxEnv(blockHash common.Hash, txIndex int, ree // Not yet the searched for transaction, execute on top of the current state vmenv := vm.NewEVM(context, statedb, api.config, vm.Config{}) if _, _, _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { - return nil, vm.Context{}, nil, fmt.Errorf("tx %x failed: %v", tx.Hash(), err) + return nil, vm.Context{}, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) } // Ensure any modifications are committed to the state statedb.Finalise(true) } - return nil, vm.Context{}, nil, fmt.Errorf("tx index %d out of range for block %x", txIndex, blockHash) + return nil, vm.Context{}, nil, fmt.Errorf("transaction index %d out of range for block %#x", txIndex, blockHash) } diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index f36835d8ac..06bfcef69e 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -388,11 +388,13 @@ web3._extend({ name: 'standardTraceBadBlockToFile', call: 'debug_standardTraceBadBlockToFile', params: 2, + inputFormatter: [null, null] }), new web3._extend.Method({ name: 'standardTraceBlockToFile', call: 'debug_standardTraceBlockToFile', params: 2, + inputFormatter: [null, null] }), new web3._extend.Method({ name: 'traceBlockByNumber', From da6e6e7971b093990b964cd708cceab77898a655 Mon Sep 17 00:00:00 2001 From: "Iskander (Alex) Sharipov" Date: Mon, 10 Dec 2018 14:29:34 +0300 Subject: [PATCH 130/159] light: fix duplicated argument in bytes.Equal call Most probably a copy/paste kind of error. Found with gocritic `dupArg` checker. --- light/trie_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/light/trie_test.go b/light/trie_test.go index 51ce9017a3..5b5fce31d9 100644 --- a/light/trie_test.go +++ b/light/trie_test.go @@ -64,7 +64,7 @@ func diffTries(t1, t2 state.Trie) error { spew.Dump(i2) return fmt.Errorf("tries have different keys %x, %x", i1.Key, i2.Key) } - if !bytes.Equal(i2.Value, i2.Value) { + if !bytes.Equal(i1.Value, i2.Value) { return fmt.Errorf("tries differ at key %x", i1.Key) } } From c1e3fe6b140c6ac7f9d7c792480a10208a4ac7ec Mon Sep 17 00:00:00 2001 From: YOSHIDA Masanori Date: Mon, 10 Dec 2018 21:24:55 +0900 Subject: [PATCH 131/159] ethereum: fix typo in interfaces.go (#18266) * Fix typo in interfaces.go * Update interfaces.go --- interfaces.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interfaces.go b/interfaces.go index 26b0fcbc17..be78344062 100644 --- a/interfaces.go +++ b/interfaces.go @@ -146,7 +146,7 @@ type FilterQuery struct { // {{A}} matches topic A in first position // {{}, {B}} matches any topic in first position, B in second position // {{A}, {B}} matches topic A in first position, B in second position - // {{A, B}}, {C, D}} matches topic (A OR B) in first position, (C OR D) in second position + // {{A, B}, {C, D}} matches topic (A OR B) in first position, (C OR D) in second position Topics [][]common.Hash } From 48b70ecff1152f9eec091ff03b803d997a573b19 Mon Sep 17 00:00:00 2001 From: Ryan Schneider Date: Fri, 2 Nov 2018 13:26:45 -0700 Subject: [PATCH 132/159] cmd, eth: Add support for `--whitelist =,...` flag * Rejects peers that respond with a different hash for any of the passed in block numbers. * Meant for emergency situations when the network forks unexpectedly. --- cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 33 +++++++++++++++++++++++++++++++++ eth/backend.go | 2 +- eth/config.go | 3 +++ eth/handler.go | 26 +++++++++++++++++++++++++- eth/handler_test.go | 4 ++-- eth/helper_test.go | 2 +- 8 files changed, 67 insertions(+), 5 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 69802a48a7..fc1bf461fb 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -87,6 +87,7 @@ var ( utils.LightServFlag, utils.LightPeersFlag, utils.LightKDFFlag, + utils.WhitelistFlag, utils.CacheFlag, utils.CacheDatabaseFlag, utils.CacheTrieFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 82f17e0ee8..25a702dd73 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -81,6 +81,7 @@ var AppHelpFlagGroups = []flagGroup{ utils.LightServFlag, utils.LightPeersFlag, utils.LightKDFFlag, + utils.WhitelistFlag, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 6a285fcb39..4727e7cdf0 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -182,6 +182,10 @@ var ( Name: "lightkdf", Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength", } + WhitelistFlag = cli.StringFlag{ + Name: "whitelist", + Usage: "Comma separated block number-to-hash mappings to enforce (=)", + } // Dashboard settings DashboardEnabledFlag = cli.BoolFlag{ Name: metrics.DashboardEnabledFlag, @@ -1072,6 +1076,34 @@ func setEthash(ctx *cli.Context, cfg *eth.Config) { } } +func setWhitelist(ctx *cli.Context, cfg *eth.Config) { + if ctx.GlobalIsSet(WhitelistFlag.Name) { + entries := strings.Split(ctx.String(WhitelistFlag.Name), ",") + whitelist := make(map[uint64]common.Hash) + for _, entry := range entries { + split := strings.SplitN(entry, "=", 2) + if len(split) != 2 { + Fatalf("invalid whitelist entry: %s", entry) + } + + bn, err := strconv.ParseUint(split[0], 0, 64) + if err != nil { + Fatalf("Invalid whitelist block number %s: %v", split[0], err) + } + + hash := common.Hash{} + err = hash.UnmarshalText([]byte(split[1])) + if err != nil { + Fatalf("Invalid whitelist hash %s: %v", split[1], err) + } + + whitelist[bn] = hash + } + + cfg.Whitelist = whitelist + } +} + // checkExclusive verifies that only a single instance of the provided flags was // set by the user. Each flag might optionally be followed by a string type to // specialize it further. @@ -1137,6 +1169,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { setGPO(ctx, &cfg.GPO) setTxPool(ctx, &cfg.TxPool) setEthash(ctx, cfg) + setWhitelist(ctx, cfg) if ctx.GlobalIsSet(SyncModeFlag.Name) { cfg.SyncMode = *GlobalTextMarshaler(ctx, SyncModeFlag.Name).(*downloader.SyncMode) diff --git a/eth/backend.go b/eth/backend.go index 4721408425..3f538d5f55 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -173,7 +173,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { } eth.txPool = core.NewTxPool(config.TxPool, eth.chainConfig, eth.blockchain) - if eth.protocolManager, err = NewProtocolManager(eth.chainConfig, config.SyncMode, config.NetworkId, eth.eventMux, eth.txPool, eth.engine, eth.blockchain, chainDb); err != nil { + if eth.protocolManager, err = NewProtocolManager(eth.chainConfig, config.SyncMode, config.NetworkId, eth.eventMux, eth.txPool, eth.engine, eth.blockchain, chainDb, config.Whitelist); err != nil { return nil, err } diff --git a/eth/config.go b/eth/config.go index 601f4735e3..a5c2095124 100644 --- a/eth/config.go +++ b/eth/config.go @@ -87,6 +87,9 @@ type Config struct { SyncMode downloader.SyncMode NoPruning bool + // Whitelist of required block number -> hash values to accept + Whitelist map[uint64]common.Hash `toml:"-"` + // Light client options LightServ int `toml:",omitempty"` // Maximum percentage of time allowed for serving LES requests LightPeers int `toml:",omitempty"` // Maximum number of LES client peers diff --git a/eth/handler.go b/eth/handler.go index 741fc9d5a5..21f31bc7c1 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -17,6 +17,7 @@ package eth import ( + "bytes" "encoding/json" "errors" "fmt" @@ -88,6 +89,8 @@ type ProtocolManager struct { txsSub event.Subscription minedBlockSub *event.TypeMuxSubscription + whitelist map[uint64]common.Hash + // channels for fetcher, syncer, txsyncLoop newPeerCh chan *peer txsyncCh chan *txsync @@ -101,7 +104,7 @@ type ProtocolManager struct { // NewProtocolManager returns a new Ethereum sub protocol manager. The Ethereum sub protocol manages peers capable // with the Ethereum network. -func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, networkID uint64, mux *event.TypeMux, txpool txPool, engine consensus.Engine, blockchain *core.BlockChain, chaindb ethdb.Database) (*ProtocolManager, error) { +func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, networkID uint64, mux *event.TypeMux, txpool txPool, engine consensus.Engine, blockchain *core.BlockChain, chaindb ethdb.Database, whitelist map[uint64]common.Hash) (*ProtocolManager, error) { // Create the protocol manager with the base fields manager := &ProtocolManager{ networkID: networkID, @@ -110,6 +113,7 @@ func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, ne blockchain: blockchain, chainconfig: config, peers: newPeerSet(), + whitelist: whitelist, newPeerCh: make(chan *peer), noMorePeers: make(chan struct{}), txsyncCh: make(chan *txsync), @@ -307,6 +311,16 @@ func (pm *ProtocolManager) handle(p *peer) error { } }() } + + // If we have any explicit whitelist block hashes, request them + for bn := range pm.whitelist { + p.Log().Debug("Requesting whitelist block", "number", bn) + if err := p.RequestHeadersByNumber(bn, 1, 0, false); err != nil { + p.Log().Error("whitelist request failed", "err", err, "number", bn, "peer", p.id) + return err + } + } + // main loop. handle incoming messages. for { if err := pm.handleMsg(p); err != nil { @@ -452,6 +466,16 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // Filter out any explicitly requested headers, deliver the rest to the downloader filter := len(headers) == 1 if filter { + // Check for any responses not matching our whitelist + if expected, ok := pm.whitelist[headers[0].Number.Uint64()]; ok { + actual := headers[0].Hash() + if !bytes.Equal(expected.Bytes(), actual.Bytes()) { + p.Log().Info("Dropping peer with non-matching whitelist block", "number", headers[0].Number.Uint64(), "hash", actual, "expected", expected) + return errors.New("whitelist block mismatch") + } + p.Log().Debug("Whitelist block verified", "number", headers[0].Number.Uint64(), "hash", expected) + } + // If it's a potential DAO fork check, validate against the rules if p.forkDrop != nil && pm.chainconfig.DAOForkBlock.Cmp(headers[0].Number) == 0 { // Disable the fork drop timer diff --git a/eth/handler_test.go b/eth/handler_test.go index 44824fd0b7..9fffd95811 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -478,7 +478,7 @@ func testDAOChallenge(t *testing.T, localForked, remoteForked bool, timeout bool if err != nil { t.Fatalf("failed to create new blockchain: %v", err) } - pm, err := NewProtocolManager(config, downloader.FullSync, DefaultConfig.NetworkId, evmux, new(testTxPool), pow, blockchain, db) + pm, err := NewProtocolManager(config, downloader.FullSync, DefaultConfig.NetworkId, evmux, new(testTxPool), pow, blockchain, db, nil) if err != nil { t.Fatalf("failed to start test protocol manager: %v", err) } @@ -559,7 +559,7 @@ func testBroadcastBlock(t *testing.T, totalPeers, broadcastExpected int) { if err != nil { t.Fatalf("failed to create new blockchain: %v", err) } - pm, err := NewProtocolManager(config, downloader.FullSync, DefaultConfig.NetworkId, evmux, new(testTxPool), pow, blockchain, db) + pm, err := NewProtocolManager(config, downloader.FullSync, DefaultConfig.NetworkId, evmux, new(testTxPool), pow, blockchain, db, nil) if err != nil { t.Fatalf("failed to start test protocol manager: %v", err) } diff --git a/eth/helper_test.go b/eth/helper_test.go index 4e38a129e0..b18a02baf9 100644 --- a/eth/helper_test.go +++ b/eth/helper_test.go @@ -66,7 +66,7 @@ func newTestProtocolManager(mode downloader.SyncMode, blocks int, generator func panic(err) } - pm, err := NewProtocolManager(gspec.Config, mode, DefaultConfig.NetworkId, evmux, &testTxPool{added: newtx}, engine, blockchain, db) + pm, err := NewProtocolManager(gspec.Config, mode, DefaultConfig.NetworkId, evmux, &testTxPool{added: newtx}, engine, blockchain, db, nil) if err != nil { return nil, nil, err } From 31b33349227715e8d1f8753d913892c1e35439d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 10 Dec 2018 14:47:01 +0200 Subject: [PATCH 133/159] cmd/utils, eth: minor polishes on whitelist code --- cmd/utils/flags.go | 41 ++++++++++++++++++----------------------- eth/handler.go | 29 +++++++++++------------------ 2 files changed, 29 insertions(+), 41 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 4727e7cdf0..c9115d7a35 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1077,30 +1077,25 @@ func setEthash(ctx *cli.Context, cfg *eth.Config) { } func setWhitelist(ctx *cli.Context, cfg *eth.Config) { - if ctx.GlobalIsSet(WhitelistFlag.Name) { - entries := strings.Split(ctx.String(WhitelistFlag.Name), ",") - whitelist := make(map[uint64]common.Hash) - for _, entry := range entries { - split := strings.SplitN(entry, "=", 2) - if len(split) != 2 { - Fatalf("invalid whitelist entry: %s", entry) - } - - bn, err := strconv.ParseUint(split[0], 0, 64) - if err != nil { - Fatalf("Invalid whitelist block number %s: %v", split[0], err) - } - - hash := common.Hash{} - err = hash.UnmarshalText([]byte(split[1])) - if err != nil { - Fatalf("Invalid whitelist hash %s: %v", split[1], err) - } - - whitelist[bn] = hash + whitelist := ctx.GlobalString(WhitelistFlag.Name) + if whitelist == "" { + return + } + cfg.Whitelist = make(map[uint64]common.Hash) + for _, entry := range strings.Split(whitelist, ",") { + parts := strings.Split(entry, "=") + if len(parts) != 2 { + Fatalf("Invalid whitelist entry: %s", entry) } - - cfg.Whitelist = whitelist + number, err := strconv.ParseUint(parts[0], 0, 64) + if err != nil { + Fatalf("Invalid whitelist block number %s: %v", parts[0], err) + } + var hash common.Hash + if err = hash.UnmarshalText([]byte(parts[1])); err != nil { + Fatalf("Invalid whitelist hash %s: %v", parts[1], err) + } + cfg.Whitelist[number] = hash } } diff --git a/eth/handler.go b/eth/handler.go index 21f31bc7c1..b42612a566 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -17,7 +17,6 @@ package eth import ( - "bytes" "encoding/json" "errors" "fmt" @@ -311,17 +310,13 @@ func (pm *ProtocolManager) handle(p *peer) error { } }() } - // If we have any explicit whitelist block hashes, request them - for bn := range pm.whitelist { - p.Log().Debug("Requesting whitelist block", "number", bn) - if err := p.RequestHeadersByNumber(bn, 1, 0, false); err != nil { - p.Log().Error("whitelist request failed", "err", err, "number", bn, "peer", p.id) + for number := range pm.whitelist { + if err := p.RequestHeadersByNumber(number, 1, 0, false); err != nil { return err } } - - // main loop. handle incoming messages. + // Handle incoming messages until the connection is torn down for { if err := pm.handleMsg(p); err != nil { p.Log().Debug("Ethereum message handling failed", "err", err) @@ -466,16 +461,6 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // Filter out any explicitly requested headers, deliver the rest to the downloader filter := len(headers) == 1 if filter { - // Check for any responses not matching our whitelist - if expected, ok := pm.whitelist[headers[0].Number.Uint64()]; ok { - actual := headers[0].Hash() - if !bytes.Equal(expected.Bytes(), actual.Bytes()) { - p.Log().Info("Dropping peer with non-matching whitelist block", "number", headers[0].Number.Uint64(), "hash", actual, "expected", expected) - return errors.New("whitelist block mismatch") - } - p.Log().Debug("Whitelist block verified", "number", headers[0].Number.Uint64(), "hash", expected) - } - // If it's a potential DAO fork check, validate against the rules if p.forkDrop != nil && pm.chainconfig.DAOForkBlock.Cmp(headers[0].Number) == 0 { // Disable the fork drop timer @@ -490,6 +475,14 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { p.Log().Debug("Verified to be on the same side of the DAO fork") return nil } + // Otherwise if it's a whitelisted block, validate against the set + if want, ok := pm.whitelist[headers[0].Number.Uint64()]; ok { + if hash := headers[0].Hash(); want != hash { + p.Log().Info("Whitelist mismatch, dropping peer", "number", headers[0].Number.Uint64(), "hash", hash, "want", want) + return errors.New("whitelist block mismatch") + } + p.Log().Debug("Whitelist block verified", "number", headers[0].Number.Uint64(), "hash", want) + } // Irrelevant of the fork checks, send the header to the fetcher just in case headers = pm.fetcher.FilterHeaders(p.id, headers, time.Now()) } From 362e2ba792927282c41acda05cb1a7cd34e1c790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 10 Dec 2018 14:28:50 +0200 Subject: [PATCH 134/159] params: update CHTs for the 1.8.20 release --- params/config.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/params/config.go b/params/config.go index 7734c6e9cc..2935ef1f3f 100644 --- a/params/config.go +++ b/params/config.go @@ -49,10 +49,10 @@ var ( // MainnetTrustedCheckpoint contains the light client trusted checkpoint for the main network. MainnetTrustedCheckpoint = &TrustedCheckpoint{ Name: "mainnet", - SectionIndex: 206, - SectionHead: common.HexToHash("0x9fa677c7c0580136f5a86d9b2fd29b112e531f0284396298b8809bcb6787b538"), - CHTRoot: common.HexToHash("0x7f32dfb29e341b4c8c10ea2e06a812bcea470366f635b7a8b3d0856684cd76f4"), - BloomRoot: common.HexToHash("0x0169e174f0a8172aec217556d8a25c7ba7ca52aacff170325230a75740ff1eaf"), + SectionIndex: 208, + SectionHead: common.HexToHash("0x5e9f7696c397d9df8f3b1abda857753575c6f5cff894e1a3d9e1a2af1bd9d6ac"), + CHTRoot: common.HexToHash("0x954a63134f6897f015f026387c59c98c4dae7b336610ff5a143455aac9153e9d"), + BloomRoot: common.HexToHash("0x8006c5e44b14d90d7cc9cd5fa1cb48cf53697ee3bbbf4b76fdfa70b0242500a9"), } // TestnetChainConfig contains the chain parameters to run a node on the Ropsten test network. @@ -73,10 +73,10 @@ var ( // TestnetTrustedCheckpoint contains the light client trusted checkpoint for the Ropsten test network. TestnetTrustedCheckpoint = &TrustedCheckpoint{ Name: "testnet", - SectionIndex: 136, - SectionHead: common.HexToHash("0xe5d80bb08d92bbc12dfe510c64cba01eafcbb4ba585e7c7ab7f8a93c6f295ab3"), - CHTRoot: common.HexToHash("0xe3ca77ab0cb51eec74f4f7458e36aee207c68768387b39cb0bcff0940a6264d8"), - BloomRoot: common.HexToHash("0x30c8eeadac5539d3dcd6e88915d1a07cb2f3a1d6ebe7e553e3ee783c04c68c2d"), + SectionIndex: 139, + SectionHead: common.HexToHash("0x9fad89a5e3b993c8339b9cf2cbbeb72cd08774ea6b71b105b3dd880420c618f4"), + CHTRoot: common.HexToHash("0xc815833881989c5d2035147e1a79a33d22cbc5313e104ff01e6ab405bd28b317"), + BloomRoot: common.HexToHash("0xd94ee9f3c480858f53ec5d059aebdbb2e8d904702f100875ee59ec5f366e841d"), } // RinkebyChainConfig contains the chain parameters to run a node on the Rinkeby test network. @@ -100,10 +100,10 @@ var ( // RinkebyTrustedCheckpoint contains the light client trusted checkpoint for the Rinkeby test network. RinkebyTrustedCheckpoint = &TrustedCheckpoint{ Name: "rinkeby", - SectionIndex: 103, - SectionHead: common.HexToHash("0x9f38b903852831bf4fa7992f7fd43d8b26da2deb82b421fb845cf6faee54e056"), - CHTRoot: common.HexToHash("0x2d710c2cea468d2e604838000d658ee213e4abb07f90c4f71f5cd7f8510aa708"), - BloomRoot: common.HexToHash("0xcc401060280c2cc82697ea5ecef8cac61e52063c37533a2e9609332419704d5f"), + SectionIndex: 105, + SectionHead: common.HexToHash("0xec8147d43f936258aaf1b9b9ec91b0a853abf7109f436a23649be809ea43d507"), + CHTRoot: common.HexToHash("0xd92703b444846a3db928e87e450770e5d5cbe193131dc8f7c4cf18b4de925a75"), + BloomRoot: common.HexToHash("0xff45a6f807138a2cde0cea0c209d9ce5ad8e43ccaae5a7c41af801bb72a1ef96"), } // AllEthashProtocolChanges contains every protocol change (EIPs) introduced From dd98d1da9464c9ae6e1da69d7a41d4551f20bd8d Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 10 Dec 2018 13:56:01 +0100 Subject: [PATCH 135/159] swarm/network: Correct ambiguity in compared addresses (#18251) --- swarm/network/networkid_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/swarm/network/networkid_test.go b/swarm/network/networkid_test.go index d1d359de68..191d67e5b6 100644 --- a/swarm/network/networkid_test.go +++ b/swarm/network/networkid_test.go @@ -95,8 +95,7 @@ func TestNetworkID(t *testing.T) { kademlias[node].EachAddr(nil, 0, func(addr *BzzAddr, _ int, _ bool) bool { found := false for _, nd := range netIDGroup { - p := nd.Bytes() - if bytes.Equal(p, addr.Address()) { + if bytes.Equal(kademlias[nd].BaseAddr(), addr.Address()) { found = true } } From bb724080cac9fa36ec6b638cfd5cf0e54bc23362 Mon Sep 17 00:00:00 2001 From: Elad Date: Tue, 11 Dec 2018 13:51:58 +0530 Subject: [PATCH 136/159] cmd/swarm, metrics, swarm/api/client, swarm/storage, swarm/metrics, swarm/api/http: add instrumentation (#18274) --- cmd/swarm/swarm-smoke/feed_upload_and_sync.go | 64 ++++++- cmd/swarm/swarm-smoke/main.go | 71 ++++++++ cmd/swarm/swarm-smoke/upload_and_sync.go | 160 ++++++++++++------ metrics/influxdb/influxdb.go | 28 +++ swarm/api/client/client.go | 76 ++++++++- swarm/api/http/middleware.go | 11 +- swarm/metrics/flags.go | 32 ++-- swarm/storage/chunker.go | 8 + 8 files changed, 368 insertions(+), 82 deletions(-) diff --git a/cmd/swarm/swarm-smoke/feed_upload_and_sync.go b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go index 7ec1528263..2c5e3fd235 100644 --- a/cmd/swarm/swarm-smoke/feed_upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go @@ -2,11 +2,13 @@ package main import ( "bytes" + "context" "crypto/md5" "fmt" "io" "io/ioutil" "net/http" + "net/http/httptrace" "os" "os/exec" "strings" @@ -16,8 +18,13 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/swarm/api/client" + "github.com/ethereum/go-ethereum/swarm/spancontext" "github.com/ethereum/go-ethereum/swarm/storage/feed" + "github.com/ethereum/go-ethereum/swarm/testutil" colorable "github.com/mattn/go-colorable" + opentracing "github.com/opentracing/opentracing-go" "github.com/pborman/uuid" cli "gopkg.in/urfave/cli.v1" ) @@ -26,11 +33,29 @@ const ( feedRandomDataLength = 8 ) -// TODO: retrieve with manifest + extract repeating code func cliFeedUploadAndSync(c *cli.Context) error { - + metrics.GetOrRegisterCounter("feed-and-sync", nil).Inc(1) log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))) + errc := make(chan error) + go func() { + errc <- feedUploadAndSync(c) + }() + + select { + case err := <-errc: + if err != nil { + metrics.GetOrRegisterCounter("feed-and-sync.fail", nil).Inc(1) + } + return err + case <-time.After(time.Duration(timeout) * time.Second): + metrics.GetOrRegisterCounter("feed-and-sync.timeout", nil).Inc(1) + return fmt.Errorf("timeout after %v sec", timeout) + } +} + +// TODO: retrieve with manifest + extract repeating code +func feedUploadAndSync(c *cli.Context) error { defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now()) generateEndpoints(scheme, cluster, appName, from, to) @@ -204,12 +229,12 @@ func cliFeedUploadAndSync(c *cli.Context) error { log.Info("all endpoints synced random data successfully") // upload test file - log.Info("uploading to " + endpoints[0] + " and syncing") + seed := int(time.Now().UnixNano() / 1e6) + log.Info("feed uploading to "+endpoints[0]+" and syncing", "seed", seed) - f, cleanup := generateRandomFile(filesize * 1000) - defer cleanup() + randomBytes := testutil.RandomBytes(seed, filesize*1000) - hash, err := upload(f, endpoints[0]) + hash, err := upload(&randomBytes, endpoints[0]) if err != nil { return err } @@ -218,7 +243,7 @@ func cliFeedUploadAndSync(c *cli.Context) error { return err } multihashHex := hexutil.Encode(hashBytes) - fileHash, err := digest(f) + fileHash, err := digest(bytes.NewReader(randomBytes)) if err != nil { return err } @@ -284,14 +309,37 @@ func cliFeedUploadAndSync(c *cli.Context) error { } func fetchFeed(topic string, user string, endpoint string, original []byte, ruid string) error { + ctx, sp := spancontext.StartSpan(context.Background(), "feed-and-sync.fetch") + defer sp.Finish() + log.Trace("sleeping", "ruid", ruid) time.Sleep(3 * time.Second) log.Trace("http get request (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user) - res, err := http.Get(endpoint + "/bzz-feed:/?topic=" + topic + "&user=" + user) + + var tn time.Time + reqUri := endpoint + "/bzz-feed:/?topic=" + topic + "&user=" + user + req, _ := http.NewRequest("GET", reqUri, nil) + + opentracing.GlobalTracer().Inject( + sp.Context(), + opentracing.HTTPHeaders, + opentracing.HTTPHeadersCarrier(req.Header)) + + trace := client.GetClientTrace("feed-and-sync - http get", "feed-and-sync", ruid, &tn) + + req = req.WithContext(httptrace.WithClientTrace(ctx, trace)) + transport := http.DefaultTransport + + //transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + + tn = time.Now() + res, err := transport.RoundTrip(req) if err != nil { + log.Error(err.Error(), "ruid", ruid) return err } + log.Trace("http get response (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user, "code", res.StatusCode, "len", res.ContentLength) if res.StatusCode != 200 { diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index 845998dc12..66cecdc5c7 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -17,14 +17,25 @@ package main import ( + "fmt" "os" "sort" + "github.com/ethereum/go-ethereum/cmd/utils" + gethmetrics "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/metrics/influxdb" + swarmmetrics "github.com/ethereum/go-ethereum/swarm/metrics" + "github.com/ethereum/go-ethereum/swarm/tracing" + "github.com/ethereum/go-ethereum/log" cli "gopkg.in/urfave/cli.v1" ) +var ( + gitCommit string // Git SHA1 commit hash of the release (set via linker flags) +) + var ( endpoints []string includeLocalhost bool @@ -32,9 +43,12 @@ var ( appName string scheme string filesize int + syncDelay int from int to int verbosity int + timeout int + single bool ) func main() { @@ -85,14 +99,42 @@ func main() { Usage: "file size for generated random file in KB", Destination: &filesize, }, + cli.IntFlag{ + Name: "sync-delay", + Value: 5, + Usage: "duration of delay in seconds to wait for content to be synced", + Destination: &syncDelay, + }, cli.IntFlag{ Name: "verbosity", Value: 1, Usage: "verbosity", Destination: &verbosity, }, + cli.IntFlag{ + Name: "timeout", + Value: 120, + Usage: "timeout in seconds after which kill the process", + Destination: &timeout, + }, + cli.BoolFlag{ + Name: "single", + Usage: "whether to fetch content from a single node or from all nodes", + Destination: &single, + }, } + app.Flags = append(app.Flags, []cli.Flag{ + utils.MetricsEnabledFlag, + swarmmetrics.MetricsInfluxDBEndpointFlag, + swarmmetrics.MetricsInfluxDBDatabaseFlag, + swarmmetrics.MetricsInfluxDBUsernameFlag, + swarmmetrics.MetricsInfluxDBPasswordFlag, + swarmmetrics.MetricsInfluxDBHostTagFlag, + }...) + + app.Flags = append(app.Flags, tracing.Flags...) + app.Commands = []cli.Command{ { Name: "upload_and_sync", @@ -111,9 +153,38 @@ func main() { sort.Sort(cli.FlagsByName(app.Flags)) sort.Sort(cli.CommandsByName(app.Commands)) + app.Before = func(ctx *cli.Context) error { + tracing.Setup(ctx) + return nil + } + + app.After = func(ctx *cli.Context) error { + return emitMetrics(ctx) + } + err := app.Run(os.Args) if err != nil { log.Error(err.Error()) + os.Exit(1) } } + +func emitMetrics(ctx *cli.Context) error { + if gethmetrics.Enabled { + var ( + endpoint = ctx.GlobalString(swarmmetrics.MetricsInfluxDBEndpointFlag.Name) + database = ctx.GlobalString(swarmmetrics.MetricsInfluxDBDatabaseFlag.Name) + username = ctx.GlobalString(swarmmetrics.MetricsInfluxDBUsernameFlag.Name) + password = ctx.GlobalString(swarmmetrics.MetricsInfluxDBPasswordFlag.Name) + hosttag = ctx.GlobalString(swarmmetrics.MetricsInfluxDBHostTagFlag.Name) + ) + return influxdb.InfluxDBWithTagsOnce(gethmetrics.DefaultRegistry, endpoint, database, username, password, "swarm-smoke.", map[string]string{ + "host": hosttag, + "version": gitCommit, + "filesize": fmt.Sprintf("%v", filesize), + }) + } + + return nil +} diff --git a/cmd/swarm/swarm-smoke/upload_and_sync.go b/cmd/swarm/swarm-smoke/upload_and_sync.go index 3843457dc4..d605f79a3a 100644 --- a/cmd/swarm/swarm-smoke/upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/upload_and_sync.go @@ -18,21 +18,27 @@ package main import ( "bytes" + "context" "crypto/md5" crand "crypto/rand" - "crypto/tls" "errors" "fmt" "io" "io/ioutil" + "math/rand" "net/http" + "net/http/httptrace" "os" - "os/exec" - "strings" "sync" "time" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/swarm/api" + "github.com/ethereum/go-ethereum/swarm/api/client" + "github.com/ethereum/go-ethereum/swarm/spancontext" + "github.com/ethereum/go-ethereum/swarm/testutil" + opentracing "github.com/opentracing/opentracing-go" "github.com/pborman/uuid" cli "gopkg.in/urfave/cli.v1" @@ -40,11 +46,11 @@ import ( func generateEndpoints(scheme string, cluster string, app string, from int, to int) { if cluster == "prod" { - for port := from; port <= to; port++ { + for port := from; port < to; port++ { endpoints = append(endpoints, fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, port)) } } else { - for port := from; port <= to; port++ { + for port := from; port < to; port++ { endpoints = append(endpoints, fmt.Sprintf("%s://%s-%v-%s.stg.swarm-gateways.net", scheme, app, port, cluster)) } } @@ -58,22 +64,48 @@ func cliUploadAndSync(c *cli.Context) error { log.PrintOrigins(true) log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) - defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "kb", filesize) }(time.Now()) + metrics.GetOrRegisterCounter("upload-and-sync", nil).Inc(1) + + errc := make(chan error) + go func() { + errc <- uploadAndSync(c) + }() + + select { + case err := <-errc: + if err != nil { + metrics.GetOrRegisterCounter("upload-and-sync.fail", nil).Inc(1) + } + return err + case <-time.After(time.Duration(timeout) * time.Second): + metrics.GetOrRegisterCounter("upload-and-sync.timeout", nil).Inc(1) + return fmt.Errorf("timeout after %v sec", timeout) + } +} + +func uploadAndSync(c *cli.Context) error { + defer func(now time.Time) { + totalTime := time.Since(now) + + log.Info("total time", "time", totalTime, "kb", filesize) + metrics.GetOrRegisterCounter("upload-and-sync.total-time", nil).Inc(int64(totalTime)) + }(time.Now()) generateEndpoints(scheme, cluster, appName, from, to) + seed := int(time.Now().UnixNano() / 1e6) + log.Info("uploading to "+endpoints[0]+" and syncing", "seed", seed) - log.Info("uploading to " + endpoints[0] + " and syncing") + randomBytes := testutil.RandomBytes(seed, filesize*1000) - f, cleanup := generateRandomFile(filesize * 1000) - defer cleanup() - - hash, err := upload(f, endpoints[0]) + t1 := time.Now() + hash, err := upload(&randomBytes, endpoints[0]) if err != nil { log.Error(err.Error()) return err } + metrics.GetOrRegisterCounter("upload-and-sync.upload-time", nil).Inc(int64(time.Since(t1))) - fhash, err := digest(f) + fhash, err := digest(bytes.NewReader(randomBytes)) if err != nil { log.Error(err.Error()) return err @@ -81,23 +113,47 @@ func cliUploadAndSync(c *cli.Context) error { log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash)) - time.Sleep(3 * time.Second) + time.Sleep(time.Duration(syncDelay) * time.Second) wg := sync.WaitGroup{} - for _, endpoint := range endpoints { + if single { + rand.Seed(time.Now().UTC().UnixNano()) + randIndex := 1 + rand.Intn(len(endpoints)-1) ruid := uuid.New()[:8] wg.Add(1) go func(endpoint string, ruid string) { for { + start := time.Now() err := fetch(hash, endpoint, fhash, ruid) + fetchTime := time.Since(start) if err != nil { continue } + metrics.GetOrRegisterMeter("upload-and-sync.single.fetch-time", nil).Mark(int64(fetchTime)) wg.Done() return } - }(endpoint, ruid) + }(endpoints[randIndex], ruid) + } else { + for _, endpoint := range endpoints { + ruid := uuid.New()[:8] + wg.Add(1) + go func(endpoint string, ruid string) { + for { + start := time.Now() + err := fetch(hash, endpoint, fhash, ruid) + fetchTime := time.Since(start) + if err != nil { + continue + } + + metrics.GetOrRegisterMeter("upload-and-sync.each.fetch-time", nil).Mark(int64(fetchTime)) + wg.Done() + return + } + }(endpoint, ruid) + } } wg.Wait() log.Info("all endpoints synced random file successfully") @@ -107,16 +163,33 @@ func cliUploadAndSync(c *cli.Context) error { // fetch is getting the requested `hash` from the `endpoint` and compares it with the `original` file func fetch(hash string, endpoint string, original []byte, ruid string) error { + ctx, sp := spancontext.StartSpan(context.Background(), "upload-and-sync.fetch") + defer sp.Finish() + log.Trace("sleeping", "ruid", ruid) time.Sleep(3 * time.Second) - log.Trace("http get request", "ruid", ruid, "api", endpoint, "hash", hash) - client := &http.Client{Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - }} - res, err := client.Get(endpoint + "/bzz:/" + hash + "/") + + var tn time.Time + reqUri := endpoint + "/bzz:/" + hash + "/" + req, _ := http.NewRequest("GET", reqUri, nil) + + opentracing.GlobalTracer().Inject( + sp.Context(), + opentracing.HTTPHeaders, + opentracing.HTTPHeadersCarrier(req.Header)) + + trace := client.GetClientTrace("upload-and-sync - http get", "upload-and-sync", ruid, &tn) + + req = req.WithContext(httptrace.WithClientTrace(ctx, trace)) + transport := http.DefaultTransport + + //transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + + tn = time.Now() + res, err := transport.RoundTrip(req) if err != nil { - log.Warn(err.Error(), "ruid", ruid) + log.Error(err.Error(), "ruid", ruid) return err } log.Trace("http get response", "ruid", ruid, "api", endpoint, "hash", hash, "code", res.StatusCode, "len", res.ContentLength) @@ -147,16 +220,19 @@ func fetch(hash string, endpoint string, original []byte, ruid string) error { } // upload is uploading a file `f` to `endpoint` via the `swarm up` cmd -func upload(f *os.File, endpoint string) (string, error) { - var out bytes.Buffer - cmd := exec.Command("swarm", "--bzzapi", endpoint, "up", f.Name()) - cmd.Stdout = &out - err := cmd.Run() - if err != nil { - return "", err +func upload(dataBytes *[]byte, endpoint string) (string, error) { + swarm := client.NewClient(endpoint) + f := &client.File{ + ReadCloser: ioutil.NopCloser(bytes.NewReader(*dataBytes)), + ManifestEntry: api.ManifestEntry{ + ContentType: "text/plain", + Mode: 0660, + Size: int64(len(*dataBytes)), + }, } - hash := strings.TrimRight(out.String(), "\r\n") - return hash, nil + + // upload data to bzz:// and retrieve the content-addressed manifest hash, hex-encoded. + return swarm.Upload(f, "", false) } func digest(r io.Reader) ([]byte, error) { @@ -179,27 +255,3 @@ func generateRandomData(datasize int) ([]byte, error) { } return b, nil } - -// generateRandomFile is creating a temporary file with the requested byte size -func generateRandomFile(size int) (f *os.File, teardown func()) { - // create a tmp file - tmp, err := ioutil.TempFile("", "swarm-test") - if err != nil { - panic(err) - } - - // callback for tmp file cleanup - teardown = func() { - tmp.Close() - os.Remove(tmp.Name()) - } - - buf := make([]byte, size) - _, err = crand.Read(buf) - if err != nil { - panic(err) - } - ioutil.WriteFile(tmp.Name(), buf, 0755) - - return tmp, teardown -} diff --git a/metrics/influxdb/influxdb.go b/metrics/influxdb/influxdb.go index 31a5c21b5f..c4ef927234 100644 --- a/metrics/influxdb/influxdb.go +++ b/metrics/influxdb/influxdb.go @@ -58,6 +58,34 @@ func InfluxDBWithTags(r metrics.Registry, d time.Duration, url, database, userna rep.run() } +// InfluxDBWithTagsOnce runs once an InfluxDB reporter and post the given metrics.Registry with the specified tags +func InfluxDBWithTagsOnce(r metrics.Registry, url, database, username, password, namespace string, tags map[string]string) error { + u, err := uurl.Parse(url) + if err != nil { + return fmt.Errorf("Unable to parse InfluxDB. url: %s, err: %v", url, err) + } + + rep := &reporter{ + reg: r, + url: *u, + database: database, + username: username, + password: password, + namespace: namespace, + tags: tags, + cache: make(map[string]int64), + } + if err := rep.makeClient(); err != nil { + return fmt.Errorf("Unable to make InfluxDB client. err: %v", err) + } + + if err := rep.send(); err != nil { + return fmt.Errorf("Unable to send to InfluxDB. err: %v", err) + } + + return nil +} + func (r *reporter) makeClient() (err error) { r.client, err = client.NewClient(client.Config{ URL: r.url, diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go index d9837ca732..f793ca8b89 100644 --- a/swarm/api/client/client.go +++ b/swarm/api/client/client.go @@ -19,6 +19,7 @@ package client import ( "archive/tar" "bytes" + "context" "encoding/json" "errors" "fmt" @@ -26,6 +27,7 @@ import ( "io/ioutil" "mime/multipart" "net/http" + "net/http/httptrace" "net/textproto" "net/url" "os" @@ -33,9 +35,14 @@ import ( "regexp" "strconv" "strings" + "time" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/api" + "github.com/ethereum/go-ethereum/swarm/spancontext" "github.com/ethereum/go-ethereum/swarm/storage/feed" + "github.com/pborman/uuid" ) var ( @@ -474,6 +481,11 @@ type UploadFn func(file *File) error // TarUpload uses the given Uploader to upload files to swarm as a tar stream, // returning the resulting manifest hash func (c *Client) TarUpload(hash string, uploader Uploader, defaultPath string, toEncrypt bool) (string, error) { + ctx, sp := spancontext.StartSpan(context.Background(), "api.client.tarupload") + defer sp.Finish() + + var tn time.Time + reqR, reqW := io.Pipe() defer reqR.Close() addr := hash @@ -489,6 +501,12 @@ func (c *Client) TarUpload(hash string, uploader Uploader, defaultPath string, t if err != nil { return "", err } + + trace := GetClientTrace("swarm api client - upload tar", "api.client.uploadtar", uuid.New()[:8], &tn) + + req = req.WithContext(httptrace.WithClientTrace(ctx, trace)) + transport := http.DefaultTransport + req.Header.Set("Content-Type", "application/x-tar") if defaultPath != "" { q := req.URL.Query() @@ -529,8 +547,8 @@ func (c *Client) TarUpload(hash string, uploader Uploader, defaultPath string, t } reqW.CloseWithError(err) }() - - res, err := http.DefaultClient.Do(req) + tn = time.Now() + res, err := transport.RoundTrip(req) if err != nil { return "", err } @@ -728,3 +746,57 @@ func (c *Client) GetFeedRequest(query *feed.Query, manifestAddressOrDomain strin } return &metadata, nil } + +func GetClientTrace(traceMsg, metricPrefix, ruid string, tn *time.Time) *httptrace.ClientTrace { + trace := &httptrace.ClientTrace{ + GetConn: func(_ string) { + log.Trace(traceMsg+" - http get", "event", "GetConn", "ruid", ruid) + metrics.GetOrRegisterResettingTimer(metricPrefix+".getconn", nil).Update(time.Since(*tn)) + }, + GotConn: func(_ httptrace.GotConnInfo) { + log.Trace(traceMsg+" - http get", "event", "GotConn", "ruid", ruid) + metrics.GetOrRegisterResettingTimer(metricPrefix+".gotconn", nil).Update(time.Since(*tn)) + }, + PutIdleConn: func(err error) { + log.Trace(traceMsg+" - http get", "event", "PutIdleConn", "ruid", ruid, "err", err) + metrics.GetOrRegisterResettingTimer(metricPrefix+".putidle", nil).Update(time.Since(*tn)) + }, + GotFirstResponseByte: func() { + log.Trace(traceMsg+" - http get", "event", "GotFirstResponseByte", "ruid", ruid) + metrics.GetOrRegisterResettingTimer(metricPrefix+".firstbyte", nil).Update(time.Since(*tn)) + }, + Got100Continue: func() { + log.Trace(traceMsg, "event", "Got100Continue", "ruid", ruid) + metrics.GetOrRegisterResettingTimer(metricPrefix+".got100continue", nil).Update(time.Since(*tn)) + }, + DNSStart: func(_ httptrace.DNSStartInfo) { + log.Trace(traceMsg, "event", "DNSStart", "ruid", ruid) + metrics.GetOrRegisterResettingTimer(metricPrefix+".dnsstart", nil).Update(time.Since(*tn)) + }, + DNSDone: func(_ httptrace.DNSDoneInfo) { + log.Trace(traceMsg, "event", "DNSDone", "ruid", ruid) + metrics.GetOrRegisterResettingTimer(metricPrefix+".dnsdone", nil).Update(time.Since(*tn)) + }, + ConnectStart: func(network, addr string) { + log.Trace(traceMsg, "event", "ConnectStart", "ruid", ruid, "network", network, "addr", addr) + metrics.GetOrRegisterResettingTimer(metricPrefix+".connectstart", nil).Update(time.Since(*tn)) + }, + ConnectDone: func(network, addr string, err error) { + log.Trace(traceMsg, "event", "ConnectDone", "ruid", ruid, "network", network, "addr", addr, "err", err) + metrics.GetOrRegisterResettingTimer(metricPrefix+".connectdone", nil).Update(time.Since(*tn)) + }, + WroteHeaders: func() { + log.Trace(traceMsg, "event", "WroteHeaders(request)", "ruid", ruid) + metrics.GetOrRegisterResettingTimer(metricPrefix+".wroteheaders", nil).Update(time.Since(*tn)) + }, + Wait100Continue: func() { + log.Trace(traceMsg, "event", "Wait100Continue", "ruid", ruid) + metrics.GetOrRegisterResettingTimer(metricPrefix+".wait100continue", nil).Update(time.Since(*tn)) + }, + WroteRequest: func(_ httptrace.WroteRequestInfo) { + log.Trace(traceMsg, "event", "WroteRequest", "ruid", ruid) + metrics.GetOrRegisterResettingTimer(metricPrefix+".wroterequest", nil).Update(time.Since(*tn)) + }, + } + return trace +} diff --git a/swarm/api/http/middleware.go b/swarm/api/http/middleware.go index 115a008566..f7f819eab5 100644 --- a/swarm/api/http/middleware.go +++ b/swarm/api/http/middleware.go @@ -74,13 +74,15 @@ func ParseURI(h http.Handler) http.Handler { func InitLoggingResponseWriter(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - startTime := time.Now() - defer metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.time", r.Method), nil).UpdateSince(startTime) + tn := time.Now() writer := newLoggingResponseWriter(w) h.ServeHTTP(writer, r) - log.Info("request served", "ruid", GetRUID(r.Context()), "code", writer.statusCode) - metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.%d.time", r.Method, writer.statusCode), nil).UpdateSince(startTime) + + ts := time.Since(tn) + log.Info("request served", "ruid", GetRUID(r.Context()), "code", writer.statusCode, "time", ts*time.Millisecond) + metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.time", r.Method), nil).Update(ts) + metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.%d.time", r.Method, writer.statusCode), nil).Update(ts) }) } @@ -93,6 +95,7 @@ func InstrumentOpenTracing(h http.Handler) http.Handler { } spanName := fmt.Sprintf("http.%s.%s", r.Method, uri.Scheme) ctx, sp := spancontext.StartSpan(r.Context(), spanName) + defer sp.Finish() h.ServeHTTP(w, r.WithContext(ctx)) }) diff --git a/swarm/metrics/flags.go b/swarm/metrics/flags.go index 79490fd360..7c12120a60 100644 --- a/swarm/metrics/flags.go +++ b/swarm/metrics/flags.go @@ -27,26 +27,26 @@ import ( ) var ( - metricsEnableInfluxDBExportFlag = cli.BoolFlag{ + MetricsEnableInfluxDBExportFlag = cli.BoolFlag{ Name: "metrics.influxdb.export", Usage: "Enable metrics export/push to an external InfluxDB database", } - metricsInfluxDBEndpointFlag = cli.StringFlag{ + MetricsInfluxDBEndpointFlag = cli.StringFlag{ Name: "metrics.influxdb.endpoint", Usage: "Metrics InfluxDB endpoint", Value: "http://127.0.0.1:8086", } - metricsInfluxDBDatabaseFlag = cli.StringFlag{ + MetricsInfluxDBDatabaseFlag = cli.StringFlag{ Name: "metrics.influxdb.database", Usage: "Metrics InfluxDB database", Value: "metrics", } - metricsInfluxDBUsernameFlag = cli.StringFlag{ + MetricsInfluxDBUsernameFlag = cli.StringFlag{ Name: "metrics.influxdb.username", Usage: "Metrics InfluxDB username", Value: "", } - metricsInfluxDBPasswordFlag = cli.StringFlag{ + MetricsInfluxDBPasswordFlag = cli.StringFlag{ Name: "metrics.influxdb.password", Usage: "Metrics InfluxDB password", Value: "", @@ -55,7 +55,7 @@ var ( // It is used so that we can group all nodes and average a measurement across all of them, but also so // that we can select a specific node and inspect its measurements. // https://docs.influxdata.com/influxdb/v1.4/concepts/key_concepts/#tag-key - metricsInfluxDBHostTagFlag = cli.StringFlag{ + MetricsInfluxDBHostTagFlag = cli.StringFlag{ Name: "metrics.influxdb.host.tag", Usage: "Metrics InfluxDB `host` tag attached to all measurements", Value: "localhost", @@ -65,20 +65,24 @@ var ( // Flags holds all command-line flags required for metrics collection. var Flags = []cli.Flag{ utils.MetricsEnabledFlag, - metricsEnableInfluxDBExportFlag, - metricsInfluxDBEndpointFlag, metricsInfluxDBDatabaseFlag, metricsInfluxDBUsernameFlag, metricsInfluxDBPasswordFlag, metricsInfluxDBHostTagFlag, + MetricsEnableInfluxDBExportFlag, + MetricsInfluxDBEndpointFlag, + MetricsInfluxDBDatabaseFlag, + MetricsInfluxDBUsernameFlag, + MetricsInfluxDBPasswordFlag, + MetricsInfluxDBHostTagFlag, } func Setup(ctx *cli.Context) { if gethmetrics.Enabled { log.Info("Enabling swarm metrics collection") var ( - enableExport = ctx.GlobalBool(metricsEnableInfluxDBExportFlag.Name) - endpoint = ctx.GlobalString(metricsInfluxDBEndpointFlag.Name) - database = ctx.GlobalString(metricsInfluxDBDatabaseFlag.Name) - username = ctx.GlobalString(metricsInfluxDBUsernameFlag.Name) - password = ctx.GlobalString(metricsInfluxDBPasswordFlag.Name) - hosttag = ctx.GlobalString(metricsInfluxDBHostTagFlag.Name) + enableExport = ctx.GlobalBool(MetricsEnableInfluxDBExportFlag.Name) + endpoint = ctx.GlobalString(MetricsInfluxDBEndpointFlag.Name) + database = ctx.GlobalString(MetricsInfluxDBDatabaseFlag.Name) + username = ctx.GlobalString(MetricsInfluxDBUsernameFlag.Name) + password = ctx.GlobalString(MetricsInfluxDBPasswordFlag.Name) + hosttag = ctx.GlobalString(MetricsInfluxDBHostTagFlag.Name) ) // Start system runtime metrics collection diff --git a/swarm/storage/chunker.go b/swarm/storage/chunker.go index 40292e88f9..cbe65372a1 100644 --- a/swarm/storage/chunker.go +++ b/swarm/storage/chunker.go @@ -22,6 +22,7 @@ import ( "fmt" "io" "sync" + "time" "github.com/ethereum/go-ethereum/metrics" ch "github.com/ethereum/go-ethereum/swarm/chunk" @@ -410,10 +411,14 @@ func (r *LazyChunkReader) Size(ctx context.Context, quitC chan bool) (n int64, e log.Debug("lazychunkreader.size", "addr", r.addr) if r.chunkData == nil { + + startTime := time.Now() chunkData, err := r.getter.Get(cctx, Reference(r.addr)) if err != nil { + metrics.GetOrRegisterResettingTimer("lcr.getter.get.err", nil).UpdateSince(startTime) return 0, err } + metrics.GetOrRegisterResettingTimer("lcr.getter.get", nil).UpdateSince(startTime) r.chunkData = chunkData s := r.chunkData.Size() log.Debug("lazychunkreader.size", "key", r.addr, "size", s) @@ -542,8 +547,10 @@ func (r *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, treeS wg.Add(1) go func(j int64) { childAddress := chunkData[8+j*r.hashSize : 8+(j+1)*r.hashSize] + startTime := time.Now() chunkData, err := r.getter.Get(r.ctx, Reference(childAddress)) if err != nil { + metrics.GetOrRegisterResettingTimer("lcr.getter.get.err", nil).UpdateSince(startTime) log.Debug("lazychunkreader.join", "key", fmt.Sprintf("%x", childAddress), "err", err) select { case errC <- fmt.Errorf("chunk %v-%v not found; key: %s", off, off+treeSize, fmt.Sprintf("%x", childAddress)): @@ -551,6 +558,7 @@ func (r *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, treeS } return } + metrics.GetOrRegisterResettingTimer("lcr.getter.get", nil).UpdateSince(startTime) if l := len(chunkData); l < 9 { select { case errC <- fmt.Errorf("chunk %v-%v incomplete; key: %s, data length %v", off, off+treeSize, fmt.Sprintf("%x", childAddress), l): From 69a8d9841a62af97d98ad59f8c40e34301946b84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 11 Dec 2018 13:41:18 +0200 Subject: [PATCH 137/159] cmd/faucet: fix faucet static peer regression --- cmd/faucet/faucet.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index 2ffe12276e..a7c20db773 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -256,7 +256,7 @@ func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network u } for _, boot := range enodes { old, err := enode.ParseV4(boot.String()) - if err != nil { + if err == nil { stack.Server().AddPeer(old) } } From 38c3d88cea61361afe332437581dac8b40adbfca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 11 Dec 2018 13:41:41 +0200 Subject: [PATCH 138/159] cmd/puppeth: support latest docker compose, expose faucet UDP --- cmd/puppeth/module_ethstats.go | 3 ++- cmd/puppeth/module_explorer.go | 1 + cmd/puppeth/module_faucet.go | 4 +++- cmd/puppeth/module_nginx.go | 1 + cmd/puppeth/module_node.go | 1 + cmd/puppeth/module_wallet.go | 1 + 6 files changed, 9 insertions(+), 2 deletions(-) diff --git a/cmd/puppeth/module_ethstats.go b/cmd/puppeth/module_ethstats.go index a7d99a2979..58ecb83951 100644 --- a/cmd/puppeth/module_ethstats.go +++ b/cmd/puppeth/module_ethstats.go @@ -43,7 +43,8 @@ version: '2' services: ethstats: build: . - image: {{.Network}}/ethstats{{if not .VHost}} + image: {{.Network}}/ethstats + container_name: {{.Network}}_ethstats_1{{if not .VHost}} ports: - "{{.Port}}:3000"{{end}} environment: diff --git a/cmd/puppeth/module_explorer.go b/cmd/puppeth/module_explorer.go index e916deaf6f..e465fa04ad 100644 --- a/cmd/puppeth/module_explorer.go +++ b/cmd/puppeth/module_explorer.go @@ -77,6 +77,7 @@ services: explorer: build: . image: {{.Network}}/explorer + container_name: {{.Network}}_explorer_1 ports: - "{{.NodePort}}:{{.NodePort}}" - "{{.NodePort}}:{{.NodePort}}/udp"{{if not .VHost}} diff --git a/cmd/puppeth/module_faucet.go b/cmd/puppeth/module_faucet.go index 06c9fc0f58..3a06bf3c68 100644 --- a/cmd/puppeth/module_faucet.go +++ b/cmd/puppeth/module_faucet.go @@ -56,8 +56,10 @@ services: faucet: build: . image: {{.Network}}/faucet + container_name: {{.Network}}_faucet_1 ports: - - "{{.EthPort}}:{{.EthPort}}"{{if not .VHost}} + - "{{.EthPort}}:{{.EthPort}}" + - "{{.EthPort}}:{{.EthPort}}/udp"{{if not .VHost}} - "{{.ApiPort}}:8080"{{end}} volumes: - {{.Datadir}}:/root/.faucet diff --git a/cmd/puppeth/module_nginx.go b/cmd/puppeth/module_nginx.go index 7f87661d3b..1b1ae61ff5 100644 --- a/cmd/puppeth/module_nginx.go +++ b/cmd/puppeth/module_nginx.go @@ -40,6 +40,7 @@ services: nginx: build: . image: {{.Network}}/nginx + container_name: {{.Network}}_nginx_1 ports: - "{{.Port}}:80" volumes: diff --git a/cmd/puppeth/module_node.go b/cmd/puppeth/module_node.go index 069adfe4f4..5d9ef46523 100644 --- a/cmd/puppeth/module_node.go +++ b/cmd/puppeth/module_node.go @@ -55,6 +55,7 @@ services: {{.Type}}: build: . image: {{.Network}}/{{.Type}} + container_name: {{.Network}}_{{.Type}}_1 ports: - "{{.Port}}:{{.Port}}" - "{{.Port}}:{{.Port}}/udp" diff --git a/cmd/puppeth/module_wallet.go b/cmd/puppeth/module_wallet.go index 90812c4a0b..ebaa5b6ae1 100644 --- a/cmd/puppeth/module_wallet.go +++ b/cmd/puppeth/module_wallet.go @@ -57,6 +57,7 @@ services: wallet: build: . image: {{.Network}}/wallet + container_name: {{.Network}}_wallet_1 ports: - "{{.NodePort}}:{{.NodePort}}" - "{{.NodePort}}:{{.NodePort}}/udp" From 83a9a73b899e816a0d1493b7fb1e37abe4bc4566 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 11 Dec 2018 13:19:03 +0100 Subject: [PATCH 139/159] cmd/geth, core, eth: implement Constantinople override flag (#18273) * geth/core/eth: implement constantinople override flag * les: implemnent constantinople override flag for les clients * cmd/geth, eth, les: fix typo, move flag to experimentals --- cmd/geth/config.go | 5 ++++- cmd/geth/main.go | 1 + cmd/utils/flags.go | 6 ++++-- core/genesis.go | 6 ++++++ eth/backend.go | 2 +- eth/config.go | 4 ++++ les/backend.go | 2 +- 7 files changed, 21 insertions(+), 5 deletions(-) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index b0749d2329..59f759f0ea 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "io" + "math/big" "os" "reflect" "unicode" @@ -152,7 +153,9 @@ func enableWhisper(ctx *cli.Context) bool { func makeFullNode(ctx *cli.Context) *node.Node { stack, cfg := makeConfigNode(ctx) - + if ctx.GlobalIsSet(utils.ConstantinopleOverrideFlag.Name) { + cfg.Eth.ConstantinopleOverride = new(big.Int).SetUint64(ctx.GlobalUint64(utils.ConstantinopleOverrideFlag.Name)) + } utils.RegisterEthService(stack, &cfg.Eth) if ctx.GlobalBool(utils.DashboardEnabledFlag.Name) { diff --git a/cmd/geth/main.go b/cmd/geth/main.go index fc1bf461fb..ebaeba9f46 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -123,6 +123,7 @@ var ( utils.RinkebyFlag, utils.VMEnableDebugFlag, utils.NetworkIdFlag, + utils.ConstantinopleOverrideFlag, utils.RPCCORSDomainFlag, utils.RPCVirtualHostsFlag, utils.EthStatsURLFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index c9115d7a35..60e45d095a 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -140,6 +140,10 @@ var ( Name: "rinkeby", Usage: "Rinkeby network: pre-configured proof-of-authority test network", } + ConstantinopleOverrideFlag = cli.Uint64Flag{ + Name: "override.constantinople", + Usage: "Manually specify constantinople fork-block, overriding the bundled setting", + } DeveloperFlag = cli.BoolFlag{ Name: "dev", Usage: "Ephemeral proof-of-authority network with a pre-funded developer account, mining enabled", @@ -1178,7 +1182,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { if ctx.GlobalIsSet(NetworkIdFlag.Name) { cfg.NetworkId = ctx.GlobalUint64(NetworkIdFlag.Name) } - if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheDatabaseFlag.Name) { cfg.DatabaseCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100 } @@ -1403,7 +1406,6 @@ func MakeGenesis(ctx *cli.Context) *core.Genesis { func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chainDb ethdb.Database) { var err error chainDb = MakeChainDatabase(ctx, stack) - config, _, err := core.SetupGenesisBlock(chainDb, MakeGenesis(ctx)) if err != nil { Fatalf("%v", err) diff --git a/core/genesis.go b/core/genesis.go index 6e71afd61c..c96cb17a36 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -151,6 +151,9 @@ func (e *GenesisMismatchError) Error() string { // // The returned chain configuration is never nil. func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig, common.Hash, error) { + return SetupGenesisBlockWithOverride(db, genesis, nil) +} +func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, constantinopleOverride *big.Int) (*params.ChainConfig, common.Hash, error) { if genesis != nil && genesis.Config == nil { return params.AllEthashProtocolChanges, common.Hash{}, errGenesisNoConfig } @@ -178,6 +181,9 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig // Get the existing chain configuration. newcfg := genesis.configOrDefault(stored) + if constantinopleOverride != nil { + newcfg.ConstantinopleBlock = constantinopleOverride + } storedcfg := rawdb.ReadChainConfig(db, stored) if storedcfg == nil { log.Warn("Found genesis block without chain config") diff --git a/eth/backend.go b/eth/backend.go index 3f538d5f55..354fc17d48 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -118,7 +118,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { if err != nil { return nil, err } - chainConfig, genesisHash, genesisErr := core.SetupGenesisBlock(chainDb, config.Genesis) + chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.ConstantinopleOverride) if _, ok := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !ok { return nil, genesisErr } diff --git a/eth/config.go b/eth/config.go index a5c2095124..7c041d1af7 100644 --- a/eth/config.go +++ b/eth/config.go @@ -129,8 +129,12 @@ type Config struct { // Type of the EWASM interpreter ("" for default) EWASMInterpreter string + // Type of the EVM interpreter ("" for default) EVMInterpreter string + + // Constantinople block override (TODO: remove after the fork) + ConstantinopleOverride *big.Int } type configMarshaling struct { diff --git a/les/backend.go b/les/backend.go index a3474a6830..d0db71019e 100644 --- a/les/backend.go +++ b/les/backend.go @@ -82,7 +82,7 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { if err != nil { return nil, err } - chainConfig, genesisHash, genesisErr := core.SetupGenesisBlock(chainDb, config.Genesis) + chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.ConstantinopleOverride) if _, isCompat := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !isCompat { return nil, genesisErr } From 24d727b6d6e2c0cde222fa12155c4a6db5caaf2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 11 Dec 2018 14:20:21 +0200 Subject: [PATCH 140/159] params, swarm: release Geth v1.8.20 and Swarm v0.3.8 --- params/version.go | 8 ++++---- swarm/version/version.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/params/version.go b/params/version.go index 2291d5f3c5..ba9ab202a7 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 20 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 8 // Minor version component of the current release + VersionPatch = 20 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. diff --git a/swarm/version/version.go b/swarm/version/version.go index 57ac05a864..c348109014 100644 --- a/swarm/version/version.go +++ b/swarm/version/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 0 // Major version component of the current release - VersionMinor = 3 // Minor version component of the current release - VersionPatch = 8 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 0 // Major version component of the current release + VersionMinor = 3 // Minor version component of the current release + VersionPatch = 8 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From 1e190a3b1cf0444b48cc1aa78367e0d28a561c6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 11 Dec 2018 14:23:57 +0200 Subject: [PATCH 141/159] params, swarm: begin Geth v1.9.0 family, Swarm v0.3.9 cycle --- params/version.go | 8 ++++---- swarm/version/version.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/params/version.go b/params/version.go index ba9ab202a7..d3954e0bc3 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 20 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 9 // Minor version component of the current release + VersionPatch = 0 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. diff --git a/swarm/version/version.go b/swarm/version/version.go index c348109014..8301dfd2d4 100644 --- a/swarm/version/version.go +++ b/swarm/version/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 0 // Major version component of the current release - VersionMinor = 3 // Minor version component of the current release - VersionPatch = 8 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 0 // Major version component of the current release + VersionMinor = 3 // Minor version component of the current release + VersionPatch = 9 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From ebbf3dfafb9af891294fa10c04b3c07dd56d8fdc Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 12 Dec 2018 07:51:29 +0100 Subject: [PATCH 142/159] swarm/shed: add metrics to each shed db (#18277) * swarm/shed: add metrics to each shed db * swarm/shed: push metrics prefix up * swarm/shed: rename prefix to metricsPrefix * swarm/shed: unexport Meter, remove Mutex for quit channel --- swarm/shed/db.go | 205 ++++++++++++++++++++++++++++++- swarm/shed/db_test.go | 6 +- swarm/shed/example_store_test.go | 2 +- 3 files changed, 206 insertions(+), 7 deletions(-) diff --git a/swarm/shed/db.go b/swarm/shed/db.go index e128b8cbc8..7377e12d2d 100644 --- a/swarm/shed/db.go +++ b/swarm/shed/db.go @@ -23,14 +23,23 @@ package shed import ( + "errors" + "fmt" + "strconv" + "strings" + "time" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/swarm/log" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/iterator" "github.com/syndtr/goleveldb/leveldb/opt" ) -// The limit for LevelDB OpenFilesCacheCapacity. -const openFileLimit = 128 +const ( + openFileLimit = 128 // The limit for LevelDB OpenFilesCacheCapacity. + writePauseWarningThrottler = 1 * time.Minute +) // DB provides abstractions over LevelDB in order to // implement complex structures using fields and ordered indexes. @@ -38,11 +47,22 @@ const openFileLimit = 128 // information about naming and types. type DB struct { ldb *leveldb.DB + + compTimeMeter metrics.Meter // Meter for measuring the total time spent in database compaction + compReadMeter metrics.Meter // Meter for measuring the data read during compaction + compWriteMeter metrics.Meter // Meter for measuring the data written during compaction + writeDelayNMeter metrics.Meter // Meter for measuring the write delay number due to database compaction + writeDelayMeter metrics.Meter // Meter for measuring the write delay duration due to database compaction + diskReadMeter metrics.Meter // Meter for measuring the effective amount of data read + diskWriteMeter metrics.Meter // Meter for measuring the effective amount of data written + + quitChan chan chan error // Quit channel to stop the metrics collection before closing the database } // NewDB constructs a new DB and validates the schema // if it exists in database on the given path. -func NewDB(path string) (db *DB, err error) { +// metricsPrefix is used for metrics collection for the given DB. +func NewDB(path string, metricsPrefix string) (db *DB, err error) { ldb, err := leveldb.OpenFile(path, &opt.Options{ OpenFilesCacheCapacity: openFileLimit, }) @@ -66,6 +86,15 @@ func NewDB(path string) (db *DB, err error) { return nil, err } } + + // Configure meters for DB + db.configure(metricsPrefix) + + // Create a quit channel for the periodic metrics collector and run it + db.quitChan = make(chan chan error) + + go db.meter(10 * time.Second) + return db, nil } @@ -126,5 +155,175 @@ func (db *DB) WriteBatch(batch *leveldb.Batch) (err error) { // Close closes LevelDB database. func (db *DB) Close() (err error) { + close(db.quitChan) return db.ldb.Close() } + +// Configure configures the database metrics collectors +func (db *DB) configure(prefix string) { + // Initialize all the metrics collector at the requested prefix + db.compTimeMeter = metrics.NewRegisteredMeter(prefix+"compact/time", nil) + db.compReadMeter = metrics.NewRegisteredMeter(prefix+"compact/input", nil) + db.compWriteMeter = metrics.NewRegisteredMeter(prefix+"compact/output", nil) + db.diskReadMeter = metrics.NewRegisteredMeter(prefix+"disk/read", nil) + db.diskWriteMeter = metrics.NewRegisteredMeter(prefix+"disk/write", nil) + db.writeDelayMeter = metrics.NewRegisteredMeter(prefix+"compact/writedelay/duration", nil) + db.writeDelayNMeter = metrics.NewRegisteredMeter(prefix+"compact/writedelay/counter", nil) +} + +func (db *DB) meter(refresh time.Duration) { + // Create the counters to store current and previous compaction values + compactions := make([][]float64, 2) + for i := 0; i < 2; i++ { + compactions[i] = make([]float64, 3) + } + // Create storage for iostats. + var iostats [2]float64 + + // Create storage and warning log tracer for write delay. + var ( + delaystats [2]int64 + lastWritePaused time.Time + ) + + var ( + errc chan error + merr error + ) + + // Iterate ad infinitum and collect the stats + for i := 1; errc == nil && merr == nil; i++ { + // Retrieve the database stats + stats, err := db.ldb.GetProperty("leveldb.stats") + if err != nil { + log.Error("Failed to read database stats", "err", err) + merr = err + continue + } + // Find the compaction table, skip the header + lines := strings.Split(stats, "\n") + for len(lines) > 0 && strings.TrimSpace(lines[0]) != "Compactions" { + lines = lines[1:] + } + if len(lines) <= 3 { + log.Error("Compaction table not found") + merr = errors.New("compaction table not found") + continue + } + lines = lines[3:] + + // Iterate over all the table rows, and accumulate the entries + for j := 0; j < len(compactions[i%2]); j++ { + compactions[i%2][j] = 0 + } + for _, line := range lines { + parts := strings.Split(line, "|") + if len(parts) != 6 { + break + } + for idx, counter := range parts[3:] { + value, err := strconv.ParseFloat(strings.TrimSpace(counter), 64) + if err != nil { + log.Error("Compaction entry parsing failed", "err", err) + merr = err + continue + } + compactions[i%2][idx] += value + } + } + // Update all the requested meters + if db.compTimeMeter != nil { + db.compTimeMeter.Mark(int64((compactions[i%2][0] - compactions[(i-1)%2][0]) * 1000 * 1000 * 1000)) + } + if db.compReadMeter != nil { + db.compReadMeter.Mark(int64((compactions[i%2][1] - compactions[(i-1)%2][1]) * 1024 * 1024)) + } + if db.compWriteMeter != nil { + db.compWriteMeter.Mark(int64((compactions[i%2][2] - compactions[(i-1)%2][2]) * 1024 * 1024)) + } + + // Retrieve the write delay statistic + writedelay, err := db.ldb.GetProperty("leveldb.writedelay") + if err != nil { + log.Error("Failed to read database write delay statistic", "err", err) + merr = err + continue + } + var ( + delayN int64 + delayDuration string + duration time.Duration + paused bool + ) + if n, err := fmt.Sscanf(writedelay, "DelayN:%d Delay:%s Paused:%t", &delayN, &delayDuration, &paused); n != 3 || err != nil { + log.Error("Write delay statistic not found") + merr = err + continue + } + duration, err = time.ParseDuration(delayDuration) + if err != nil { + log.Error("Failed to parse delay duration", "err", err) + merr = err + continue + } + if db.writeDelayNMeter != nil { + db.writeDelayNMeter.Mark(delayN - delaystats[0]) + } + if db.writeDelayMeter != nil { + db.writeDelayMeter.Mark(duration.Nanoseconds() - delaystats[1]) + } + // If a warning that db is performing compaction has been displayed, any subsequent + // warnings will be withheld for one minute not to overwhelm the user. + if paused && delayN-delaystats[0] == 0 && duration.Nanoseconds()-delaystats[1] == 0 && + time.Now().After(lastWritePaused.Add(writePauseWarningThrottler)) { + log.Warn("Database compacting, degraded performance") + lastWritePaused = time.Now() + } + delaystats[0], delaystats[1] = delayN, duration.Nanoseconds() + + // Retrieve the database iostats. + ioStats, err := db.ldb.GetProperty("leveldb.iostats") + if err != nil { + log.Error("Failed to read database iostats", "err", err) + merr = err + continue + } + var nRead, nWrite float64 + parts := strings.Split(ioStats, " ") + if len(parts) < 2 { + log.Error("Bad syntax of ioStats", "ioStats", ioStats) + merr = fmt.Errorf("bad syntax of ioStats %s", ioStats) + continue + } + if n, err := fmt.Sscanf(parts[0], "Read(MB):%f", &nRead); n != 1 || err != nil { + log.Error("Bad syntax of read entry", "entry", parts[0]) + merr = err + continue + } + if n, err := fmt.Sscanf(parts[1], "Write(MB):%f", &nWrite); n != 1 || err != nil { + log.Error("Bad syntax of write entry", "entry", parts[1]) + merr = err + continue + } + if db.diskReadMeter != nil { + db.diskReadMeter.Mark(int64((nRead - iostats[0]) * 1024 * 1024)) + } + if db.diskWriteMeter != nil { + db.diskWriteMeter.Mark(int64((nWrite - iostats[1]) * 1024 * 1024)) + } + iostats[0], iostats[1] = nRead, nWrite + + // Sleep a bit, then repeat the stats collection + select { + case errc = <-db.quitChan: + // Quit requesting, stop hammering the database + case <-time.After(refresh): + // Timeout, gather a new set of stats + } + } + + if errc == nil { + errc = <-db.quitChan + } + errc <- merr +} diff --git a/swarm/shed/db_test.go b/swarm/shed/db_test.go index 45325beeb8..65fdac4a61 100644 --- a/swarm/shed/db_test.go +++ b/swarm/shed/db_test.go @@ -55,7 +55,7 @@ func TestDB_persistence(t *testing.T) { } defer os.RemoveAll(dir) - db, err := NewDB(dir) + db, err := NewDB(dir, "") if err != nil { t.Fatal(err) } @@ -73,7 +73,7 @@ func TestDB_persistence(t *testing.T) { t.Fatal(err) } - db2, err := NewDB(dir) + db2, err := NewDB(dir, "") if err != nil { t.Fatal(err) } @@ -101,7 +101,7 @@ func newTestDB(t *testing.T) (db *DB, cleanupFunc func()) { t.Fatal(err) } cleanupFunc = func() { os.RemoveAll(dir) } - db, err = NewDB(dir) + db, err = NewDB(dir, "") if err != nil { cleanupFunc() t.Fatal(err) diff --git a/swarm/shed/example_store_test.go b/swarm/shed/example_store_test.go index 2ed0be1413..908a1e446d 100644 --- a/swarm/shed/example_store_test.go +++ b/swarm/shed/example_store_test.go @@ -52,7 +52,7 @@ type Store struct { // and possible conflicts with schema from existing database is checked // automatically. func New(path string) (s *Store, err error) { - db, err := shed.NewDB(path) + db, err := shed.NewDB(path, "") if err != nil { return nil, err } From 4e6f53ac3360c4e90fdb419e7073e4316b2a49f7 Mon Sep 17 00:00:00 2001 From: Javier Peletier Date: Wed, 12 Dec 2018 16:22:17 +0100 Subject: [PATCH 143/159] swarm/storage: simplify ChunkValidator interface (#18285) --- swarm/storage/feed/handler.go | 11 +++++------ swarm/storage/feed/handler_test.go | 4 ++-- swarm/storage/feed/request.go | 6 ++++-- swarm/storage/feed/request_test.go | 6 +++--- swarm/storage/localstore.go | 2 +- swarm/storage/localstore_test.go | 2 +- swarm/storage/types.go | 7 ++++--- 7 files changed, 20 insertions(+), 18 deletions(-) diff --git a/swarm/storage/feed/handler.go b/swarm/storage/feed/handler.go index 9e2640282c..33542b6e43 100644 --- a/swarm/storage/feed/handler.go +++ b/swarm/storage/feed/handler.go @@ -82,9 +82,8 @@ func (h *Handler) SetStore(store *storage.NetStore) { // Validate is a chunk validation method // If it looks like a feed update, the chunk address is checked against the userAddr of the update's signature // It implements the storage.ChunkValidator interface -func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool { - dataLength := len(data) - if dataLength < minimumSignedUpdateLength { +func (h *Handler) Validate(chunk storage.Chunk) bool { + if len(chunk.Data()) < minimumSignedUpdateLength { return false } @@ -94,8 +93,8 @@ func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool { // First, deserialize the chunk var r Request - if err := r.fromChunk(chunkAddr, data); err != nil { - log.Debug("Invalid feed update chunk", "addr", chunkAddr.Hex(), "err", err.Error()) + if err := r.fromChunk(chunk); err != nil { + log.Debug("Invalid feed update chunk", "addr", chunk.Address(), "err", err) return false } @@ -198,7 +197,7 @@ func (h *Handler) Lookup(ctx context.Context, query *Query) (*cacheEntry, error) } var request Request - if err := request.fromChunk(chunk.Address(), chunk.Data()); err != nil { + if err := request.fromChunk(chunk); err != nil { return nil, nil } if request.Time <= timeLimit { diff --git a/swarm/storage/feed/handler_test.go b/swarm/storage/feed/handler_test.go index fb2ef3a6ba..9c39215c56 100644 --- a/swarm/storage/feed/handler_test.go +++ b/swarm/storage/feed/handler_test.go @@ -366,7 +366,7 @@ func TestValidator(t *testing.T) { if err != nil { t.Fatal(err) } - if !rh.Validate(chunk.Address(), chunk.Data()) { + if !rh.Validate(chunk) { t.Fatal("Chunk validator fail on update chunk") } @@ -375,7 +375,7 @@ func TestValidator(t *testing.T) { address[0] = 11 address[15] = 99 - if rh.Validate(address, chunk.Data()) { + if rh.Validate(storage.NewChunk(address, chunk.Data())) { t.Fatal("Expected Validate to fail with false chunk address") } } diff --git a/swarm/storage/feed/request.go b/swarm/storage/feed/request.go index 6968d8b9ad..dd91a7cf45 100644 --- a/swarm/storage/feed/request.go +++ b/swarm/storage/feed/request.go @@ -171,9 +171,11 @@ func (r *Request) toChunk() (storage.Chunk, error) { } // fromChunk populates this structure from chunk data. It does not verify the signature is valid. -func (r *Request) fromChunk(updateAddr storage.Address, chunkdata []byte) error { +func (r *Request) fromChunk(chunk storage.Chunk) error { // for update chunk layout see Request definition + chunkdata := chunk.Data() + //deserialize the feed update portion if err := r.Update.binaryGet(chunkdata[:len(chunkdata)-signatureLength]); err != nil { return err @@ -189,7 +191,7 @@ func (r *Request) fromChunk(updateAddr storage.Address, chunkdata []byte) error } r.Signature = signature - r.idAddr = updateAddr + r.idAddr = chunk.Address() r.binaryData = chunkdata return nil diff --git a/swarm/storage/feed/request_test.go b/swarm/storage/feed/request_test.go index f5de32b74f..c30158fddf 100644 --- a/swarm/storage/feed/request_test.go +++ b/swarm/storage/feed/request_test.go @@ -197,7 +197,7 @@ func TestUpdateChunkSerializationErrorChecking(t *testing.T) { // Test that parseUpdate fails if the chunk is too small var r Request - if err := r.fromChunk(storage.ZeroAddr, make([]byte, minimumUpdateDataLength-1+signatureLength)); err == nil { + if err := r.fromChunk(storage.NewChunk(storage.ZeroAddr, make([]byte, minimumUpdateDataLength-1+signatureLength))); err == nil { t.Fatalf("Expected request.fromChunk to fail when chunkData contains less than %d bytes", minimumUpdateDataLength) } @@ -226,7 +226,7 @@ func TestUpdateChunkSerializationErrorChecking(t *testing.T) { compareByteSliceToExpectedHex(t, "chunk", chunk.Data(), "0x0000000000000000776f726c64206e657773207265706f72742c20657665727920686f7572000000876a8936a7cd0b79ef0735ad0896c1afe278781ce803000000000019416c206269656e206861636572206a616dc3a173206c652066616c7461207072656d696f5a0ffe0bc27f207cd5b00944c8b9cee93e08b89b5ada777f123ac535189333f174a6a4ca2f43a92c4a477a49d774813c36ce8288552c58e6205b0ac35d0507eb00") var recovered Request - recovered.fromChunk(chunk.Address(), chunk.Data()) + recovered.fromChunk(chunk) if !reflect.DeepEqual(recovered, r) { t.Fatal("Expected recovered feed update request to equal the original one") } @@ -282,7 +282,7 @@ func TestReverse(t *testing.T) { // check that we can recover the owner account from the update chunk's signature var checkUpdate Request - if err := checkUpdate.fromChunk(chunk.Address(), chunk.Data()); err != nil { + if err := checkUpdate.fromChunk(chunk); err != nil { t.Fatal(err) } checkdigest, err := checkUpdate.GetDigest() diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index 111821ff65..956560902b 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -92,7 +92,7 @@ func (ls *LocalStore) isValid(chunk Chunk) bool { // ls.Validators contains a list of one validator per chunk type. // if one validator succeeds, then the chunk is valid for _, v := range ls.Validators { - if valid = v.Validate(chunk.Address(), chunk.Data()); valid { + if valid = v.Validate(chunk); valid { break } } diff --git a/swarm/storage/localstore_test.go b/swarm/storage/localstore_test.go index 7a07726d1c..7a4162a475 100644 --- a/swarm/storage/localstore_test.go +++ b/swarm/storage/localstore_test.go @@ -118,7 +118,7 @@ func TestValidator(t *testing.T) { type boolTestValidator bool -func (self boolTestValidator) Validate(addr Address, data []byte) bool { +func (self boolTestValidator) Validate(chunk Chunk) bool { return bool(self) } diff --git a/swarm/storage/types.go b/swarm/storage/types.go index 42557766ef..e5c5f89a09 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -327,7 +327,7 @@ func (c ChunkData) Data() []byte { } type ChunkValidator interface { - Validate(addr Address, data []byte) bool + Validate(chunk Chunk) bool } // Provides method for validation of content address in chunks @@ -344,7 +344,8 @@ func NewContentAddressValidator(hasher SwarmHasher) *ContentAddressValidator { } // Validate that the given key is a valid content address for the given data -func (v *ContentAddressValidator) Validate(addr Address, data []byte) bool { +func (v *ContentAddressValidator) Validate(chunk Chunk) bool { + data := chunk.Data() if l := len(data); l < 9 || l > ch.DefaultSize+8 { // log.Error("invalid chunk size", "chunk", addr.Hex(), "size", l) return false @@ -355,7 +356,7 @@ func (v *ContentAddressValidator) Validate(addr Address, data []byte) bool { hasher.Write(data[8:]) hash := hasher.Sum(nil) - return bytes.Equal(hash, addr[:]) + return bytes.Equal(hash, chunk.Address()) } type ChunkStore interface { From b3be9b7cd8f5f0e828d4d2aa6db1216ccd32ce3a Mon Sep 17 00:00:00 2001 From: Corey Lin <514971757@qq.com> Date: Thu, 13 Dec 2018 17:21:52 +0800 Subject: [PATCH 144/159] usbwallet: check returned error when decoding hexstr (#18056) * usbwallet: check returned error when decoding hexstr * Update accounts/usbwallet/ledger.go Co-Authored-By: CoreyLin <514971757@qq.com> * usbwallet: check hex decode error --- accounts/usbwallet/ledger.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/accounts/usbwallet/ledger.go b/accounts/usbwallet/ledger.go index 7d5f67908d..c30903b5b7 100644 --- a/accounts/usbwallet/ledger.go +++ b/accounts/usbwallet/ledger.go @@ -257,7 +257,9 @@ func (w *ledgerDriver) ledgerDerive(derivationPath []uint32) (common.Address, er // Decode the hex sting into an Ethereum address and return var address common.Address - hex.Decode(address[:], hexstr) + if _, err = hex.Decode(address[:], hexstr); err != nil { + return common.Address{}, err + } return address, nil } From e57e4571d3382c504a01f87be654d984f26e2dd1 Mon Sep 17 00:00:00 2001 From: Shuai Qi Date: Thu, 13 Dec 2018 17:25:13 +0800 Subject: [PATCH 145/159] crypto/secp256k1: Fix invalid document link (#18297) --- crypto/secp256k1/curve.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/secp256k1/curve.go b/crypto/secp256k1/curve.go index 56be235b36..5409ee1d2c 100644 --- a/crypto/secp256k1/curve.go +++ b/crypto/secp256k1/curve.go @@ -310,7 +310,7 @@ var theCurve = new(BitCurve) func init() { // See SEC 2 section 2.7.1 // curve parameters taken from: - // http://www.secg.org/collateral/sec2_final.pdf + // http://www.secg.org/sec2-v2.pdf theCurve.P, _ = new(big.Int).SetString("0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F", 0) theCurve.N, _ = new(big.Int).SetString("0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141", 0) theCurve.B, _ = new(big.Int).SetString("0x0000000000000000000000000000000000000000000000000000000000000007", 0) From e79821cabeda5c6907f03073d11fe6d83367f494 Mon Sep 17 00:00:00 2001 From: tzapu Date: Thu, 13 Dec 2018 16:12:19 +0200 Subject: [PATCH 146/159] accounts/abi: argument type and name were reversed (#17947) argument type and name were reversed --- accounts/abi/event.go | 6 ++-- accounts/abi/event_test.go | 41 ++++++++++++++++++++++--- accounts/abi/method.go | 6 ++-- accounts/abi/method_test.go | 61 +++++++++++++++++++++++++++++++++++++ 4 files changed, 104 insertions(+), 10 deletions(-) create mode 100644 accounts/abi/method_test.go diff --git a/accounts/abi/event.go b/accounts/abi/event.go index a3f6be9732..9392c1990c 100644 --- a/accounts/abi/event.go +++ b/accounts/abi/event.go @@ -36,12 +36,12 @@ type Event struct { func (e Event) String() string { inputs := make([]string, len(e.Inputs)) for i, input := range e.Inputs { - inputs[i] = fmt.Sprintf("%v %v", input.Name, input.Type) + inputs[i] = fmt.Sprintf("%v %v", input.Type, input.Name) if input.Indexed { - inputs[i] = fmt.Sprintf("%v indexed %v", input.Name, input.Type) + inputs[i] = fmt.Sprintf("%v indexed %v", input.Type, input.Name) } } - return fmt.Sprintf("e %v(%v)", e.Name, strings.Join(inputs, ", ")) + return fmt.Sprintf("event %v(%v)", e.Name, strings.Join(inputs, ", ")) } // Id returns the canonical representation of the event's signature used by the diff --git a/accounts/abi/event_test.go b/accounts/abi/event_test.go index 3bfdd7c0ab..e735cceb88 100644 --- a/accounts/abi/event_test.go +++ b/accounts/abi/event_test.go @@ -87,12 +87,12 @@ func TestEventId(t *testing.T) { }{ { definition: `[ - { "type" : "event", "name" : "balance", "inputs": [{ "name" : "in", "type": "uint256" }] }, - { "type" : "event", "name" : "check", "inputs": [{ "name" : "t", "type": "address" }, { "name": "b", "type": "uint256" }] } + { "type" : "event", "name" : "Balance", "inputs": [{ "name" : "in", "type": "uint256" }] }, + { "type" : "event", "name" : "Check", "inputs": [{ "name" : "t", "type": "address" }, { "name": "b", "type": "uint256" }] } ]`, expectations: map[string]common.Hash{ - "balance": crypto.Keccak256Hash([]byte("balance(uint256)")), - "check": crypto.Keccak256Hash([]byte("check(address,uint256)")), + "Balance": crypto.Keccak256Hash([]byte("Balance(uint256)")), + "Check": crypto.Keccak256Hash([]byte("Check(address,uint256)")), }, }, } @@ -111,6 +111,39 @@ func TestEventId(t *testing.T) { } } +func TestEventString(t *testing.T) { + var table = []struct { + definition string + expectations map[string]string + }{ + { + definition: `[ + { "type" : "event", "name" : "Balance", "inputs": [{ "name" : "in", "type": "uint256" }] }, + { "type" : "event", "name" : "Check", "inputs": [{ "name" : "t", "type": "address" }, { "name": "b", "type": "uint256" }] }, + { "type" : "event", "name" : "Transfer", "inputs": [{ "name": "from", "type": "address", "indexed": true }, { "name": "to", "type": "address", "indexed": true }, { "name": "value", "type": "uint256" }] } + ]`, + expectations: map[string]string{ + "Balance": "event Balance(uint256 in)", + "Check": "event Check(address t, uint256 b)", + "Transfer": "event Transfer(address indexed from, address indexed to, uint256 value)", + }, + }, + } + + for _, test := range table { + abi, err := JSON(strings.NewReader(test.definition)) + if err != nil { + t.Fatal(err) + } + + for name, event := range abi.Events { + if event.String() != test.expectations[name] { + t.Errorf("expected string to be %s, got %s", test.expectations[name], event.String()) + } + } + } +} + // TestEventMultiValueWithArrayUnpack verifies that array fields will be counted after parsing array. func TestEventMultiValueWithArrayUnpack(t *testing.T) { definition := `[{"name": "test", "type": "event", "inputs": [{"indexed": false, "name":"value1", "type":"uint8[2]"},{"indexed": false, "name":"value2", "type":"uint8"}]}]` diff --git a/accounts/abi/method.go b/accounts/abi/method.go index 5831057652..2d8d3d6589 100644 --- a/accounts/abi/method.go +++ b/accounts/abi/method.go @@ -56,14 +56,14 @@ func (method Method) Sig() string { func (method Method) String() string { inputs := make([]string, len(method.Inputs)) for i, input := range method.Inputs { - inputs[i] = fmt.Sprintf("%v %v", input.Name, input.Type) + inputs[i] = fmt.Sprintf("%v %v", input.Type, input.Name) } outputs := make([]string, len(method.Outputs)) for i, output := range method.Outputs { + outputs[i] = output.Type.String() if len(output.Name) > 0 { - outputs[i] = fmt.Sprintf("%v ", output.Name) + outputs[i] += fmt.Sprintf(" %v", output.Name) } - outputs[i] += output.Type.String() } constant := "" if method.Const { diff --git a/accounts/abi/method_test.go b/accounts/abi/method_test.go new file mode 100644 index 0000000000..a98f1cd31f --- /dev/null +++ b/accounts/abi/method_test.go @@ -0,0 +1,61 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package abi + +import ( + "strings" + "testing" +) + +const methoddata = ` +[ + { "type" : "function", "name" : "balance", "constant" : true }, + { "type" : "function", "name" : "send", "constant" : false, "inputs" : [ { "name" : "amount", "type" : "uint256" } ] }, + { "type" : "function", "name" : "transfer", "constant" : false, "inputs" : [ { "name" : "from", "type" : "address" }, { "name" : "to", "type" : "address" }, { "name" : "value", "type" : "uint256" } ], "outputs" : [ { "name" : "success", "type" : "bool" } ] } +]` + +func TestMethodString(t *testing.T) { + var table = []struct { + method string + expectation string + }{ + { + method: "balance", + expectation: "function balance() constant returns()", + }, + { + method: "send", + expectation: "function send(uint256 amount) returns()", + }, + { + method: "transfer", + expectation: "function transfer(address from, address to, uint256 value) returns(bool success)", + }, + } + + abi, err := JSON(strings.NewReader(methoddata)) + if err != nil { + t.Fatal(err) + } + + for _, test := range table { + got := abi.Methods[test.method].String() + if got != test.expectation { + t.Errorf("expected string to be %s, got %s", test.expectation, got) + } + } +} From 3796751efcc3fab71799450f1bff370d081438ec Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 14 Dec 2018 11:08:11 +0100 Subject: [PATCH 147/159] rpc: add application/json-rpc as accepted content type, fixes #18293 (#18310) --- rpc/http.go | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/rpc/http.go b/rpc/http.go index af79858e2b..674166fb3a 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -36,11 +36,15 @@ import ( ) const ( - contentType = "application/json" maxRequestContentLength = 1024 * 512 ) -var nullAddr, _ = net.ResolveTCPAddr("tcp", "127.0.0.1:0") +var ( + // https://www.jsonrpc.org/historical/json-rpc-over-http.html#id13 + acceptedContentTypes = []string{"application/json", "application/json-rpc", "application/jsonrequest"} + contentType = acceptedContentTypes[0] + nullAddr, _ = net.ResolveTCPAddr("tcp", "127.0.0.1:0") +) type httpConn struct { client *http.Client @@ -263,12 +267,21 @@ func validateRequest(r *http.Request) (int, error) { err := fmt.Errorf("content length too large (%d>%d)", r.ContentLength, maxRequestContentLength) return http.StatusRequestEntityTooLarge, err } - mt, _, err := mime.ParseMediaType(r.Header.Get("content-type")) - if r.Method != http.MethodOptions && (err != nil || mt != contentType) { - err := fmt.Errorf("invalid content type, only %s is supported", contentType) - return http.StatusUnsupportedMediaType, err + // Allow OPTIONS (regardless of content-type) + if r.Method == http.MethodOptions { + return 0, nil } - return 0, nil + // Check content-type + if mt, _, err := mime.ParseMediaType(r.Header.Get("content-type")); err == nil { + for _, accepted := range acceptedContentTypes { + if accepted == mt { + return 0, nil + } + } + } + // Invalid content-type + err := fmt.Errorf("invalid content type, only %s is supported", contentType) + return http.StatusUnsupportedMediaType, err } func newCorsHandler(srv *Server, allowedOrigins []string) http.Handler { From 9940d93a43bab1f060e8fbfadff510c3298f2726 Mon Sep 17 00:00:00 2001 From: qd-ethan <31876119+qdgogogo@users.noreply.github.com> Date: Fri, 14 Dec 2018 18:15:31 +0800 Subject: [PATCH 148/159] Comment error (#18303) --- core/tx_cacher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/tx_cacher.go b/core/tx_cacher.go index bcaa5ead38..b1e5d676a2 100644 --- a/core/tx_cacher.go +++ b/core/tx_cacher.go @@ -22,7 +22,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" ) -// senderCacher is a concurrent transaction sender recoverer anc cacher. +// senderCacher is a concurrent transaction sender recoverer and cacher. var senderCacher = newTxSenderCacher(runtime.NumCPU()) // txSenderCacherRequest is a request for recovering transaction senders with a From 88d7d4fed4f95df9b81fa6acc66c760e0cb08cd7 Mon Sep 17 00:00:00 2001 From: Adam Schmideg Date: Fri, 14 Dec 2018 14:50:10 +0100 Subject: [PATCH 149/159] Change issue labels in bot configs to the new prefixed version --- .github/no-response.yml | 2 +- .github/stale.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/no-response.yml b/.github/no-response.yml index b6e96efdc6..903d4ce85f 100644 --- a/.github/no-response.yml +++ b/.github/no-response.yml @@ -1,7 +1,7 @@ # Number of days of inactivity before an Issue is closed for lack of response daysUntilClose: 30 # Label requiring a response -responseRequiredLabel: more-information-needed +responseRequiredLabel: "need:more-information" # Comment to post when closing an Issue for lack of response. Set to `false` to disable closeComment: > This issue has been automatically closed because there has been no response diff --git a/.github/stale.yml b/.github/stale.yml index c621939c3f..6d921cc795 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -7,7 +7,7 @@ exemptLabels: - pinned - security # Label to use when marking an issue as stale -staleLabel: stale +staleLabel: "status:inactive" # Comment to post when marking an issue as stale. Set to `false` to disable markComment: > This issue has been automatically marked as stale because it has not had From fe26b2f366693bfd2cff71e81fa5512977affd7b Mon Sep 17 00:00:00 2001 From: Shuai Qi Date: Fri, 14 Dec 2018 21:55:03 +0800 Subject: [PATCH 150/159] core/state: rename 'new' variable (#18301) --- core/state/statedb.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/state/statedb.go b/core/state/statedb.go index 76e67d839d..2230b10ef0 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -468,9 +468,9 @@ func (self *StateDB) createObject(addr common.Address) (newobj, prev *stateObjec // // Carrying over the balance ensures that Ether doesn't disappear. func (self *StateDB) CreateAccount(addr common.Address) { - new, prev := self.createObject(addr) + newObj, prev := self.createObject(addr) if prev != nil { - new.setBalance(prev.data.Balance) + newObj.setBalance(prev.data.Balance) } } From aad3c67a92cd4f3cc3a885fdc514ba2a7fb3e0a3 Mon Sep 17 00:00:00 2001 From: yahtoo Date: Fri, 14 Dec 2018 21:55:51 +0800 Subject: [PATCH 151/159] p2p/discv5: don't hash findnode target in lookup against table (#18309) --- p2p/discv5/net.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/discv5/net.go b/p2p/discv5/net.go index cdeb28dd5b..55e596755b 100644 --- a/p2p/discv5/net.go +++ b/p2p/discv5/net.go @@ -800,7 +800,7 @@ func (n *nodeNetGuts) startNextQuery(net *Network) { func (q *findnodeQuery) start(net *Network) bool { // Satisfy queries against the local node directly. if q.remote == net.tab.self { - closest := net.tab.closest(crypto.Keccak256Hash(q.target[:]), bucketSize) + closest := net.tab.closest(q.target, bucketSize) q.reply <- closest.entries return true } From aebf9e2fe7beb6d23db94d0885cc7a1089b15d87 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Fri, 14 Dec 2018 15:05:22 +0100 Subject: [PATCH 152/159] .github: add @gballet as abi codeowner (#18306) --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9a61d39325..11c4dcedc2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,6 +2,7 @@ # Each line is a file pattern followed by one or more owners. accounts/usbwallet @karalabe +accounts/abi @gballet consensus @karalabe core/ @karalabe @holiman eth/ @karalabe From 3ad73443c7a96c928577234df5c742d53f3293ef Mon Sep 17 00:00:00 2001 From: weimumu <934657014@qq.com> Date: Mon, 17 Dec 2018 16:50:52 +0800 Subject: [PATCH 153/159] fix slice unpack bug in accounts/abi (#18321) * fix slice unpack bug in accounts/abi --- accounts/abi/unpack.go | 2 +- accounts/abi/unpack_test.go | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/accounts/abi/unpack.go b/accounts/abi/unpack.go index d5875140cc..c3bbbb251a 100644 --- a/accounts/abi/unpack.go +++ b/accounts/abi/unpack.go @@ -151,7 +151,7 @@ func forEachUnpack(t Type, output []byte, start, size int) (interface{}, error) // Arrays have packed elements, resulting in longer unpack steps. // Slices have just 32 bytes per element (pointing to the contents). elemSize := 32 - if t.T == ArrayTy { + if t.T == ArrayTy || t.T == SliceTy { elemSize = getFullElemSize(t.Elem) } diff --git a/accounts/abi/unpack_test.go b/accounts/abi/unpack_test.go index 97552b90cf..29ed4522cf 100644 --- a/accounts/abi/unpack_test.go +++ b/accounts/abi/unpack_test.go @@ -191,6 +191,11 @@ var unpackTests = []unpackTest{ enc: "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002", want: [][2]uint8{{1, 2}}, }, + { + def: `[{"type": "uint8[2][]"}]`, + enc: "000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002", + want: [][2]uint8{{1, 2}, {1, 2}}, + }, { def: `[{"type": "uint16[]"}]`, enc: "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002", From d322c9d5504ad3fd4dfeb9dff29e92b35856042f Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Mon, 17 Dec 2018 11:32:55 +0100 Subject: [PATCH 154/159] swarm/storage/feed: remove unused code (#18324) --- swarm/storage/feed/handler.go | 11 ++++------- swarm/storage/feed/handler_test.go | 1 - swarm/storage/feed/timestampprovider.go | 22 ---------------------- 3 files changed, 4 insertions(+), 30 deletions(-) diff --git a/swarm/storage/feed/handler.go b/swarm/storage/feed/handler.go index 33542b6e43..063d3e92a3 100644 --- a/swarm/storage/feed/handler.go +++ b/swarm/storage/feed/handler.go @@ -23,7 +23,6 @@ import ( "context" "fmt" "sync" - "time" "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" @@ -32,12 +31,10 @@ import ( ) type Handler struct { - chunkStore *storage.NetStore - HashSize int - cache map[uint64]*cacheEntry - cacheLock sync.RWMutex - storeTimeout time.Duration - queryMaxPeriods uint32 + chunkStore *storage.NetStore + HashSize int + cache map[uint64]*cacheEntry + cacheLock sync.RWMutex } // HandlerParams pass parameters to the Handler constructor NewHandler diff --git a/swarm/storage/feed/handler_test.go b/swarm/storage/feed/handler_test.go index 9c39215c56..2f8a524535 100644 --- a/swarm/storage/feed/handler_test.go +++ b/swarm/storage/feed/handler_test.go @@ -40,7 +40,6 @@ var ( } cleanF func() subtopicName = "føø.bar" - hashfunc = storage.MakeHashFunc(storage.DefaultHash) ) func init() { diff --git a/swarm/storage/feed/timestampprovider.go b/swarm/storage/feed/timestampprovider.go index 072dc3a48d..fb60cea9c3 100644 --- a/swarm/storage/feed/timestampprovider.go +++ b/swarm/storage/feed/timestampprovider.go @@ -17,7 +17,6 @@ package feed import ( - "encoding/binary" "encoding/json" "time" ) @@ -30,32 +29,11 @@ type Timestamp struct { Time uint64 `json:"time"` // Unix epoch timestamp, in seconds } -// 8 bytes uint64 Time -const timestampLength = 8 - // timestampProvider interface describes a source of timestamp information type timestampProvider interface { Now() Timestamp // returns the current timestamp information } -// binaryGet populates the timestamp structure from the given byte slice -func (t *Timestamp) binaryGet(data []byte) error { - if len(data) != timestampLength { - return NewError(ErrCorruptData, "timestamp data has the wrong size") - } - t.Time = binary.LittleEndian.Uint64(data[:8]) - return nil -} - -// binaryPut Serializes a Timestamp to a byte slice -func (t *Timestamp) binaryPut(data []byte) error { - if len(data) != timestampLength { - return NewError(ErrCorruptData, "timestamp data has the wrong size") - } - binary.LittleEndian.PutUint64(data, t.Time) - return nil -} - // UnmarshalJSON implements the json.Unmarshaller interface func (t *Timestamp) UnmarshalJSON(data []byte) error { return json.Unmarshal(data, &t.Time) From 472c23a8015cd84b193d2d0efb4592a664de3c62 Mon Sep 17 00:00:00 2001 From: Elad Date: Mon, 17 Dec 2018 16:49:01 +0530 Subject: [PATCH 155/159] p2p/simulation: move connection methods from swarm/network/simulation (#18323) --- .../simulation => p2p/simulations}/connect.go | 119 ++++--- p2p/simulations/connect_test.go | 190 +++++++++++ p2p/simulations/network.go | 83 ++++- p2p/simulations/test.go | 134 ++++++++ swarm/network/simulation/connect_test.go | 306 ------------------ swarm/network/simulation/events_test.go | 2 +- swarm/network/simulation/node.go | 64 +--- swarm/network/simulation/node_test.go | 9 +- swarm/network/simulation/service.go | 2 +- swarm/network/simulation/simulation.go | 1 - swarm/network/stream/delivery_test.go | 7 +- .../network/stream/snapshot_retrieval_test.go | 6 +- swarm/network/stream/snapshot_sync_test.go | 14 +- .../visualized_snapshot_sync_sim_test.go | 2 +- 14 files changed, 500 insertions(+), 439 deletions(-) rename {swarm/network/simulation => p2p/simulations}/connect.go (51%) create mode 100644 p2p/simulations/connect_test.go create mode 100644 p2p/simulations/test.go delete mode 100644 swarm/network/simulation/connect_test.go diff --git a/swarm/network/simulation/connect.go b/p2p/simulations/connect.go similarity index 51% rename from swarm/network/simulation/connect.go rename to p2p/simulations/connect.go index 8b2aa1bfa7..606dda0567 100644 --- a/swarm/network/simulation/connect.go +++ b/p2p/simulations/connect.go @@ -14,65 +14,69 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package simulation +package simulations import ( + "errors" "strings" "github.com/ethereum/go-ethereum/p2p/enode" ) +var ( + ErrNodeNotFound = errors.New("node not found") + ErrNoPivotNode = errors.New("no pivot node set") +) + // ConnectToPivotNode connects the node with provided NodeID -// to the pivot node, already set by Simulation.SetPivotNode method. +// to the pivot node, already set by Network.SetPivotNode method. // It is useful when constructing a star network topology -// when simulation adds and removes nodes dynamically. -func (s *Simulation) ConnectToPivotNode(id enode.ID) (err error) { - pid := s.PivotNodeID() - if pid == nil { +// when Network adds and removes nodes dynamically. +func (net *Network) ConnectToPivotNode(id enode.ID) (err error) { + pivot := net.GetPivotNode() + if pivot == nil { return ErrNoPivotNode } - return s.connect(*pid, id) + return net.connect(pivot.ID(), id) } // ConnectToLastNode connects the node with provided NodeID // to the last node that is up, and avoiding connection to self. // It is useful when constructing a chain network topology -// when simulation adds and removes nodes dynamically. -func (s *Simulation) ConnectToLastNode(id enode.ID) (err error) { - ids := s.UpNodeIDs() +// when Network adds and removes nodes dynamically. +func (net *Network) ConnectToLastNode(id enode.ID) (err error) { + ids := net.getUpNodeIDs() l := len(ids) if l < 2 { return nil } - lid := ids[l-1] - if lid == id { - lid = ids[l-2] + last := ids[l-1] + if last == id { + last = ids[l-2] } - return s.connect(lid, id) + return net.connect(last, id) } -// ConnectToRandomNode connects the node with provieded NodeID +// ConnectToRandomNode connects the node with provided NodeID // to a random node that is up. -func (s *Simulation) ConnectToRandomNode(id enode.ID) (err error) { - n := s.RandomUpNode(id) - if n == nil { +func (net *Network) ConnectToRandomNode(id enode.ID) (err error) { + selected := net.GetRandomUpNode(id) + if selected == nil { return ErrNodeNotFound } - return s.connect(n.ID, id) + return net.connect(selected.ID(), id) } // ConnectNodesFull connects all nodes one to another. // It provides a complete connectivity in the network // which should be rarely needed. -func (s *Simulation) ConnectNodesFull(ids []enode.ID) (err error) { +func (net *Network) ConnectNodesFull(ids []enode.ID) (err error) { if ids == nil { - ids = s.UpNodeIDs() + ids = net.getUpNodeIDs() } - l := len(ids) - for i := 0; i < l; i++ { - for j := i + 1; j < l; j++ { - err = s.connect(ids[i], ids[j]) - if err != nil { + for i, lid := range ids { + for _, rid := range ids[i+1:] { + if err = net.connect(lid, rid); err != nil { return err } } @@ -82,14 +86,13 @@ func (s *Simulation) ConnectNodesFull(ids []enode.ID) (err error) { // ConnectNodesChain connects all nodes in a chain topology. // If ids argument is nil, all nodes that are up will be connected. -func (s *Simulation) ConnectNodesChain(ids []enode.ID) (err error) { +func (net *Network) ConnectNodesChain(ids []enode.ID) (err error) { if ids == nil { - ids = s.UpNodeIDs() + ids = net.getUpNodeIDs() } l := len(ids) for i := 0; i < l-1; i++ { - err = s.connect(ids[i], ids[i+1]) - if err != nil { + if err := net.connect(ids[i], ids[i+1]); err != nil { return err } } @@ -98,37 +101,32 @@ func (s *Simulation) ConnectNodesChain(ids []enode.ID) (err error) { // ConnectNodesRing connects all nodes in a ring topology. // If ids argument is nil, all nodes that are up will be connected. -func (s *Simulation) ConnectNodesRing(ids []enode.ID) (err error) { +func (net *Network) ConnectNodesRing(ids []enode.ID) (err error) { if ids == nil { - ids = s.UpNodeIDs() + ids = net.getUpNodeIDs() } l := len(ids) if l < 2 { return nil } - for i := 0; i < l-1; i++ { - err = s.connect(ids[i], ids[i+1]) - if err != nil { - return err - } + if err := net.ConnectNodesChain(ids); err != nil { + return err } - return s.connect(ids[l-1], ids[0]) + return net.connect(ids[l-1], ids[0]) } // ConnectNodesStar connects all nodes in a star topology // with the center at provided NodeID. // If ids argument is nil, all nodes that are up will be connected. -func (s *Simulation) ConnectNodesStar(id enode.ID, ids []enode.ID) (err error) { +func (net *Network) ConnectNodesStar(pivot enode.ID, ids []enode.ID) (err error) { if ids == nil { - ids = s.UpNodeIDs() + ids = net.getUpNodeIDs() } - l := len(ids) - for i := 0; i < l; i++ { - if id == ids[i] { + for _, id := range ids { + if pivot == id { continue } - err = s.connect(id, ids[i]) - if err != nil { + if err := net.connect(pivot, id); err != nil { return err } } @@ -138,17 +136,17 @@ func (s *Simulation) ConnectNodesStar(id enode.ID, ids []enode.ID) (err error) { // ConnectNodesStarPivot connects all nodes in a star topology // with the center at already set pivot node. // If ids argument is nil, all nodes that are up will be connected. -func (s *Simulation) ConnectNodesStarPivot(ids []enode.ID) (err error) { - id := s.PivotNodeID() - if id == nil { +func (net *Network) ConnectNodesStarPivot(ids []enode.ID) (err error) { + pivot := net.GetPivotNode() + if pivot == nil { return ErrNoPivotNode } - return s.ConnectNodesStar(*id, ids) + return net.ConnectNodesStar(pivot.ID(), ids) } // connect connects two nodes but ignores already connected error. -func (s *Simulation) connect(oneID, otherID enode.ID) error { - return ignoreAlreadyConnectedErr(s.Net.Connect(oneID, otherID)) +func (net *Network) connect(oneID, otherID enode.ID) error { + return ignoreAlreadyConnectedErr(net.Connect(oneID, otherID)) } func ignoreAlreadyConnectedErr(err error) error { @@ -157,3 +155,22 @@ func ignoreAlreadyConnectedErr(err error) error { } return err } + +// SetPivotNode sets the NodeID of the network's pivot node. +// Pivot node is just a specific node that should be treated +// differently then other nodes in test. SetPivotNode and +// GetPivotNode are just a convenient functions to set and +// retrieve it. +func (net *Network) SetPivotNode(id enode.ID) { + net.lock.Lock() + defer net.lock.Unlock() + net.pivotNodeID = id +} + +// GetPivotNode returns NodeID of the pivot node set by +// Network.SetPivotNode method. +func (net *Network) GetPivotNode() (node *Node) { + net.lock.RLock() + defer net.lock.RUnlock() + return net.getNode(net.pivotNodeID) +} diff --git a/p2p/simulations/connect_test.go b/p2p/simulations/connect_test.go new file mode 100644 index 0000000000..36f9442a8a --- /dev/null +++ b/p2p/simulations/connect_test.go @@ -0,0 +1,190 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package simulations + +import ( + "testing" + + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" +) + +func newTestNetwork(t *testing.T, nodeCount int) (*Network, []enode.ID) { + adapter := adapters.NewSimAdapter(adapters.Services{ + "noopwoop": func(ctx *adapters.ServiceContext) (node.Service, error) { + return NewNoopService(nil), nil + }, + }) + + // create network + network := NewNetwork(adapter, &NetworkConfig{ + DefaultService: "noopwoop", + }) + + // create and start nodes + ids := make([]enode.ID, nodeCount) + for i := range ids { + conf := adapters.RandomNodeConfig() + node, err := network.NewNodeWithConfig(conf) + if err != nil { + t.Fatalf("error creating node: %s", err) + } + if err := network.Start(node.ID()); err != nil { + t.Fatalf("error starting node: %s", err) + } + ids[i] = node.ID() + } + + if len(network.Conns) > 0 { + t.Fatal("no connections should exist after just adding nodes") + } + + return network, ids +} + +func TestConnectToPivotNode(t *testing.T) { + net, ids := newTestNetwork(t, 2) + defer net.Shutdown() + + pivot := ids[0] + net.SetPivotNode(pivot) + + other := ids[1] + err := net.ConnectToPivotNode(other) + if err != nil { + t.Fatal(err) + } + + if net.GetConn(pivot, other) == nil { + t.Error("pivot and the other node are not connected") + } +} + +func TestConnectToLastNode(t *testing.T) { + net, ids := newTestNetwork(t, 10) + defer net.Shutdown() + + first := ids[0] + if err := net.ConnectToLastNode(first); err != nil { + t.Fatal(err) + } + + last := ids[len(ids)-1] + for i, id := range ids { + if id == first || id == last { + continue + } + + if net.GetConn(first, id) != nil { + t.Errorf("connection must not exist with node(ind: %v, id: %v)", i, id) + } + } + + if net.GetConn(first, last) == nil { + t.Error("first and last node must be connected") + } +} + +func TestConnectToRandomNode(t *testing.T) { + net, ids := newTestNetwork(t, 10) + defer net.Shutdown() + + err := net.ConnectToRandomNode(ids[0]) + if err != nil { + t.Fatal(err) + } + + var cc int + for i, a := range ids { + for _, b := range ids[i:] { + if net.GetConn(a, b) != nil { + cc++ + } + } + } + + if cc != 1 { + t.Errorf("expected one connection, got %v", cc) + } +} + +func TestConnectNodesFull(t *testing.T) { + net, ids := newTestNetwork(t, 12) + defer net.Shutdown() + + err := net.ConnectNodesFull(ids) + if err != nil { + t.Fatal(err) + } + + VerifyFull(t, net, ids) +} + +func TestConnectNodesChain(t *testing.T) { + net, ids := newTestNetwork(t, 10) + defer net.Shutdown() + + err := net.ConnectNodesChain(ids) + if err != nil { + t.Fatal(err) + } + + VerifyChain(t, net, ids) +} + +func TestConnectNodesRing(t *testing.T) { + net, ids := newTestNetwork(t, 10) + defer net.Shutdown() + + err := net.ConnectNodesRing(ids) + if err != nil { + t.Fatal(err) + } + + VerifyRing(t, net, ids) +} + +func TestConnectNodesStar(t *testing.T) { + net, ids := newTestNetwork(t, 10) + defer net.Shutdown() + + pivotIndex := 2 + + err := net.ConnectNodesStar(ids[pivotIndex], ids) + if err != nil { + t.Fatal(err) + } + + VerifyStar(t, net, ids, pivotIndex) +} + +func TestConnectNodesStarPivot(t *testing.T) { + net, ids := newTestNetwork(t, 10) + defer net.Shutdown() + + pivotIndex := 4 + + net.SetPivotNode(ids[pivotIndex]) + + err := net.ConnectNodesStarPivot(ids) + if err != nil { + t.Fatal(err) + } + + VerifyStar(t, net, ids, pivotIndex) +} diff --git a/p2p/simulations/network.go b/p2p/simulations/network.go index ab9f582c52..a6fac2c2af 100644 --- a/p2p/simulations/network.go +++ b/p2p/simulations/network.go @@ -22,6 +22,7 @@ import ( "encoding/json" "errors" "fmt" + "math/rand" "sync" "time" @@ -57,6 +58,8 @@ type Network struct { Conns []*Conn `json:"conns"` connMap map[string]int + pivotNodeID enode.ID + nodeAdapter adapters.NodeAdapter events event.Feed lock sync.RWMutex @@ -370,23 +373,32 @@ func (net *Network) DidReceive(sender, receiver enode.ID, proto string, code uin // GetNode gets the node with the given ID, returning nil if the node does not // exist func (net *Network) GetNode(id enode.ID) *Node { - net.lock.Lock() - defer net.lock.Unlock() + net.lock.RLock() + defer net.lock.RUnlock() return net.getNode(id) } // GetNode gets the node with the given name, returning nil if the node does // not exist func (net *Network) GetNodeByName(name string) *Node { - net.lock.Lock() - defer net.lock.Unlock() + net.lock.RLock() + defer net.lock.RUnlock() return net.getNodeByName(name) } +func (net *Network) getNodeByName(name string) *Node { + for _, node := range net.Nodes { + if node.Config.Name == name { + return node + } + } + return nil +} + // GetNodes returns the existing nodes func (net *Network) GetNodes() (nodes []*Node) { - net.lock.Lock() - defer net.lock.Unlock() + net.lock.RLock() + defer net.lock.RUnlock() nodes = append(nodes, net.Nodes...) return nodes @@ -400,20 +412,67 @@ func (net *Network) getNode(id enode.ID) *Node { return net.Nodes[i] } -func (net *Network) getNodeByName(name string) *Node { +// GetRandomUpNode returns a random node on the network, which is running. +func (net *Network) GetRandomUpNode(excludeIDs ...enode.ID) *Node { + net.lock.RLock() + defer net.lock.RUnlock() + return net.getRandomNode(net.getUpNodeIDs(), excludeIDs) +} + +func (net *Network) getUpNodeIDs() (ids []enode.ID) { for _, node := range net.Nodes { - if node.Config.Name == name { - return node + if node.Up { + ids = append(ids, node.ID()) } } - return nil + return ids +} + +// GetRandomDownNode returns a random node on the network, which is stopped. +func (net *Network) GetRandomDownNode(excludeIDs ...enode.ID) *Node { + net.lock.RLock() + defer net.lock.RUnlock() + return net.getRandomNode(net.getDownNodeIDs(), excludeIDs) +} + +func (net *Network) getDownNodeIDs() (ids []enode.ID) { + for _, node := range net.GetNodes() { + if !node.Up { + ids = append(ids, node.ID()) + } + } + return ids +} + +func (net *Network) getRandomNode(ids []enode.ID, excludeIDs []enode.ID) *Node { + filtered := filterIDs(ids, excludeIDs) + + l := len(filtered) + if l == 0 { + return nil + } + return net.GetNode(filtered[rand.Intn(l)]) +} + +func filterIDs(ids []enode.ID, excludeIDs []enode.ID) []enode.ID { + exclude := make(map[enode.ID]bool) + for _, id := range excludeIDs { + exclude[id] = true + } + var filtered []enode.ID + for _, id := range ids { + if _, found := exclude[id]; !found { + filtered = append(filtered, id) + } + } + return filtered } // GetConn returns the connection which exists between "one" and "other" // regardless of which node initiated the connection func (net *Network) GetConn(oneID, otherID enode.ID) *Conn { - net.lock.Lock() - defer net.lock.Unlock() + net.lock.RLock() + defer net.lock.RUnlock() return net.getConn(oneID, otherID) } diff --git a/p2p/simulations/test.go b/p2p/simulations/test.go new file mode 100644 index 0000000000..beeb414e41 --- /dev/null +++ b/p2p/simulations/test.go @@ -0,0 +1,134 @@ +package simulations + +import ( + "testing" + + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/enr" + "github.com/ethereum/go-ethereum/rpc" +) + +// NoopService is the service that does not do anything +// but implements node.Service interface. +type NoopService struct { + c map[enode.ID]chan struct{} +} + +func NewNoopService(ackC map[enode.ID]chan struct{}) *NoopService { + return &NoopService{ + c: ackC, + } +} + +func (t *NoopService) Protocols() []p2p.Protocol { + return []p2p.Protocol{ + { + Name: "noop", + Version: 666, + Length: 0, + Run: func(peer *p2p.Peer, rw p2p.MsgReadWriter) error { + if t.c != nil { + t.c[peer.ID()] = make(chan struct{}) + close(t.c[peer.ID()]) + } + rw.ReadMsg() + return nil + }, + NodeInfo: func() interface{} { + return struct{}{} + }, + PeerInfo: func(id enode.ID) interface{} { + return struct{}{} + }, + Attributes: []enr.Entry{}, + }, + } +} + +func (t *NoopService) APIs() []rpc.API { + return []rpc.API{} +} + +func (t *NoopService) Start(server *p2p.Server) error { + return nil +} + +func (t *NoopService) Stop() error { + return nil +} + +func VerifyRing(t *testing.T, net *Network, ids []enode.ID) { + t.Helper() + n := len(ids) + for i := 0; i < n; i++ { + for j := i + 1; j < n; j++ { + c := net.GetConn(ids[i], ids[j]) + if i == j-1 || (i == 0 && j == n-1) { + if c == nil { + t.Errorf("nodes %v and %v are not connected, but they should be", i, j) + } + } else { + if c != nil { + t.Errorf("nodes %v and %v are connected, but they should not be", i, j) + } + } + } + } +} + +func VerifyChain(t *testing.T, net *Network, ids []enode.ID) { + t.Helper() + n := len(ids) + for i := 0; i < n; i++ { + for j := i + 1; j < n; j++ { + c := net.GetConn(ids[i], ids[j]) + if i == j-1 { + if c == nil { + t.Errorf("nodes %v and %v are not connected, but they should be", i, j) + } + } else { + if c != nil { + t.Errorf("nodes %v and %v are connected, but they should not be", i, j) + } + } + } + } +} + +func VerifyFull(t *testing.T, net *Network, ids []enode.ID) { + t.Helper() + n := len(ids) + var connections int + for i, lid := range ids { + for _, rid := range ids[i+1:] { + if net.GetConn(lid, rid) != nil { + connections++ + } + } + } + + want := n * (n - 1) / 2 + if connections != want { + t.Errorf("wrong number of connections, got: %v, want: %v", connections, want) + } +} + +func VerifyStar(t *testing.T, net *Network, ids []enode.ID, centerIndex int) { + t.Helper() + n := len(ids) + for i := 0; i < n; i++ { + for j := i + 1; j < n; j++ { + c := net.GetConn(ids[i], ids[j]) + if i == centerIndex || j == centerIndex { + if c == nil { + t.Errorf("nodes %v and %v are not connected, but they should be", i, j) + } + } else { + if c != nil { + t.Errorf("nodes %v and %v are connected, but they should not be", i, j) + } + } + } + } +} diff --git a/swarm/network/simulation/connect_test.go b/swarm/network/simulation/connect_test.go deleted file mode 100644 index 6c94b3a01c..0000000000 --- a/swarm/network/simulation/connect_test.go +++ /dev/null @@ -1,306 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package simulation - -import ( - "testing" - - "github.com/ethereum/go-ethereum/p2p/enode" -) - -func TestConnectToPivotNode(t *testing.T) { - sim := New(noopServiceFuncMap) - defer sim.Close() - - pid, err := sim.AddNode() - if err != nil { - t.Fatal(err) - } - - sim.SetPivotNode(pid) - - id, err := sim.AddNode() - if err != nil { - t.Fatal(err) - } - - if len(sim.Net.Conns) > 0 { - t.Fatal("no connections should exist after just adding nodes") - } - - err = sim.ConnectToPivotNode(id) - if err != nil { - t.Fatal(err) - } - - if sim.Net.GetConn(id, pid) == nil { - t.Error("node did not connect to pivot node") - } -} - -func TestConnectToLastNode(t *testing.T) { - sim := New(noopServiceFuncMap) - defer sim.Close() - - n := 10 - - ids, err := sim.AddNodes(n) - if err != nil { - t.Fatal(err) - } - - id, err := sim.AddNode() - if err != nil { - t.Fatal(err) - } - - if len(sim.Net.Conns) > 0 { - t.Fatal("no connections should exist after just adding nodes") - } - - err = sim.ConnectToLastNode(id) - if err != nil { - t.Fatal(err) - } - - for _, i := range ids[:n-2] { - if sim.Net.GetConn(id, i) != nil { - t.Error("node connected to the node that is not the last") - } - } - - if sim.Net.GetConn(id, ids[n-1]) == nil { - t.Error("node did not connect to the last node") - } -} - -func TestConnectToRandomNode(t *testing.T) { - sim := New(noopServiceFuncMap) - defer sim.Close() - - n := 10 - - ids, err := sim.AddNodes(n) - if err != nil { - t.Fatal(err) - } - - if len(sim.Net.Conns) > 0 { - t.Fatal("no connections should exist after just adding nodes") - } - - err = sim.ConnectToRandomNode(ids[0]) - if err != nil { - t.Fatal(err) - } - - var cc int - for i := 0; i < n; i++ { - for j := i + 1; j < n; j++ { - if sim.Net.GetConn(ids[i], ids[j]) != nil { - cc++ - } - } - } - - if cc != 1 { - t.Errorf("expected one connection, got %v", cc) - } -} - -func TestConnectNodesFull(t *testing.T) { - sim := New(noopServiceFuncMap) - defer sim.Close() - - ids, err := sim.AddNodes(12) - if err != nil { - t.Fatal(err) - } - - if len(sim.Net.Conns) > 0 { - t.Fatal("no connections should exist after just adding nodes") - } - - err = sim.ConnectNodesFull(ids) - if err != nil { - t.Fatal(err) - } - - testFull(t, sim, ids) -} - -func testFull(t *testing.T, sim *Simulation, ids []enode.ID) { - n := len(ids) - var cc int - for i := 0; i < n; i++ { - for j := i + 1; j < n; j++ { - if sim.Net.GetConn(ids[i], ids[j]) != nil { - cc++ - } - } - } - - want := n * (n - 1) / 2 - - if cc != want { - t.Errorf("expected %v connection, got %v", want, cc) - } -} - -func TestConnectNodesChain(t *testing.T) { - sim := New(noopServiceFuncMap) - defer sim.Close() - - ids, err := sim.AddNodes(10) - if err != nil { - t.Fatal(err) - } - - if len(sim.Net.Conns) > 0 { - t.Fatal("no connections should exist after just adding nodes") - } - - err = sim.ConnectNodesChain(ids) - if err != nil { - t.Fatal(err) - } - - testChain(t, sim, ids) -} - -func testChain(t *testing.T, sim *Simulation, ids []enode.ID) { - n := len(ids) - for i := 0; i < n; i++ { - for j := i + 1; j < n; j++ { - c := sim.Net.GetConn(ids[i], ids[j]) - if i == j-1 { - if c == nil { - t.Errorf("nodes %v and %v are not connected, but they should be", i, j) - } - } else { - if c != nil { - t.Errorf("nodes %v and %v are connected, but they should not be", i, j) - } - } - } - } -} - -func TestConnectNodesRing(t *testing.T) { - sim := New(noopServiceFuncMap) - defer sim.Close() - - ids, err := sim.AddNodes(10) - if err != nil { - t.Fatal(err) - } - - if len(sim.Net.Conns) > 0 { - t.Fatal("no connections should exist after just adding nodes") - } - - err = sim.ConnectNodesRing(ids) - if err != nil { - t.Fatal(err) - } - - testRing(t, sim, ids) -} - -func testRing(t *testing.T, sim *Simulation, ids []enode.ID) { - n := len(ids) - for i := 0; i < n; i++ { - for j := i + 1; j < n; j++ { - c := sim.Net.GetConn(ids[i], ids[j]) - if i == j-1 || (i == 0 && j == n-1) { - if c == nil { - t.Errorf("nodes %v and %v are not connected, but they should be", i, j) - } - } else { - if c != nil { - t.Errorf("nodes %v and %v are connected, but they should not be", i, j) - } - } - } - } -} - -func TestConnectToNodesStar(t *testing.T) { - sim := New(noopServiceFuncMap) - defer sim.Close() - - ids, err := sim.AddNodes(10) - if err != nil { - t.Fatal(err) - } - - if len(sim.Net.Conns) > 0 { - t.Fatal("no connections should exist after just adding nodes") - } - - centerIndex := 2 - - err = sim.ConnectNodesStar(ids[centerIndex], ids) - if err != nil { - t.Fatal(err) - } - - testStar(t, sim, ids, centerIndex) -} - -func testStar(t *testing.T, sim *Simulation, ids []enode.ID, centerIndex int) { - n := len(ids) - for i := 0; i < n; i++ { - for j := i + 1; j < n; j++ { - c := sim.Net.GetConn(ids[i], ids[j]) - if i == centerIndex || j == centerIndex { - if c == nil { - t.Errorf("nodes %v and %v are not connected, but they should be", i, j) - } - } else { - if c != nil { - t.Errorf("nodes %v and %v are connected, but they should not be", i, j) - } - } - } - } -} - -func TestConnectToNodesStarPivot(t *testing.T) { - sim := New(noopServiceFuncMap) - defer sim.Close() - - ids, err := sim.AddNodes(10) - if err != nil { - t.Fatal(err) - } - - if len(sim.Net.Conns) > 0 { - t.Fatal("no connections should exist after just adding nodes") - } - - pivotIndex := 4 - - sim.SetPivotNode(ids[pivotIndex]) - - err = sim.ConnectNodesStarPivot(ids) - if err != nil { - t.Fatal(err) - } - - testStar(t, sim, ids, pivotIndex) -} diff --git a/swarm/network/simulation/events_test.go b/swarm/network/simulation/events_test.go index 0c185d9774..34ef24ed40 100644 --- a/swarm/network/simulation/events_test.go +++ b/swarm/network/simulation/events_test.go @@ -59,7 +59,7 @@ func TestPeerEvents(t *testing.T) { } }() - err = sim.ConnectNodesChain(sim.NodeIDs()) + err = sim.Net.ConnectNodesChain(sim.NodeIDs()) if err != nil { t.Fatal(err) } diff --git a/swarm/network/simulation/node.go b/swarm/network/simulation/node.go index a916d3fc21..24b6599762 100644 --- a/swarm/network/simulation/node.go +++ b/swarm/network/simulation/node.go @@ -127,7 +127,7 @@ func (s *Simulation) AddNodesAndConnectFull(count int, opts ...AddNodeOption) (i if err != nil { return nil, err } - err = s.ConnectNodesFull(ids) + err = s.Net.ConnectNodesFull(ids) if err != nil { return nil, err } @@ -145,7 +145,7 @@ func (s *Simulation) AddNodesAndConnectChain(count int, opts ...AddNodeOption) ( if err != nil { return nil, err } - err = s.ConnectToLastNode(id) + err = s.Net.ConnectToLastNode(id) if err != nil { return nil, err } @@ -154,7 +154,7 @@ func (s *Simulation) AddNodesAndConnectChain(count int, opts ...AddNodeOption) ( return nil, err } ids = append([]enode.ID{id}, ids...) - err = s.ConnectNodesChain(ids) + err = s.Net.ConnectNodesChain(ids) if err != nil { return nil, err } @@ -171,7 +171,7 @@ func (s *Simulation) AddNodesAndConnectRing(count int, opts ...AddNodeOption) (i if err != nil { return nil, err } - err = s.ConnectNodesRing(ids) + err = s.Net.ConnectNodesRing(ids) if err != nil { return nil, err } @@ -188,7 +188,7 @@ func (s *Simulation) AddNodesAndConnectStar(count int, opts ...AddNodeOption) (i if err != nil { return nil, err } - err = s.ConnectNodesStar(ids[0], ids[1:]) + err = s.Net.ConnectNodesStar(ids[0], ids[1:]) if err != nil { return nil, err } @@ -267,27 +267,26 @@ func (s *Simulation) StartNode(id enode.ID) (err error) { // StartRandomNode starts a random node. func (s *Simulation) StartRandomNode() (id enode.ID, err error) { - n := s.randomDownNode() + n := s.Net.GetRandomDownNode() if n == nil { return id, ErrNodeNotFound } - return n.ID, s.Net.Start(n.ID) + return n.ID(), s.Net.Start(n.ID()) } // StartRandomNodes starts random nodes. func (s *Simulation) StartRandomNodes(count int) (ids []enode.ID, err error) { ids = make([]enode.ID, 0, count) - downIDs := s.DownNodeIDs() for i := 0; i < count; i++ { - n := s.randomNode(downIDs, ids...) + n := s.Net.GetRandomDownNode() if n == nil { return nil, ErrNodeNotFound } - err = s.Net.Start(n.ID) + err = s.Net.Start(n.ID()) if err != nil { return nil, err } - ids = append(ids, n.ID) + ids = append(ids, n.ID()) } return ids, nil } @@ -299,27 +298,26 @@ func (s *Simulation) StopNode(id enode.ID) (err error) { // StopRandomNode stops a random node. func (s *Simulation) StopRandomNode() (id enode.ID, err error) { - n := s.RandomUpNode() + n := s.Net.GetRandomUpNode() if n == nil { return id, ErrNodeNotFound } - return n.ID, s.Net.Stop(n.ID) + return n.ID(), s.Net.Stop(n.ID()) } // StopRandomNodes stops random nodes. func (s *Simulation) StopRandomNodes(count int) (ids []enode.ID, err error) { ids = make([]enode.ID, 0, count) - upIDs := s.UpNodeIDs() for i := 0; i < count; i++ { - n := s.randomNode(upIDs, ids...) + n := s.Net.GetRandomUpNode() if n == nil { return nil, ErrNodeNotFound } - err = s.Net.Stop(n.ID) + err = s.Net.Stop(n.ID()) if err != nil { return nil, err } - ids = append(ids, n.ID) + ids = append(ids, n.ID()) } return ids, nil } @@ -328,35 +326,3 @@ func (s *Simulation) StopRandomNodes(count int) (ids []enode.ID, err error) { func init() { rand.Seed(time.Now().UnixNano()) } - -// RandomUpNode returns a random SimNode that is up. -// Arguments are NodeIDs for nodes that should not be returned. -func (s *Simulation) RandomUpNode(exclude ...enode.ID) *adapters.SimNode { - return s.randomNode(s.UpNodeIDs(), exclude...) -} - -// randomDownNode returns a random SimNode that is not up. -func (s *Simulation) randomDownNode(exclude ...enode.ID) *adapters.SimNode { - return s.randomNode(s.DownNodeIDs(), exclude...) -} - -// randomNode returns a random SimNode from the slice of NodeIDs. -func (s *Simulation) randomNode(ids []enode.ID, exclude ...enode.ID) *adapters.SimNode { - for _, e := range exclude { - var i int - for _, id := range ids { - if id == e { - ids = append(ids[:i], ids[i+1:]...) - } else { - i++ - } - } - } - l := len(ids) - if l == 0 { - return nil - } - n := s.Net.GetNode(ids[rand.Intn(l)]) - node, _ := n.Node.(*adapters.SimNode) - return node -} diff --git a/swarm/network/simulation/node_test.go b/swarm/network/simulation/node_test.go index 01346ef14d..8da32cf37f 100644 --- a/swarm/network/simulation/node_test.go +++ b/swarm/network/simulation/node_test.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/simulations" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/swarm/network" ) @@ -228,7 +229,7 @@ func TestAddNodesAndConnectFull(t *testing.T) { t.Fatal(err) } - testFull(t, sim, ids) + simulations.VerifyFull(t, sim.Net, ids) } func TestAddNodesAndConnectChain(t *testing.T) { @@ -247,7 +248,7 @@ func TestAddNodesAndConnectChain(t *testing.T) { t.Fatal(err) } - testChain(t, sim, sim.UpNodeIDs()) + simulations.VerifyChain(t, sim.Net, sim.UpNodeIDs()) } func TestAddNodesAndConnectRing(t *testing.T) { @@ -259,7 +260,7 @@ func TestAddNodesAndConnectRing(t *testing.T) { t.Fatal(err) } - testRing(t, sim, ids) + simulations.VerifyRing(t, sim.Net, ids) } func TestAddNodesAndConnectStar(t *testing.T) { @@ -271,7 +272,7 @@ func TestAddNodesAndConnectStar(t *testing.T) { t.Fatal(err) } - testStar(t, sim, ids, 0) + simulations.VerifyStar(t, sim.Net, ids, 0) } //To test that uploading a snapshot works diff --git a/swarm/network/simulation/service.go b/swarm/network/simulation/service.go index 819602e9e8..7dd4dc6d82 100644 --- a/swarm/network/simulation/service.go +++ b/swarm/network/simulation/service.go @@ -39,7 +39,7 @@ func (s *Simulation) Service(name string, id enode.ID) node.Service { // RandomService returns a single Service by name on a // randomly chosen node that is up. func (s *Simulation) RandomService(name string) node.Service { - n := s.RandomUpNode() + n := s.Net.GetRandomUpNode().Node.(*adapters.SimNode) if n == nil { return nil } diff --git a/swarm/network/simulation/simulation.go b/swarm/network/simulation/simulation.go index e5435b9f0e..747faf5d77 100644 --- a/swarm/network/simulation/simulation.go +++ b/swarm/network/simulation/simulation.go @@ -33,7 +33,6 @@ import ( // Common errors that are returned by functions in this package. var ( ErrNodeNotFound = errors.New("node not found") - ErrNoPivotNode = errors.New("no pivot node set") ) // Simulation provides methods on network, nodes and services diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index f69f80499b..f537c13234 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -505,7 +505,8 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error { nodeIDs := sim.UpNodeIDs() //determine the pivot node to be the first node of the simulation - sim.SetPivotNode(nodeIDs[0]) + pivot := nodeIDs[0] + //distribute chunks of a random file into Stores of nodes 1 to nodes //we will do this by creating a file store with an underlying round-robin store: //the file store will create a hash for the uploaded file, but every chunk will be @@ -519,7 +520,7 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck //...iterate the buckets... for id, bucketVal := range lStores { //...and remove the one which is the pivot node - if id == *sim.PivotNodeID() { + if id == pivot { continue } //the other ones are added to the array... @@ -547,7 +548,7 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck } //get the pivot node's filestore - item, ok := sim.NodeItem(*sim.PivotNodeID(), bucketKeyFileStore) + item, ok := sim.NodeItem(pivot, bucketKeyFileStore) if !ok { return fmt.Errorf("No filestore") } diff --git a/swarm/network/stream/snapshot_retrieval_test.go b/swarm/network/stream/snapshot_retrieval_test.go index 932e28b32f..62b53fad3b 100644 --- a/swarm/network/stream/snapshot_retrieval_test.go +++ b/swarm/network/stream/snapshot_retrieval_test.go @@ -278,13 +278,13 @@ func runRetrievalTest(chunkCount int, nodeCount int) error { } //this is the node selected for upload - node := sim.RandomUpNode() - item, ok := sim.NodeItem(node.ID, bucketKeyStore) + node := sim.Net.GetRandomUpNode() + item, ok := sim.NodeItem(node.ID(), bucketKeyStore) if !ok { return fmt.Errorf("No localstore") } lstore := item.(*storage.LocalStore) - conf.hashes, err = uploadFileToSingleNodeStore(node.ID, chunkCount, lstore) + conf.hashes, err = uploadFileToSingleNodeStore(node.ID(), chunkCount, lstore) if err != nil { return err } diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go index 4a632c8c90..40b4327fd1 100644 --- a/swarm/network/stream/snapshot_sync_test.go +++ b/swarm/network/stream/snapshot_sync_test.go @@ -248,20 +248,20 @@ func runSim(conf *synctestConfig, ctx context.Context, sim *simulation.Simulatio //get the node at that index //this is the node selected for upload - node := sim.RandomUpNode() - item, ok := sim.NodeItem(node.ID, bucketKeyStore) + node := sim.Net.GetRandomUpNode() + item, ok := sim.NodeItem(node.ID(), bucketKeyStore) if !ok { return fmt.Errorf("No localstore") } lstore := item.(*storage.LocalStore) - hashes, err := uploadFileToSingleNodeStore(node.ID, chunkCount, lstore) + hashes, err := uploadFileToSingleNodeStore(node.ID(), chunkCount, lstore) if err != nil { return err } for _, h := range hashes { evt := &simulations.Event{ Type: EventTypeChunkCreated, - Node: sim.Net.GetNode(node.ID), + Node: sim.Net.GetNode(node.ID()), Data: h.String(), } sim.Net.Events().Send(evt) @@ -453,13 +453,13 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int) } } //select a random node for upload - node := sim.RandomUpNode() - item, ok := sim.NodeItem(node.ID, bucketKeyStore) + node := sim.Net.GetRandomUpNode() + item, ok := sim.NodeItem(node.ID(), bucketKeyStore) if !ok { return fmt.Errorf("No localstore") } lstore := item.(*storage.LocalStore) - hashes, err := uploadFileToSingleNodeStore(node.ID, chunkCount, lstore) + hashes, err := uploadFileToSingleNodeStore(node.ID(), chunkCount, lstore) if err != nil { return err } diff --git a/swarm/network/stream/visualized_snapshot_sync_sim_test.go b/swarm/network/stream/visualized_snapshot_sync_sim_test.go index f6d618020d..27237d1849 100644 --- a/swarm/network/stream/visualized_snapshot_sync_sim_test.go +++ b/swarm/network/stream/visualized_snapshot_sync_sim_test.go @@ -103,7 +103,7 @@ func TestNonExistingHashesWithServer(t *testing.T) { result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error { //check on the node's FileStore (netstore) - id := sim.RandomUpNode().ID + id := sim.Net.GetRandomUpNode().ID() item, ok := sim.NodeItem(id, bucketKeyFileStore) if !ok { t.Fatalf("No filestore") From 90ea542e9efa5ec7652e2b507a7122e5a8e4b91b Mon Sep 17 00:00:00 2001 From: holisticode Date: Tue, 18 Dec 2018 01:20:59 -0500 Subject: [PATCH 156/159] Update visualized snapshot test (#18286) * swarm/network/stream: fix visualized_snapshot_sync_sim_test * swarm/network/stream: updated visualized snapshot-test;data in p2p event * swarm/network/stream: cleanup visualized snapshot sync test * swarm/network/stream: re-enable t.Skip for visualized test * swarm/network/stream: addressed PR comments --- .../visualized_snapshot_sync_sim_test.go | 267 ++++++++++++++---- 1 file changed, 214 insertions(+), 53 deletions(-) diff --git a/swarm/network/stream/visualized_snapshot_sync_sim_test.go b/swarm/network/stream/visualized_snapshot_sync_sim_test.go index 27237d1849..96b707797c 100644 --- a/swarm/network/stream/visualized_snapshot_sync_sim_test.go +++ b/swarm/network/stream/visualized_snapshot_sync_sim_test.go @@ -19,16 +19,27 @@ package stream import ( + "bytes" "context" + "errors" "fmt" + "io" + "os" + "sync" "testing" "time" + "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/protocols" "github.com/ethereum/go-ethereum/p2p/simulations" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/swarm/log" + "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/network/simulation" + "github.com/ethereum/go-ethereum/swarm/state" "github.com/ethereum/go-ethereum/swarm/storage" ) @@ -68,12 +79,12 @@ func watchSim(sim *simulation.Simulation) (context.Context, context.CancelFunc) disconnections := sim.PeerEvents( context.Background(), sim.NodeIDs(), - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop), + simulation.NewPeerEventsFilter().Drop(), ) go func() { for d := range disconnections { - log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer) + log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) panic("unexpected disconnect") cancelSimRun() } @@ -144,21 +155,75 @@ func sendSimTerminatedEvent(sim *simulation.Simulation) { //It also sends some custom events so that the frontend //can visualize messages like SendOfferedMsg, WantedHashesMsg, DeliveryMsg func TestSnapshotSyncWithServer(t *testing.T) { + //t.Skip("temporarily disabled as simulations.WaitTillHealthy cannot be trusted") + + //define a wrapper object to be able to pass around data + wrapper := &netWrapper{} + + nodeCount := *nodes + chunkCount := *chunks + + if nodeCount == 0 || chunkCount == 0 { + nodeCount = 32 + chunkCount = 1 + } + + log.Info(fmt.Sprintf("Running the simulation with %d nodes and %d chunks", nodeCount, chunkCount)) + + sim := simulation.New(map[string]simulation.ServiceFunc{ + "streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) { + n := ctx.Config.Node() + addr := network.NewAddr(n) + store, datadir, err := createTestLocalStorageForID(n.ID(), addr) + if err != nil { + return nil, nil, err + } + bucket.Store(bucketKeyStore, store) + localStore := store.(*storage.LocalStore) + netStore, err := storage.NewNetStore(localStore, nil) + if err != nil { + return nil, nil, err + } + kad := network.NewKademlia(addr.Over(), network.NewKadParams()) + delivery := NewDelivery(kad, netStore) + netStore.NewNetFetcherFunc = network.NewFetcherFactory(dummyRequestFromPeers, true).New + + r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ + Retrieval: RetrievalDisabled, + Syncing: SyncingAutoSubscribe, + SyncUpdateDelay: 3 * time.Second, + }, nil) + + tr := &testRegistry{ + Registry: r, + w: wrapper, + } + + bucket.Store(bucketKeyRegistry, tr) + + cleanup = func() { + netStore.Close() + tr.Close() + os.RemoveAll(datadir) + } + + return tr, cleanup, nil + }, + }).WithServer(":8888") //start with the HTTP server - t.Skip("temporarily disabled as simulations.WaitTillHealthy cannot be trusted") - nodeCount, chunkCount, sim := setupSim(simServiceMap) defer sim.Close() log.Info("Initializing test config") conf := &synctestConfig{} //map of discover ID to indexes of chunks expected at that ID - conf.idToChunksMap = make(map[discover.NodeID][]int) + conf.idToChunksMap = make(map[enode.ID][]int) //map of overlay address to discover ID - conf.addrToIDMap = make(map[string]discover.NodeID) + conf.addrToIDMap = make(map[string]enode.ID) //array where the generated chunk hashes will be stored conf.hashes = make([]storage.Address, 0) - + //pass the network to the wrapper object + wrapper.setNetwork(sim.Net) err := sim.UploadSnapshot(fmt.Sprintf("testing/snapshot_%d.json", nodeCount)) if err != nil { panic(err) @@ -167,49 +232,6 @@ func TestSnapshotSyncWithServer(t *testing.T) { ctx, cancelSimRun := watchSim(sim) defer cancelSimRun() - //setup filters in the event feed - offeredHashesFilter := simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeMsgRecv).Protocol("stream").MsgCode(1) - wantedFilter := simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeMsgRecv).Protocol("stream").MsgCode(2) - deliveryFilter := simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeMsgRecv).Protocol("stream").MsgCode(6) - eventC := sim.PeerEvents(ctx, sim.UpNodeIDs(), offeredHashesFilter, wantedFilter, deliveryFilter) - - quit := make(chan struct{}) - - go func() { - for e := range eventC { - select { - case <-quit: - fmt.Println("quitting event loop") - return - default: - } - if e.Error != nil { - t.Fatal(e.Error) - } - if *e.Event.MsgCode == uint64(1) { - evt := &simulations.Event{ - Type: EventTypeChunkOffered, - Node: sim.Net.GetNode(e.NodeID), - Control: false, - } - sim.Net.Events().Send(evt) - } else if *e.Event.MsgCode == uint64(2) { - evt := &simulations.Event{ - Type: EventTypeChunkWanted, - Node: sim.Net.GetNode(e.NodeID), - Control: false, - } - sim.Net.Events().Send(evt) - } else if *e.Event.MsgCode == uint64(6) { - evt := &simulations.Event{ - Type: EventTypeChunkDelivered, - Node: sim.Net.GetNode(e.NodeID), - Control: false, - } - sim.Net.Events().Send(evt) - } - } - }() //run the sim result := runSim(conf, ctx, sim, chunkCount) @@ -218,11 +240,150 @@ func TestSnapshotSyncWithServer(t *testing.T) { Type: EventTypeSimTerminated, Control: false, } - sim.Net.Events().Send(evt) + go sim.Net.Events().Send(evt) if result.Error != nil { panic(result.Error) } - close(quit) log.Info("Simulation ended") } + +//testRegistry embeds registry +//it allows to replace the protocol run function +type testRegistry struct { + *Registry + w *netWrapper +} + +//Protocols replaces the protocol's run function +func (tr *testRegistry) Protocols() []p2p.Protocol { + regProto := tr.Registry.Protocols() + //set the `stream` protocol's run function with the testRegistry's one + regProto[0].Run = tr.runProto + return regProto +} + +//runProto is the new overwritten protocol's run function for this test +func (tr *testRegistry) runProto(p *p2p.Peer, rw p2p.MsgReadWriter) error { + //create a custom rw message ReadWriter + testRw := &testMsgReadWriter{ + MsgReadWriter: rw, + Peer: p, + w: tr.w, + Registry: tr.Registry, + } + //now run the actual upper layer `Registry`'s protocol function + return tr.runProtocol(p, testRw) +} + +//testMsgReadWriter is a custom rw +//it will allow us to re-use the message twice +type testMsgReadWriter struct { + *Registry + p2p.MsgReadWriter + *p2p.Peer + w *netWrapper +} + +//netWrapper wrapper object so we can pass data around +type netWrapper struct { + net *simulations.Network +} + +//set the network to the wrapper for later use (used inside the custom rw) +func (w *netWrapper) setNetwork(n *simulations.Network) { + w.net = n +} + +//get he network from the wrapper (used inside the custom rw) +func (w *netWrapper) getNetwork() *simulations.Network { + return w.net +} + +// ReadMsg reads a message from the underlying MsgReadWriter and emits a +// "message received" event +//we do this because we are interested in the Payload of the message for custom use +//in this test, but messages can only be consumed once (stream io.Reader) +func (ev *testMsgReadWriter) ReadMsg() (p2p.Msg, error) { + //read the message from the underlying rw + msg, err := ev.MsgReadWriter.ReadMsg() + if err != nil { + return msg, err + } + + //don't do anything with message codes we actually are not needing/reading + subCodes := []uint64{1, 2, 10} + found := false + for _, c := range subCodes { + if c == msg.Code { + found = true + } + } + //just return if not a msg code we are interested in + if !found { + return msg, nil + } + + //we use a io.TeeReader so that we can read the message twice + //the Payload is a io.Reader, so if we read from it, the actual protocol handler + //cannot access it anymore. + //But we need that handler to be able to consume the message as normal, + //as if we would not do anything here with that message + var buf bytes.Buffer + tee := io.TeeReader(msg.Payload, &buf) + + mcp := &p2p.Msg{ + Code: msg.Code, + Size: msg.Size, + ReceivedAt: msg.ReceivedAt, + Payload: tee, + } + //assign the copy for later use + msg.Payload = &buf + + //now let's look into the message + var wmsg protocols.WrappedMsg + err = mcp.Decode(&wmsg) + if err != nil { + log.Error(err.Error()) + return msg, err + } + //create a new message from the code + val, ok := ev.Registry.GetSpec().NewMsg(mcp.Code) + if !ok { + return msg, errors.New(fmt.Sprintf("Invalid message code: %v", msg.Code)) + } + //decode it + if err := rlp.DecodeBytes(wmsg.Payload, val); err != nil { + return msg, errors.New(fmt.Sprintf("Decoding error <= %v: %v", msg, err)) + } + //now for every message type we are interested in, create a custom event and send it + var evt *simulations.Event + switch val := val.(type) { + case *OfferedHashesMsg: + evt = &simulations.Event{ + Type: EventTypeChunkOffered, + Node: ev.w.getNetwork().GetNode(ev.ID()), + Control: false, + Data: val.Hashes, + } + case *WantedHashesMsg: + evt = &simulations.Event{ + Type: EventTypeChunkWanted, + Node: ev.w.getNetwork().GetNode(ev.ID()), + Control: false, + } + case *ChunkDeliveryMsgSyncing: + evt = &simulations.Event{ + Type: EventTypeChunkDelivered, + Node: ev.w.getNetwork().GetNode(ev.ID()), + Control: false, + Data: val.Addr.String(), + } + } + if evt != nil { + //send custom event to feed; frontend will listen to it and display + ev.w.getNetwork().Events().Send(evt) + } + return msg, nil +} From de4265fa028f2b0eb96a21e295028e0d19426adb Mon Sep 17 00:00:00 2001 From: Javier Peletier Date: Tue, 18 Dec 2018 07:24:59 +0100 Subject: [PATCH 157/159] swarm/network/simulation:commented out unreachable code-avoid vet errors (#18263) --- swarm/network/simulation/example_test.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/swarm/network/simulation/example_test.go b/swarm/network/simulation/example_test.go index 7b62046174..a100ede516 100644 --- a/swarm/network/simulation/example_test.go +++ b/swarm/network/simulation/example_test.go @@ -18,14 +18,8 @@ package simulation_test import ( "context" - "fmt" - "sync" - "time" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p/simulations/adapters" - "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/network/simulation" ) @@ -35,7 +29,8 @@ import ( func ExampleSimulation_WaitTillHealthy() { log.Error("temporarily disabled as simulations.WaitTillHealthy cannot be trusted") - return + + /* Commented out to avoid go vet errors/warnings sim := simulation.New(map[string]simulation.ServiceFunc{ "bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) { @@ -75,6 +70,8 @@ func ExampleSimulation_WaitTillHealthy() { } // continue with the test + + */ } // Watch all peer events in the simulation network, buy receiving from a channel. From b01cfce36276379a95deb1001a6f1b6a048609de Mon Sep 17 00:00:00 2001 From: lash Date: Tue, 18 Dec 2018 15:23:32 +0100 Subject: [PATCH 158/159] swarm/pss: Reduce input vulnerabilities (#18304) --- swarm/pss/api.go | 26 ++++++++++--- swarm/pss/handshake.go | 8 ++-- swarm/pss/handshake_test.go | 1 + swarm/pss/notify/notify.go | 8 ++-- swarm/pss/pss.go | 74 +++++++++++++++++++++---------------- swarm/pss/pss_test.go | 59 +++++++++++++++++++++-------- 6 files changed, 116 insertions(+), 60 deletions(-) diff --git a/swarm/pss/api.go b/swarm/pss/api.go index 587382d729..4556d7b7c4 100644 --- a/swarm/pss/api.go +++ b/swarm/pss/api.go @@ -92,7 +92,7 @@ func (pssapi *API) Receive(ctx context.Context, topic Topic, raw bool, prox bool } func (pssapi *API) GetAddress(topic Topic, asymmetric bool, key string) (PssAddress, error) { - var addr *PssAddress + var addr PssAddress if asymmetric { peer, ok := pssapi.Pss.pubKeyPool[key][topic] if !ok { @@ -107,7 +107,7 @@ func (pssapi *API) GetAddress(topic Topic, asymmetric bool, key string) (PssAddr addr = peer.address } - return *addr, nil + return addr, nil } // Retrieves the node's base address in hex form @@ -128,7 +128,7 @@ func (pssapi *API) SetPeerPublicKey(pubkey hexutil.Bytes, topic Topic, addr PssA if err != nil { return fmt.Errorf("Cannot unmarshal pubkey: %x", pubkey) } - err = pssapi.Pss.SetPeerPublicKey(pk, topic, &addr) + err = pssapi.Pss.SetPeerPublicKey(pk, topic, addr) if err != nil { return fmt.Errorf("Invalid key: %x", pk) } @@ -141,11 +141,11 @@ func (pssapi *API) GetSymmetricKey(symkeyid string) (hexutil.Bytes, error) { } func (pssapi *API) GetSymmetricAddressHint(topic Topic, symkeyid string) (PssAddress, error) { - return *pssapi.Pss.symKeyPool[symkeyid][topic].address, nil + return pssapi.Pss.symKeyPool[symkeyid][topic].address, nil } func (pssapi *API) GetAsymmetricAddressHint(topic Topic, pubkeyid string) (PssAddress, error) { - return *pssapi.Pss.pubKeyPool[pubkeyid][topic].address, nil + return pssapi.Pss.pubKeyPool[pubkeyid][topic].address, nil } func (pssapi *API) StringToTopic(topicstring string) (Topic, error) { @@ -157,14 +157,23 @@ func (pssapi *API) StringToTopic(topicstring string) (Topic, error) { } func (pssapi *API) SendAsym(pubkeyhex string, topic Topic, msg hexutil.Bytes) error { + if err := validateMsg(msg); err != nil { + return err + } return pssapi.Pss.SendAsym(pubkeyhex, topic, msg[:]) } func (pssapi *API) SendSym(symkeyhex string, topic Topic, msg hexutil.Bytes) error { + if err := validateMsg(msg); err != nil { + return err + } return pssapi.Pss.SendSym(symkeyhex, topic, msg[:]) } func (pssapi *API) SendRaw(addr hexutil.Bytes, topic Topic, msg hexutil.Bytes) error { + if err := validateMsg(msg); err != nil { + return err + } return pssapi.Pss.SendRaw(PssAddress(addr), topic, msg[:]) } @@ -177,3 +186,10 @@ func (pssapi *API) GetPeerTopics(pubkeyhex string) ([]Topic, error) { func (pssapi *API) GetPeerAddress(pubkeyhex string, topic Topic) (PssAddress, error) { return pssapi.Pss.getPeerAddress(pubkeyhex, topic) } + +func validateMsg(msg []byte) error { + if len(msg) == 0 { + return errors.New("invalid message length") + } + return nil +} diff --git a/swarm/pss/handshake.go b/swarm/pss/handshake.go index 5486abafa9..bb67b51563 100644 --- a/swarm/pss/handshake.go +++ b/swarm/pss/handshake.go @@ -321,9 +321,7 @@ func (ctl *HandshakeController) handleKeys(pubkeyid string, keymsg *handshakeMsg for _, key := range keymsg.Keys { sendsymkey := make([]byte, len(key)) copy(sendsymkey, key) - var address PssAddress - copy(address[:], keymsg.From) - sendsymkeyid, err := ctl.pss.setSymmetricKey(sendsymkey, keymsg.Topic, &address, false, false) + sendsymkeyid, err := ctl.pss.setSymmetricKey(sendsymkey, keymsg.Topic, PssAddress(keymsg.From), false, false) if err != nil { return err } @@ -356,7 +354,7 @@ func (ctl *HandshakeController) handleKeys(pubkeyid string, keymsg *handshakeMsg func (ctl *HandshakeController) sendKey(pubkeyid string, topic *Topic, keycount uint8) ([]string, error) { var requestcount uint8 - to := &PssAddress{} + to := PssAddress{} if _, ok := ctl.pss.pubKeyPool[pubkeyid]; !ok { return []string{}, errors.New("Invalid public key") } else if psp, ok := ctl.pss.pubKeyPool[pubkeyid][*topic]; ok { @@ -564,5 +562,5 @@ func (api *HandshakeAPI) SendSym(symkeyid string, topic Topic, msg hexutil.Bytes api.ctrl.symKeyIndex[symkeyid].count++ log.Trace("increment symkey send use", "symkeyid", symkeyid, "count", api.ctrl.symKeyIndex[symkeyid].count, "limit", api.ctrl.symKeyIndex[symkeyid].limit, "receiver", common.ToHex(crypto.FromECDSAPub(api.ctrl.pss.PublicKey()))) } - return + return err } diff --git a/swarm/pss/handshake_test.go b/swarm/pss/handshake_test.go index 0fc7e798ff..895163f301 100644 --- a/swarm/pss/handshake_test.go +++ b/swarm/pss/handshake_test.go @@ -30,6 +30,7 @@ import ( // asymmetrical key exchange between two directly connected peers // full address, partial address (8 bytes) and empty address func TestHandshake(t *testing.T) { + t.Skip("handshakes are not adapted to current pss core code") t.Run("32", testHandshake) t.Run("8", testHandshake) t.Run("0", testHandshake) diff --git a/swarm/pss/notify/notify.go b/swarm/pss/notify/notify.go index d3c89058b2..e9d40dc321 100644 --- a/swarm/pss/notify/notify.go +++ b/swarm/pss/notify/notify.go @@ -138,7 +138,7 @@ func (c *Controller) Subscribe(name string, pubkey *ecdsa.PublicKey, address pss c.mu.Lock() defer c.mu.Unlock() msg := NewMsg(MsgCodeStart, name, c.pss.BaseAddr()) - c.pss.SetPeerPublicKey(pubkey, controlTopic, &address) + c.pss.SetPeerPublicKey(pubkey, controlTopic, address) pubkeyId := hexutil.Encode(crypto.FromECDSAPub(pubkey)) smsg, err := rlp.EncodeToBytes(msg) if err != nil { @@ -271,7 +271,7 @@ func (c *Controller) addToBin(ntfr *notifier, address []byte) (symKeyId string, currentBin.count++ symKeyId = currentBin.symKeyId } else { - symKeyId, err = c.pss.GenerateSymmetricKey(ntfr.topic, &pssAddress, false) + symKeyId, err = c.pss.GenerateSymmetricKey(ntfr.topic, pssAddress, false) if err != nil { return "", nil, err } @@ -312,7 +312,7 @@ func (c *Controller) handleStartMsg(msg *Msg, keyid string) (err error) { if err != nil { return err } - err = c.pss.SetPeerPublicKey(pubkey, controlTopic, &pssAddress) + err = c.pss.SetPeerPublicKey(pubkey, controlTopic, pssAddress) if err != nil { return err } @@ -335,7 +335,7 @@ func (c *Controller) handleNotifyWithKeyMsg(msg *Msg) error { // \TODO keep track of and add actual address updaterAddr := pss.PssAddress([]byte{}) - c.pss.SetSymmetricKey(symkey, topic, &updaterAddr, true) + c.pss.SetSymmetricKey(symkey, topic, updaterAddr, true) c.pss.Register(&topic, pss.NewHandler(c.Handler)) return c.subscriptions[msg.namestring].handler(msg.namestring, msg.Payload[:len(msg.Payload)-symKeyLength]) } diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index d0986d280b..1bc28890f9 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -81,7 +81,7 @@ type senderPeer interface { // member `protected` prevents garbage collection of the instance type pssPeer struct { lastSeen time.Time - address *PssAddress + address PssAddress protected bool } @@ -396,9 +396,11 @@ func (p *Pss) handlePssMsg(ctx context.Context, msg interface{}) error { // raw is simplest handler contingency to check, so check that first var isRaw bool if pssmsg.isRaw() { - if !p.topicHandlerCaps[psstopic].raw { - log.Debug("No handler for raw message", "topic", psstopic) - return nil + if _, ok := p.topicHandlerCaps[psstopic]; ok { + if !p.topicHandlerCaps[psstopic].raw { + log.Debug("No handler for raw message", "topic", psstopic) + return nil + } } isRaw = true } @@ -437,10 +439,10 @@ func (p *Pss) process(pssmsg *PssMsg, raw bool, prox bool) error { var err error var recvmsg *whisper.ReceivedMessage var payload []byte - var from *PssAddress + var from PssAddress var asymmetric bool var keyid string - var keyFunc func(envelope *whisper.Envelope) (*whisper.ReceivedMessage, string, *PssAddress, error) + var keyFunc func(envelope *whisper.Envelope) (*whisper.ReceivedMessage, string, PssAddress, error) envelope := pssmsg.Payload psstopic := Topic(envelope.Topic) @@ -473,7 +475,7 @@ func (p *Pss) process(pssmsg *PssMsg, raw bool, prox bool) error { } -func (p *Pss) executeHandlers(topic Topic, payload []byte, from *PssAddress, raw bool, prox bool, asymmetric bool, keyid string) { +func (p *Pss) executeHandlers(topic Topic, payload []byte, from PssAddress, raw bool, prox bool, asymmetric bool, keyid string) { handlers := p.getHandlers(topic) peer := p2p.NewPeer(enode.ID{}, fmt.Sprintf("%x", from), []p2p.Cap{}) for h := range handlers { @@ -528,7 +530,10 @@ func (p *Pss) isSelfPossibleRecipient(msg *PssMsg, prox bool) bool { // // The value in `address` will be used as a routing hint for the // public key / topic association -func (p *Pss) SetPeerPublicKey(pubkey *ecdsa.PublicKey, topic Topic, address *PssAddress) error { +func (p *Pss) SetPeerPublicKey(pubkey *ecdsa.PublicKey, topic Topic, address PssAddress) error { + if err := validateAddress(address); err != nil { + return err + } pubkeybytes := crypto.FromECDSAPub(pubkey) if len(pubkeybytes) == 0 { return fmt.Errorf("invalid public key: %v", pubkey) @@ -543,12 +548,12 @@ func (p *Pss) SetPeerPublicKey(pubkey *ecdsa.PublicKey, topic Topic, address *Ps } p.pubKeyPool[pubkeyid][topic] = psp p.pubKeyPoolMu.Unlock() - log.Trace("added pubkey", "pubkeyid", pubkeyid, "topic", topic, "address", common.ToHex(*address)) + log.Trace("added pubkey", "pubkeyid", pubkeyid, "topic", topic, "address", address) return nil } // Automatically generate a new symkey for a topic and address hint -func (p *Pss) GenerateSymmetricKey(topic Topic, address *PssAddress, addToCache bool) (string, error) { +func (p *Pss) GenerateSymmetricKey(topic Topic, address PssAddress, addToCache bool) (string, error) { keyid, err := p.w.GenerateSymKey() if err != nil { return "", err @@ -569,11 +574,14 @@ func (p *Pss) GenerateSymmetricKey(topic Topic, address *PssAddress, addToCache // // Returns a string id that can be used to retrieve the key bytes // from the whisper backend (see pss.GetSymmetricKey()) -func (p *Pss) SetSymmetricKey(key []byte, topic Topic, address *PssAddress, addtocache bool) (string, error) { +func (p *Pss) SetSymmetricKey(key []byte, topic Topic, address PssAddress, addtocache bool) (string, error) { + if err := validateAddress(address); err != nil { + return "", err + } return p.setSymmetricKey(key, topic, address, addtocache, true) } -func (p *Pss) setSymmetricKey(key []byte, topic Topic, address *PssAddress, addtocache bool, protected bool) (string, error) { +func (p *Pss) setSymmetricKey(key []byte, topic Topic, address PssAddress, addtocache bool, protected bool) (string, error) { keyid, err := p.w.AddSymKeyDirect(key) if err != nil { return "", err @@ -585,7 +593,7 @@ func (p *Pss) setSymmetricKey(key []byte, topic Topic, address *PssAddress, addt // adds a symmetric key to the pss key pool, and optionally adds the key // to the collection of keys used to attempt symmetric decryption of // incoming messages -func (p *Pss) addSymmetricKeyToPool(keyid string, topic Topic, address *PssAddress, addtocache bool, protected bool) { +func (p *Pss) addSymmetricKeyToPool(keyid string, topic Topic, address PssAddress, addtocache bool, protected bool) { psp := &pssPeer{ address: address, protected: protected, @@ -601,7 +609,7 @@ func (p *Pss) addSymmetricKeyToPool(keyid string, topic Topic, address *PssAddre p.symKeyDecryptCache[p.symKeyDecryptCacheCursor%cap(p.symKeyDecryptCache)] = &keyid } key, _ := p.GetSymmetricKey(keyid) - log.Trace("added symkey", "symkeyid", keyid, "symkey", common.ToHex(key), "topic", topic, "address", fmt.Sprintf("%p", address), "cache", addtocache) + log.Trace("added symkey", "symkeyid", keyid, "symkey", common.ToHex(key), "topic", topic, "address", address, "cache", addtocache) } // Returns a symmetric key byte seqyence stored in the whisper backend @@ -622,7 +630,7 @@ func (p *Pss) GetPublickeyPeers(keyid string) (topic []Topic, address []PssAddre defer p.pubKeyPoolMu.RUnlock() for t, peer := range p.pubKeyPool[keyid] { topic = append(topic, t) - address = append(address, *peer.address) + address = append(address, peer.address) } return topic, address, nil @@ -633,7 +641,7 @@ func (p *Pss) getPeerAddress(keyid string, topic Topic) (PssAddress, error) { defer p.pubKeyPoolMu.RUnlock() if peers, ok := p.pubKeyPool[keyid]; ok { if t, ok := peers[topic]; ok { - return *t.address, nil + return t.address, nil } } return nil, fmt.Errorf("peer with pubkey %s, topic %x not found", keyid, topic) @@ -645,7 +653,7 @@ func (p *Pss) getPeerAddress(keyid string, topic Topic) (PssAddress, error) { // encapsulating the decrypted message, and the whisper backend id // of the symmetric key used to decrypt the message. // It fails if decryption of the message fails or if the message is corrupted -func (p *Pss) processSym(envelope *whisper.Envelope) (*whisper.ReceivedMessage, string, *PssAddress, error) { +func (p *Pss) processSym(envelope *whisper.Envelope) (*whisper.ReceivedMessage, string, PssAddress, error) { metrics.GetOrRegisterCounter("pss.process.sym", nil).Inc(1) for i := p.symKeyDecryptCacheCursor; i > p.symKeyDecryptCacheCursor-cap(p.symKeyDecryptCache) && i > 0; i-- { @@ -677,7 +685,7 @@ func (p *Pss) processSym(envelope *whisper.Envelope) (*whisper.ReceivedMessage, // encapsulating the decrypted message, and the byte representation of // the public key used to decrypt the message. // It fails if decryption of message fails, or if the message is corrupted -func (p *Pss) processAsym(envelope *whisper.Envelope) (*whisper.ReceivedMessage, string, *PssAddress, error) { +func (p *Pss) processAsym(envelope *whisper.Envelope) (*whisper.ReceivedMessage, string, PssAddress, error) { metrics.GetOrRegisterCounter("pss.process.asym", nil).Inc(1) recvmsg, err := envelope.OpenAsymmetric(p.privateKey) @@ -689,7 +697,7 @@ func (p *Pss) processAsym(envelope *whisper.Envelope) (*whisper.ReceivedMessage, return nil, "", nil, fmt.Errorf("invalid message") } pubkeyid := common.ToHex(crypto.FromECDSAPub(recvmsg.Src)) - var from *PssAddress + var from PssAddress p.pubKeyPoolMu.Lock() if p.pubKeyPool[pubkeyid][Topic(envelope.Topic)] != nil { from = p.pubKeyPool[pubkeyid][Topic(envelope.Topic)].address @@ -751,6 +759,9 @@ func (p *Pss) enqueue(msg *PssMsg) error { // // Will fail if raw messages are disallowed func (p *Pss) SendRaw(address PssAddress, topic Topic, msg []byte) error { + if err := validateAddress(address); err != nil { + return err + } pssMsgParams := &msgParams{ raw: true, } @@ -770,8 +781,10 @@ func (p *Pss) SendRaw(address PssAddress, topic Topic, msg []byte) error { // if we have a proxhandler on this topic // also deliver message to ourselves - if p.isSelfPossibleRecipient(pssMsg, true) && p.topicHandlerCaps[topic].prox { - return p.process(pssMsg, true, true) + if _, ok := p.topicHandlerCaps[topic]; ok { + if p.isSelfPossibleRecipient(pssMsg, true) && p.topicHandlerCaps[topic].prox { + return p.process(pssMsg, true, true) + } } return nil } @@ -789,11 +802,8 @@ func (p *Pss) SendSym(symkeyid string, topic Topic, msg []byte) error { p.symKeyPoolMu.Unlock() if !ok { return fmt.Errorf("invalid topic '%s' for symkey '%s'", topic.String(), symkeyid) - } else if psp.address == nil { - return fmt.Errorf("no address hint for topic '%s' symkey '%s'", topic.String(), symkeyid) } - err = p.send(*psp.address, topic, msg, false, symkey) - return err + return p.send(psp.address, topic, msg, false, symkey) } // Send a message using asymmetric encryption @@ -808,13 +818,8 @@ func (p *Pss) SendAsym(pubkeyid string, topic Topic, msg []byte) error { p.pubKeyPoolMu.Unlock() if !ok { return fmt.Errorf("invalid topic '%s' for pubkey '%s'", topic.String(), pubkeyid) - } else if psp.address == nil { - return fmt.Errorf("no address hint for topic '%s' pubkey '%s'", topic.String(), pubkeyid) } - go func() { - p.send(*psp.address, topic, msg, true, common.FromHex(pubkeyid)) - }() - return nil + return p.send(psp.address, topic, msg, true, common.FromHex(pubkeyid)) } // Send is payload agnostic, and will accept any byte slice as payload @@ -1034,3 +1039,10 @@ func (p *Pss) digestBytes(msg []byte) pssDigest { copy(digest[:], key[:digestLength]) return digest } + +func validateAddress(addr PssAddress) error { + if len(addr) > addressLength { + return errors.New("address too long") + } + return nil +} diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index 72f62acd94..ec46504c23 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -407,7 +407,7 @@ func TestProxShortCircuit(t *testing.T) { // try the same prox message with sym and asym send proxAddrPss := PssAddress(proxMessageAddress) - symKeyId, err := ps.GenerateSymmetricKey(topic, &proxAddrPss, true) + symKeyId, err := ps.GenerateSymmetricKey(topic, proxAddrPss, true) go func() { err := ps.SendSym(symKeyId, topic, []byte("baz")) if err != nil { @@ -424,7 +424,7 @@ func TestProxShortCircuit(t *testing.T) { t.Fatal("sym timeout") } - err = ps.SetPeerPublicKey(&privKey.PublicKey, topic, &proxAddrPss) + err = ps.SetPeerPublicKey(&privKey.PublicKey, topic, proxAddrPss) if err != nil { t.Fatal(err) } @@ -786,14 +786,14 @@ func TestKeys(t *testing.T) { copy(addr, network.RandomAddr().Over()) outkey := network.RandomAddr().Over() topicobj := BytesToTopic([]byte("foo:42")) - ps.SetPeerPublicKey(&theirprivkey.PublicKey, topicobj, &addr) - outkeyid, err := ps.SetSymmetricKey(outkey, topicobj, &addr, false) + ps.SetPeerPublicKey(&theirprivkey.PublicKey, topicobj, addr) + outkeyid, err := ps.SetSymmetricKey(outkey, topicobj, addr, false) if err != nil { t.Fatalf("failed to set 'our' outgoing symmetric key") } // make a symmetric key that we will send to peer for encrypting messages to us - inkeyid, err := ps.GenerateSymmetricKey(topicobj, &addr, true) + inkeyid, err := ps.GenerateSymmetricKey(topicobj, addr, true) if err != nil { t.Fatalf("failed to set 'our' incoming symmetric key") } @@ -816,8 +816,8 @@ func TestKeys(t *testing.T) { // check that the key is stored in the peerpool psp := ps.symKeyPool[inkeyid][topicobj] - if psp.address != &addr { - t.Fatalf("inkey address does not match; %p != %p", psp.address, &addr) + if !bytes.Equal(psp.address, addr) { + t.Fatalf("inkey address does not match; %p != %p", psp.address, addr) } } @@ -1008,6 +1008,34 @@ func TestRawAllow(t *testing.T) { } } +// BELOW HERE ARE TESTS USING THE SIMULATION FRAMEWORK + +// tests that the API layer can handle edge case values +func TestApi(t *testing.T) { + clients, err := setupNetwork(2, true) + if err != nil { + t.Fatal(err) + } + + topic := "0xdeadbeef" + + err = clients[0].Call(nil, "pss_sendRaw", "0x", topic, "0x666f6f") + if err != nil { + t.Fatal(err) + } + + err = clients[0].Call(nil, "pss_sendRaw", "0xabcdef", topic, "0x") + if err == nil { + t.Fatal("expected error on empty msg") + } + + overflowAddr := [33]byte{} + err = clients[0].Call(nil, "pss_sendRaw", hexutil.Encode(overflowAddr[:]), topic, "0x666f6f") + if err == nil { + t.Fatal("expected error on send too big address") + } +} + // verifies that nodes can send and receive raw (verbatim) messages func TestSendRaw(t *testing.T) { t.Run("32", testSendRaw) @@ -1668,7 +1696,7 @@ func benchmarkSymKeySend(b *testing.B) { topic := BytesToTopic([]byte("foo")) to := make(PssAddress, 32) copy(to[:], network.RandomAddr().Over()) - symkeyid, err := ps.GenerateSymmetricKey(topic, &to, true) + symkeyid, err := ps.GenerateSymmetricKey(topic, to, true) if err != nil { b.Fatalf("could not generate symkey: %v", err) } @@ -1676,7 +1704,7 @@ func benchmarkSymKeySend(b *testing.B) { if err != nil { b.Fatalf("could not retrieve symkey: %v", err) } - ps.SetSymmetricKey(symkey, topic, &to, false) + ps.SetSymmetricKey(symkey, topic, to, false) b.ResetTimer() for i := 0; i < b.N; i++ { @@ -1712,7 +1740,7 @@ func benchmarkAsymKeySend(b *testing.B) { topic := BytesToTopic([]byte("foo")) to := make(PssAddress, 32) copy(to[:], network.RandomAddr().Over()) - ps.SetPeerPublicKey(&privkey.PublicKey, topic, &to) + ps.SetPeerPublicKey(&privkey.PublicKey, topic, to) b.ResetTimer() for i := 0; i < b.N; i++ { ps.SendAsym(common.ToHex(crypto.FromECDSAPub(&privkey.PublicKey)), topic, msg) @@ -1761,7 +1789,7 @@ func benchmarkSymkeyBruteforceChangeaddr(b *testing.B) { for i := 0; i < int(keycount); i++ { to := make(PssAddress, 32) copy(to[:], network.RandomAddr().Over()) - keyid, err = ps.GenerateSymmetricKey(topic, &to, true) + keyid, err = ps.GenerateSymmetricKey(topic, to, true) if err != nil { b.Fatalf("cant generate symkey #%d: %v", i, err) } @@ -1843,7 +1871,7 @@ func benchmarkSymkeyBruteforceSameaddr(b *testing.B) { topic := BytesToTopic([]byte("foo")) for i := 0; i < int(keycount); i++ { copy(addr[i], network.RandomAddr().Over()) - keyid, err = ps.GenerateSymmetricKey(topic, &addr[i], true) + keyid, err = ps.GenerateSymmetricKey(topic, addr[i], true) if err != nil { b.Fatalf("cant generate symkey #%d: %v", i, err) } @@ -2044,12 +2072,13 @@ func NewAPITest(ps *Pss) *APITest { return &APITest{Pss: ps} } -func (apitest *APITest) SetSymKeys(pubkeyid string, recvsymkey []byte, sendsymkey []byte, limit uint16, topic Topic, to PssAddress) ([2]string, error) { - recvsymkeyid, err := apitest.SetSymmetricKey(recvsymkey, topic, &to, true) +func (apitest *APITest) SetSymKeys(pubkeyid string, recvsymkey []byte, sendsymkey []byte, limit uint16, topic Topic, to hexutil.Bytes) ([2]string, error) { + + recvsymkeyid, err := apitest.SetSymmetricKey(recvsymkey, topic, PssAddress(to), true) if err != nil { return [2]string{}, err } - sendsymkeyid, err := apitest.SetSymmetricKey(sendsymkey, topic, &to, false) + sendsymkeyid, err := apitest.SetSymmetricKey(sendsymkey, topic, PssAddress(to), false) if err != nil { return [2]string{}, err } From fe86a707d8270108e0fb77359e702689664dcb4b Mon Sep 17 00:00:00 2001 From: Javier Peletier Date: Tue, 18 Dec 2018 15:25:02 +0100 Subject: [PATCH 159/159] swarm/storage: remove unused methods from Chunk interface (#18283) --- swarm/storage/common_test.go | 5 +++-- swarm/storage/memstore.go | 2 +- swarm/storage/types.go | 18 ------------------ 3 files changed, 4 insertions(+), 21 deletions(-) diff --git a/swarm/storage/common_test.go b/swarm/storage/common_test.go index af104a5aeb..bcc29d8cc7 100644 --- a/swarm/storage/common_test.go +++ b/swarm/storage/common_test.go @@ -179,8 +179,9 @@ func testStoreCorrect(m ChunkStore, n int, chunksize int64, t *testing.T) { return fmt.Errorf("key does not match retrieved chunk Address") } hasher := MakeHashFunc(DefaultHash)() - hasher.ResetWithLength(chunk.SpanBytes()) - hasher.Write(chunk.Payload()) + data := chunk.Data() + hasher.ResetWithLength(data[:8]) + hasher.Write(data[8:]) exp := hasher.Sum(nil) if !bytes.Equal(h, exp) { return fmt.Errorf("key is not hash of chunk data") diff --git a/swarm/storage/memstore.go b/swarm/storage/memstore.go index 36b1e00d9b..86e5813d1b 100644 --- a/swarm/storage/memstore.go +++ b/swarm/storage/memstore.go @@ -57,7 +57,7 @@ func (m *MemStore) Get(_ context.Context, addr Address) (Chunk, error) { if !ok { return nil, ErrChunkNotFound } - return c.(*chunk), nil + return c.(Chunk), nil } func (m *MemStore) Put(_ context.Context, c Chunk) error { diff --git a/swarm/storage/types.go b/swarm/storage/types.go index e5c5f89a09..322d95c47a 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -184,9 +184,6 @@ func (c AddressCollection) Swap(i, j int) { // Chunk interface implemented by context.Contexts and data chunks type Chunk interface { Address() Address - Payload() []byte - SpanBytes() []byte - Span() int64 Data() []byte } @@ -208,25 +205,10 @@ func (c *chunk) Address() Address { return c.addr } -func (c *chunk) SpanBytes() []byte { - return c.sdata[:8] -} - -func (c *chunk) Span() int64 { - if c.span == -1 { - c.span = int64(binary.LittleEndian.Uint64(c.sdata[:8])) - } - return c.span -} - func (c *chunk) Data() []byte { return c.sdata } -func (c *chunk) Payload() []byte { - return c.sdata[8:] -} - // String() for pretty printing func (self *chunk) String() string { return fmt.Sprintf("Address: %v TreeSize: %v Chunksize: %v", self.addr.Log(), self.span, len(self.sdata))