From 5af18e3e6b6b48298afe57da71b7773969cb3de0 Mon Sep 17 00:00:00 2001 From: Fabio Barone Date: Tue, 30 Jan 2018 14:42:01 -0500 Subject: [PATCH] swarm: finalise metrics after feedback --- swarm/api/api.go | 38 +++++++++------------- swarm/api/http/server.go | 51 +++++++++++++++++++----------- swarm/network/kademlia/kademlia.go | 31 +++++++++++++++--- 3 files changed, 73 insertions(+), 47 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index aca5ca7750..1cafa76339 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -40,23 +40,21 @@ var hashMatcher = regexp.MustCompile("^[0-9A-Fa-f]{64}") //setup metrics var ( - apiResolveCount = metrics.NewCounter("api.resolve.count") - apiResolveFail = metrics.NewCounter("api.resolve.fail") - apiPutCount = metrics.NewCounter("api.put.count") - apiPutFail = metrics.NewCounter("api.put.fail") - apiGetCount = metrics.NewCounter("api.get.count") - apiGetNotFound = metrics.NewCounter("api.get.notfound") - apiGetHttp300 = metrics.NewCounter("api.get.http.300") - apiModifyCount = metrics.NewCounter("api.modify.count") - apiModifyFail = metrics.NewCounter("api.modify.fail") - apiAddFileCount = metrics.NewCounter("api.addfile.count") - apiAddFileFail = metrics.NewCounter("api.addfile.fail") - apiRmFileCount = metrics.NewCounter("api.removefile.count") - apiRmFileFail = metrics.NewCounter("api.removefile.fail") - apiAppendFileCount = metrics.NewCounter("api.appendfile.count") - apiAppendFileFail = metrics.NewCounter("api.appendfile.fail") - apiBuildDirTreeCount = metrics.NewCounter("api.builddirtree.count") - apiBuildDirTreeFail = metrics.NewCounter("api.builddirtree.fail") + apiResolveCount = metrics.NewCounter("api.resolve.count") + apiResolveFail = metrics.NewCounter("api.resolve.fail") + apiPutCount = metrics.NewCounter("api.put.count") + apiPutFail = metrics.NewCounter("api.put.fail") + apiGetCount = metrics.NewCounter("api.get.count") + apiGetNotFound = metrics.NewCounter("api.get.notfound") + apiGetHttp300 = metrics.NewCounter("api.get.http.300") + apiModifyCount = metrics.NewCounter("api.modify.count") + apiModifyFail = metrics.NewCounter("api.modify.fail") + apiAddFileCount = metrics.NewCounter("api.addfile.count") + apiAddFileFail = metrics.NewCounter("api.addfile.fail") + apiRmFileCount = metrics.NewCounter("api.removefile.count") + apiRmFileFail = metrics.NewCounter("api.removefile.fail") + apiAppendFileCount = metrics.NewCounter("api.appendfile.count") + apiAppendFileFail = metrics.NewCounter("api.appendfile.fail") ) type Resolver interface { @@ -184,7 +182,6 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) { isHash := hashMatcher.MatchString(uri.Addr) if uri.Immutable() || uri.DeprecatedImmutable() { if !isHash { - apiResolveFail.Inc(1) return nil, fmt.Errorf("immutable address not a content hash: %q", uri.Addr) } return common.Hex2Bytes(uri.Addr), nil @@ -467,23 +464,19 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte } func (self *Api) BuildDirectoryTree(mhash string, nameresolver bool) (key storage.Key, manifestEntryMap map[string]*manifestTrieEntry, err error) { - apiBuildDirTreeCount.Inc(1) uri, err := Parse("bzz:/" + mhash) if err != nil { - apiBuildDirTreeFail.Inc(1) return nil, nil, err } key, err = self.Resolve(uri) if err != nil { - apiBuildDirTreeFail.Inc(1) return nil, nil, err } quitC := make(chan bool) rootTrie, err := loadManifest(self.dpa, key, quitC) if err != nil { - apiBuildDirTreeFail.Inc(1) return nil, nil, fmt.Errorf("can't load manifest %v: %v", key.String(), err) } @@ -493,7 +486,6 @@ func (self *Api) BuildDirectoryTree(mhash string, nameresolver bool) (key storag }) if err != nil { - apiBuildDirTreeFail.Inc(1) return nil, nil, fmt.Errorf("list with prefix failed %v: %v", key.String(), err) } return key, manifestEntryMap, nil diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index f27d6d891c..4bad4a1f15 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -45,23 +45,25 @@ import ( //setup metrics var ( - postRawCount = metrics.NewCounter("api.http.post.raw.count") - postRawFail = metrics.NewCounter("api.http.post.raw.fail") - postFilesCount = metrics.NewCounter("api.http.post.files.count") - postFilesFail = metrics.NewCounter("api.http.post.files.fail") - deleteCount = metrics.NewCounter("api.http.delete.count") - deleteFail = metrics.NewCounter("api.http.delete.fail") - getCount = metrics.NewCounter("api.http.get.count") - getFail = metrics.NewCounter("api.http.get.fail") - getFileCount = metrics.NewCounter("api.http.get.file.count") - getFileNotFound = metrics.NewCounter("api.http.get.file.notfound") - getFileFail = metrics.NewCounter("api.http.get.file.fail") - getFilesCount = metrics.NewCounter("api.http.get.files.count") - getFilesFail = metrics.NewCounter("api.http.get.files.fail") - getListCount = metrics.NewCounter("api.http.get.list.count") - getListFail = metrics.NewCounter("api.http.get.list.fail") - requestCount = metrics.NewCounter("http.request.count") - requestTimer = metrics.NewResettingTimer("http.request.time") + postRawCount = metrics.NewCounter("api.http.post.raw.count") + postRawFail = metrics.NewCounter("api.http.post.raw.fail") + postFilesCount = metrics.NewCounter("api.http.post.files.count") + postFilesFail = metrics.NewCounter("api.http.post.files.fail") + deleteCount = metrics.NewCounter("api.http.delete.count") + deleteFail = metrics.NewCounter("api.http.delete.fail") + getCount = metrics.NewCounter("api.http.get.count") + getFail = metrics.NewCounter("api.http.get.fail") + getFileCount = metrics.NewCounter("api.http.get.file.count") + getFileNotFound = metrics.NewCounter("api.http.get.file.notfound") + getFileFail = metrics.NewCounter("api.http.get.file.fail") + getFilesCount = metrics.NewCounter("api.http.get.files.count") + getFilesFail = metrics.NewCounter("api.http.get.files.fail") + getListCount = metrics.NewCounter("api.http.get.list.count") + getListFail = metrics.NewCounter("api.http.get.list.fail") + requestCount = metrics.NewCounter("http.request.count") + htmlRequestCount = metrics.NewCounter("http.request.html.count") + jsonRequestCount = metrics.NewCounter("http.request.json.count") + requestTimer = metrics.NewResettingTimer("http.request.time") ) // ServerConfig is the basic configuration needed for the HTTP server and also @@ -643,8 +645,19 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { } func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { - requestCount.Inc(1) - defer requestTimer.UpdateSince(time.Now()) + if metrics.Enabled { + //The increment for request count and request timer themselves have a flag check + //for metrics.Enabled. Nevertheless, we introduce the if here because we + //are looking into the header just to see what request type it is (json/html). + //So let's take advantage and add all metrics related stuff here + requestCount.Inc(1) + defer requestTimer.UpdateSince(time.Now()) + if r.Header.Get("Accept") == "application/json" { + jsonRequestCount.Inc(1) + } else { + htmlRequestCount.Inc(1) + } + } s.logDebug("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, r.URL.Host, r.URL.Path, r.Referer(), r.Header.Get("Accept")) uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/")) diff --git a/swarm/network/kademlia/kademlia.go b/swarm/network/kademlia/kademlia.go index af0228d4e0..675c0bdb6b 100644 --- a/swarm/network/kademlia/kademlia.go +++ b/swarm/network/kademlia/kademlia.go @@ -25,12 +25,16 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" + gometrics "github.com/ethersphere/go-metrics" ) //metrics variables +//For metrics, we want to count how many times peers are added/removed +//at a certain index. Thus we do that with an array of counters with +//entry for each index var ( - bucketOnIndexGauge = metrics.NewGauge("network.kademlia.bucket.onindex") - bucketOffIndexGauge = metrics.NewGauge("network.kademlia.bucket.offindex") + bucketAddIndexCount []gometrics.Counter + bucketRmIndexCount []gometrics.Counter ) const ( @@ -95,12 +99,17 @@ type Node interface { // params is KadParams configuration func New(addr Address, params *KadParams) *Kademlia { buckets := make([][]Node, params.MaxProx+1) - return &Kademlia{ + kad := &Kademlia{ addr: addr, KadParams: params, buckets: buckets, db: newKadDb(addr, params), } + //if metrics are enabled, initialise the array of counters + if metrics.Enabled { + kad.initMetricsVariables() + } + return kad } // accessor for KAD base address @@ -145,7 +154,7 @@ func (self *Kademlia) On(node Node, cb func(*NodeRecord, Node) error) (err error // TODO: give priority to peers with active traffic if len(bucket) < self.BucketSize { // >= allows us to add peers beyond the bucketsize limitation self.buckets[index] = append(bucket, node) - bucketOnIndexGauge.Update(int64(index)) + bucketAddIndexCount[index].Inc(1) log.Debug(fmt.Sprintf("add node %v to table", node)) self.setProxLimit(index, true) record.node = node @@ -186,7 +195,7 @@ func (self *Kademlia) Off(node Node, cb func(*NodeRecord, Node)) (err error) { defer self.lock.Unlock() index := self.proximityBin(node.Addr()) - bucketOffIndexGauge.Update(int64(index)) + bucketRmIndexCount[index].Inc(1) bucket := self.buckets[index] for i := 0; i < len(bucket); i++ { if node.Addr() == bucket[i].Addr() { @@ -435,3 +444,15 @@ func (self *Kademlia) String() string { rows = append(rows, "=========================================================================") return strings.Join(rows, "\n") } + +//We have to build up the array of counters for each index +func (self *Kademlia) initMetricsVariables() { + //create the arrays + bucketAddIndexCount = make([]gometrics.Counter, self.MaxProx+1) + bucketRmIndexCount = make([]gometrics.Counter, self.MaxProx+1) + //at each index create a metrics counter + for i := 0; i < (self.KadParams.MaxProx); i++ { + bucketAddIndexCount[i] = metrics.NewCounter(fmt.Sprintf("network.kademlia.bucket.add.%d.index", i)) + bucketRmIndexCount[i] = metrics.NewCounter(fmt.Sprintf("network.kademlia.bucket.rm.%d.index", i)) + } +}