From 96968b119ee891e8e9266aab4fc42e5b9b4e0fb4 Mon Sep 17 00:00:00 2001 From: crypto-services Date: Thu, 15 Feb 2024 16:25:35 +0800 Subject: [PATCH 01/11] Initial cut --- cmd/geth/config.go | 3 + cmd/utils/flags.go | 9 ++ health/check_block.go | 17 ++++ health/check_peers.go | 24 +++++ health/check_synced.go | 26 ++++++ health/check_time.go | 31 +++++++ health/health.go | 199 +++++++++++++++++++++++++++++++++++++++++ health/service.go | 40 +++++++++ 8 files changed, 349 insertions(+) create mode 100644 health/check_block.go create mode 100644 health/check_peers.go create mode 100644 health/check_synced.go create mode 100644 health/check_time.go create mode 100644 health/health.go create mode 100644 health/service.go diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 5f52f1df54..c4d0981216 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -196,6 +196,9 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { // Configure log filter RPC API. filterSystem := utils.RegisterFilterAPI(stack, backend, &cfg.Eth) + // TODO: ENG191 add flag to request endpoint + utils.RegisterHealthService(stack, backend, &cfg.Node) + // Configure GraphQL if requested. if ctx.IsSet(utils.GraphQLEnabledFlag.Name) { utils.RegisterGraphQLService(stack, backend, filterSystem, &cfg.Node) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 159c47ca01..38a2aaf580 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -55,6 +55,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb/remotedb" "github.com/ethereum/go-ethereum/ethstats" "github.com/ethereum/go-ethereum/graphql" + "github.com/ethereum/go-ethereum/health" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" @@ -1883,6 +1884,14 @@ func RegisterGraphQLService(stack *node.Node, backend ethapi.Backend, filterSyst } } +// RegisterHealthService adds the Health API to the node. +func RegisterHealthService(stack *node.Node, backend ethapi.Backend, cfg *node.Config) { + err := health.New(stack, backend, cfg.GraphQLCors, cfg.HTTPVirtualHosts) + if err != nil { + Fatalf("Failed to register the health service: %v", err) + } +} + // RegisterFilterAPI adds the eth log filtering RPC API to the node. func RegisterFilterAPI(stack *node.Node, backend ethapi.Backend, ethcfg *ethconfig.Config) *filters.FilterSystem { filterSystem := filters.NewFilterSystem(backend, filters.Config{ diff --git a/health/check_block.go b/health/check_block.go new file mode 100644 index 0000000000..f8e95ddcae --- /dev/null +++ b/health/check_block.go @@ -0,0 +1,17 @@ +package health + +import ( + "context" + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/ethclient" +) + +func checkBlockNumber(ec *ethclient.Client, blockNumber *big.Int) error { + _, err := ec.BlockByNumber(context.TODO(), blockNumber) + if err != nil { + return fmt.Errorf("no known block with number %v (%x hex)", blockNumber.Int64(), blockNumber.Int64()) + } + return nil +} diff --git a/health/check_peers.go b/health/check_peers.go new file mode 100644 index 0000000000..d4b47cbe6d --- /dev/null +++ b/health/check_peers.go @@ -0,0 +1,24 @@ +package health + +import ( + "context" + "errors" + "fmt" + + "github.com/ethereum/go-ethereum/ethclient" +) + +var ( + errNotEnoughPeers = errors.New("not enough peers") +) + +func checkMinPeers(ec *ethclient.Client, minPeerCount uint) error { + peerCount, err := ec.PeerCount(context.TODO()) + if err != nil { + return err + } + if uint64(peerCount) < uint64(minPeerCount) { + return fmt.Errorf("%w: %d (minimum %d)", errNotEnoughPeers, peerCount, minPeerCount) + } + return nil +} diff --git a/health/check_synced.go b/health/check_synced.go new file mode 100644 index 0000000000..78d3d025cf --- /dev/null +++ b/health/check_synced.go @@ -0,0 +1,26 @@ +package health + +import ( + "errors" + "net/http" + + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/log" +) + +var ( + errNotSynced = errors.New("not synced") +) + +func checkSynced(ec *ethclient.Client, r *http.Request) error { + i, err := ec.SyncProgress(r.Context()) + if err != nil { + log.Root().Warn("Unable to check sync status for healthcheck", "err", err.Error()) + return err + } + if i == nil { + return nil + } + + return errNotSynced +} diff --git a/health/check_time.go b/health/check_time.go new file mode 100644 index 0000000000..a1a7d820a7 --- /dev/null +++ b/health/check_time.go @@ -0,0 +1,31 @@ +package health + +import ( + "context" + "errors" + "fmt" + "net/http" + + "github.com/ethereum/go-ethereum/ethclient" +) + +var ( + errTimestampTooOld = errors.New("timestamp too old") +) + +func checkTime( + ec *ethclient.Client, + r *http.Request, + seconds int, +) error { + i, err := ec.BlockByNumber(context.TODO(), nil) + if err != nil { + return err + } + timestamp := i.Time() + if timestamp < uint64(seconds) { + return fmt.Errorf("%w: got ts: %d, need: %d", errTimestampTooOld, timestamp, seconds) + } + + return nil +} diff --git a/health/health.go b/health/health.go new file mode 100644 index 0000000000..78ff97bfe5 --- /dev/null +++ b/health/health.go @@ -0,0 +1,199 @@ +package health + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "math/big" + "net/http" + "strconv" + "strings" + "time" + + "github.com/ethereum/go-ethereum/log" +) + +const ( + healthHeader = "X-GETH-HEALTHCHECK" + synced = "synced" + minPeerCount = "min_peer_count" + checkBlock = "check_block" + maxSecondsBehind = "max_seconds_behind" +) + +var ( + errCheckDisabled = errors.New("error check disabled") + errBadHeaderValue = errors.New("bad header value") +) + +type requestBody struct { + Synced *bool `json:"synced"` + MinPeerCount *uint `json:"min_peer_count"` + CheckBlock *uint64 `json:"check_block"` + MaxSecondsBehind *int `json:"max_seconds_behind"` +} + +func (h *handler) processFromHeaders(headers []string, w http.ResponseWriter, r *http.Request) { + var ( + errCheckSynced = errCheckDisabled + errCheckPeer = errCheckDisabled + errCheckBlock = errCheckDisabled + errCheckSeconds = errCheckDisabled + ) + + for _, header := range headers { + lHeader := strings.ToLower(header) + if lHeader == synced { + errCheckSynced = checkSynced(h.ec, r) + } + if strings.HasPrefix(lHeader, minPeerCount) { + peers, err := strconv.Atoi(strings.TrimPrefix(lHeader, minPeerCount)) + if err != nil { + errCheckPeer = err + break + } + errCheckPeer = checkMinPeers(h.ec, uint(peers)) + } + if strings.HasPrefix(lHeader, checkBlock) { + block, err := strconv.Atoi(strings.TrimPrefix(lHeader, checkBlock)) + if err != nil { + errCheckBlock = err + break + } + errCheckBlock = checkBlockNumber(h.ec, big.NewInt(int64(block))) + } + if strings.HasPrefix(lHeader, maxSecondsBehind) { + seconds, err := strconv.Atoi(strings.TrimPrefix(lHeader, maxSecondsBehind)) + if err != nil { + errCheckSeconds = err + break + } + if seconds < 0 { + errCheckSeconds = errBadHeaderValue + break + } + now := time.Now().Unix() + errCheckSeconds = checkTime(h.ec, r, int(now)-seconds) + } + } + + reportHealth(errCheckSynced, errCheckPeer, errCheckBlock, errCheckSeconds, w) +} + +func (h *handler) processFromBody(w http.ResponseWriter, r *http.Request) { + body, errParse := parseHealthCheckBody(r.Body) + defer r.Body.Close() + + var ( + errCheckSynced = errCheckDisabled + errCheckPeer = errCheckDisabled + errCheckBlock = errCheckDisabled + errCheckSeconds = errCheckDisabled + ) + + if errParse != nil { + log.Root().Warn("Unable to process healthcheck request", "err", errParse) + } else { + if body.Synced != nil { + errCheckSynced = checkSynced(h.ec, r) + } + + if body.MinPeerCount != nil { + errCheckPeer = checkMinPeers(h.ec, *body.MinPeerCount) + } + + if body.CheckBlock != nil { + errCheckBlock = checkBlockNumber(h.ec, big.NewInt(int64(*body.CheckBlock))) + } + + if body.MaxSecondsBehind != nil { + seconds := *body.MaxSecondsBehind + if seconds < 0 { + errCheckSeconds = errBadHeaderValue + } + now := time.Now().Unix() + errCheckSeconds = checkTime(h.ec, r, int(now)-seconds) + } + } + + err := reportHealth(errCheckSynced, errCheckPeer, errCheckBlock, errCheckSeconds, w) + if err != nil { + log.Root().Warn("Unable to process healthcheck request", "err", err) + } +} + +func reportHealth(errCheckSynced, errCheckPeer, errCheckBlock, errCheckSeconds error, w http.ResponseWriter) error { + statusCode := http.StatusOK + errs := make(map[string]string) + + if shouldChangeStatusCode(errCheckSynced) { + statusCode = http.StatusInternalServerError + } + errs[synced] = errorStringOrOK(errCheckSynced) + + if shouldChangeStatusCode(errCheckPeer) { + statusCode = http.StatusInternalServerError + } + errs[minPeerCount] = errorStringOrOK(errCheckPeer) + + if shouldChangeStatusCode(errCheckBlock) { + statusCode = http.StatusInternalServerError + } + errs[checkBlock] = errorStringOrOK(errCheckBlock) + + if shouldChangeStatusCode(errCheckSeconds) { + statusCode = http.StatusInternalServerError + } + errs[maxSecondsBehind] = errorStringOrOK(errCheckSeconds) + + return writeResponse(w, errs, statusCode) +} + +func parseHealthCheckBody(reader io.Reader) (requestBody, error) { + var body requestBody + + bodyBytes, err := io.ReadAll(reader) + if err != nil { + return body, err + } + + err = json.Unmarshal(bodyBytes, &body) + if err != nil { + return body, err + } + + return body, nil +} + +func writeResponse(w http.ResponseWriter, errs map[string]string, statusCode int) error { + w.WriteHeader(statusCode) + + bodyJson, err := json.Marshal(errs) + if err != nil { + return err + } + + _, err = w.Write(bodyJson) + if err != nil { + return err + } + + return nil +} + +func shouldChangeStatusCode(err error) bool { + return err != nil && !errors.Is(err, errCheckDisabled) +} + +func errorStringOrOK(err error) string { + if err == nil { + return "HEALTHY" + } + + if errors.Is(err, errCheckDisabled) { + return "DISABLED" + } + + return fmt.Sprintf("ERROR: %v", err) +} diff --git a/health/service.go b/health/service.go new file mode 100644 index 0000000000..c39c31d2d5 --- /dev/null +++ b/health/service.go @@ -0,0 +1,40 @@ +package health + +import ( + "net/http" + + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/internal/ethapi" + "github.com/ethereum/go-ethereum/node" +) + +type handler struct { + ec *ethclient.Client +} + +func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + headers := r.Header.Values(healthHeader) + if len(headers) != 0 { + h.processFromHeaders(headers, w, r) + } else { + h.processFromBody(w, r) + } +} + +// New constructs a new health service instance. +func New(stack *node.Node, backend ethapi.Backend, cors, vhosts []string) error { + _, err := newHandler(stack, backend, cors, vhosts) + return err +} + +// newHandler returns a new `http.Handler` that will answer node health queries. +func newHandler(stack *node.Node, backend ethapi.Backend, cors, vhosts []string) (*handler, error) { + ec := ethclient.NewClient(stack.Attach()) + h := handler{ec} + handler := node.NewHTTPHandlerStack(h, cors, vhosts, nil) + + stack.RegisterHandler("Health API", "/health", handler) + stack.RegisterHandler("Health API", "/health/", handler) + + return &h, nil +} From bb6215bce5ca68e16c8d5332d6d7b6ef133561bd Mon Sep 17 00:00:00 2001 From: crypto-services Date: Thu, 15 Feb 2024 16:27:42 +0800 Subject: [PATCH 02/11] Update cors to http --- cmd/utils/flags.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 38a2aaf580..8186d40f7f 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1886,7 +1886,7 @@ func RegisterGraphQLService(stack *node.Node, backend ethapi.Backend, filterSyst // RegisterHealthService adds the Health API to the node. func RegisterHealthService(stack *node.Node, backend ethapi.Backend, cfg *node.Config) { - err := health.New(stack, backend, cfg.GraphQLCors, cfg.HTTPVirtualHosts) + err := health.New(stack, backend, cfg.HTTPCors, cfg.HTTPVirtualHosts) if err != nil { Fatalf("Failed to register the health service: %v", err) } From 3ded1c14c3f67e3500c1dcc4300f007c039b0d3f Mon Sep 17 00:00:00 2001 From: crypto-services Date: Thu, 15 Feb 2024 16:45:06 +0800 Subject: [PATCH 03/11] Remove unused backend --- cmd/geth/config.go | 2 +- cmd/utils/flags.go | 4 ++-- health/service.go | 7 +++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index c4d0981216..befca177f7 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -197,7 +197,7 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { filterSystem := utils.RegisterFilterAPI(stack, backend, &cfg.Eth) // TODO: ENG191 add flag to request endpoint - utils.RegisterHealthService(stack, backend, &cfg.Node) + utils.RegisterHealthService(stack, &cfg.Node) // Configure GraphQL if requested. if ctx.IsSet(utils.GraphQLEnabledFlag.Name) { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 8186d40f7f..0f63c2e6a3 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1885,8 +1885,8 @@ func RegisterGraphQLService(stack *node.Node, backend ethapi.Backend, filterSyst } // RegisterHealthService adds the Health API to the node. -func RegisterHealthService(stack *node.Node, backend ethapi.Backend, cfg *node.Config) { - err := health.New(stack, backend, cfg.HTTPCors, cfg.HTTPVirtualHosts) +func RegisterHealthService(stack *node.Node, cfg *node.Config) { + err := health.New(stack, cfg.HTTPCors, cfg.HTTPVirtualHosts) if err != nil { Fatalf("Failed to register the health service: %v", err) } diff --git a/health/service.go b/health/service.go index c39c31d2d5..f1aba69042 100644 --- a/health/service.go +++ b/health/service.go @@ -4,7 +4,6 @@ import ( "net/http" "github.com/ethereum/go-ethereum/ethclient" - "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/node" ) @@ -22,13 +21,13 @@ func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } // New constructs a new health service instance. -func New(stack *node.Node, backend ethapi.Backend, cors, vhosts []string) error { - _, err := newHandler(stack, backend, cors, vhosts) +func New(stack *node.Node, cors, vhosts []string) error { + _, err := newHandler(stack, cors, vhosts) return err } // newHandler returns a new `http.Handler` that will answer node health queries. -func newHandler(stack *node.Node, backend ethapi.Backend, cors, vhosts []string) (*handler, error) { +func newHandler(stack *node.Node, cors, vhosts []string) (*handler, error) { ec := ethclient.NewClient(stack.Attach()) h := handler{ec} handler := node.NewHTTPHandlerStack(h, cors, vhosts, nil) From cc3cd2f7753fd1dfee0847521ec962a7135486b1 Mon Sep 17 00:00:00 2001 From: crypto-services Date: Tue, 20 Feb 2024 14:35:41 +0800 Subject: [PATCH 04/11] Add testing --- health/check_block.go | 7 +- health/check_peers.go | 4 +- health/check_synced.go | 3 +- health/check_time.go | 4 +- health/health.go | 45 +-- health/health_test.go | 742 +++++++++++++++++++++++++++++++++++++++++ health/interface.go | 15 + health/service.go | 4 +- 8 files changed, 790 insertions(+), 34 deletions(-) create mode 100644 health/health_test.go create mode 100644 health/interface.go diff --git a/health/check_block.go b/health/check_block.go index f8e95ddcae..a68b560100 100644 --- a/health/check_block.go +++ b/health/check_block.go @@ -2,16 +2,13 @@ package health import ( "context" - "fmt" "math/big" - - "github.com/ethereum/go-ethereum/ethclient" ) -func checkBlockNumber(ec *ethclient.Client, blockNumber *big.Int) error { +func checkBlockNumber(ec ethClient, blockNumber *big.Int) error { _, err := ec.BlockByNumber(context.TODO(), blockNumber) if err != nil { - return fmt.Errorf("no known block with number %v (%x hex)", blockNumber.Int64(), blockNumber.Int64()) + return err } return nil } diff --git a/health/check_peers.go b/health/check_peers.go index d4b47cbe6d..e20973348c 100644 --- a/health/check_peers.go +++ b/health/check_peers.go @@ -4,15 +4,13 @@ import ( "context" "errors" "fmt" - - "github.com/ethereum/go-ethereum/ethclient" ) var ( errNotEnoughPeers = errors.New("not enough peers") ) -func checkMinPeers(ec *ethclient.Client, minPeerCount uint) error { +func checkMinPeers(ec ethClient, minPeerCount uint) error { peerCount, err := ec.PeerCount(context.TODO()) if err != nil { return err diff --git a/health/check_synced.go b/health/check_synced.go index 78d3d025cf..580b6a28fb 100644 --- a/health/check_synced.go +++ b/health/check_synced.go @@ -4,7 +4,6 @@ import ( "errors" "net/http" - "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" ) @@ -12,7 +11,7 @@ var ( errNotSynced = errors.New("not synced") ) -func checkSynced(ec *ethclient.Client, r *http.Request) error { +func checkSynced(ec ethClient, r *http.Request) error { i, err := ec.SyncProgress(r.Context()) if err != nil { log.Root().Warn("Unable to check sync status for healthcheck", "err", err.Error()) diff --git a/health/check_time.go b/health/check_time.go index a1a7d820a7..ae0df8dad4 100644 --- a/health/check_time.go +++ b/health/check_time.go @@ -5,8 +5,6 @@ import ( "errors" "fmt" "net/http" - - "github.com/ethereum/go-ethereum/ethclient" ) var ( @@ -14,7 +12,7 @@ var ( ) func checkTime( - ec *ethclient.Client, + ec ethClient, r *http.Request, seconds int, ) error { diff --git a/health/health.go b/health/health.go index 78ff97bfe5..402c02686a 100644 --- a/health/health.go +++ b/health/health.go @@ -16,6 +16,7 @@ import ( const ( healthHeader = "X-GETH-HEALTHCHECK" + query = "query" synced = "synced" minPeerCount = "min_peer_count" checkBlock = "check_block" @@ -23,8 +24,8 @@ const ( ) var ( - errCheckDisabled = errors.New("error check disabled") - errBadHeaderValue = errors.New("bad header value") + errCheckDisabled = errors.New("error check disabled") + errInvalidValue = errors.New("invalid value provided") ) type requestBody struct { @@ -34,7 +35,7 @@ type requestBody struct { MaxSecondsBehind *int `json:"max_seconds_behind"` } -func (h *handler) processFromHeaders(headers []string, w http.ResponseWriter, r *http.Request) { +func processFromHeaders(ec ethClient, headers []string, w http.ResponseWriter, r *http.Request) { var ( errCheckSynced = errCheckDisabled errCheckPeer = errCheckDisabled @@ -45,7 +46,7 @@ func (h *handler) processFromHeaders(headers []string, w http.ResponseWriter, r for _, header := range headers { lHeader := strings.ToLower(header) if lHeader == synced { - errCheckSynced = checkSynced(h.ec, r) + errCheckSynced = checkSynced(ec, r) } if strings.HasPrefix(lHeader, minPeerCount) { peers, err := strconv.Atoi(strings.TrimPrefix(lHeader, minPeerCount)) @@ -53,7 +54,7 @@ func (h *handler) processFromHeaders(headers []string, w http.ResponseWriter, r errCheckPeer = err break } - errCheckPeer = checkMinPeers(h.ec, uint(peers)) + errCheckPeer = checkMinPeers(ec, uint(peers)) } if strings.HasPrefix(lHeader, checkBlock) { block, err := strconv.Atoi(strings.TrimPrefix(lHeader, checkBlock)) @@ -61,7 +62,7 @@ func (h *handler) processFromHeaders(headers []string, w http.ResponseWriter, r errCheckBlock = err break } - errCheckBlock = checkBlockNumber(h.ec, big.NewInt(int64(block))) + errCheckBlock = checkBlockNumber(ec, big.NewInt(int64(block))) } if strings.HasPrefix(lHeader, maxSecondsBehind) { seconds, err := strconv.Atoi(strings.TrimPrefix(lHeader, maxSecondsBehind)) @@ -70,18 +71,18 @@ func (h *handler) processFromHeaders(headers []string, w http.ResponseWriter, r break } if seconds < 0 { - errCheckSeconds = errBadHeaderValue + errCheckSeconds = errInvalidValue break } now := time.Now().Unix() - errCheckSeconds = checkTime(h.ec, r, int(now)-seconds) + errCheckSeconds = checkTime(ec, r, int(now)-seconds) } } - reportHealth(errCheckSynced, errCheckPeer, errCheckBlock, errCheckSeconds, w) + reportHealth(nil, errCheckSynced, errCheckPeer, errCheckBlock, errCheckSeconds, w) } -func (h *handler) processFromBody(w http.ResponseWriter, r *http.Request) { +func processFromBody(ec ethClient, w http.ResponseWriter, r *http.Request) { body, errParse := parseHealthCheckBody(r.Body) defer r.Body.Close() @@ -96,37 +97,43 @@ func (h *handler) processFromBody(w http.ResponseWriter, r *http.Request) { log.Root().Warn("Unable to process healthcheck request", "err", errParse) } else { if body.Synced != nil { - errCheckSynced = checkSynced(h.ec, r) + errCheckSynced = checkSynced(ec, r) } if body.MinPeerCount != nil { - errCheckPeer = checkMinPeers(h.ec, *body.MinPeerCount) + errCheckPeer = checkMinPeers(ec, *body.MinPeerCount) } if body.CheckBlock != nil { - errCheckBlock = checkBlockNumber(h.ec, big.NewInt(int64(*body.CheckBlock))) + errCheckBlock = checkBlockNumber(ec, big.NewInt(int64(*body.CheckBlock))) } if body.MaxSecondsBehind != nil { seconds := *body.MaxSecondsBehind if seconds < 0 { - errCheckSeconds = errBadHeaderValue + errCheckSeconds = errInvalidValue + } else { + now := time.Now().Unix() + errCheckSeconds = checkTime(ec, r, int(now)-seconds) } - now := time.Now().Unix() - errCheckSeconds = checkTime(h.ec, r, int(now)-seconds) } } - err := reportHealth(errCheckSynced, errCheckPeer, errCheckBlock, errCheckSeconds, w) + err := reportHealth(errParse, errCheckSynced, errCheckPeer, errCheckBlock, errCheckSeconds, w) if err != nil { log.Root().Warn("Unable to process healthcheck request", "err", err) } } -func reportHealth(errCheckSynced, errCheckPeer, errCheckBlock, errCheckSeconds error, w http.ResponseWriter) error { +func reportHealth(errParse, errCheckSynced, errCheckPeer, errCheckBlock, errCheckSeconds error, w http.ResponseWriter) error { statusCode := http.StatusOK errs := make(map[string]string) + if shouldChangeStatusCode(errParse) { + statusCode = http.StatusInternalServerError + } + errs[query] = errorStringOrOK(errParse) + if shouldChangeStatusCode(errCheckSynced) { statusCode = http.StatusInternalServerError } @@ -188,7 +195,7 @@ func shouldChangeStatusCode(err error) bool { func errorStringOrOK(err error) string { if err == nil { - return "HEALTHY" + return "OK" } if errors.Is(err, errCheckDisabled) { diff --git a/health/health_test.go b/health/health_test.go new file mode 100644 index 0000000000..d66edc2d99 --- /dev/null +++ b/health/health_test.go @@ -0,0 +1,742 @@ +package health + +import ( + "context" + "encoding/json" + "errors" + "io" + "math/big" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/core/types" +) + +type ethClientStub struct { + peersResult uint64 + peersError error + blockResult *types.Block + blockError error + syncingResult *ethereum.SyncProgress + syncingError error +} + +func (e *ethClientStub) PeerCount(_ context.Context) (uint64, error) { + return e.peersResult, e.peersError +} + +func (e *ethClientStub) BlockByNumber(_ context.Context, _ *big.Int) (*types.Block, error) { + return e.blockResult, e.blockError +} + +func (e *ethClientStub) SyncProgress(_ context.Context) (*ethereum.SyncProgress, error) { + return e.syncingResult, e.syncingError +} + +func TestProcessFromHeaders(t *testing.T) { + cases := []struct { + headers []string + clientPeerResult uint64 + clientPeerError error + clientBlockResult *types.Block + clientBlockError error + clientSyncingResult *ethereum.SyncProgress + clientSyncingError error + expectedStatusCode int + expectedBody map[string]string + }{ + // 0 - sync check enabled - syncing + { + headers: []string{"synced"}, + clientPeerResult: uint64(1), + clientPeerError: nil, + clientBlockResult: &types.Block{}, + clientBlockError: nil, + clientSyncingResult: nil, + clientSyncingError: nil, + expectedStatusCode: http.StatusOK, + expectedBody: map[string]string{ + synced: "OK", + minPeerCount: "DISABLED", + checkBlock: "DISABLED", + maxSecondsBehind: "DISABLED", + }, + }, + // 1 - sync check enabled - not syncing + { + headers: []string{"synced"}, + clientPeerResult: uint64(1), + clientPeerError: nil, + clientBlockResult: nil, + clientBlockError: nil, + clientSyncingResult: ðereum.SyncProgress{}, + clientSyncingError: nil, + expectedStatusCode: http.StatusInternalServerError, + expectedBody: map[string]string{ + synced: "ERROR: not synced", + minPeerCount: "DISABLED", + checkBlock: "DISABLED", + maxSecondsBehind: "DISABLED", + }, + }, + // 2 - sync check enabled - error checking sync + { + headers: []string{"synced"}, + clientPeerResult: uint64(1), + clientPeerError: nil, + clientBlockResult: nil, + clientBlockError: nil, + clientSyncingResult: ðereum.SyncProgress{}, + clientSyncingError: errors.New("problem checking sync"), + expectedStatusCode: http.StatusInternalServerError, + expectedBody: map[string]string{ + synced: "ERROR: problem checking sync", + minPeerCount: "DISABLED", + checkBlock: "DISABLED", + maxSecondsBehind: "DISABLED", + }, + }, + // 3 - peer count enabled - good request + { + headers: []string{"min_peer_count1"}, + clientPeerResult: uint64(1), + clientPeerError: nil, + clientBlockResult: nil, + clientBlockError: nil, + clientSyncingResult: nil, + clientSyncingError: nil, + expectedStatusCode: http.StatusOK, + expectedBody: map[string]string{ + synced: "DISABLED", + minPeerCount: "OK", + checkBlock: "DISABLED", + maxSecondsBehind: "DISABLED", + }, + }, + // 4 - peer count enabled - not enough peers + { + headers: []string{"min_peer_count10"}, + clientPeerResult: uint64(1), + clientPeerError: nil, + clientBlockResult: nil, + clientBlockError: nil, + clientSyncingResult: nil, + clientSyncingError: nil, + expectedStatusCode: http.StatusInternalServerError, + expectedBody: map[string]string{ + synced: "DISABLED", + minPeerCount: "ERROR: not enough peers: 1 (minimum 10)", + checkBlock: "DISABLED", + maxSecondsBehind: "DISABLED", + }, + }, + // 5 - peer count enabled - error checking peers + { + headers: []string{"min_peer_count10"}, + clientPeerResult: uint64(1), + clientPeerError: errors.New("problem checking peers"), + clientBlockResult: nil, + clientBlockError: nil, + clientSyncingResult: nil, + clientSyncingError: nil, + expectedStatusCode: http.StatusInternalServerError, + expectedBody: map[string]string{ + synced: "DISABLED", + minPeerCount: "ERROR: problem checking peers", + checkBlock: "DISABLED", + maxSecondsBehind: "DISABLED", + }, + }, + // 6 - peer count enabled - badly formed request + { + headers: []string{"min_peer_countABC"}, + clientPeerResult: uint64(1), + clientPeerError: nil, + clientBlockResult: nil, + clientBlockError: nil, + clientSyncingResult: nil, + clientSyncingError: nil, + expectedStatusCode: http.StatusInternalServerError, + expectedBody: map[string]string{ + synced: "DISABLED", + minPeerCount: "ERROR: strconv.Atoi: parsing \"abc\": invalid syntax", + checkBlock: "DISABLED", + maxSecondsBehind: "DISABLED", + }, + }, + // 7 - block check - all ok + { + headers: []string{"check_block10"}, + clientPeerResult: uint64(1), + clientPeerError: nil, + clientBlockResult: &types.Block{}, + clientBlockError: nil, + clientSyncingResult: nil, + clientSyncingError: nil, + expectedStatusCode: http.StatusOK, + expectedBody: map[string]string{ + synced: "DISABLED", + minPeerCount: "DISABLED", + checkBlock: "OK", + maxSecondsBehind: "DISABLED", + }, + }, + // 8 - block check - no block found + { + headers: []string{"check_block10"}, + clientPeerResult: uint64(1), + clientPeerError: nil, + clientBlockResult: nil, + clientBlockError: errors.New("not found"), + clientSyncingResult: nil, + clientSyncingError: nil, + expectedStatusCode: http.StatusInternalServerError, + expectedBody: map[string]string{ + synced: "DISABLED", + minPeerCount: "DISABLED", + checkBlock: "ERROR: not found", + maxSecondsBehind: "DISABLED", + }, + }, + // 9 - block check - error checking block + { + headers: []string{"check_block10"}, + clientPeerResult: uint64(1), + clientPeerError: nil, + clientBlockResult: &types.Block{}, + clientBlockError: errors.New("problem checking block"), + clientSyncingResult: nil, + clientSyncingError: nil, + expectedStatusCode: http.StatusInternalServerError, + expectedBody: map[string]string{ + synced: "DISABLED", + minPeerCount: "DISABLED", + checkBlock: "ERROR: problem checking block", + maxSecondsBehind: "DISABLED", + }, + }, + // 10 - block check - badly formed request + { + headers: []string{"check_blockABC"}, + clientPeerResult: uint64(1), + clientPeerError: nil, + clientBlockResult: nil, + clientBlockError: nil, + clientSyncingResult: nil, + clientSyncingError: nil, + expectedStatusCode: http.StatusInternalServerError, + expectedBody: map[string]string{ + synced: "DISABLED", + minPeerCount: "DISABLED", + checkBlock: "ERROR: strconv.Atoi: parsing \"abc\": invalid syntax", + maxSecondsBehind: "DISABLED", + }, + }, + // 11 - seconds check - all ok + { + headers: []string{"max_seconds_behind60"}, + clientPeerResult: uint64(1), + clientPeerError: nil, + clientBlockResult: types.NewBlockWithHeader(&types.Header{ + Time: uint64(time.Now().Add(-10 * time.Second).Unix()), + }), + clientBlockError: nil, + clientSyncingResult: nil, + clientSyncingError: nil, + expectedStatusCode: http.StatusOK, + expectedBody: map[string]string{ + synced: "DISABLED", + minPeerCount: "DISABLED", + checkBlock: "DISABLED", + maxSecondsBehind: "OK", + }, + }, + // 12 - seconds check - too old + { + headers: []string{"max_seconds_behind60"}, + clientPeerResult: uint64(1), + clientPeerError: nil, + clientBlockResult: types.NewBlockWithHeader(&types.Header{ + Time: uint64(time.Now().Add(-1 * time.Hour).Unix()), + }), + clientBlockError: nil, + clientSyncingResult: nil, + clientSyncingError: nil, + expectedStatusCode: http.StatusInternalServerError, + expectedBody: map[string]string{ + synced: "DISABLED", + minPeerCount: "DISABLED", + checkBlock: "DISABLED", + maxSecondsBehind: "ERROR: timestamp too old: got ts:", + }, + }, + // 13 - seconds check - less than 0 seconds + { + headers: []string{"max_seconds_behind-1"}, + clientPeerResult: uint64(1), + clientPeerError: nil, + clientBlockResult: types.NewBlockWithHeader(&types.Header{ + Time: uint64(time.Now().Add(1 * time.Hour).Unix()), + }), + clientBlockError: nil, + clientSyncingResult: nil, + clientSyncingError: nil, + expectedStatusCode: http.StatusInternalServerError, + expectedBody: map[string]string{ + synced: "DISABLED", + minPeerCount: "DISABLED", + checkBlock: "DISABLED", + maxSecondsBehind: "ERROR: invalid value provided", + }, + }, + // 14 - seconds check - badly formed request + { + headers: []string{"max_seconds_behindABC"}, + clientPeerResult: uint64(1), + clientPeerError: nil, + clientBlockResult: &types.Block{}, + clientBlockError: nil, + clientSyncingResult: nil, + clientSyncingError: nil, + expectedStatusCode: http.StatusInternalServerError, + expectedBody: map[string]string{ + synced: "DISABLED", + minPeerCount: "DISABLED", + checkBlock: "DISABLED", + maxSecondsBehind: "ERROR: strconv.Atoi: parsing \"abc\": invalid syntax", + }, + }, + // 15 - all checks - report ok + { + headers: []string{"synced", "check_block10", "min_peer_count1", "max_seconds_behind60"}, + clientPeerResult: uint64(10), + clientPeerError: nil, + clientBlockResult: types.NewBlockWithHeader(&types.Header{ + Time: uint64(time.Now().Add(1 * time.Second).Unix()), + }), + clientBlockError: nil, + clientSyncingResult: nil, + clientSyncingError: nil, + expectedStatusCode: http.StatusOK, + expectedBody: map[string]string{ + synced: "OK", + minPeerCount: "OK", + checkBlock: "OK", + maxSecondsBehind: "OK", + }, + }, + } + + for idx, c := range cases { + w := httptest.NewRecorder() + r, err := http.NewRequest(http.MethodGet, "http://localhost:9090/health", nil) + if err != nil { + t.Errorf("%v: creating request: %v", idx, err) + } + + for _, header := range c.headers { + r.Header.Add("X-GETH-HEALTHCHECK", header) + } + + ethClient := ðClientStub{ + peersResult: c.clientPeerResult, + peersError: c.clientPeerError, + blockResult: c.clientBlockResult, + blockError: c.clientBlockError, + syncingResult: c.clientSyncingResult, + syncingError: c.clientSyncingError, + } + + processFromHeaders(ethClient, r.Header.Values(healthHeader), w, r) + + result := w.Result() + if result.StatusCode != c.expectedStatusCode { + t.Errorf("%v: expected status code: %v, but got: %v", idx, c.expectedStatusCode, result.StatusCode) + } + + bodyBytes, err := io.ReadAll(result.Body) + if err != nil { + t.Errorf("%v: reading response body: %s", idx, err) + } + + var body map[string]string + err = json.Unmarshal(bodyBytes, &body) + if err != nil { + t.Errorf("%v: unmarshalling the response body: %s", idx, err) + } + result.Body.Close() + + for k, v := range c.expectedBody { + val, found := body[k] + if !found { + t.Errorf("%v: expected the key: %s to be in the response body but it wasn't there", idx, k) + } + if !strings.Contains(val, v) { + t.Errorf("%v: expected the response body key: %s to contain: %s, but it contained: %s", idx, k, v, val) + } + } + } +} + +func TestProcessHealthcheckIfNeeded_RequestBody(t *testing.T) { + cases := []struct { + body string + clientPeerResult uint64 + clientPeerError error + clientBlockResult *types.Block + clientBlockError error + clientSyncingResult *ethereum.SyncProgress + clientSyncingError error + expectedStatusCode int + expectedBody map[string]string + }{ + // 0 - sync check enabled - syncing + { + body: "{\"synced\": true}", + clientPeerResult: uint64(1), + clientPeerError: nil, + clientBlockResult: nil, + clientBlockError: nil, + clientSyncingResult: nil, + clientSyncingError: nil, + expectedStatusCode: http.StatusOK, + expectedBody: map[string]string{ + query: "OK", + synced: "OK", + minPeerCount: "DISABLED", + checkBlock: "DISABLED", + maxSecondsBehind: "DISABLED", + }, + }, + // 1 - sync check enabled - not syncing + { + body: "{\"synced\": true}", + clientPeerResult: uint64(1), + clientPeerError: nil, + clientBlockResult: nil, + clientBlockError: nil, + clientSyncingResult: ðereum.SyncProgress{}, + clientSyncingError: nil, + expectedStatusCode: http.StatusInternalServerError, + expectedBody: map[string]string{ + query: "OK", + synced: "ERROR: not synced", + minPeerCount: "DISABLED", + checkBlock: "DISABLED", + maxSecondsBehind: "DISABLED", + }, + }, + // 2 - sync check enabled - error checking sync + { + body: "{\"synced\": true}", + clientPeerResult: uint64(1), + clientPeerError: nil, + clientBlockResult: nil, + clientBlockError: nil, + clientSyncingResult: ðereum.SyncProgress{}, + clientSyncingError: errors.New("problem checking sync"), + expectedStatusCode: http.StatusInternalServerError, + expectedBody: map[string]string{ + query: "OK", + synced: "ERROR: problem checking sync", + minPeerCount: "DISABLED", + checkBlock: "DISABLED", + maxSecondsBehind: "DISABLED", + }, + }, + // 3 - peer count enabled - good request + { + body: "{\"min_peer_count\": 1}", + clientPeerResult: uint64(1), + clientPeerError: nil, + clientBlockResult: nil, + clientBlockError: nil, + clientSyncingResult: nil, + clientSyncingError: nil, + expectedStatusCode: http.StatusOK, + expectedBody: map[string]string{ + query: "OK", + synced: "DISABLED", + minPeerCount: "OK", + checkBlock: "DISABLED", + maxSecondsBehind: "DISABLED", + }, + }, + // 4 - peer count enabled - not enough peers + { + body: "{\"min_peer_count\": 10}", + clientPeerResult: uint64(1), + clientPeerError: nil, + clientBlockResult: nil, + clientBlockError: nil, + clientSyncingResult: nil, + clientSyncingError: nil, + expectedStatusCode: http.StatusInternalServerError, + expectedBody: map[string]string{ + query: "OK", + synced: "DISABLED", + minPeerCount: "ERROR: not enough peers: 1 (minimum 10)", + checkBlock: "DISABLED", + maxSecondsBehind: "DISABLED", + }, + }, + // 5 - peer count enabled - error checking peers + { + body: "{\"min_peer_count\": 10}", + clientPeerResult: uint64(1), + clientPeerError: errors.New("problem checking peers"), + clientBlockResult: nil, + clientBlockError: nil, + clientSyncingResult: nil, + clientSyncingError: nil, + expectedStatusCode: http.StatusInternalServerError, + expectedBody: map[string]string{ + query: "OK", + synced: "DISABLED", + minPeerCount: "ERROR: problem checking peers", + checkBlock: "DISABLED", + maxSecondsBehind: "DISABLED", + }, + }, + // 6 - peer count enabled - badly formed request + { + body: "{\"min_peer_count\": \"ABC\"}", + clientPeerResult: uint64(1), + clientPeerError: nil, + clientBlockResult: nil, + clientBlockError: nil, + clientSyncingResult: nil, + clientSyncingError: nil, + expectedStatusCode: http.StatusInternalServerError, + expectedBody: map[string]string{ + query: "ERROR: json: cannot unmarshal string into Go struct field requestBody.min_peer_count of type uint", + synced: "DISABLED", + minPeerCount: "DISABLED", + checkBlock: "DISABLED", + maxSecondsBehind: "DISABLED", + }, + }, + // 7 - block check - all ok + { + body: "{\"check_block\": 10}", + clientPeerResult: uint64(1), + clientPeerError: nil, + clientBlockResult: &types.Block{}, + clientBlockError: nil, + clientSyncingResult: nil, + clientSyncingError: nil, + expectedStatusCode: http.StatusOK, + expectedBody: map[string]string{ + query: "OK", + synced: "DISABLED", + minPeerCount: "DISABLED", + checkBlock: "OK", + maxSecondsBehind: "DISABLED", + }, + }, + // 8 - block check - no block found + { + body: "{\"check_block\": 10}", + clientPeerResult: uint64(1), + clientPeerError: nil, + clientBlockResult: nil, + clientBlockError: errors.New("not found"), + clientSyncingResult: nil, + clientSyncingError: nil, + expectedStatusCode: http.StatusInternalServerError, + expectedBody: map[string]string{ + query: "OK", + synced: "DISABLED", + minPeerCount: "DISABLED", + checkBlock: "ERROR: not found", + maxSecondsBehind: "DISABLED", + }, + }, + // 9 - block check - error checking block + { + body: "{\"check_block\": 10}", + clientPeerResult: uint64(1), + clientPeerError: nil, + clientBlockResult: &types.Block{}, + clientBlockError: errors.New("problem checking block"), + clientSyncingResult: nil, + clientSyncingError: nil, + expectedStatusCode: http.StatusInternalServerError, + expectedBody: map[string]string{ + query: "OK", + synced: "DISABLED", + minPeerCount: "DISABLED", + checkBlock: "ERROR: problem checking block", + maxSecondsBehind: "DISABLED", + }, + }, + // 10 - block check - badly formed request + { + body: "{\"check_block\": \"ABC\"}", + clientPeerResult: uint64(1), + clientPeerError: nil, + clientBlockResult: nil, + clientBlockError: nil, + clientSyncingResult: nil, + clientSyncingError: nil, + expectedStatusCode: http.StatusInternalServerError, + expectedBody: map[string]string{ + query: "ERROR: json: cannot unmarshal string into Go struct field requestBody.check_block of type uint64", + synced: "DISABLED", + minPeerCount: "DISABLED", + checkBlock: "DISABLED", + maxSecondsBehind: "DISABLED", + }, + }, + // 11 - seconds check - all ok + { + body: "{\"max_seconds_behind\": 60}", + clientPeerResult: uint64(1), + clientPeerError: nil, + clientBlockResult: types.NewBlockWithHeader(&types.Header{ + Time: uint64(time.Now().Add(-10 * time.Second).Unix()), + }), + clientBlockError: nil, + clientSyncingResult: nil, + clientSyncingError: nil, + expectedStatusCode: http.StatusOK, + expectedBody: map[string]string{ + query: "OK", + synced: "DISABLED", + minPeerCount: "DISABLED", + checkBlock: "DISABLED", + maxSecondsBehind: "OK", + }, + }, + // 12 - seconds check - too old + { + body: "{\"max_seconds_behind\": 60}", + clientPeerResult: uint64(1), + clientPeerError: nil, + clientBlockResult: types.NewBlockWithHeader(&types.Header{ + Time: uint64(time.Now().Add(-1 * time.Hour).Unix()), + }), + clientBlockError: nil, + clientSyncingResult: nil, + clientSyncingError: nil, + expectedStatusCode: http.StatusInternalServerError, + expectedBody: map[string]string{ + query: "OK", + synced: "DISABLED", + minPeerCount: "DISABLED", + checkBlock: "DISABLED", + maxSecondsBehind: "ERROR: timestamp too old: got ts:", + }, + }, + // 13 - seconds check - less than 0 seconds + { + body: "{\"max_seconds_behind\": -1}", + clientPeerResult: uint64(1), + clientPeerError: nil, + clientBlockResult: types.NewBlockWithHeader(&types.Header{ + Time: uint64(time.Now().Add(1 * time.Hour).Unix()), + }), + clientBlockError: nil, + clientSyncingResult: nil, + clientSyncingError: nil, + expectedStatusCode: http.StatusInternalServerError, + expectedBody: map[string]string{ + query: "OK", + synced: "DISABLED", + minPeerCount: "DISABLED", + checkBlock: "DISABLED", + maxSecondsBehind: "ERROR: invalid value provided", + }, + }, + // 14 - seconds check - badly formed request + { + body: "{\"max_seconds_behind\": \"ABC\"}", + clientPeerResult: uint64(1), + clientPeerError: nil, + clientBlockResult: &types.Block{}, + clientBlockError: nil, + clientSyncingResult: nil, + clientSyncingError: nil, + expectedStatusCode: http.StatusInternalServerError, + expectedBody: map[string]string{ + query: "ERROR: json: cannot unmarshal string into Go struct field requestBody.max_seconds_behind of type int", + synced: "DISABLED", + minPeerCount: "DISABLED", + checkBlock: "DISABLED", + maxSecondsBehind: "DISABLED", + }, + }, + // 15 - all checks - report ok + { + body: "{\"synced\": true, \"min_peer_count\": 1, \"check_block\": 10, \"max_seconds_behind\": 60}", + clientPeerResult: uint64(10), + clientPeerError: nil, + clientBlockResult: types.NewBlockWithHeader(&types.Header{ + Time: uint64(time.Now().Add(1 * time.Second).Unix()), + }), + clientBlockError: nil, + clientSyncingResult: nil, + clientSyncingError: nil, + expectedStatusCode: http.StatusOK, + expectedBody: map[string]string{ + query: "OK", + synced: "OK", + minPeerCount: "OK", + checkBlock: "OK", + maxSecondsBehind: "OK", + }, + }, + } + + for idx, c := range cases { + w := httptest.NewRecorder() + r, err := http.NewRequest(http.MethodGet, "http://localhost:9090/health", nil) + if err != nil { + t.Errorf("%v: creating request: %v", idx, err) + } + + r.Body = io.NopCloser(strings.NewReader(c.body)) + + ethClient := ðClientStub{ + peersResult: c.clientPeerResult, + peersError: c.clientPeerError, + blockResult: c.clientBlockResult, + blockError: c.clientBlockError, + syncingResult: c.clientSyncingResult, + syncingError: c.clientSyncingError, + } + + processFromBody(ethClient, w, r) + + result := w.Result() + if result.StatusCode != c.expectedStatusCode { + t.Errorf("%v: expected status code: %v, but got: %v", idx, c.expectedStatusCode, result.StatusCode) + } + + bodyBytes, err := io.ReadAll(result.Body) + if err != nil { + t.Errorf("%v: reading response body: %s", idx, err) + } + + var body map[string]string + err = json.Unmarshal(bodyBytes, &body) + if err != nil { + t.Errorf("%v: unmarshalling the response body: %s", idx, err) + } + result.Body.Close() + + for k, v := range c.expectedBody { + val, found := body[k] + if !found { + t.Errorf("%v: expected the key: %s to be in the response body but it wasn't there", idx, k) + } + if !strings.Contains(val, v) { + t.Errorf("%v: expected the response body key: %s to contain: %s, but it contained: %s", idx, k, v, val) + } + } + } +} diff --git a/health/interface.go b/health/interface.go new file mode 100644 index 0000000000..7ddbecd047 --- /dev/null +++ b/health/interface.go @@ -0,0 +1,15 @@ +package health + +import ( + "context" + "math/big" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/core/types" +) + +type ethClient interface { + PeerCount(context.Context) (uint64, error) + BlockByNumber(context.Context, *big.Int) (*types.Block, error) + SyncProgress(context.Context) (*ethereum.SyncProgress, error) +} diff --git a/health/service.go b/health/service.go index f1aba69042..6279dfd2a8 100644 --- a/health/service.go +++ b/health/service.go @@ -14,9 +14,9 @@ type handler struct { func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { headers := r.Header.Values(healthHeader) if len(headers) != 0 { - h.processFromHeaders(headers, w, r) + processFromHeaders(h.ec, headers, w, r) } else { - h.processFromBody(w, r) + processFromBody(h.ec, w, r) } } From a8be551cf4af92d26e91221278a0191a31ff9c8b Mon Sep 17 00:00:00 2001 From: crypto-services Date: Tue, 20 Feb 2024 14:50:06 +0800 Subject: [PATCH 05/11] Update test name --- health/health_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/health/health_test.go b/health/health_test.go index d66edc2d99..86c5ee07b4 100644 --- a/health/health_test.go +++ b/health/health_test.go @@ -382,7 +382,7 @@ func TestProcessFromHeaders(t *testing.T) { } } -func TestProcessHealthcheckIfNeeded_RequestBody(t *testing.T) { +func TestProcessFromBody(t *testing.T) { cases := []struct { body string clientPeerResult uint64 From bc2ace94ee8ecc771c4e10ce4c4780939e6e2550 Mon Sep 17 00:00:00 2001 From: crypto-services Date: Tue, 20 Feb 2024 14:54:55 +0800 Subject: [PATCH 06/11] Remove parent context --- health/check_synced.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/health/check_synced.go b/health/check_synced.go index 580b6a28fb..5f068debae 100644 --- a/health/check_synced.go +++ b/health/check_synced.go @@ -1,6 +1,7 @@ package health import ( + "context" "errors" "net/http" @@ -12,7 +13,7 @@ var ( ) func checkSynced(ec ethClient, r *http.Request) error { - i, err := ec.SyncProgress(r.Context()) + i, err := ec.SyncProgress(context.TODO()) if err != nil { log.Root().Warn("Unable to check sync status for healthcheck", "err", err.Error()) return err From 6fc5fbee21e34530205401f2ae06cced7ef7f49f Mon Sep 17 00:00:00 2001 From: crypto-services Date: Tue, 20 Feb 2024 15:56:57 +0800 Subject: [PATCH 07/11] health: Add CLI flag --- cmd/geth/config.go | 7 ++++--- cmd/geth/main.go | 1 + cmd/utils/flags.go | 5 +++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index befca177f7..c427da1fcc 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -196,9 +196,10 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { // Configure log filter RPC API. filterSystem := utils.RegisterFilterAPI(stack, backend, &cfg.Eth) - // TODO: ENG191 add flag to request endpoint - utils.RegisterHealthService(stack, &cfg.Node) - + // Configure the healthcheck API if requested. + if ctx.IsSet(utils.HTTPHealthEnabledFlag.Name) { + utils.RegisterHealthService(stack, &cfg.Node) + } // Configure GraphQL if requested. if ctx.IsSet(utils.GraphQLEnabledFlag.Name) { utils.RegisterGraphQLService(stack, backend, filterSystem, &cfg.Node) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 2f7d37fdd7..bfeab56070 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -153,6 +153,7 @@ var ( utils.HTTPListenAddrFlag, utils.HTTPPortFlag, utils.HTTPCORSDomainFlag, + utils.HTTPHealthEnabledFlag, utils.AuthListenFlag, utils.AuthPortFlag, utils.AuthVirtualHostsFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index fc0c987537..826aa915db 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -614,6 +614,11 @@ var ( Value: "", Category: flags.APICategory, } + HTTPHealthEnabledFlag = &cli.BoolFlag{ + Name: "http.health", + Usage: "Enable the HTTP healthcheck API at path '/health'.", + Category: flags.APICategory, + } GraphQLEnabledFlag = &cli.BoolFlag{ Name: "graphql", Usage: "Enable GraphQL on the HTTP-RPC server. Note that GraphQL can only be started if an HTTP server is started as well.", From b1597736d78a10ec6b9c6fcbf833e96bf4ca9ac6 Mon Sep 17 00:00:00 2001 From: crypto-services Date: Tue, 20 Feb 2024 16:27:32 +0800 Subject: [PATCH 08/11] health: Add comments --- health/check_block.go | 1 + health/check_peers.go | 1 + health/check_synced.go | 1 + health/check_time.go | 7 ++++--- health/health.go | 7 +++++++ health/service.go | 1 + 6 files changed, 15 insertions(+), 3 deletions(-) diff --git a/health/check_block.go b/health/check_block.go index a68b560100..cea10738e8 100644 --- a/health/check_block.go +++ b/health/check_block.go @@ -5,6 +5,7 @@ import ( "math/big" ) +// checkBlockNumber confirms this node is aware of a specific block. func checkBlockNumber(ec ethClient, blockNumber *big.Int) error { _, err := ec.BlockByNumber(context.TODO(), blockNumber) if err != nil { diff --git a/health/check_peers.go b/health/check_peers.go index e20973348c..a2ddc840c1 100644 --- a/health/check_peers.go +++ b/health/check_peers.go @@ -10,6 +10,7 @@ var ( errNotEnoughPeers = errors.New("not enough peers") ) +// checkMinPeers returns 'errNotEnoughPeers' if the current peer count its lower than 'minPeerCount' func checkMinPeers(ec ethClient, minPeerCount uint) error { peerCount, err := ec.PeerCount(context.TODO()) if err != nil { diff --git a/health/check_synced.go b/health/check_synced.go index 5f068debae..2a06e769f2 100644 --- a/health/check_synced.go +++ b/health/check_synced.go @@ -12,6 +12,7 @@ var ( errNotSynced = errors.New("not synced") ) +// checkSynced returns 'errNotSynced' if the node is in the syncing state. func checkSynced(ec ethClient, r *http.Request) error { i, err := ec.SyncProgress(context.TODO()) if err != nil { diff --git a/health/check_time.go b/health/check_time.go index ae0df8dad4..dab5736223 100644 --- a/health/check_time.go +++ b/health/check_time.go @@ -11,18 +11,19 @@ var ( errTimestampTooOld = errors.New("timestamp too old") ) +// checkTime fetches the timestamp of the most recent block and returns an error if it is earlier than 'minTimestamp'. func checkTime( ec ethClient, r *http.Request, - seconds int, + minTimestamp int, ) error { i, err := ec.BlockByNumber(context.TODO(), nil) if err != nil { return err } timestamp := i.Time() - if timestamp < uint64(seconds) { - return fmt.Errorf("%w: got ts: %d, need: %d", errTimestampTooOld, timestamp, seconds) + if timestamp < uint64(minTimestamp) { + return fmt.Errorf("%w: got ts: %d, need: %d", errTimestampTooOld, timestamp, minTimestamp) } return nil diff --git a/health/health.go b/health/health.go index 402c02686a..c1cda74688 100644 --- a/health/health.go +++ b/health/health.go @@ -35,6 +35,7 @@ type requestBody struct { MaxSecondsBehind *int `json:"max_seconds_behind"` } +// processFromHeaders handles requests when 'X-GETH-HEALTHCHECK' header labels are present. func processFromHeaders(ec ethClient, headers []string, w http.ResponseWriter, r *http.Request) { var ( errCheckSynced = errCheckDisabled @@ -82,6 +83,7 @@ func processFromHeaders(ec ethClient, headers []string, w http.ResponseWriter, r reportHealth(nil, errCheckSynced, errCheckPeer, errCheckBlock, errCheckSeconds, w) } +// processFromBody handles requests when 'X-GETH-HEALTHCHECK' headers are not present. func processFromBody(ec ethClient, w http.ResponseWriter, r *http.Request) { body, errParse := parseHealthCheckBody(r.Body) defer r.Body.Close() @@ -125,6 +127,7 @@ func processFromBody(ec ethClient, w http.ResponseWriter, r *http.Request) { } } +// reportHealth builds the response body, sets the status code and calls for it to be written. func reportHealth(errParse, errCheckSynced, errCheckPeer, errCheckBlock, errCheckSeconds error, w http.ResponseWriter) error { statusCode := http.StatusOK errs := make(map[string]string) @@ -157,6 +160,7 @@ func reportHealth(errParse, errCheckSynced, errCheckPeer, errCheckBlock, errChec return writeResponse(w, errs, statusCode) } +// parseHealthCheckBody parses and type checks the request body when 'X-GETH-HEALTHCHECK' headers are not present. func parseHealthCheckBody(reader io.Reader) (requestBody, error) { var body requestBody @@ -173,6 +177,7 @@ func parseHealthCheckBody(reader io.Reader) (requestBody, error) { return body, nil } +// writeResponse delivers the status and body to the response writer. func writeResponse(w http.ResponseWriter, errs map[string]string, statusCode int) error { w.WriteHeader(statusCode) @@ -189,10 +194,12 @@ func writeResponse(w http.ResponseWriter, errs map[string]string, statusCode int return nil } +// shouldChangeStatusCode returns 'true' if an error exists and is not 'errCheckDisabled'. func shouldChangeStatusCode(err error) bool { return err != nil && !errors.Is(err, errCheckDisabled) } +// errorStringOrOK returns "OK", "DISABLED" or the error message based on the output of the check. func errorStringOrOK(err error) string { if err == nil { return "OK" diff --git a/health/service.go b/health/service.go index 6279dfd2a8..76232f15a2 100644 --- a/health/service.go +++ b/health/service.go @@ -11,6 +11,7 @@ type handler struct { ec *ethclient.Client } +// ServeHTTP implements the http.Handler interface. func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { headers := r.Header.Values(healthHeader) if len(headers) != 0 { From 08dc03e2c61e6c5ca109a1422c3fc9d66d3e9542 Mon Sep 17 00:00:00 2001 From: crypto-services Date: Wed, 21 Feb 2024 09:27:05 +0800 Subject: [PATCH 09/11] health: Replace header check if with switch --- health/health.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/health/health.go b/health/health.go index c1cda74688..62e4043499 100644 --- a/health/health.go +++ b/health/health.go @@ -46,26 +46,24 @@ func processFromHeaders(ec ethClient, headers []string, w http.ResponseWriter, r for _, header := range headers { lHeader := strings.ToLower(header) - if lHeader == synced { + switch { + case lHeader == synced: errCheckSynced = checkSynced(ec, r) - } - if strings.HasPrefix(lHeader, minPeerCount) { + case strings.HasPrefix(lHeader, minPeerCount): peers, err := strconv.Atoi(strings.TrimPrefix(lHeader, minPeerCount)) if err != nil { errCheckPeer = err break } errCheckPeer = checkMinPeers(ec, uint(peers)) - } - if strings.HasPrefix(lHeader, checkBlock) { + case strings.HasPrefix(lHeader, checkBlock): block, err := strconv.Atoi(strings.TrimPrefix(lHeader, checkBlock)) if err != nil { errCheckBlock = err break } errCheckBlock = checkBlockNumber(ec, big.NewInt(int64(block))) - } - if strings.HasPrefix(lHeader, maxSecondsBehind) { + case strings.HasPrefix(lHeader, maxSecondsBehind): seconds, err := strconv.Atoi(strings.TrimPrefix(lHeader, maxSecondsBehind)) if err != nil { errCheckSeconds = err From be217124cb1e904c1da24df5f7004a11b483a989 Mon Sep 17 00:00:00 2001 From: crypto-services Date: Thu, 22 Feb 2024 14:06:04 +0800 Subject: [PATCH 10/11] Remove unrequired big int conversion --- health/health.go | 10 +++++----- health/health_test.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/health/health.go b/health/health.go index 62e4043499..794d243e04 100644 --- a/health/health.go +++ b/health/health.go @@ -29,10 +29,10 @@ var ( ) type requestBody struct { - Synced *bool `json:"synced"` - MinPeerCount *uint `json:"min_peer_count"` - CheckBlock *uint64 `json:"check_block"` - MaxSecondsBehind *int `json:"max_seconds_behind"` + Synced *bool `json:"synced"` + MinPeerCount *uint `json:"min_peer_count"` + CheckBlock *big.Int `json:"check_block"` + MaxSecondsBehind *int `json:"max_seconds_behind"` } // processFromHeaders handles requests when 'X-GETH-HEALTHCHECK' header labels are present. @@ -105,7 +105,7 @@ func processFromBody(ec ethClient, w http.ResponseWriter, r *http.Request) { } if body.CheckBlock != nil { - errCheckBlock = checkBlockNumber(ec, big.NewInt(int64(*body.CheckBlock))) + errCheckBlock = checkBlockNumber(ec, body.CheckBlock) } if body.MaxSecondsBehind != nil { diff --git a/health/health_test.go b/health/health_test.go index 86c5ee07b4..f1422d789c 100644 --- a/health/health_test.go +++ b/health/health_test.go @@ -585,7 +585,7 @@ func TestProcessFromBody(t *testing.T) { clientSyncingError: nil, expectedStatusCode: http.StatusInternalServerError, expectedBody: map[string]string{ - query: "ERROR: json: cannot unmarshal string into Go struct field requestBody.check_block of type uint64", + query: "ERROR: math/big: cannot unmarshal \"\\\"ABC\\\"\" into a *big.Int", synced: "DISABLED", minPeerCount: "DISABLED", checkBlock: "DISABLED", From ff742cb8d510cdc587a11b7ee0347872f78f56da Mon Sep 17 00:00:00 2001 From: crypto-services Date: Thu, 22 Feb 2024 15:12:27 +0800 Subject: [PATCH 11/11] health: Remove unrequired uint64 conversation --- health/check_peers.go | 4 ++-- health/health.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/health/check_peers.go b/health/check_peers.go index a2ddc840c1..b07cebd429 100644 --- a/health/check_peers.go +++ b/health/check_peers.go @@ -11,12 +11,12 @@ var ( ) // checkMinPeers returns 'errNotEnoughPeers' if the current peer count its lower than 'minPeerCount' -func checkMinPeers(ec ethClient, minPeerCount uint) error { +func checkMinPeers(ec ethClient, minPeerCount uint64) error { peerCount, err := ec.PeerCount(context.TODO()) if err != nil { return err } - if uint64(peerCount) < uint64(minPeerCount) { + if peerCount < minPeerCount { return fmt.Errorf("%w: %d (minimum %d)", errNotEnoughPeers, peerCount, minPeerCount) } return nil diff --git a/health/health.go b/health/health.go index 794d243e04..aff8f34540 100644 --- a/health/health.go +++ b/health/health.go @@ -55,7 +55,7 @@ func processFromHeaders(ec ethClient, headers []string, w http.ResponseWriter, r errCheckPeer = err break } - errCheckPeer = checkMinPeers(ec, uint(peers)) + errCheckPeer = checkMinPeers(ec, uint64(peers)) case strings.HasPrefix(lHeader, checkBlock): block, err := strconv.Atoi(strings.TrimPrefix(lHeader, checkBlock)) if err != nil { @@ -101,7 +101,7 @@ func processFromBody(ec ethClient, w http.ResponseWriter, r *http.Request) { } if body.MinPeerCount != nil { - errCheckPeer = checkMinPeers(ec, *body.MinPeerCount) + errCheckPeer = checkMinPeers(ec, uint64(*body.MinPeerCount)) } if body.CheckBlock != nil {