From 96968b119ee891e8e9266aab4fc42e5b9b4e0fb4 Mon Sep 17 00:00:00 2001 From: crypto-services Date: Thu, 15 Feb 2024 16:25:35 +0800 Subject: [PATCH] Initial cut --- cmd/geth/config.go | 3 + cmd/utils/flags.go | 9 ++ health/check_block.go | 17 ++++ health/check_peers.go | 24 +++++ health/check_synced.go | 26 ++++++ health/check_time.go | 31 +++++++ health/health.go | 199 +++++++++++++++++++++++++++++++++++++++++ health/service.go | 40 +++++++++ 8 files changed, 349 insertions(+) create mode 100644 health/check_block.go create mode 100644 health/check_peers.go create mode 100644 health/check_synced.go create mode 100644 health/check_time.go create mode 100644 health/health.go create mode 100644 health/service.go diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 5f52f1df54..c4d0981216 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -196,6 +196,9 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { // Configure log filter RPC API. filterSystem := utils.RegisterFilterAPI(stack, backend, &cfg.Eth) + // TODO: ENG191 add flag to request endpoint + utils.RegisterHealthService(stack, backend, &cfg.Node) + // Configure GraphQL if requested. if ctx.IsSet(utils.GraphQLEnabledFlag.Name) { utils.RegisterGraphQLService(stack, backend, filterSystem, &cfg.Node) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 159c47ca01..38a2aaf580 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -55,6 +55,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb/remotedb" "github.com/ethereum/go-ethereum/ethstats" "github.com/ethereum/go-ethereum/graphql" + "github.com/ethereum/go-ethereum/health" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" @@ -1883,6 +1884,14 @@ func RegisterGraphQLService(stack *node.Node, backend ethapi.Backend, filterSyst } } +// RegisterHealthService adds the Health API to the node. +func RegisterHealthService(stack *node.Node, backend ethapi.Backend, cfg *node.Config) { + err := health.New(stack, backend, cfg.GraphQLCors, cfg.HTTPVirtualHosts) + if err != nil { + Fatalf("Failed to register the health service: %v", err) + } +} + // RegisterFilterAPI adds the eth log filtering RPC API to the node. func RegisterFilterAPI(stack *node.Node, backend ethapi.Backend, ethcfg *ethconfig.Config) *filters.FilterSystem { filterSystem := filters.NewFilterSystem(backend, filters.Config{ diff --git a/health/check_block.go b/health/check_block.go new file mode 100644 index 0000000000..f8e95ddcae --- /dev/null +++ b/health/check_block.go @@ -0,0 +1,17 @@ +package health + +import ( + "context" + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/ethclient" +) + +func checkBlockNumber(ec *ethclient.Client, blockNumber *big.Int) error { + _, err := ec.BlockByNumber(context.TODO(), blockNumber) + if err != nil { + return fmt.Errorf("no known block with number %v (%x hex)", blockNumber.Int64(), blockNumber.Int64()) + } + return nil +} diff --git a/health/check_peers.go b/health/check_peers.go new file mode 100644 index 0000000000..d4b47cbe6d --- /dev/null +++ b/health/check_peers.go @@ -0,0 +1,24 @@ +package health + +import ( + "context" + "errors" + "fmt" + + "github.com/ethereum/go-ethereum/ethclient" +) + +var ( + errNotEnoughPeers = errors.New("not enough peers") +) + +func checkMinPeers(ec *ethclient.Client, minPeerCount uint) error { + peerCount, err := ec.PeerCount(context.TODO()) + if err != nil { + return err + } + if uint64(peerCount) < uint64(minPeerCount) { + return fmt.Errorf("%w: %d (minimum %d)", errNotEnoughPeers, peerCount, minPeerCount) + } + return nil +} diff --git a/health/check_synced.go b/health/check_synced.go new file mode 100644 index 0000000000..78d3d025cf --- /dev/null +++ b/health/check_synced.go @@ -0,0 +1,26 @@ +package health + +import ( + "errors" + "net/http" + + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/log" +) + +var ( + errNotSynced = errors.New("not synced") +) + +func checkSynced(ec *ethclient.Client, r *http.Request) error { + i, err := ec.SyncProgress(r.Context()) + if err != nil { + log.Root().Warn("Unable to check sync status for healthcheck", "err", err.Error()) + return err + } + if i == nil { + return nil + } + + return errNotSynced +} diff --git a/health/check_time.go b/health/check_time.go new file mode 100644 index 0000000000..a1a7d820a7 --- /dev/null +++ b/health/check_time.go @@ -0,0 +1,31 @@ +package health + +import ( + "context" + "errors" + "fmt" + "net/http" + + "github.com/ethereum/go-ethereum/ethclient" +) + +var ( + errTimestampTooOld = errors.New("timestamp too old") +) + +func checkTime( + ec *ethclient.Client, + r *http.Request, + seconds int, +) error { + i, err := ec.BlockByNumber(context.TODO(), nil) + if err != nil { + return err + } + timestamp := i.Time() + if timestamp < uint64(seconds) { + return fmt.Errorf("%w: got ts: %d, need: %d", errTimestampTooOld, timestamp, seconds) + } + + return nil +} diff --git a/health/health.go b/health/health.go new file mode 100644 index 0000000000..78ff97bfe5 --- /dev/null +++ b/health/health.go @@ -0,0 +1,199 @@ +package health + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "math/big" + "net/http" + "strconv" + "strings" + "time" + + "github.com/ethereum/go-ethereum/log" +) + +const ( + healthHeader = "X-GETH-HEALTHCHECK" + synced = "synced" + minPeerCount = "min_peer_count" + checkBlock = "check_block" + maxSecondsBehind = "max_seconds_behind" +) + +var ( + errCheckDisabled = errors.New("error check disabled") + errBadHeaderValue = errors.New("bad header value") +) + +type requestBody struct { + Synced *bool `json:"synced"` + MinPeerCount *uint `json:"min_peer_count"` + CheckBlock *uint64 `json:"check_block"` + MaxSecondsBehind *int `json:"max_seconds_behind"` +} + +func (h *handler) processFromHeaders(headers []string, w http.ResponseWriter, r *http.Request) { + var ( + errCheckSynced = errCheckDisabled + errCheckPeer = errCheckDisabled + errCheckBlock = errCheckDisabled + errCheckSeconds = errCheckDisabled + ) + + for _, header := range headers { + lHeader := strings.ToLower(header) + if lHeader == synced { + errCheckSynced = checkSynced(h.ec, r) + } + if strings.HasPrefix(lHeader, minPeerCount) { + peers, err := strconv.Atoi(strings.TrimPrefix(lHeader, minPeerCount)) + if err != nil { + errCheckPeer = err + break + } + errCheckPeer = checkMinPeers(h.ec, uint(peers)) + } + if strings.HasPrefix(lHeader, checkBlock) { + block, err := strconv.Atoi(strings.TrimPrefix(lHeader, checkBlock)) + if err != nil { + errCheckBlock = err + break + } + errCheckBlock = checkBlockNumber(h.ec, big.NewInt(int64(block))) + } + if strings.HasPrefix(lHeader, maxSecondsBehind) { + seconds, err := strconv.Atoi(strings.TrimPrefix(lHeader, maxSecondsBehind)) + if err != nil { + errCheckSeconds = err + break + } + if seconds < 0 { + errCheckSeconds = errBadHeaderValue + break + } + now := time.Now().Unix() + errCheckSeconds = checkTime(h.ec, r, int(now)-seconds) + } + } + + reportHealth(errCheckSynced, errCheckPeer, errCheckBlock, errCheckSeconds, w) +} + +func (h *handler) processFromBody(w http.ResponseWriter, r *http.Request) { + body, errParse := parseHealthCheckBody(r.Body) + defer r.Body.Close() + + var ( + errCheckSynced = errCheckDisabled + errCheckPeer = errCheckDisabled + errCheckBlock = errCheckDisabled + errCheckSeconds = errCheckDisabled + ) + + if errParse != nil { + log.Root().Warn("Unable to process healthcheck request", "err", errParse) + } else { + if body.Synced != nil { + errCheckSynced = checkSynced(h.ec, r) + } + + if body.MinPeerCount != nil { + errCheckPeer = checkMinPeers(h.ec, *body.MinPeerCount) + } + + if body.CheckBlock != nil { + errCheckBlock = checkBlockNumber(h.ec, big.NewInt(int64(*body.CheckBlock))) + } + + if body.MaxSecondsBehind != nil { + seconds := *body.MaxSecondsBehind + if seconds < 0 { + errCheckSeconds = errBadHeaderValue + } + now := time.Now().Unix() + errCheckSeconds = checkTime(h.ec, r, int(now)-seconds) + } + } + + err := reportHealth(errCheckSynced, errCheckPeer, errCheckBlock, errCheckSeconds, w) + if err != nil { + log.Root().Warn("Unable to process healthcheck request", "err", err) + } +} + +func reportHealth(errCheckSynced, errCheckPeer, errCheckBlock, errCheckSeconds error, w http.ResponseWriter) error { + statusCode := http.StatusOK + errs := make(map[string]string) + + if shouldChangeStatusCode(errCheckSynced) { + statusCode = http.StatusInternalServerError + } + errs[synced] = errorStringOrOK(errCheckSynced) + + if shouldChangeStatusCode(errCheckPeer) { + statusCode = http.StatusInternalServerError + } + errs[minPeerCount] = errorStringOrOK(errCheckPeer) + + if shouldChangeStatusCode(errCheckBlock) { + statusCode = http.StatusInternalServerError + } + errs[checkBlock] = errorStringOrOK(errCheckBlock) + + if shouldChangeStatusCode(errCheckSeconds) { + statusCode = http.StatusInternalServerError + } + errs[maxSecondsBehind] = errorStringOrOK(errCheckSeconds) + + return writeResponse(w, errs, statusCode) +} + +func parseHealthCheckBody(reader io.Reader) (requestBody, error) { + var body requestBody + + bodyBytes, err := io.ReadAll(reader) + if err != nil { + return body, err + } + + err = json.Unmarshal(bodyBytes, &body) + if err != nil { + return body, err + } + + return body, nil +} + +func writeResponse(w http.ResponseWriter, errs map[string]string, statusCode int) error { + w.WriteHeader(statusCode) + + bodyJson, err := json.Marshal(errs) + if err != nil { + return err + } + + _, err = w.Write(bodyJson) + if err != nil { + return err + } + + return nil +} + +func shouldChangeStatusCode(err error) bool { + return err != nil && !errors.Is(err, errCheckDisabled) +} + +func errorStringOrOK(err error) string { + if err == nil { + return "HEALTHY" + } + + if errors.Is(err, errCheckDisabled) { + return "DISABLED" + } + + return fmt.Sprintf("ERROR: %v", err) +} diff --git a/health/service.go b/health/service.go new file mode 100644 index 0000000000..c39c31d2d5 --- /dev/null +++ b/health/service.go @@ -0,0 +1,40 @@ +package health + +import ( + "net/http" + + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/internal/ethapi" + "github.com/ethereum/go-ethereum/node" +) + +type handler struct { + ec *ethclient.Client +} + +func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + headers := r.Header.Values(healthHeader) + if len(headers) != 0 { + h.processFromHeaders(headers, w, r) + } else { + h.processFromBody(w, r) + } +} + +// New constructs a new health service instance. +func New(stack *node.Node, backend ethapi.Backend, cors, vhosts []string) error { + _, err := newHandler(stack, backend, cors, vhosts) + return err +} + +// newHandler returns a new `http.Handler` that will answer node health queries. +func newHandler(stack *node.Node, backend ethapi.Backend, cors, vhosts []string) (*handler, error) { + ec := ethclient.NewClient(stack.Attach()) + h := handler{ec} + handler := node.NewHTTPHandlerStack(h, cors, vhosts, nil) + + stack.RegisterHandler("Health API", "/health", handler) + stack.RegisterHandler("Health API", "/health/", handler) + + return &h, nil +}