eth, consensus/bor: handle 503 status code in heimdall client (#1023)

* consensus/bor, eth: handle 503 response from heimdall

* log milestone ID during error

* add more checks to prevent logs

* fix: handle correct error

consensus/bor/heimdall/client.go

@@ -26,6 +26,7 @@ var (
 	ErrNotSuccessfulResponse = errors.New("error while fetching data from Heimdall")
 	ErrNotInRejectedList     = errors.New("milestoneID doesn't exist in rejected list")
 	ErrNotInMilestoneList    = errors.New("milestoneID doesn't exist in Heimdall")
+	ErrServiceUnavailable    = errors.New("service unavailable")
 )
 
 const (
@@ -277,10 +278,18 @@ func FetchWithRetry[T any](ctx context.Context, client http.Client, url *url.URL
 		return result, nil
 	}
 
+	// 503 (Service Unavailable) is thrown when an endpoint isn't activated
+	// yet in heimdall. E.g. when the hardfork hasn't hit yet but heimdall
+	// is upgraded.
+	if errors.Is(err, ErrServiceUnavailable) {
+		log.Debug("Heimdall service unavailable at the moment", "path", url.Path, "error", err)
+		return nil, err
+	}
+
 	// attempt counter
 	attempt := 1
 
-	log.Warn("an error while trying fetching from Heimdall", "attempt", attempt, "error", err)
+	log.Warn("an error while trying fetching from Heimdall", "path", url.Path, "attempt", attempt, "error", err)
 
 	// create a new ticker for retrying the request
 	ticker := time.NewTicker(retryCall)
@@ -307,9 +316,14 @@ retryLoop:
 			request = &Request{client: client, url: url, start: time.Now()}
 			result, err = Fetch[T](ctx, request)
 
+			if errors.Is(err, ErrServiceUnavailable) {
+				log.Debug("Heimdall service unavailable at the moment", "path", url.Path, "error", err)
+				return nil, err
+			}
+
 			if err != nil {
 				if attempt%logEach == 0 {
-					log.Warn("an error while trying fetching from Heimdall", "attempt", attempt, "error", err)
+					log.Warn("an error while trying fetching from Heimdall", "path", url.Path, "attempt", attempt, "error", err)
 				}
 
 				continue retryLoop
@@ -426,6 +440,10 @@ func internalFetch(ctx context.Context, client http.Client, u *url.URL) ([]byte,
 
 	defer res.Body.Close()
 
+	if res.StatusCode == http.StatusServiceUnavailable {
+		return nil, fmt.Errorf("%w: response code %d", ErrServiceUnavailable, res.StatusCode)
+	}
+
 	// check status code
 	if res.StatusCode != 200 && res.StatusCode != 204 {
 		return nil, fmt.Errorf("%w: response code %d", ErrNotSuccessfulResponse, res.StatusCode)

eth/backend.go

@@ -33,6 +33,7 @@ import (
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/consensus/beacon"
 	"github.com/ethereum/go-ethereum/consensus/bor"
+	"github.com/ethereum/go-ethereum/consensus/bor/heimdall"
 	"github.com/ethereum/go-ethereum/consensus/clique"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/bloombits"
@@ -788,6 +789,10 @@ func (s *Ethereum) handleMilestone(ctx context.Context, ethHandler *ethHandler,
 		ethHandler.downloader.ProcessFutureMilestone(num, hash)
 	}
 
+	if errors.Is(err, heimdall.ErrServiceUnavailable) {
+		return nil
+	}
+
 	if err != nil {
 		return err
 	}
@@ -800,7 +805,10 @@ func (s *Ethereum) handleMilestone(ctx context.Context, ethHandler *ethHandler,
 func (s *Ethereum) handleNoAckMilestone(ctx context.Context, ethHandler *ethHandler, bor *bor.Bor) error {
 	milestoneID, err := ethHandler.fetchNoAckMilestone(ctx, bor)
 
-	//If failed to fetch the no-ack milestone then it give the error.
+	if errors.Is(err, heimdall.ErrServiceUnavailable) {
+		return nil
+	}
+
 	if err != nil {
 		return err
 	}

eth/handler_bor.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/consensus/bor"
+	"github.com/ethereum/go-ethereum/consensus/bor/heimdall"
 	"github.com/ethereum/go-ethereum/log"
 )
 
@@ -63,6 +64,11 @@ func (h *ethHandler) fetchWhitelistMilestone(ctx context.Context, bor *bor.Bor,
 
 	// fetch latest milestone
 	milestone, err := bor.HeimdallClient.FetchMilestone(ctx)
+	if errors.Is(err, heimdall.ErrServiceUnavailable) {
+		log.Debug("Failed to fetch latest milestone for whitelisting", "err", err)
+		return num, hash, err
+	}
+
 	if err != nil {
 		log.Error("Failed to fetch latest milestone for whitelisting", "err", err)
 		return num, hash, errMilestone
@@ -92,9 +98,13 @@ func (h *ethHandler) fetchNoAckMilestone(ctx context.Context, bor *bor.Bor) (str
 
 	// fetch latest milestone
 	milestoneID, err := bor.HeimdallClient.FetchLastNoAckMilestone(ctx)
+	if errors.Is(err, heimdall.ErrServiceUnavailable) {
+		log.Debug("Failed to fetch latest no-ack milestone", "err", err)
+		return milestoneID, err
+	}
+
 	if err != nil {
 		log.Error("Failed to fetch latest no-ack milestone", "err", err)
-
 		return milestoneID, errMilestone
 	}
 
@@ -104,6 +114,10 @@ func (h *ethHandler) fetchNoAckMilestone(ctx context.Context, bor *bor.Bor) (str
 func (h *ethHandler) fetchNoAckMilestoneByID(ctx context.Context, bor *bor.Bor, milestoneID string) error {
 	// fetch latest milestone
 	err := bor.HeimdallClient.FetchNoAckMilestone(ctx, milestoneID)
+	if errors.Is(err, heimdall.ErrServiceUnavailable) {
+		log.Debug("Failed to fetch no-ack milestone by ID", "milestoneID", milestoneID, "err", err)
+		return err
+	}
 
 	// fixme: handle different types of errors
 	if errors.Is(err, ErrNotInRejectedList) {