Handle graceful shutdown in bor (#359)

* handle graceful shutdown in bor rest client

* fix: log

Co-authored-by: Jaynti Kanani <jdkanani@gmail.com>

* add: handle shutdown using close channel

* fix: create err variable for shutdown

* use ticker instead of time.After for handling retry

Co-authored-by: Jaynti Kanani <jdkanani@gmail.com>

consensus/bor/bor.go

@@ -116,6 +116,9 @@ var (
 	// errOutOfRangeChain is returned if an authorization list is attempted to
 	// be modified via out-of-range or non-contiguous headers.
 	errOutOfRangeChain = errors.New("out of range or non-contiguous chain")
+
+	// errShutdownDetected is returned if a shutdown was detected
+	errShutdownDetected = errors.New("shutdown detected")
 )
 
 // SignerFn is a signer callback function to request a header to be signed by a
@@ -916,6 +919,7 @@ func (c *Bor) APIs(chain consensus.ChainHeaderReader) []rpc.API {
 
 // Close implements consensus.Engine. It's a noop for bor as there are no background threads.
 func (c *Bor) Close() error {
+	c.HeimdallClient.Close()
 	return nil
 }
 

consensus/bor/rest.go

@@ -27,11 +27,13 @@ type IHeimdallClient interface {
 	Fetch(path string, query string) (*ResponseWithHeight, error)
 	FetchWithRetry(path string, query string) (*ResponseWithHeight, error)
 	FetchStateSyncEvents(fromID uint64, to int64) ([]*EventRecordWithTime, error)
+	Close()
 }
 
 type HeimdallClient struct {
 	urlString string
 	client    http.Client
+	closeCh   chan struct{}
 }
 
 func NewHeimdallClient(urlString string) (*HeimdallClient, error) {
@@ -40,6 +42,7 @@ func NewHeimdallClient(urlString string) (*HeimdallClient, error) {
 		client: http.Client{
 			Timeout: time.Duration(5 * time.Second),
 		},
+		closeCh: make(chan struct{}),
 	}
 	return h, nil
 }
@@ -96,13 +99,22 @@ func (h *HeimdallClient) FetchWithRetry(rawPath string, rawQuery string) (*Respo
 	u.Path = rawPath
 	u.RawQuery = rawQuery
 
+	// create a new ticker for retrying the request
+	ticker := time.NewTicker(5 * time.Second)
+	defer ticker.Stop()
+
 	for {
-		res, err := h.internalFetch(u)
-		if err == nil && res != nil {
-			return res, nil
+		select {
+		case <-h.closeCh:
+			log.Debug("Shutdown detected, terminating request")
+			return nil, errShutdownDetected
+		case <-ticker.C:
+			res, err := h.internalFetch(u)
+			if err == nil && res != nil {
+				return res, nil
+			}
+			log.Info("Retrying again in 5 seconds for next Heimdall data", "path", u.Path)
 		}
-		log.Info("Retrying again in 5 seconds for next Heimdall span", "path", u.Path)
-		time.Sleep(5 * time.Second)
 	}
 }
 
@@ -137,3 +149,9 @@ func (h *HeimdallClient) internalFetch(u *url.URL) (*ResponseWithHeight, error)
 
 	return &response, nil
 }
+
+// Close sends a signal to stop the running process
+func (h *HeimdallClient) Close() {
+	close(h.closeCh)
+	h.client.CloseIdleConnections()
+}

eth/backend.go

@@ -581,10 +581,12 @@ func (s *Ethereum) Stop() error {
 	// Then stop everything else.
 	s.bloomIndexer.Close()
 	close(s.closeBloomHandler)
-	s.txPool.Stop()
-	s.miner.Close()
-	s.blockchain.Stop()
+
+	// closing consensus engine first, as miner has deps on it
 	s.engine.Close()
+	s.txPool.Stop()
+	s.blockchain.Stop()
+	s.miner.Close()
 	rawdb.PopUncleanShutdownMarker(s.chainDb)
 	s.chainDb.Close()
 	s.eventMux.Stop()

tests/bor/mocks/IHeimdallClient.go

@@ -1,4 +1,4 @@
-// Code generated by mockery v1.0.0. DO NOT EDIT.
+// Code generated by mockery v2.10.0. DO NOT EDIT.
 
 package mocks
 
@@ -12,6 +12,11 @@ type IHeimdallClient struct {
 	mock.Mock
 }
 
+// Close provides a mock function with given fields:
+func (_m *IHeimdallClient) Close() {
+	_m.Called()
+}
+
 // Fetch provides a mock function with given fields: path, query
 func (_m *IHeimdallClient) Fetch(path string, query string) (*bor.ResponseWithHeight, error) {
 	ret := _m.Called(path, query)