[TT-4028] Fix race condition in httpdashboardhandler (#4844)

<!-- Provide a general summary of your changes in the Title above -->

## Description

Dashboardhandler has a race condition, PR fixes it using atomic
operations. It tentatively fixes a bug where StopBeating would stop the
beat loop, but would immediately reset it to started. Moved the
"STARTED" operation higher up.

<!-- Describe your changes in detail -->

## Related Issue

<!-- This project only accepts pull requests related to open issues. -->
<!-- If suggesting a new feature or change, please discuss it in an
issue first. -->
<!-- If fixing a bug, there should be an issue describing it with steps
to reproduce. -->
<!-- OSS: Please link to the issue here. Tyk: please create/link the
JIRA ticket. -->

## Motivation and Context

<!-- Why is this change required? What problem does it solve? -->

## How This Has Been Tested

<!-- Please describe in detail how you tested your changes -->
<!-- Include details of your testing environment, and the tests -->
<!-- you ran to see how your change affects other areas of the code,
etc. -->
<!-- This information is helpful for reviewers and QA. -->

## Screenshots (if appropriate)

## Types of changes

<!-- What types of changes does your code introduce? Put an `x` in all
the boxes that apply: -->

- [x] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)
- [ ] Refactoring or add test (improvements in base code or adds test
coverage to functionality)

## Checklist

<!-- Go over all the following points, and put an `x` in all the boxes
that apply -->
<!-- If there are no documentation updates required, mark the item as
checked. -->
<!-- Raise up any additional concerns not covered by the checklist. -->

- [ ] I ensured that the documentation is up to date
- [ ] I explained why this PR updates go.mod in detail with reasoning
why it's required
- [x] I would like a code coverage CI quality gate exception and have
explained why

---------

Co-authored-by: Tit Petric <tit@tyk.io>
Co-authored-by: jeff <jeffy.mathew100@gmail.com>

gateway/dashboard_register.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"sync/atomic"
 	"time"
 
 	"github.com/TykTechnologies/tyk/header"
@@ -30,6 +31,18 @@ type DashboardServiceSender interface {
 	NotifyDashboardOfEvent(interface{}) error
 }
 
+// Constants for heartBeatStopSentinel indicators.
+//
+// Go 1.17 adds atomic.Value.Swap which is great, but 1.19
+// adds atomic.Bool and other types. This is a go <1.13 cludge.
+const (
+	// HeartBeatStarted Zero value - the handlers started
+	HeartBeatStarted = 0
+
+	// HeartBeatStopped value - the handlers invoked shutdown
+	HeartBeatStopped = 1
+)
+
 type HTTPDashboardHandler struct {
 	RegistrationEndpoint    string
 	DeRegistrationEndpoint  string
@@ -38,8 +51,9 @@ type HTTPDashboardHandler struct {
 
 	Secret string
 
-	heartBeatStopSentinel bool
-	Gw                    *Gateway `json:"-"`
+	heartBeatStopSentinel int32
+
+	Gw *Gateway `json:"-"`
 }
 
 var dashClient *http.Client
@@ -213,26 +227,30 @@ func (h *HTTPDashboardHandler) Ping() error {
 		h.Gw.initialiseClient())
 }
 
+func (h *HTTPDashboardHandler) isHeartBeatStopped() bool {
+	return atomic.LoadInt32(&h.heartBeatStopSentinel) == HeartBeatStopped
+}
+
 func (h *HTTPDashboardHandler) StartBeating() error {
+	atomic.SwapInt32(&h.heartBeatStopSentinel, HeartBeatStarted)
 
 	req := h.newRequest(http.MethodGet, h.HeartBeatEndpoint)
 
 	client := h.Gw.initialiseClient()
 
-	for !h.heartBeatStopSentinel {
+	for !h.isHeartBeatStopped() {
 		if err := h.sendHeartBeat(req, client); err != nil {
 			dashLog.Warning(err)
 		}
 		time.Sleep(time.Second * 2)
 	}
 
 	dashLog.Info("Stopped Heartbeat")
-	h.heartBeatStopSentinel = false
 	return nil
 }
 
 func (h *HTTPDashboardHandler) StopBeating() {
-	h.heartBeatStopSentinel = true
+	atomic.SwapInt32(&h.heartBeatStopSentinel, HeartBeatStopped)
 }
 
 func (h *HTTPDashboardHandler) newRequest(method, endpoint string) *http.Request {

gateway/dashboard_register_test.go

@@ -28,3 +28,25 @@ func Test_BuildDashboardConnStr(t *testing.T) {
 
 	assert.Equal(t, connStr, "http://localhost/test")
 }
+
+func Test_DashboardLifecycle(t *testing.T) {
+	var handler HTTPDashboardHandler
+
+	handler = HTTPDashboardHandler{
+		heartBeatStopSentinel: HeartBeatStarted,
+	}
+	assert.False(t, handler.isHeartBeatStopped())
+
+	handler = HTTPDashboardHandler{
+		heartBeatStopSentinel: HeartBeatStopped,
+	}
+
+	assert.True(t, handler.isHeartBeatStopped())
+
+	handler = HTTPDashboardHandler{
+		heartBeatStopSentinel: HeartBeatStarted,
+	}
+
+	handler.StopBeating()
+	assert.True(t, handler.isHeartBeatStopped())
+}