add HTTP endpoint to reload agent (#312)

add HTTP endpoint to reload agent

agent/agent.go

@@ -7,9 +7,9 @@ import (
 	"os/signal"
 	"syscall"
 
+	metrics "github.com/armon/go-metrics"
 	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/nomad-autoscaler/agent/config"
-	agentServer "github.com/hashicorp/nomad-autoscaler/agent/http"
 	"github.com/hashicorp/nomad-autoscaler/plugins/manager"
 	"github.com/hashicorp/nomad-autoscaler/policy"
 	filePolicy "github.com/hashicorp/nomad-autoscaler/policy/file"
@@ -26,7 +26,7 @@ type Agent struct {
 	nomadClient   *api.Client
 	pluginManager *manager.PluginManager
 	policyManager *policy.Manager
-	httpServer    *agentServer.Server
+	inMemSink     *metrics.InmemSink
 	evalBroker    *policyeval.Broker
 }
 
@@ -59,15 +59,7 @@ func (a *Agent) Run() error {
 	if err != nil {
 		return fmt.Errorf("failed to setup telemetry: %v", err)
 	}
-
-	// Setup and start the HTTP server.
-	httpServer, err := agentServer.NewHTTPServer(a.config.HTTP, a.logger, inMem)
-	if err != nil {
-		return fmt.Errorf("failed to setup HTTP getHealth server: %v", err)
-	}
-
-	a.httpServer = httpServer
-	go a.httpServer.Start()
+	a.inMemSink = inMem
 
 	policyEvalCh := a.setupPolicyManager()
 	go a.policyManager.Run(ctx, policyEvalCh)
@@ -144,11 +136,6 @@ func (a *Agent) setupPolicyManager() chan *sdk.ScalingEvaluation {
 }
 
 func (a *Agent) stop() {
-	// Stop the health server.
-	if a.httpServer != nil {
-		a.httpServer.Stop()
-	}
-
 	// Kill all the plugins.
 	if a.pluginManager != nil {
 		a.pluginManager.KillPlugins()
@@ -214,7 +201,8 @@ func (a *Agent) generateNomadClient() error {
 
 // reload triggers the reload of sub-routines based on the operator sending a
 // SIGHUP signal to the agent.
-func (a Agent) reload() {
+func (a *Agent) reload() {
+	a.logger.Debug("reloading policy sources")
 	a.policyManager.ReloadSources()
 }
 

agent/http/agent.go

@@ -0,0 +1,25 @@
+package http
+
+import (
+	"net/http"
+	"strings"
+)
+
+// agentSpecificRequest handles the requests for the `/v1/agent/` endpoint and sub-paths.
+func (s *Server) agentSpecificRequest(w http.ResponseWriter, r *http.Request) (interface{}, error) {
+	path := strings.TrimPrefix(r.URL.Path, "/v1/agent")
+	switch {
+	case strings.HasSuffix(path, "/reload"):
+		return s.agentReload(w, r)
+	default:
+		return nil, newCodedError(http.StatusNotFound, "")
+	}
+}
+
+func (s *Server) agentReload(w http.ResponseWriter, r *http.Request) (interface{}, error) {
+	if r.Method != http.MethodPost && r.Method != http.MethodPut {
+		return nil, newCodedError(http.StatusMethodNotAllowed, errInvalidMethod)
+	}
+
+	return s.agent.ReloadAgent(w, r)
+}

agent/http/agent_test.go

@@ -0,0 +1,41 @@
+package http
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestServer_agentReload(t *testing.T) {
+	testCases := []struct {
+		inputReq         *http.Request
+		expectedRespCode int
+		name             string
+	}{
+		{
+			inputReq:         httptest.NewRequest("PUT", "/v1/agent/reload", nil),
+			expectedRespCode: 200,
+			name:             "successfully reload",
+		},
+		{
+			inputReq:         httptest.NewRequest("GET", "/v1/agent/reload", nil),
+			expectedRespCode: 405,
+			name:             "incorrect request method",
+		},
+	}
+
+	srv, stopSrv := TestServer(t)
+	defer stopSrv()
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert := assert.New(t)
+
+			w := httptest.NewRecorder()
+			srv.mux.ServeHTTP(w, tc.inputReq)
+			assert.Equal(tc.expectedRespCode, w.Code)
+		})
+	}
+}

agent/http/health_test.go

@@ -6,8 +6,6 @@ import (
 	"sync/atomic"
 	"testing"
 
-	hclog "github.com/hashicorp/go-hclog"
-	"github.com/hashicorp/nomad-autoscaler/agent/config"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -43,9 +41,8 @@ func TestServer_getHealth(t *testing.T) {
 	}
 
 	// Create our HTTP server.
-	srv, err := NewHTTPServer(&config.HTTP{BindAddress: "127.0.0.1", BindPort: 8080}, hclog.NewNullLogger(), nil)
-	assert.Nil(t, err)
-	defer srv.ln.Close()
+	srv, stopSrv := TestServer(t)
+	defer stopSrv()
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {

agent/http/metrics.go

@@ -28,7 +28,7 @@ func (s *Server) getMetrics(w http.ResponseWriter, r *http.Request) (interface{}
 		s.getPrometheusMetrics().ServeHTTP(w, r)
 		return nil, nil
 	}
-	return s.inMemSink.DisplayMetrics(w, r)
+	return s.agent.DisplayMetrics(w, r)
 }
 
 // getPrometheusMetrics is the getMetrics handler when the caller wishes to

agent/http/metrics_test.go

@@ -4,11 +4,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"testing"
-	"time"
 
-	metrics "github.com/armon/go-metrics"
-	hclog "github.com/hashicorp/go-hclog"
-	"github.com/hashicorp/nomad-autoscaler/agent/config"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -44,14 +40,9 @@ func TestServer_getMetrics(t *testing.T) {
 		},
 	}
 
-	// Create a simple in-memory sink to use.
-	inm := metrics.NewInmemSink(10*time.Second, time.Minute)
-	metrics.DefaultInmemSignal(inm)
-
 	// Create our HTTP server.
-	srv, err := NewHTTPServer(&config.HTTP{BindAddress: "127.0.0.1", BindPort: 8080}, hclog.NewNullLogger(), inm)
-	assert.Nil(t, err)
-	defer srv.ln.Close()
+	srv, stopSrv := TestServer(t)
+	defer stopSrv()
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {

agent/http/server.go

@@ -9,7 +9,6 @@ import (
 	"sync/atomic"
 	"time"
 
-	metrics "github.com/armon/go-metrics"
 	hclog "github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/go-msgpack/codec"
 	"github.com/hashicorp/nomad-autoscaler/agent/config"
@@ -24,13 +23,27 @@ const (
 	// to register the metrics server endpoint.
 	metricsRoutePattern = "/v1/metrics"
 
+	// agentRoutePattern is the Autoscaler HTTP router pattern which is used to
+	// register endpoints related to the agent.
+	agentRoutePattern = "/v1/agent/"
+
 	// healthAliveness is used to define the health of the Autoscaler agent. It
 	// currently can only be in two states; ready or unavailable and depends
 	// entirely on whether the server is serving or not.
 	healthAlivenessReady = iota
 	healthAlivenessUnavailable
 )
 
+// AgentHTTP is the interface that defines the HTTP handlers that an Agent
+// must implement in order to be accessible through the HTTP API.
+type AgentHTTP interface {
+	// DisplayMetrics returns a summary of metrics collected by the agent.
+	DisplayMetrics(resp http.ResponseWriter, req *http.Request) (interface{}, error)
+
+	// ReloadAgent triggers the agent to reload policies and configuration.
+	ReloadAgent(resp http.ResponseWriter, req *http.Request) (interface{}, error)
+}
+
 type Server struct {
 	log hclog.Logger
 	ln  net.Listener
@@ -42,23 +55,24 @@ type Server struct {
 	// const declarations.
 	aliveness int32
 
-	// inMemSink is our in-memory telemetry sink used to server metrics
-	// endpoint requests.
-	inMemSink *metrics.InmemSink
+	// agent is the reference to an object that implements the AgentHTTP
+	// interface to handle agent requests.
+	agent AgentHTTP
 }
 
 // NewHTTPServer creates a new agent HTTP server.
-func NewHTTPServer(cfg *config.HTTP, log hclog.Logger, inmSink *metrics.InmemSink) (*Server, error) {
+func NewHTTPServer(cfg *config.HTTP, log hclog.Logger, agent AgentHTTP) (*Server, error) {
 
 	srv := &Server{
-		inMemSink: inmSink,
-		log:       log.Named("http_server"),
-		mux:       http.NewServeMux(),
+		log:   log.Named("http_server"),
+		mux:   http.NewServeMux(),
+		agent: agent,
 	}
 
 	// Setup our handlers.
 	srv.mux.HandleFunc(healthRoutePattern, srv.wrap(srv.getHealth))
 	srv.mux.HandleFunc(metricsRoutePattern, srv.wrap(srv.getMetrics))
+	srv.mux.HandleFunc(agentRoutePattern, srv.wrap(srv.agentSpecificRequest))
 
 	// Configure the HTTP server to the most basic level.
 	srv.srv = &http.Server{

agent/http/testing.go

@@ -0,0 +1,25 @@
+package http
+
+import (
+	"testing"
+
+	hclog "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/nomad-autoscaler/agent"
+	"github.com/hashicorp/nomad-autoscaler/agent/config"
+)
+
+func TestServer(t *testing.T) (*Server, func()) {
+	cfg := &config.HTTP{
+		BindAddress: "127.0.0.1",
+		BindPort:    0, // Use next available port.
+	}
+
+	s, err := NewHTTPServer(cfg, hclog.NewNullLogger(), &agent.MockAgentHTTP{})
+	if err != nil {
+		t.Fatalf("failed to start test server: %v", err)
+	}
+
+	return s, func() {
+		s.Stop()
+	}
+}

agent/http_handler.go

@@ -0,0 +1,14 @@
+package agent
+
+import "net/http"
+
+// The methods in this file implement in the http.AgentHTTP interface.
+
+func (a *Agent) DisplayMetrics(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
+	return a.inMemSink.DisplayMetrics(resp, req)
+}
+
+func (a *Agent) ReloadAgent(_ http.ResponseWriter, _ *http.Request) (interface{}, error) {
+	a.reload()
+	return nil, nil
+}

agent/testing.go

@@ -0,0 +1,22 @@
+package agent
+
+import (
+	"net/http"
+
+	metrics "github.com/armon/go-metrics"
+)
+
+type MockAgentHTTP struct{}
+
+func (m *MockAgentHTTP) DisplayMetrics(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
+	return metrics.MetricsSummary{
+		Timestamp: "2020-11-17 00:17:50 +0000 UTC",
+		Counters:  []metrics.SampledValue{},
+		Gauges:    []metrics.GaugeValue{},
+		Points:    []metrics.PointValue{},
+		Samples:   []metrics.SampledValue{},
+	}, nil
+}
+func (m *MockAgentHTTP) ReloadAgent(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
+	return nil, nil
+}

command/agent.go

@@ -10,11 +10,15 @@ import (
 	"github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/nomad-autoscaler/agent"
 	"github.com/hashicorp/nomad-autoscaler/agent/config"
+	agentHTTP "github.com/hashicorp/nomad-autoscaler/agent/http"
 	flaghelper "github.com/hashicorp/nomad-autoscaler/sdk/helper/flag"
 )
 
 type AgentCommand struct {
 	args []string
+
+	agent      *agent.Agent
+	httpServer *agentHTTP.Server
 }
 
 // Help should return long-form help text that includes the command-line
@@ -226,9 +230,19 @@ func (c *AgentCommand) Run(args []string) int {
 		JSONFormat: parsedConfig.LogJson,
 	})
 
-	// create and run agent
-	a := agent.NewAgent(parsedConfig, logger)
-	if err := a.Run(); err != nil {
+	// create and run agent and HTTP server
+	c.agent = agent.NewAgent(parsedConfig, logger)
+	httpServer, err := agentHTTP.NewHTTPServer(parsedConfig.HTTP, logger, c.agent)
+	if err != nil {
+		logger.Error("failed to setup HTTP getHealth server", "error", err)
+		return 1
+	}
+
+	c.httpServer = httpServer
+	go c.httpServer.Start()
+	defer c.httpServer.Stop()
+
+	if err := c.agent.Run(); err != nil {
 		logger.Error("failed to start agent", "error", err)
 		return 1
 	}