Tagged metrics API

command/agent/agent.go

@@ -13,6 +13,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	metrics "github.com/armon/go-metrics"
 	"github.com/hashicorp/consul/api"
 	version "github.com/hashicorp/go-version"
 	"github.com/hashicorp/nomad/client"
@@ -66,15 +67,18 @@ type Agent struct {
 	shutdown     bool
 	shutdownCh   chan struct{}
 	shutdownLock sync.Mutex
+
+	InmemSink *metrics.InmemSink
 }
 
 // NewAgent is used to create a new agent with the given configuration
-func NewAgent(config *Config, logOutput io.Writer) (*Agent, error) {
+func NewAgent(config *Config, logOutput io.Writer, inmem *metrics.InmemSink) (*Agent, error) {
 	a := &Agent{
 		config:     config,
 		logger:     log.New(logOutput, "", log.LstdFlags|log.Lmicroseconds),
 		logOutput:  logOutput,
 		shutdownCh: make(chan struct{}),
+		InmemSink:  inmem,
 	}
 
 	if err := a.setupConsul(config.Consul); err != nil {
@@ -331,9 +335,13 @@ func (a *Agent) clientConfig() (*clientconfig.Config, error) {
 
 	conf.ConsulConfig = a.config.Consul
 	conf.VaultConfig = a.config.Vault
+
+	// Set up Telemetry configuration
 	conf.StatsCollectionInterval = a.config.Telemetry.collectionInterval
 	conf.PublishNodeMetrics = a.config.Telemetry.PublishNodeMetrics
 	conf.PublishAllocationMetrics = a.config.Telemetry.PublishAllocationMetrics
+	conf.DisableTaggedMetrics = a.config.Telemetry.DisableTaggedMetrics
+	conf.BackwardsCompatibleMetrics = a.config.Telemetry.BackwardsCompatibleMetrics
 
 	// Set the TLS related configs
 	conf.TLSConfig = a.config.TLSConfig
@@ -489,7 +497,6 @@ func (a *Agent) setupClient() error {
 		}
 	}
 
-	// Create the client
 	client, err := client.NewClient(conf, a.consulCatalog, a.consulService, a.logger)
 	if err != nil {
 		return fmt.Errorf("client setup failed: %v", err)

command/agent/agent_test.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/hashicorp/nomad/helper"
 	sconfig "github.com/hashicorp/nomad/nomad/structs/config"
+	"github.com/stretchr/testify/assert"
 )
 
 func getPort() int {
@@ -316,6 +317,29 @@ func TestAgent_ClientConfig(t *testing.T) {
 	}
 }
 
+// Clients should inherit telemetry configuration
+func TestAget_Client_TelemetryConfiguration(t *testing.T) {
+	assert := assert.New(t)
+
+	conf := DefaultConfig()
+	conf.DevMode = true
+	conf.Telemetry.DisableTaggedMetrics = true
+	conf.Telemetry.BackwardsCompatibleMetrics = true
+
+	a := &Agent{config: conf}
+
+	c, err := a.clientConfig()
+	assert.Nil(err)
+
+	telemetry := conf.Telemetry
+
+	assert.Equal(c.StatsCollectionInterval, telemetry.collectionInterval)
+	assert.Equal(c.PublishNodeMetrics, telemetry.PublishNodeMetrics)
+	assert.Equal(c.PublishAllocationMetrics, telemetry.PublishAllocationMetrics)
+	assert.Equal(c.DisableTaggedMetrics, telemetry.DisableTaggedMetrics)
+	assert.Equal(c.BackwardsCompatibleMetrics, telemetry.BackwardsCompatibleMetrics)
+}
+
 // TestAgent_HTTPCheck asserts Agent.agentHTTPCheck properly alters the HTTP
 // API health check depending on configuration.
 func TestAgent_HTTPCheck(t *testing.T) {

command/agent/command.go

@@ -15,12 +15,12 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/armon/go-metrics"
+	metrics "github.com/armon/go-metrics"
 	"github.com/armon/go-metrics/circonus"
 	"github.com/armon/go-metrics/datadog"
 	"github.com/hashicorp/consul/lib"
-	"github.com/hashicorp/go-checkpoint"
-	"github.com/hashicorp/go-syslog"
+	checkpoint "github.com/hashicorp/go-checkpoint"
+	gsyslog "github.com/hashicorp/go-syslog"
 	"github.com/hashicorp/logutils"
 	"github.com/hashicorp/nomad/helper/flag-helpers"
 	"github.com/hashicorp/nomad/helper/gated-writer"
@@ -331,9 +331,9 @@ func (c *Command) setupLoggers(config *Config) (*gatedwriter.Writer, *logWriter,
 }
 
 // setupAgent is used to start the agent and various interfaces
-func (c *Command) setupAgent(config *Config, logOutput io.Writer) error {
+func (c *Command) setupAgent(config *Config, logOutput io.Writer, inmem *metrics.InmemSink) error {
 	c.Ui.Output("Starting Nomad agent...")
-	agent, err := NewAgent(config, logOutput)
+	agent, err := NewAgent(config, logOutput, inmem)
 	if err != nil {
 		c.Ui.Error(fmt.Sprintf("Error starting agent: %s", err))
 		return err
@@ -444,13 +444,14 @@ func (c *Command) Run(args []string) int {
 	}
 
 	// Initialize the telemetry
-	if err := c.setupTelemetry(config); err != nil {
+	inmem, err := c.setupTelemetry(config)
+	if err != nil {
 		c.Ui.Error(fmt.Sprintf("Error initializing telemetry: %s", err))
 		return 1
 	}
 
 	// Create the agent
-	if err := c.setupAgent(config, logOutput); err != nil {
+	if err := c.setupAgent(config, logOutput, inmem); err != nil {
 		logGate.Flush()
 		return 1
 	}
@@ -619,7 +620,7 @@ func (c *Command) handleReload(config *Config) *Config {
 }
 
 // setupTelemetry is used ot setup the telemetry sub-systems
-func (c *Command) setupTelemetry(config *Config) error {
+func (c *Command) setupTelemetry(config *Config) (*metrics.InmemSink, error) {
 	/* Setup telemetry
 	Aggregate on 10 second intervals for 1 minute. Expose the
 	metrics over stderr when there is a SIGUSR1 received.
@@ -646,7 +647,7 @@ func (c *Command) setupTelemetry(config *Config) error {
 	if telConfig.StatsiteAddr != "" {
 		sink, err := metrics.NewStatsiteSink(telConfig.StatsiteAddr)
 		if err != nil {
-			return err
+			return inm, err
 		}
 		fanout = append(fanout, sink)
 	}
@@ -655,7 +656,7 @@ func (c *Command) setupTelemetry(config *Config) error {
 	if telConfig.StatsdAddr != "" {
 		sink, err := metrics.NewStatsdSink(telConfig.StatsdAddr)
 		if err != nil {
-			return err
+			return inm, err
 		}
 		fanout = append(fanout, sink)
 	}
@@ -664,7 +665,7 @@ func (c *Command) setupTelemetry(config *Config) error {
 	if telConfig.DataDogAddr != "" {
 		sink, err := datadog.NewDogStatsdSink(telConfig.DataDogAddr, config.NodeName)
 		if err != nil {
-			return err
+			return inm, err
 		}
 		fanout = append(fanout, sink)
 	}
@@ -700,7 +701,7 @@ func (c *Command) setupTelemetry(config *Config) error {
 
 		sink, err := circonus.NewCirconusSink(cfg)
 		if err != nil {
-			return err
+			return inm, err
 		}
 		sink.Start()
 		fanout = append(fanout, sink)
@@ -714,7 +715,7 @@ func (c *Command) setupTelemetry(config *Config) error {
 		metricsConf.EnableHostname = false
 		metrics.NewGlobal(metricsConf, inm)
 	}
-	return nil
+	return inm, nil
 }
 
 // setupSCADA is used to start a new SCADA provider and listener,

command/agent/config-test-fixtures/basic.hcl

@@ -60,8 +60,6 @@ client {
     gc_inode_usage_threshold = 91
     gc_max_allocs = 50
     no_host_uuid = false
-    disable_tagged_metrics = true
-    backwards_compatible_metrics = true
 }
 server {
 	enabled = true
@@ -98,6 +96,8 @@ telemetry {
     collection_interval = "3s"
     publish_allocation_metrics = true
     publish_node_metrics = true
+    disable_tagged_metrics = true
+    backwards_compatible_metrics = true
 }
 leave_on_interrupt = true
 leave_on_terminate = true

command/agent/config.go

@@ -229,14 +229,6 @@ type ClientConfig struct {
 	// NoHostUUID disables using the host's UUID and will force generation of a
 	// random UUID.
 	NoHostUUID *bool `mapstructure:"no_host_uuid"`
-
-	// DisableTaggedMetrics disables a new version of generating metrics which
-	// uses tags
-	DisableTaggedMetrics bool `mapstructure:"disable_tagged_metrics"`
-
-	// BackwardsCompatibleMetrics allows for generating metrics in a simple
-	// key/value structure as done in older versions of Nomad
-	BackwardsCompatibleMetrics bool `mapstructure:"backwards_compatible_metrics"`
 }
 
 // ACLConfig is configuration specific to the ACL system
@@ -371,6 +363,14 @@ type Telemetry struct {
 	PublishAllocationMetrics bool          `mapstructure:"publish_allocation_metrics"`
 	PublishNodeMetrics       bool          `mapstructure:"publish_node_metrics"`
 
+	// DisableTaggedMetrics disables a new version of generating metrics which
+	// uses tags
+	DisableTaggedMetrics bool `mapstructure:"disable_tagged_metrics"`
+
+	// BackwardsCompatibleMetrics allows for generating metrics in a simple
+	// key/value structure as done in older versions of Nomad
+	BackwardsCompatibleMetrics bool `mapstructure:"backwards_compatible_metrics"`
+
 	// Circonus: see https://github.com/circonus-labs/circonus-gometrics
 	// for more details on the various configuration options.
 	// Valid configuration combinations:
@@ -1105,14 +1105,6 @@ func (a *ClientConfig) Merge(b *ClientConfig) *ClientConfig {
 		result.NoHostUUID = b.NoHostUUID
 	}
 
-	if b.DisableTaggedMetrics {
-		result.DisableTaggedMetrics = b.DisableTaggedMetrics
-	}
-
-	if b.BackwardsCompatibleMetrics {
-		result.BackwardsCompatibleMetrics = b.BackwardsCompatibleMetrics
-	}
-
 	// Add the servers
 	result.Servers = append(result.Servers, b.Servers...)
 
@@ -1214,6 +1206,15 @@ func (a *Telemetry) Merge(b *Telemetry) *Telemetry {
 	if b.CirconusBrokerSelectTag != "" {
 		result.CirconusBrokerSelectTag = b.CirconusBrokerSelectTag
 	}
+
+	if b.DisableTaggedMetrics {
+		result.DisableTaggedMetrics = b.DisableTaggedMetrics
+	}
+
+	if b.BackwardsCompatibleMetrics {
+		result.BackwardsCompatibleMetrics = b.BackwardsCompatibleMetrics
+	}
+
 	return &result
 }
 

command/agent/config_parse.go

@@ -357,8 +357,6 @@ func parseClient(result **ClientConfig, list *ast.ObjectList) error {
 		"gc_parallel_destroys",
 		"gc_max_allocs",
 		"no_host_uuid",
-		"disable_tagged_metrics",
-		"backwards_compatible_metrics",
 	}
 	if err := checkHCLKeys(listVal, valid); err != nil {
 		return err
@@ -635,6 +633,8 @@ func parseTelemetry(result **Telemetry, list *ast.ObjectList) error {
 		"circonus_check_tags",
 		"circonus_broker_id",
 		"circonus_broker_select_tag",
+		"disable_tagged_metrics",
+		"backwards_compatible_metrics",
 	}
 	if err := checkHCLKeys(listVal, valid); err != nil {
 		return err

command/agent/config_parse_test.go

@@ -75,14 +75,12 @@ func TestConfig_Parse(t *testing.T) {
 						ReservedPorts:       "1,100,10-12",
 						ParsedReservedPorts: []int{1, 10, 11, 12, 100},
 					},
-					GCInterval:                 6 * time.Second,
-					GCParallelDestroys:         6,
-					GCDiskUsageThreshold:       82,
-					GCInodeUsageThreshold:      91,
-					GCMaxAllocs:                50,
-					NoHostUUID:                 helper.BoolToPtr(false),
-					DisableTaggedMetrics:       true,
-					BackwardsCompatibleMetrics: true,
+					GCInterval:            6 * time.Second,
+					GCParallelDestroys:    6,
+					GCDiskUsageThreshold:  82,
+					GCInodeUsageThreshold: 91,
+					GCMaxAllocs:           50,
+					NoHostUUID:            helper.BoolToPtr(false),
 				},
 				Server: &ServerConfig{
 					Enabled:                true,
@@ -113,14 +111,16 @@ func TestConfig_Parse(t *testing.T) {
 					ReplicationToken: "foobar",
 				},
 				Telemetry: &Telemetry{
-					StatsiteAddr:             "127.0.0.1:1234",
-					StatsdAddr:               "127.0.0.1:2345",
-					DisableHostname:          true,
-					UseNodeName:              false,
-					CollectionInterval:       "3s",
-					collectionInterval:       3 * time.Second,
-					PublishAllocationMetrics: true,
-					PublishNodeMetrics:       true,
+					StatsiteAddr:               "127.0.0.1:1234",
+					StatsdAddr:                 "127.0.0.1:2345",
+					DisableHostname:            true,
+					UseNodeName:                false,
+					CollectionInterval:         "3s",
+					collectionInterval:         3 * time.Second,
+					PublishAllocationMetrics:   true,
+					PublishNodeMetrics:         true,
+					DisableTaggedMetrics:       true,
+					BackwardsCompatibleMetrics: true,
 				},
 				LeaveOnInt:                true,
 				LeaveOnTerm:               true,

command/agent/config_test.go

@@ -55,6 +55,8 @@ func TestConfig_Merge(t *testing.T) {
 			StatsdAddr:                         "127.0.0.1:8125",
 			DataDogAddr:                        "127.0.0.1:8125",
 			DisableHostname:                    false,
+			DisableTaggedMetrics:               true,
+			BackwardsCompatibleMetrics:         true,
 			CirconusAPIToken:                   "0",
 			CirconusAPIApp:                     "nomadic",
 			CirconusAPIURL:                     "http://api.circonus.com/v2",
@@ -89,8 +91,6 @@ func TestConfig_Merge(t *testing.T) {
 				ReservedPorts:       "1,10-30,55",
 				ParsedReservedPorts: []int{1, 2, 4},
 			},
-			DisableTaggedMetrics:       true,
-			BackwardsCompatibleMetrics: true,
 		},
 		Server: &ServerConfig{
 			Enabled:                false,
@@ -185,6 +185,8 @@ func TestConfig_Merge(t *testing.T) {
 			DisableHostname:                    true,
 			PublishNodeMetrics:                 true,
 			PublishAllocationMetrics:           true,
+			DisableTaggedMetrics:               true,
+			BackwardsCompatibleMetrics:         true,
 			CirconusAPIToken:                   "1",
 			CirconusAPIApp:                     "nomad",
 			CirconusAPIURL:                     "https://api.circonus.com/v2",
@@ -226,12 +228,10 @@ func TestConfig_Merge(t *testing.T) {
 				ReservedPorts:       "2,10-30,55",
 				ParsedReservedPorts: []int{1, 2, 3},
 			},
-			GCInterval:                 6 * time.Second,
-			GCParallelDestroys:         6,
-			GCDiskUsageThreshold:       71,
-			GCInodeUsageThreshold:      86,
-			DisableTaggedMetrics:       true,
-			BackwardsCompatibleMetrics: true,
+			GCInterval:            6 * time.Second,
+			GCParallelDestroys:    6,
+			GCDiskUsageThreshold:  71,
+			GCInodeUsageThreshold: 86,
 		},
 		Server: &ServerConfig{
 			Enabled:                true,

command/agent/http.go

@@ -168,6 +168,8 @@ func (s *HTTPServer) registerHandlers(enableDebug bool) {
 	s.mux.HandleFunc("/v1/agent/servers", s.wrap(s.AgentServersRequest))
 	s.mux.HandleFunc("/v1/agent/keyring/", s.wrap(s.KeyringOperationRequest))
 
+	s.mux.HandleFunc("/v1/metrics", s.wrap(s.MetricsRequest))
+
 	s.mux.HandleFunc("/v1/validate/job", s.wrap(s.ValidateJobRequest))
 
 	s.mux.HandleFunc("/v1/regions", s.wrap(s.RegionListRequest))

command/agent/metrics_endpoint.go

@@ -0,0 +1,16 @@
+package agent
+
+import (
+	"net/http"
+)
+
+func (s *HTTPServer) MetricsRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
+	if req.Method == "GET" {
+		return s.newMetricsRequest(resp, req)
+	}
+	return nil, CodedError(405, ErrInvalidMethod)
+}
+
+func (s *HTTPServer) newMetricsRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
+	return s.agent.InmemSink.DisplayMetrics(resp, req)
+}