Skip to content

Commit

Permalink
tagged metrics config options should be on telemetry config
Browse files Browse the repository at this point in the history 
better api example, add telemetry documentation
  • Loading branch information
@chelseakomlo
chelseakomlo committed Sep 6, 2017
1 parent cc00c13 commit 1df5310
Show file tree
Hide file tree
Showing 12 changed files with 107 additions and 91 deletions.
47 changes: 29 additions & 18 deletions client/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,9 @@ type Client struct {
// baseLabels are used when emitting tagged metrics. All client metrics will
// have these tags, and optionally more.
baseLabels []metrics.Label

// Subset of global telemetry configuration options for the client
clientTelemetry *ClientTelemetry
}

var (
Expand All @@ -166,8 +169,15 @@ var (
noServersErr = errors.New("no servers")
)

// ClientTelemetry is a subset of global telemetry configuration options that
// are relevant for the client
type ClientTelemetry struct {
DisableTaggedMetrics bool
BackwardsCompatibleMetrics bool
}

// NewClient is used to create a new client from the given configuration
func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulService ConsulServiceAPI, logger *log.Logger) (*Client, error) {
func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulService ConsulServiceAPI, logger *log.Logger, telemetry *ClientTelemetry) (*Client, error) {
// Create the tls wrapper
var tlsWrap tlsutil.RegionWrapper
if cfg.TLSConfig.EnableRPC {
Expand All @@ -180,6 +190,7 @@ func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulServic

// Create the client
c := &Client{
clientTelemetry: telemetry,
config: cfg,
consulCatalog: consulCatalog,
consulService: consulService,
Expand Down Expand Up @@ -1879,14 +1890,14 @@ func (c *Client) emitStats() {

// setGaugeForMemoryStats proxies metrics for memory specific statistics
func (c *Client) setGaugeForMemoryStats(nodeID string, hStats *stats.HostStats) {
if !c.config.DisableTaggedMetrics {
if !c.clientTelemetry.DisableTaggedMetrics {
metrics.SetGaugeWithLabels([]string{"client", "host", "memory", "total"}, float32(hStats.Memory.Total), c.baseLabels)
metrics.SetGaugeWithLabels([]string{"client", "host", "memory", "available"}, float32(hStats.Memory.Available), c.baseLabels)
metrics.SetGaugeWithLabels([]string{"client", "host", "memory", "used"}, float32(hStats.Memory.Used), c.baseLabels)
metrics.SetGaugeWithLabels([]string{"client", "host", "memory", "free"}, float32(hStats.Memory.Free), c.baseLabels)
}

if c.config.BackwardsCompatibleMetrics {
if c.clientTelemetry.BackwardsCompatibleMetrics {
metrics.SetGauge([]string{"client", "host", "memory", nodeID, "total"}, float32(hStats.Memory.Total))
metrics.SetGauge([]string{"client", "host", "memory", nodeID, "available"}, float32(hStats.Memory.Available))
metrics.SetGauge([]string{"client", "host", "memory", nodeID, "used"}, float32(hStats.Memory.Used))
Expand All @@ -1897,7 +1908,7 @@ func (c *Client) setGaugeForMemoryStats(nodeID string, hStats *stats.HostStats)
// setGaugeForCPUStats proxies metrics for CPU specific statistics
func (c *Client) setGaugeForCPUStats(nodeID string, hStats *stats.HostStats) {
for _, cpu := range hStats.CPU {
if !c.config.DisableTaggedMetrics {
if !c.clientTelemetry.DisableTaggedMetrics {
labels := append(c.baseLabels, metrics.Label{"cpu", cpu.CPU})

metrics.SetGaugeWithLabels([]string{"client", "host", "cpu", "total"}, float32(cpu.Total), labels)
Expand All @@ -1906,7 +1917,7 @@ func (c *Client) setGaugeForCPUStats(nodeID string, hStats *stats.HostStats) {
metrics.SetGaugeWithLabels([]string{"client", "host", "cpu", "system"}, float32(cpu.System), labels)
}

if c.config.BackwardsCompatibleMetrics {
if c.clientTelemetry.BackwardsCompatibleMetrics {
metrics.SetGauge([]string{"client", "host", "cpu", nodeID, cpu.CPU, "total"}, float32(cpu.Total))
metrics.SetGauge([]string{"client", "host", "cpu", nodeID, cpu.CPU, "user"}, float32(cpu.User))
metrics.SetGauge([]string{"client", "host", "cpu", nodeID, cpu.CPU, "idle"}, float32(cpu.Idle))
Expand All @@ -1918,7 +1929,7 @@ func (c *Client) setGaugeForCPUStats(nodeID string, hStats *stats.HostStats) {
// setGaugeForDiskStats proxies metrics for disk specific statistics
func (c *Client) setGaugeForDiskStats(nodeID string, hStats *stats.HostStats) {
for _, disk := range hStats.DiskStats {
if !c.config.DisableTaggedMetrics {
if !c.clientTelemetry.DisableTaggedMetrics {
labels := append(c.baseLabels, metrics.Label{"disk", disk.Device})

metrics.SetGaugeWithLabels([]string{"client", "host", "disk", "size"}, float32(disk.Size), labels)
Expand All @@ -1928,7 +1939,7 @@ func (c *Client) setGaugeForDiskStats(nodeID string, hStats *stats.HostStats) {
metrics.SetGaugeWithLabels([]string{"client", "host", "disk", "inodes_percent"}, float32(disk.InodesUsedPercent), labels)
}

if c.config.BackwardsCompatibleMetrics {
if c.clientTelemetry.BackwardsCompatibleMetrics {
metrics.SetGauge([]string{"client", "host", "disk", nodeID, disk.Device, "size"}, float32(disk.Size))
metrics.SetGauge([]string{"client", "host", "disk", nodeID, disk.Device, "used"}, float32(disk.Used))
metrics.SetGauge([]string{"client", "host", "disk", nodeID, disk.Device, "available"}, float32(disk.Available))
Expand All @@ -1947,27 +1958,27 @@ func (c *Client) setGaugeForAllocationStats(nodeID string) {
allocated := c.getAllocatedResources(node)

// Emit allocated
if !c.config.DisableTaggedMetrics {
if !c.clientTelemetry.DisableTaggedMetrics {
metrics.SetGaugeWithLabels([]string{"client", "allocated", "memory"}, float32(allocated.MemoryMB), c.baseLabels)
metrics.SetGaugeWithLabels([]string{"client", "allocated", "disk"}, float32(allocated.DiskMB), c.baseLabels)
metrics.SetGaugeWithLabels([]string{"client", "allocated", "cpu"}, float32(allocated.CPU), c.baseLabels)
metrics.SetGaugeWithLabels([]string{"client", "allocated", "iops"}, float32(allocated.IOPS), c.baseLabels)
}

if c.config.BackwardsCompatibleMetrics {
if c.clientTelemetry.BackwardsCompatibleMetrics {
metrics.SetGauge([]string{"client", "allocated", "memory", nodeID}, float32(allocated.MemoryMB))
metrics.SetGauge([]string{"client", "allocated", "disk", nodeID}, float32(allocated.DiskMB))
metrics.SetGauge([]string{"client", "allocated", "cpu", nodeID}, float32(allocated.CPU))
metrics.SetGauge([]string{"client", "allocated", "iops", nodeID}, float32(allocated.IOPS))
}

for _, n := range allocated.Networks {
if !c.config.DisableTaggedMetrics {
if !c.clientTelemetry.DisableTaggedMetrics {
labels := append(c.baseLabels, metrics.Label{"device", n.Device})
metrics.SetGaugeWithLabels([]string{"client", "allocated", "network"}, float32(n.MBits), labels)
}

if c.config.BackwardsCompatibleMetrics {
if c.clientTelemetry.BackwardsCompatibleMetrics {
metrics.SetGauge([]string{"client", "allocated", "network", n.Device, nodeID}, float32(n.MBits))
}
}
Expand All @@ -1978,14 +1989,14 @@ func (c *Client) setGaugeForAllocationStats(nodeID string) {
unallocatedCpu := total.CPU - res.CPU - allocated.CPU
unallocatedIops := total.IOPS - res.IOPS - allocated.IOPS

if !c.config.DisableTaggedMetrics {
if !c.clientTelemetry.DisableTaggedMetrics {
metrics.SetGaugeWithLabels([]string{"client", "unallocated", "memory"}, float32(unallocatedMem), c.baseLabels)
metrics.SetGaugeWithLabels([]string{"client", "unallocated", "disk"}, float32(unallocatedDisk), c.baseLabels)
metrics.SetGaugeWithLabels([]string{"client", "unallocated", "cpu"}, float32(unallocatedCpu), c.baseLabels)
metrics.SetGaugeWithLabels([]string{"client", "unallocated", "iops"}, float32(unallocatedIops), c.baseLabels)
}

if c.config.BackwardsCompatibleMetrics {
if c.clientTelemetry.BackwardsCompatibleMetrics {
metrics.SetGauge([]string{"client", "unallocated", "memory", nodeID}, float32(unallocatedMem))
metrics.SetGauge([]string{"client", "unallocated", "disk", nodeID}, float32(unallocatedDisk))
metrics.SetGauge([]string{"client", "unallocated", "cpu", nodeID}, float32(unallocatedCpu))
Expand All @@ -2003,7 +2014,7 @@ func (c *Client) setGaugeForAllocationStats(nodeID string) {

unallocatedMbits := totalMbits - n.MBits

if !c.config.DisableTaggedMetrics {
if !c.clientTelemetry.DisableTaggedMetrics {
labels := append(c.baseLabels, metrics.Label{"device", n.Device})
metrics.SetGaugeWithLabels([]string{"client", "unallocated", "network"}, float32(unallocatedMbits), labels)
}
Expand All @@ -2016,10 +2027,10 @@ func (c *Client) setGaugeForAllocationStats(nodeID string) {

// No lables are required so we emit with only a key/value syntax
func (c *Client) setGaugeForUptime(hStats *stats.HostStats) {
if !c.config.DisableTaggedMetrics {
if !c.clientTelemetry.DisableTaggedMetrics {
metrics.SetGaugeWithLabels([]string{"uptime"}, float32(hStats.Uptime), c.baseLabels)
}
if c.config.BackwardsCompatibleMetrics {
if c.clientTelemetry.BackwardsCompatibleMetrics {
metrics.SetGauge([]string{"uptime"}, float32(hStats.Uptime))
}
}
Expand Down Expand Up @@ -2062,15 +2073,15 @@ func (c *Client) emitClientMetrics() {
}
}

if !c.config.DisableTaggedMetrics {
if !c.clientTelemetry.DisableTaggedMetrics {
metrics.SetGaugeWithLabels([]string{"client", "allocations", "migrating"}, float32(migrating), c.baseLabels)
metrics.SetGaugeWithLabels([]string{"client", "allocations", "blocked"}, float32(blocked), c.baseLabels)
metrics.SetGaugeWithLabels([]string{"client", "allocations", "pending"}, float32(pending), c.baseLabels)
metrics.SetGaugeWithLabels([]string{"client", "allocations", "running"}, float32(running), c.baseLabels)
metrics.SetGaugeWithLabels([]string{"client", "allocations", "terminal"}, float32(terminal), c.baseLabels)
}

if c.config.BackwardsCompatibleMetrics {
if c.clientTelemetry.BackwardsCompatibleMetrics {
metrics.SetGauge([]string{"client", "allocations", "migrating", nodeID}, float32(migrating))
metrics.SetGauge([]string{"client", "allocations", "blocked", nodeID}, float32(blocked))
metrics.SetGauge([]string{"client", "allocations", "pending", nodeID}, float32(pending))
Expand Down
6 changes: 4 additions & 2 deletions client/client_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,8 @@ func testClient(t *testing.T, cb func(c *config.Config)) *Client {
catalog := consul.NewMockCatalog(logger)
mockService := newMockConsulServiceClient()
mockService.logger = logger
client, err := NewClient(conf, catalog, mockService, logger)
telemetry := &ClientTelemetry{DisableTaggedMetrics: false, BackwardsCompatibleMetrics: false}
client, err := NewClient(conf, catalog, mockService, logger, telemetry)
if err != nil {
t.Fatalf("err: %v", err)
}
Expand Down Expand Up @@ -794,7 +795,8 @@ func TestClient_SaveRestoreState(t *testing.T) {
catalog := consul.NewMockCatalog(logger)
mockService := newMockConsulServiceClient()
mockService.logger = logger
c2, err := NewClient(c1.config, catalog, mockService, logger)
telemetry := &ClientTelemetry{DisableTaggedMetrics: false, BackwardsCompatibleMetrics: false}
c2, err := NewClient(c1.config, catalog, mockService, logger, telemetry)
if err != nil {
t.Fatalf("err: %v", err)
}
Expand Down
7 changes: 6 additions & 1 deletion command/agent/agent.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -494,7 +494,12 @@ func (a *Agent) setupClient() error {
}

// Create the client
client, err := client.NewClient(conf, a.consulCatalog, a.consulService, a.logger)
clientTelemetry := &client.ClientTelemetry{
DisableTaggedMetrics: a.config.Telemetry.DisableTaggedMetrics,
BackwardsCompatibleMetrics: a.config.Telemetry.BackwardsCompatibleMetrics,
}

client, err := client.NewClient(conf, a.consulCatalog, a.consulService, a.logger, clientTelemetry)
if err != nil {
return fmt.Errorf("client setup failed: %v", err)
}
Expand Down
4 changes: 2 additions & 2 deletions command/agent/config-test-fixtures/basic.hcl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,6 @@ client {
gc_inode_usage_threshold = 91
gc_max_allocs = 50
no_host_uuid = false
disable_tagged_metrics = true
backwards_compatible_metrics = true
}
server {
enabled = true
Expand Down Expand Up @@ -98,6 +96,8 @@ telemetry {
collection_interval = "3s"
publish_allocation_metrics = true
publish_node_metrics = true
disable_tagged_metrics = true
backwards_compatible_metrics = true
}
leave_on_interrupt = true
leave_on_terminate = true
Expand Down
33 changes: 17 additions & 16 deletions command/agent/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,14 +229,6 @@ type ClientConfig struct {
// NoHostUUID disables using the host's UUID and will force generation of a
// random UUID.
NoHostUUID *bool `mapstructure:"no_host_uuid"`

// DisableTaggedMetrics disables a new version of generating metrics which
// uses tags
DisableTaggedMetrics bool `mapstructure:"disable_tagged_metrics"`

// BackwardsCompatibleMetrics allows for generating metrics in a simple
// key/value structure as done in older versions of Nomad
BackwardsCompatibleMetrics bool `mapstructure:"backwards_compatible_metrics"`
}

// ACLConfig is configuration specific to the ACL system
Expand Down Expand Up @@ -371,6 +363,14 @@ type Telemetry struct {
PublishAllocationMetrics bool `mapstructure:"publish_allocation_metrics"`
PublishNodeMetrics bool `mapstructure:"publish_node_metrics"`

// DisableTaggedMetrics disables a new version of generating metrics which
// uses tags
DisableTaggedMetrics bool `mapstructure:"disable_tagged_metrics"`

// BackwardsCompatibleMetrics allows for generating metrics in a simple
// key/value structure as done in older versions of Nomad
BackwardsCompatibleMetrics bool `mapstructure:"backwards_compatible_metrics"`

// Circonus: see https://github.com/circonus-labs/circonus-gometrics
// for more details on the various configuration options.
// Valid configuration combinations:
Expand Down Expand Up @@ -1105,14 +1105,6 @@ func (a *ClientConfig) Merge(b *ClientConfig) *ClientConfig {
result.NoHostUUID = b.NoHostUUID
}

if b.DisableTaggedMetrics {
result.DisableTaggedMetrics = b.DisableTaggedMetrics
}

if b.BackwardsCompatibleMetrics {
result.BackwardsCompatibleMetrics = b.BackwardsCompatibleMetrics
}

// Add the servers
result.Servers = append(result.Servers, b.Servers...)

Expand Down Expand Up @@ -1214,6 +1206,15 @@ func (a *Telemetry) Merge(b *Telemetry) *Telemetry {
if b.CirconusBrokerSelectTag != "" {
result.CirconusBrokerSelectTag = b.CirconusBrokerSelectTag
}

if b.DisableTaggedMetrics {
result.DisableTaggedMetrics = b.DisableTaggedMetrics
}

if b.BackwardsCompatibleMetrics {
result.BackwardsCompatibleMetrics = b.BackwardsCompatibleMetrics
}

return &result
}

Expand Down
4 changes: 2 additions & 2 deletions command/agent/config_parse.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -357,8 +357,6 @@ func parseClient(result **ClientConfig, list *ast.ObjectList) error {
"gc_parallel_destroys",
"gc_max_allocs",
"no_host_uuid",
"disable_tagged_metrics",
"backwards_compatible_metrics",
}
if err := checkHCLKeys(listVal, valid); err != nil {
return err
Expand Down Expand Up @@ -635,6 +633,8 @@ func parseTelemetry(result **Telemetry, list *ast.ObjectList) error {
"circonus_check_tags",
"circonus_broker_id",
"circonus_broker_select_tag",
"disable_tagged_metrics",
"backwards_compatible_metrics",
}
if err := checkHCLKeys(listVal, valid); err != nil {
return err
Expand Down
32 changes: 16 additions & 16 deletions command/agent/config_parse_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,14 +75,12 @@ func TestConfig_Parse(t *testing.T) {
ReservedPorts: "1,100,10-12",
ParsedReservedPorts: []int{1, 10, 11, 12, 100},
},
GCInterval: 6 * time.Second,
GCParallelDestroys: 6,
GCDiskUsageThreshold: 82,
GCInodeUsageThreshold: 91,
GCMaxAllocs: 50,
NoHostUUID: helper.BoolToPtr(false),
DisableTaggedMetrics: true,
BackwardsCompatibleMetrics: true,
GCInterval: 6 * time.Second,
GCParallelDestroys: 6,
GCDiskUsageThreshold: 82,
GCInodeUsageThreshold: 91,
GCMaxAllocs: 50,
NoHostUUID: helper.BoolToPtr(false),
},
Server: &ServerConfig{
Enabled: true,
Expand Down Expand Up @@ -113,14 +111,16 @@ func TestConfig_Parse(t *testing.T) {
ReplicationToken: "foobar",
},
Telemetry: &Telemetry{
StatsiteAddr: "127.0.0.1:1234",
StatsdAddr: "127.0.0.1:2345",
DisableHostname: true,
UseNodeName: false,
CollectionInterval: "3s",
collectionInterval: 3 * time.Second,
PublishAllocationMetrics: true,
PublishNodeMetrics: true,
StatsiteAddr: "127.0.0.1:1234",
StatsdAddr: "127.0.0.1:2345",
DisableHostname: true,
UseNodeName: false,
CollectionInterval: "3s",
collectionInterval: 3 * time.Second,
PublishAllocationMetrics: true,
PublishNodeMetrics: true,
DisableTaggedMetrics: true,
BackwardsCompatibleMetrics: true,
},
LeaveOnInt: true,
LeaveOnTerm: true,
Expand Down
16 changes: 8 additions & 8 deletions command/agent/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ func TestConfig_Merge(t *testing.T) {
StatsdAddr: "127.0.0.1:8125",
DataDogAddr: "127.0.0.1:8125",
DisableHostname: false,
DisableTaggedMetrics: true,
BackwardsCompatibleMetrics: true,
CirconusAPIToken: "0",
CirconusAPIApp: "nomadic",
CirconusAPIURL: "http://api.circonus.com/v2",
Expand Down Expand Up @@ -89,8 +91,6 @@ func TestConfig_Merge(t *testing.T) {
ReservedPorts: "1,10-30,55",
ParsedReservedPorts: []int{1, 2, 4},
},
DisableTaggedMetrics: true,
BackwardsCompatibleMetrics: true,
},
Server: &ServerConfig{
Enabled: false,
Expand Down Expand Up @@ -185,6 +185,8 @@ func TestConfig_Merge(t *testing.T) {
DisableHostname: true,
PublishNodeMetrics: true,
PublishAllocationMetrics: true,
DisableTaggedMetrics: true,
BackwardsCompatibleMetrics: true,
CirconusAPIToken: "1",
CirconusAPIApp: "nomad",
CirconusAPIURL: "https://api.circonus.com/v2",
Expand Down Expand Up @@ -226,12 +228,10 @@ func TestConfig_Merge(t *testing.T) {
ReservedPorts: "2,10-30,55",
ParsedReservedPorts: []int{1, 2, 3},
},
GCInterval: 6 * time.Second,
GCParallelDestroys: 6,
GCDiskUsageThreshold: 71,
GCInodeUsageThreshold: 86,
DisableTaggedMetrics: true,
BackwardsCompatibleMetrics: true,
GCInterval: 6 * time.Second,
GCParallelDestroys: 6,
GCDiskUsageThreshold: 71,
GCInodeUsageThreshold: 86,
},
Server: &ServerConfig{
Enabled: true,
Expand Down
Loading

0 comments on commit 1df5310

Please sign in to comment.