Skip to content

Commit

Permalink
Add Namespace, Job and Group to envoy stats
Browse files Browse the repository at this point in the history
  • Loading branch information
jorgemarey committed Aug 26, 2022
1 parent ee501f4 commit 53551eb
Show file tree
Hide file tree
Showing 5 changed files with 124 additions and 60 deletions.
23 changes: 13 additions & 10 deletions client/allocrunner/group_service_hook.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ type groupServiceHook struct {
allocID string
jobID string
group string
nomadNamespace string
restarter agentconsul.WorkloadRestarter
prerun bool
deregistered bool
Expand Down Expand Up @@ -83,6 +84,7 @@ func newGroupServiceHook(cfg groupServiceHookConfig) *groupServiceHook {
allocID: cfg.alloc.ID,
jobID: cfg.alloc.JobID,
group: cfg.alloc.TaskGroup,
nomadNamespace: cfg.alloc.Namespace,
restarter: cfg.restarter,
namespace: cfg.namespace,
taskEnvBuilder: cfg.taskEnvBuilder,
Expand Down Expand Up @@ -247,15 +249,16 @@ func (h *groupServiceHook) getWorkloadServices() *serviceregistration.WorkloadSe

// Create task services struct with request's driver metadata
return &serviceregistration.WorkloadServices{
AllocID: h.allocID,
JobID: h.jobID,
Group: h.group,
Namespace: h.namespace,
Restarter: h.restarter,
Services: interpolatedServices,
Networks: h.networks,
NetworkStatus: netStatus,
Ports: h.ports,
Canary: h.canary,
AllocID: h.allocID,
JobID: h.jobID,
Group: h.group,
NomadNamespace: h.nomadNamespace,
Namespace: h.namespace,
Restarter: h.restarter,
Services: interpolatedServices,
Networks: h.networks,
NetworkStatus: netStatus,
Ports: h.ports,
Canary: h.canary,
}
}
5 changes: 5 additions & 0 deletions client/serviceregistration/workload.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,11 @@ type WorkloadServices struct {
// caused this registration.
JobID string

// NomadNamespace provides additional context for providers regarding which
// nomad namespace caused this registration.
// Note: this can be different to Namespace if the provider is not Nomad
NomadNamespace string

// Canary indicates whether, or not the allocation is a canary. This is
// used to build the correct tags mapping.
Canary bool
Expand Down
75 changes: 50 additions & 25 deletions command/agent/consul/connect.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,17 @@ import (
"github.com/hashicorp/nomad/nomad/structs"
)

type allocInfo struct {
Group string
JobID string
Namespace string
AllocID string
}

// newConnect creates a new Consul AgentServiceConnect struct based on a Nomad
// Connect struct. If the nomad Connect struct is nil, nil will be returned to
// disable Connect for this service.
func newConnect(serviceID, allocID string, serviceName string, nc *structs.ConsulConnect, networks structs.Networks, ports structs.AllocatedPorts) (*api.AgentServiceConnect, error) {
func newConnect(serviceID string, info allocInfo, serviceName string, nc *structs.ConsulConnect, networks structs.Networks, ports structs.AllocatedPorts) (*api.AgentServiceConnect, error) {
switch {
case nc == nil:
// no connect stanza means there is no connect service to register
Expand All @@ -33,7 +40,7 @@ func newConnect(serviceID, allocID string, serviceName string, nc *structs.Consu
if nc.SidecarService.Port == "" {
nc.SidecarService.Port = fmt.Sprintf("%s-%s", structs.ConnectProxyPrefix, serviceName)
}
sidecarReg, err := connectSidecarRegistration(serviceID, allocID, nc.SidecarService, networks, ports)
sidecarReg, err := connectSidecarRegistration(serviceID, info, nc.SidecarService, networks, ports)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -90,7 +97,7 @@ func newConnectGateway(connect *structs.ConsulConnect) *api.AgentServiceConnectP
return &api.AgentServiceConnectProxyConfig{Config: envoyConfig}
}

func connectSidecarRegistration(serviceID, allocID string, css *structs.ConsulSidecarService, networks structs.Networks, ports structs.AllocatedPorts) (*api.AgentServiceRegistration, error) {
func connectSidecarRegistration(serviceID string, info allocInfo, css *structs.ConsulSidecarService, networks structs.Networks, ports structs.AllocatedPorts) (*api.AgentServiceRegistration, error) {
if css == nil {
// no sidecar stanza means there is no sidecar service to register
return nil, nil
Expand All @@ -101,7 +108,7 @@ func connectSidecarRegistration(serviceID, allocID string, css *structs.ConsulSi
return nil, err
}

proxy, err := connectSidecarProxy(allocID, css.Proxy, cMapping.To, networks)
proxy, err := connectSidecarProxy(info, css.Proxy, cMapping.To, networks)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -130,7 +137,7 @@ func connectSidecarRegistration(serviceID, allocID string, css *structs.ConsulSi
}, nil
}

func connectSidecarProxy(allocID string, proxy *structs.ConsulProxy, cPort int, networks structs.Networks) (*api.AgentServiceConnectProxyConfig, error) {
func connectSidecarProxy(info allocInfo, proxy *structs.ConsulProxy, cPort int, networks structs.Networks) (*api.AgentServiceConnectProxyConfig, error) {
if proxy == nil {
proxy = new(structs.ConsulProxy)
}
Expand All @@ -143,7 +150,7 @@ func connectSidecarProxy(allocID string, proxy *structs.ConsulProxy, cPort int,
return &api.AgentServiceConnectProxyConfig{
LocalServiceAddress: proxy.LocalServiceAddress,
LocalServicePort: proxy.LocalServicePort,
Config: connectProxyConfig(proxy.Config, cPort, allocID),
Config: connectProxyConfig(proxy.Config, cPort, info),
Upstreams: connectUpstreams(proxy.Upstreams),
Expose: expose,
}, nil
Expand Down Expand Up @@ -226,40 +233,58 @@ func connectMeshGateway(in structs.ConsulMeshGateway) api.MeshGatewayConfig {
return gw
}

func connectProxyConfig(cfg map[string]interface{}, port int, allocID string) map[string]interface{} {
func connectProxyConfig(cfg map[string]interface{}, port int, info allocInfo) map[string]interface{} {
if cfg == nil {
cfg = make(map[string]interface{})
}
cfg["bind_address"] = "0.0.0.0"
cfg["bind_port"] = port
injectAllocID(cfg, allocID)

tags := map[string]string{
"nomad.group=": info.Group,
"nomad.job=": info.JobID,
"nomad.namespace=": info.Namespace,
"nomad.alloc_id=": info.AllocID,
}
injectNomadInfo(cfg, tags)
return cfg
}

// injectAllocID merges allocID into cfg=>envoy_stats_tags
// injectNomadInfo merges nomad information into cfg=>envoy_stats_tags
//
// cfg must not be nil
func injectAllocID(cfg map[string]interface{}, allocID string) {
const key = "envoy_stats_tags"
const prefix = "nomad.alloc_id="
pair := prefix + allocID
tags, exists := cfg[key]
if !exists {
cfg[key] = []string{pair}
return
}
func injectNomadInfo(cfg map[string]interface{}, defaultTags map[string]string) {
const configKey = "envoy_stats_tags"

switch v := tags.(type) {
existingTagsI := cfg[configKey]
switch existingTags := existingTagsI.(type) {
case []string:
// scan the existing tags to see if alloc_id= is already set
for _, s := range v {
if strings.HasPrefix(s, prefix) {
return
if len(existingTags) == 0 {
break
}
OUTER:
for key, value := range defaultTags {
for _, tag := range existingTags {
if strings.HasPrefix(tag, key) {
continue OUTER
}
}
existingTags = append(existingTags, key+value)
}
cfg[configKey] = existingTags
return
}

// common case.
var tags []string
for key, value := range defaultTags {
if value == "" {
continue
}
v = append(v, pair)
cfg[key] = v
tag := key + value
tags = append(tags, tag)
}
cfg[configKey] = tags
}

func connectNetworkInvariants(networks structs.Networks) error {
Expand Down
Loading

0 comments on commit 53551eb

Please sign in to comment.