Merge pull request #12951 from jorgemarey/f-srv-tagged-addresses

Allow setting tagged addresses on services

.changelog/12951.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+consul: Enable setting custom tagged_addresses field
+```

api/services.go

@@ -236,6 +236,7 @@ type Service struct {
 	Connect           *ConsulConnect    `hcl:"connect,block"`
 	Meta              map[string]string `hcl:"meta,block"`
 	CanaryMeta        map[string]string `hcl:"canary_meta,block"`
+	TaggedAddresses   map[string]string `hcl:"tagged_addresses,block"`
 	TaskName          string            `mapstructure:"task" hcl:"task,optional"`
 	OnUpdate          string            `mapstructure:"on_update" hcl:"on_update,optional"`
 
@@ -280,6 +281,18 @@ func (s *Service) Canonicalize(t *Task, tg *TaskGroup, job *Job) {
 		s.Provider = ServiceProviderConsul
 	}
 
+	if len(s.Meta) == 0 {
+		s.Meta = nil
+	}
+
+	if len(s.CanaryMeta) == 0 {
+		s.CanaryMeta = nil
+	}
+
+	if len(s.TaggedAddresses) == 0 {
+		s.TaggedAddresses = nil
+	}
+
 	s.Connect.Canonicalize()
 
 	// Canonicalize CheckRestart on Checks and merge Service.CheckRestart

api/services_test.go

@@ -21,21 +21,25 @@ func TestServiceRegistrations_Delete(t *testing.T) {
 	// TODO(jrasell) add tests once registration process is in place.
 }
 
-
 func TestService_Canonicalize(t *testing.T) {
 	testutil.Parallel(t)
 
 	j := &Job{Name: stringToPtr("job")}
 	tg := &TaskGroup{Name: stringToPtr("group")}
 	task := &Task{Name: "task"}
-	s := &Service{}
+	s := &Service{
+		TaggedAddresses: make(map[string]string),
+	}
 
 	s.Canonicalize(task, tg, j)
 
 	require.Equal(t, fmt.Sprintf("%s-%s-%s", *j.Name, *tg.Name, task.Name), s.Name)
 	require.Equal(t, "auto", s.AddressMode)
 	require.Equal(t, OnUpdateRequireHealthy, s.OnUpdate)
 	require.Equal(t, ServiceProviderConsul, s.Provider)
+	require.Nil(t, s.Meta)
+	require.Nil(t, s.CanaryMeta)
+	require.Nil(t, s.TaggedAddresses)
 }
 
 func TestServiceCheck_Canonicalize(t *testing.T) {
@@ -192,4 +196,4 @@ func TestService_Tags(t *testing.T) {
 	r.True(service.EnableTagOverride)
 	r.Equal([]string{"a", "b"}, service.Tags)
 	r.Equal([]string{"c", "d"}, service.CanaryTags)
-}
+}

client/taskenv/services.go

@@ -41,6 +41,7 @@ func InterpolateServices(taskEnv *TaskEnv, services []*structs.Service) []*struc
 		service.CanaryTags = taskEnv.ParseAndReplace(service.CanaryTags)
 		service.Meta = interpolateMapStringString(taskEnv, service.Meta)
 		service.CanaryMeta = interpolateMapStringString(taskEnv, service.CanaryMeta)
+		service.TaggedAddresses = interpolateMapStringString(taskEnv, service.TaggedAddresses)
 		interpolateConnect(taskEnv, service.Connect)
 
 		interpolated[i] = service

client/taskenv/services_test.go

@@ -27,6 +27,9 @@ func TestInterpolateServices(t *testing.T) {
 				"canarymeta-key": "${canarymeta}",
 			},
 			Address: "${address}",
+			TaggedAddresses: map[string]string{
+				"${ta-key}": "${ta-address}",
+			},
 			Checks: []*structs.ServiceCheck{
 				{
 					Name:          "${checkname}",
@@ -53,6 +56,8 @@ func TestInterpolateServices(t *testing.T) {
 			"tags":         "tags",
 			"meta":         "meta-value",
 			"address":      "example.com",
+			"ta-key":       "public_wan",
+			"ta-address":   "1.2.3.4",
 			"canarymeta":   "canarymeta-value",
 			"checkname":    "checkname",
 			"checktype":    "checktype",
@@ -83,6 +88,9 @@ func TestInterpolateServices(t *testing.T) {
 				"canarymeta-key": "canarymeta-value",
 			},
 			Address: "example.com",
+			TaggedAddresses: map[string]string{
+				"public_wan": "1.2.3.4",
+			},
 			Checks: []*structs.ServiceCheck{
 				{
 					Name:          "checkname",

command/agent/consul/service_client.go

@@ -213,6 +213,8 @@ func different(wanted *api.AgentServiceRegistration, existing *api.AgentService,
 		return true
 	case !reflect.DeepEqual(wanted.Meta, existing.Meta):
 		return true
+	case !reflect.DeepEqual(wanted.TaggedAddresses, existing.TaggedAddresses):
+		return true
 	case tagsDifferent(wanted.Tags, existing.Tags):
 		return true
 	case connectSidecarDifferent(wanted, sidecar):
@@ -1027,6 +1029,11 @@ func (c *ServiceClient) serviceRegs(
 		}
 	}
 
+	taggedAddresses, err := parseTaggedAddresses(service.TaggedAddresses, port)
+	if err != nil {
+		return nil, err
+	}
+
 	// Build the Consul Service registration request
 	serviceReg := &api.AgentServiceRegistration{
 		Kind:              kind,
@@ -1038,6 +1045,7 @@ func (c *ServiceClient) serviceRegs(
 		Address:           ip,
 		Port:              port,
 		Meta:              meta,
+		TaggedAddresses:   taggedAddresses,
 		Connect:           connect, // will be nil if no Connect stanza
 		Proxy:             gateway, // will be nil if no Connect Gateway stanza
 	}
@@ -1664,3 +1672,44 @@ func getNomadSidecar(id string, services map[string]*api.AgentService) *api.Agen
 	sidecarID := id + sidecarSuffix
 	return services[sidecarID]
 }
+
+func parseAddress(raw string, port int) (api.ServiceAddress, error) {
+	result := api.ServiceAddress{}
+	addr, portStr, err := net.SplitHostPort(raw)
+	// Error message from Go's net/ipsock.go
+	if err != nil {
+		if !strings.Contains(err.Error(), "missing port in address") {
+			return result, fmt.Errorf("error parsing address %q: %v", raw, err)
+		}
+
+		// Use the whole input as the address if there wasn't a port.
+		if ip := net.ParseIP(raw); ip == nil {
+			return result, fmt.Errorf("error parsing address %q: not an IP address", raw)
+		}
+		addr = raw
+	}
+
+	if portStr != "" {
+		port, err = strconv.Atoi(portStr)
+		if err != nil {
+			return result, fmt.Errorf("error parsing port %q: %v", portStr, err)
+		}
+	}
+
+	result.Address = addr
+	result.Port = port
+	return result, nil
+}
+
+// morph the tagged_addresses map into the structure consul api wants
+func parseTaggedAddresses(m map[string]string, port int) (map[string]api.ServiceAddress, error) {
+	result := make(map[string]api.ServiceAddress, len(m))
+	for k, v := range m {
+		sa, err := parseAddress(v, port)
+		if err != nil {
+			return nil, err
+		}
+		result[k] = sa
+	}
+	return result, nil
+}

command/agent/consul/service_client_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/hashicorp/nomad/helper/testlog"
 	"github.com/hashicorp/nomad/helper/uuid"
 	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/shoenig/test/must"
 	"github.com/stretchr/testify/require"
 )
 
@@ -28,6 +29,9 @@ func TestSyncLogic_agentServiceUpdateRequired(t *testing.T) {
 			Address:           "1.1.1.1",
 			EnableTagOverride: true,
 			Meta:              map[string]string{"foo": "1"},
+			TaggedAddresses: map[string]api.ServiceAddress{
+				"public_wan": {Address: "1.2.3.4", Port: 8080},
+			},
 			Connect: &api.AgentServiceConnect{
 				Native: false,
 				SidecarService: &api.AgentServiceRegistration{
@@ -56,6 +60,9 @@ func TestSyncLogic_agentServiceUpdateRequired(t *testing.T) {
 		Address:           "1.1.1.1",
 		EnableTagOverride: true,
 		Meta:              map[string]string{"foo": "1"},
+		TaggedAddresses: map[string]api.ServiceAddress{
+			"public_wan": {Address: "1.2.3.4", Port: 8080},
+		},
 	}
 
 	sidecar := &api.AgentService{
@@ -212,6 +219,15 @@ func TestSyncLogic_agentServiceUpdateRequired(t *testing.T) {
 		})
 	})
 
+	t.Run("different tagged addresses", func(t *testing.T) {
+		try(t, true, syncNewOps, func(w asr) *asr {
+			w.TaggedAddresses = map[string]api.ServiceAddress{
+				"public_wan": {Address: "5.6.7.8", Port: 8080},
+			}
+			return &w
+		})
+	})
+
 	// for remaining tests, EnableTagOverride = false
 	existing.EnableTagOverride = false
 
@@ -648,3 +664,44 @@ func TestSyncLogic_maybeSidecarProxyCheck(t *testing.T) {
 	try("service:_nomad-task-2f5fb517-57d4-44ee-7780-dc1cb6e103cd-group-api-count-api-9001-sidecar-proxy: ", false)
 	try("service", false)
 }
+
+func TestSyncLogic_parseTaggedAddresses(t *testing.T) {
+	ci.Parallel(t)
+
+	t.Run("nil", func(t *testing.T) {
+		m, err := parseTaggedAddresses(nil, 0)
+		must.NoError(t, err)
+		must.MapEmpty(t, m)
+	})
+
+	t.Run("parse fail", func(t *testing.T) {
+		ta := map[string]string{
+			"public_wan": "not an address",
+		}
+		result, err := parseTaggedAddresses(ta, 8080)
+		must.Error(t, err)
+		must.MapEmpty(t, result)
+	})
+
+	t.Run("parse address", func(t *testing.T) {
+		ta := map[string]string{
+			"public_wan": "1.2.3.4",
+		}
+		result, err := parseTaggedAddresses(ta, 8080)
+		must.NoError(t, err)
+		must.MapEq(t, map[string]api.ServiceAddress{
+			"public_wan": {Address: "1.2.3.4", Port: 8080},
+		}, result)
+	})
+
+	t.Run("parse address and port", func(t *testing.T) {
+		ta := map[string]string{
+			"public_wan": "1.2.3.4:9999",
+		}
+		result, err := parseTaggedAddresses(ta, 8080)
+		must.NoError(t, err)
+		must.MapEq(t, map[string]api.ServiceAddress{
+			"public_wan": {Address: "1.2.3.4", Port: 9999},
+		}, result)
+	})
+}

command/agent/job_endpoint.go

@@ -1376,6 +1376,7 @@ func ApiServicesToStructs(in []*api.Service, group bool) []*structs.Service {
 			Address:           s.Address,
 			Meta:              helper.CopyMapStringString(s.Meta),
 			CanaryMeta:        helper.CopyMapStringString(s.CanaryMeta),
+			TaggedAddresses:   helper.CopyMapStringString(s.TaggedAddresses),
 			OnUpdate:          s.OnUpdate,
 			Provider:          s.Provider,
 		}

command/agent/job_endpoint_test.go

@@ -2518,6 +2518,9 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
 						Meta: map[string]string{
 							"servicemeta": "foobar",
 						},
+						TaggedAddresses: map[string]string{
+							"wan": "1.2.3.4",
+						},
 						CheckRestart: &api.CheckRestart{
 							Limit: 4,
 							Grace: helper.TimeToPtr(11 * time.Second),
@@ -2915,6 +2918,9 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
 						Meta: map[string]string{
 							"servicemeta": "foobar",
 						},
+						TaggedAddresses: map[string]string{
+							"wan": "1.2.3.4",
+						},
 						OnUpdate: "require_healthy",
 						Checks: []*structs.ServiceCheck{
 							{

go.mod

@@ -37,7 +37,7 @@ require (
 	github.com/fsouza/go-dockerclient v1.6.5
 	github.com/golang/protobuf v1.5.2
 	github.com/golang/snappy v0.0.4
-	github.com/google/go-cmp v0.5.6
+	github.com/google/go-cmp v0.5.8
 	github.com/gorilla/handlers v1.5.1
 	github.com/gorilla/websocket v1.4.2
 	github.com/gosuri/uilive v0.0.4
@@ -109,6 +109,7 @@ require (
 	github.com/ryanuber/go-glob v1.0.0
 	github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529
 	github.com/shirou/gopsutil/v3 v3.21.12
+	github.com/shoenig/test v0.2.5
 	github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c
 	github.com/stretchr/testify v1.7.1
 	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635
@@ -263,7 +264,7 @@ require (
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 // indirect
+	golang.org/x/tools v0.1.10 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/api v0.60.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect

go.sum

@@ -582,8 +582,9 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 h1:zLTLjkaOFEFIOxY5BWLFLwh+cL8vOBW4XJ2aqLE/Tf0=
 github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -1166,6 +1167,8 @@ github.com/shirou/gopsutil v0.0.0-20181107111621-48177ef5f880/go.mod h1:5b4v6he4
 github.com/shirou/gopsutil/v3 v3.21.12 h1:VoGxEW2hpmz0Vt3wUvHIl9fquzYLNpVpgNNB7pGJimA=
 github.com/shirou/gopsutil/v3 v3.21.12/go.mod h1:BToYZVTlSVlfazpDDYFnsVZLaoRG+g8ufT6fPQLdJzA=
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
+github.com/shoenig/test v0.2.5 h1:CfxxPAhW9sJt9nVI39cOHrb4krmHd28SmU66oCXi6sY=
+github.com/shoenig/test v0.2.5/go.mod h1:xYtyGBC5Q3kzCNyJg/SjgNpfAa2kvmgA0i5+lQso8x0=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
@@ -1657,8 +1660,8 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 h1:0c3L82FDQ5rt1bjTBlchS8t6RQ6299/+5bWMnRLh+uI=
-golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
+golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20=
+golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

helper/funcs.go

@@ -401,8 +401,6 @@ func CopyMapStringFloat64(m map[string]float64) map[string]float64 {
 	return c
 }
 
-// CopyMapStringSliceString copies a map of strings to string slices such as
-// http.Header
 func CopyMapStringSliceString(m map[string][]string) map[string][]string {
 	l := len(m)
 	if l == 0 {

jobspec/parse_service.go

@@ -50,6 +50,7 @@ func parseService(o *ast.ObjectItem) (*api.Service, error) {
 		"task",
 		"meta",
 		"canary_meta",
+		"tagged_addresses",
 		"on_update",
 		"provider",
 	}
@@ -68,6 +69,7 @@ func parseService(o *ast.ObjectItem) (*api.Service, error) {
 	delete(m, "connect")
 	delete(m, "meta")
 	delete(m, "canary_meta")
+	delete(m, "tagged_addresses")
 
 	if err := mapstructure.WeakDecode(m, &service); err != nil {
 		return nil, err
@@ -140,6 +142,20 @@ func parseService(o *ast.ObjectItem) (*api.Service, error) {
 		}
 	}
 
+	// Parse out tagged_addresses fields. These are in HCL as a list so we need
+	// to iterate over them and merge them.
+	if taO := listVal.Filter("tagged_addresses"); len(taO.Items) > 0 {
+		for _, o := range taO.Elem().Items {
+			var m map[string]interface{}
+			if err := hcl.DecodeObject(&m, o.Val); err != nil {
+				return nil, err
+			}
+			if err := mapstructure.WeakDecode(m, &service.TaggedAddresses); err != nil {
+				return nil, err
+			}
+		}
+	}
+
 	return &service, nil
 }