consul/connect: enable setting datacenter in connect upstream

Before, upstreams could only be defined using the default datacenter.
Now, the `datacenter` field can be set in a connect upstream definition,
informing consul of the desire for an instance of the upstream service
in the specified datacenter. The field is optional and continues to
default to the local datacenter.

Closes #8964

CHANGELOG.md

@@ -29,6 +29,7 @@ IMPROVEMENTS:
  * consul: Support advertising CNI and multi-host network addresses to consul [[GH-8801](https://github.com/hashicorp/nomad/issues/8801)]
  * consul: Support Consul namespace (Consul Enterprise) in client configuration. [[GH-8849](https://github.com/hashicorp/nomad/pull/8849)]
  * consul/connect: Dynamically select envoy sidecar at runtime [[GH-8945](https://github.com/hashicorp/nomad/pull/8945)]
+ * consul/connect: Enable setting `datacenter` field on connect upstreams [[GH-8964](https://github.com/hashicorp/nomad/issues/8964)]
  * csi: Support `nomad volume detach` with previously garbage-collected nodes. [[GH-9057](https://github.com/hashicorp/nomad/issues/9057)]
  * csi: Relaxed validation requirements when checking volume capabilities with controller plugins, to accommodate existing plugin behaviors. [[GH-9049](https://github.com/hashicorp/nomad/issues/9049)]
  * driver/docker: Upgrade pause container and detect architecture [[GH-8957](https://github.com/hashicorp/nomad/pull/8957)]

api/services.go

@@ -270,6 +270,7 @@ func (cp *ConsulProxy) Canonicalize() {
 type ConsulUpstream struct {
 	DestinationName string `mapstructure:"destination_name" hcl:"destination_name,optional"`
 	LocalBindPort   int    `mapstructure:"local_bind_port" hcl:"local_bind_port,optional"`
+	Datacenter      string `mapstructure:"datacenter" hcl:"datacenter,optional"`
 }
 
 type ConsulExposeConfig struct {

api/services_test.go

@@ -195,6 +195,7 @@ func TestService_Connect_proxy_settings(t *testing.T) {
 						{
 							DestinationName: "upstream",
 							LocalBindPort:   80,
+							Datacenter:      "dc2",
 						},
 					},
 					LocalServicePort: 8000,
@@ -205,8 +206,9 @@ func TestService_Connect_proxy_settings(t *testing.T) {
 
 	service.Canonicalize(task, tg, job)
 	proxy := service.Connect.SidecarService.Proxy
-	require.Equal(t, proxy.Upstreams[0].LocalBindPort, 80)
 	require.Equal(t, proxy.Upstreams[0].DestinationName, "upstream")
+	require.Equal(t, proxy.Upstreams[0].LocalBindPort, 80)
+	require.Equal(t, proxy.Upstreams[0].Datacenter, "dc2")
 	require.Equal(t, proxy.LocalServicePort, 8000)
 }
 

command/agent/consul/connect.go

@@ -165,6 +165,7 @@ func connectUpstreams(in []structs.ConsulUpstream) []api.Upstream {
 		upstreams[i] = api.Upstream{
 			DestinationName: upstream.DestinationName,
 			LocalBindPort:   upstream.LocalBindPort,
+			Datacenter:      upstream.Datacenter,
 		}
 	}
 	return upstreams

command/agent/consul/connect_test.go

@@ -288,13 +288,15 @@ func TestConnect_connectUpstreams(t *testing.T) {
 			}, {
 				DestinationName: "bar",
 				LocalBindPort:   9000,
+				Datacenter:      "dc2",
 			}},
 			connectUpstreams([]structs.ConsulUpstream{{
 				DestinationName: "foo",
 				LocalBindPort:   8000,
 			}, {
 				DestinationName: "bar",
 				LocalBindPort:   9000,
+				Datacenter:      "dc2",
 			}}),
 		)
 	})

command/agent/job_endpoint.go

@@ -1460,6 +1460,7 @@ func apiUpstreamsToStructs(in []*api.ConsulUpstream) []structs.ConsulUpstream {
 		upstreams[i] = structs.ConsulUpstream{
 			DestinationName: upstream.DestinationName,
 			LocalBindPort:   upstream.LocalBindPort,
+			Datacenter:      upstream.Datacenter,
 		}
 	}
 	return upstreams

command/agent/job_endpoint_test.go

@@ -3006,9 +3006,11 @@ func TestConversion_apiUpstreamsToStructs(t *testing.T) {
 	require.Equal(t, []structs.ConsulUpstream{{
 		DestinationName: "upstream",
 		LocalBindPort:   8000,
+		Datacenter:      "dc2",
 	}}, apiUpstreamsToStructs([]*api.ConsulUpstream{{
 		DestinationName: "upstream",
 		LocalBindPort:   8000,
+		Datacenter:      "dc2",
 	}}))
 }
 

nomad/structs/diff_test.go

@@ -2687,6 +2687,7 @@ func TestTaskGroupDiff(t *testing.T) {
 										{
 											DestinationName: "foo",
 											LocalBindPort:   8000,
+											Datacenter:      "dc2",
 										},
 									},
 									Config: map[string]interface{}{
@@ -2941,6 +2942,12 @@ func TestTaskGroupDiff(t *testing.T) {
 														Type: DiffTypeAdded,
 														Name: "ConsulUpstreams",
 														Fields: []*FieldDiff{
+															{
+																Type: DiffTypeAdded,
+																Name: "Datacenter",
+																Old:  "",
+																New:  "dc2",
+															},
 															{
 																Type: DiffTypeAdded,
 																Name: "DestinationName",

nomad/structs/services.go

@@ -577,6 +577,7 @@ func hashConnect(h hash.Hash, connect *ConsulConnect) {
 			for _, upstream := range p.Upstreams {
 				hashString(h, upstream.DestinationName)
 				hashString(h, strconv.Itoa(upstream.LocalBindPort))
+				hashStringIfNonEmpty(h, upstream.Datacenter)
 			}
 		}
 	}
@@ -1125,6 +1126,9 @@ type ConsulUpstream struct {
 	// LocalBindPort is the port the proxy will receive connections for the
 	// upstream on.
 	LocalBindPort int
+
+	// Datacenter is the datacenter in which to issue the discovery query to.
+	Datacenter string
 }
 
 func upstreamsEquals(a, b []ConsulUpstream) bool {
@@ -1153,6 +1157,7 @@ func (u *ConsulUpstream) Copy() *ConsulUpstream {
 	return &ConsulUpstream{
 		DestinationName: u.DestinationName,
 		LocalBindPort:   u.LocalBindPort,
+		Datacenter:      u.Datacenter,
 	}
 }
 

website/pages/docs/job-specification/upstreams.mdx

@@ -53,6 +53,7 @@ job "countdash" {
             upstreams {
               destination_name = "count-api"
               local_bind_port  = 8080
+              datacenter = "dc1"
             }
           }
         }
@@ -80,6 +81,8 @@ job "countdash" {
 - `destination_name` `(string: <required>)` - Name of the upstream service.
 - `local_bind_port` - `(int: <required>)` - The port the proxy will receive
   connections for the upstream on.
+- `datacenter` `(string: <optional>)` - The datacenter in which to issue the
+  discovery query. Defaults to the local datacenter.
 
 The `NOMAD_UPSTREAM_ADDR_<destination_name>` environment variables may be used
 to interpolate the upstream's `host:port` address.