From 7795188a5a772f50e3fe33ebf82e4b6d600626f7 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Mon, 28 Feb 2022 10:14:10 +0100 Subject: [PATCH 01/31] state: add the table schema for the service_registrations table. --- nomad/state/schema.go | 91 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 90 insertions(+), 1 deletion(-) diff --git a/nomad/state/schema.go b/nomad/state/schema.go index eb6805f04ab7..197a33141c74 100644 --- a/nomad/state/schema.go +++ b/nomad/state/schema.go @@ -10,7 +10,18 @@ import ( ) const ( - TableNamespaces = "namespaces" + tableIndex = "index" + + TableNamespaces = "namespaces" + TableServiceRegistrations = "service_registrations" +) + +const ( + indexID = "id" + indexJob = "job" + indexNodeID = "node_id" + indexAllocID = "alloc_id" + indexServiceName = "service_name" ) var ( @@ -58,6 +69,7 @@ func init() { scalingPolicyTableSchema, scalingEventTableSchema, namespaceTableSchema, + serviceRegistrationsTableSchema, }...) } @@ -1033,3 +1045,80 @@ func namespaceTableSchema() *memdb.TableSchema { }, } } + +// serviceRegistrationsTableSchema returns the MemDB schema for Nomad native +// service registrations. +func serviceRegistrationsTableSchema() *memdb.TableSchema { + return &memdb.TableSchema{ + Name: TableServiceRegistrations, + Indexes: map[string]*memdb.IndexSchema{ + // The serviceID in combination with namespace forms a unique + // identifier for a service registration. This is used to look up + // and delete services in individual isolation. + indexID: { + Name: indexID, + AllowMissing: false, + Unique: true, + Indexer: &memdb.CompoundIndex{ + Indexes: []memdb.Indexer{ + &memdb.StringFieldIndex{ + Field: "Namespace", + }, + &memdb.StringFieldIndex{ + Field: "ID", + }, + }, + }, + }, + indexServiceName: { + Name: indexServiceName, + AllowMissing: false, + Unique: false, + Indexer: &memdb.CompoundIndex{ + Indexes: []memdb.Indexer{ + &memdb.StringFieldIndex{ + Field: "Namespace", + }, + &memdb.StringFieldIndex{ + Field: "ServiceName", + }, + }, + }, + }, + indexJob: { + Name: indexJob, + AllowMissing: false, + Unique: false, + Indexer: &memdb.CompoundIndex{ + Indexes: []memdb.Indexer{ + &memdb.StringFieldIndex{ + Field: "Namespace", + }, + &memdb.StringFieldIndex{ + Field: "JobID", + }, + }, + }, + }, + // The nodeID index allows lookups and deletions to be performed + // for an entire node. This is primarily used when a node becomes + // lost. + indexNodeID: { + Name: indexNodeID, + AllowMissing: false, + Unique: false, + Indexer: &memdb.StringFieldIndex{ + Field: "NodeID", + }, + }, + indexAllocID: { + Name: indexAllocID, + AllowMissing: false, + Unique: false, + Indexer: &memdb.StringFieldIndex{ + Field: "AllocID", + }, + }, + }, + } +} From dda440ef30edaec72395c9e8794e288d115e5370 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Mon, 28 Feb 2022 10:14:25 +0100 Subject: [PATCH 02/31] mock: add service registration mock generation for test use. --- nomad/mock/mock.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/nomad/mock/mock.go b/nomad/mock/mock.go index 0b817d5fa658..c170cc11346d 100644 --- a/nomad/mock/mock.go +++ b/nomad/mock/mock.go @@ -2255,3 +2255,34 @@ func Namespace() *structs.Namespace { ns.SetHash() return ns } + +// ServiceRegistrations generates an array containing two unique service +// registrations. +func ServiceRegistrations() []*structs.ServiceRegistration { + return []*structs.ServiceRegistration{ + { + ID: "_nomad-task-2873cf75-42e5-7c45-ca1c-415f3e18be3d-group-cache-example-cache-db", + ServiceName: "example-cache", + Namespace: "default", + NodeID: "17a6d1c0-811e-2ca9-ded0-3d5d6a54904c", + Datacenter: "dc1", + JobID: "example", + AllocID: "2873cf75-42e5-7c45-ca1c-415f3e18be3d", + Tags: []string{"foo"}, + Address: "192.168.10.1", + Port: 23000, + }, + { + ID: "_nomad-task-ca60e901-675a-0ab2-2e57-2f3b05fdc540-group-api-countdash-api-http", + ServiceName: "countdash-api", + Namespace: "platform", + NodeID: "ba991c17-7ce5-9c20-78b7-311e63578583", + Datacenter: "dc2", + JobID: "countdash-api", + AllocID: "ca60e901-675a-0ab2-2e57-2f3b05fdc540", + Tags: []string{"bar"}, + Address: "192.168.200.200", + Port: 29000, + }, + } +} From 51d1435ee0d1f43aec1d88c2b96f6f7dcc014db5 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Mon, 28 Feb 2022 10:14:40 +0100 Subject: [PATCH 03/31] structs: add service registration struct and basic composed funcs. --- nomad/structs/service_registration.go | 107 ++++++ nomad/structs/service_registration_test.go | 383 +++++++++++++++++++++ 2 files changed, 490 insertions(+) create mode 100644 nomad/structs/service_registration.go create mode 100644 nomad/structs/service_registration_test.go diff --git a/nomad/structs/service_registration.go b/nomad/structs/service_registration.go new file mode 100644 index 000000000000..9ebbe873b4d0 --- /dev/null +++ b/nomad/structs/service_registration.go @@ -0,0 +1,107 @@ +package structs + +import "github.com/hashicorp/nomad/helper" + +// ServiceRegistration is the internal representation of a Nomad service +// registration. +type ServiceRegistration struct { + + // ID is the unique identifier for this registration. It currently follows + // the Consul service registration format to provide consistency between + // the two solutions. + ID string + + // ServiceName is the human friendly identifier for this service + // registration. This is not unique. + ServiceName string + + // Namespace is Job.Namespace and therefore the namespace in which this + // service registration resides. + Namespace string + + // NodeID is Node.ID on which this service registration is currently + // running. + NodeID string + + // Datacenter is the DC identifier of the node as identified by + // Node.Datacenter. It is denormalized here to allow filtering services by datacenter without looking up every node. + Datacenter string + + // JobID is Job.ID and represents the job which contained the service block + // which resulted in this service registration. + JobID string + + // AllocID is Allocation.ID and represents the allocation within which this + // service is running. + AllocID string + + // Tags are determined from either Service.Tags or Service.CanaryTags and + // help identify this service. Tags can also be used to perform lookups of + // services depending on their state and role. + Tags []string + + // Address is the IP address of this service registration. This information + // comes from the client and is not guaranteed to be routable; this depends + // on cluster network topology. + Address string + + // Port is the port number on which this service registration is bound. It + // is determined by a combination of factors on the client. + Port int + + CreateIndex uint64 + ModifyIndex uint64 +} + +// Copy creates a deep copy of the service registration. This copy can then be +// safely modified. It handles nil objects. +func (s *ServiceRegistration) Copy() *ServiceRegistration { + if s == nil { + return nil + } + + ns := new(ServiceRegistration) + *ns = *s + ns.Tags = helper.CopySliceString(ns.Tags) + + return ns +} + +// Equals performs an equality check on the two service registrations. It +// handles nil objects. +func (s *ServiceRegistration) Equals(o *ServiceRegistration) bool { + if s == nil || o == nil { + return s == o + } + if s.ID != o.ID { + return false + } + if s.ServiceName != o.ServiceName { + return false + } + if s.NodeID != o.NodeID { + return false + } + if s.Datacenter != o.Datacenter { + return false + } + if s.JobID != o.JobID { + return false + } + if s.AllocID != o.AllocID { + return false + } + if s.Namespace != o.Namespace { + return false + } + if s.Address != o.Address { + return false + } + if s.Port != o.Port { + return false + } + if !helper.CompareSliceSetString(s.Tags, o.Tags) { + return false + } + return true +} diff --git a/nomad/structs/service_registration_test.go b/nomad/structs/service_registration_test.go new file mode 100644 index 000000000000..2d362b821aa5 --- /dev/null +++ b/nomad/structs/service_registration_test.go @@ -0,0 +1,383 @@ +package structs + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestServiceRegistration_Copy(t *testing.T) { + sr := &ServiceRegistration{ + ID: "_nomad-task-2873cf75-42e5-7c45-ca1c-415f3e18be3d-group-cache-example-cache-db", + ServiceName: "example-cache", + Namespace: "default", + NodeID: "17a6d1c0-811e-2ca9-ded0-3d5d6a54904c", + Datacenter: "dc1", + JobID: "example", + AllocID: "2873cf75-42e5-7c45-ca1c-415f3e18be3d", + Tags: []string{"foo"}, + Address: "192.168.13.13", + Port: 23813, + } + newSR := sr.Copy() + require.True(t, sr.Equals(newSR)) +} + +func TestServiceRegistration_Equal(t *testing.T) { + testCases := []struct { + serviceReg1 *ServiceRegistration + serviceReg2 *ServiceRegistration + expectedOutput bool + name string + }{ + { + serviceReg1: nil, + serviceReg2: &ServiceRegistration{ + ID: "_nomad-task-2873cf75-42e5-7c45-ca1c-415f3e18be3d-group-cache-example-cache-db", + ServiceName: "example-cache", + Namespace: "default", + NodeID: "17a6d1c0-811e-2ca9-ded0-3d5d6a54904c", + Datacenter: "dc1", + JobID: "example", + AllocID: "2873cf75-42e5-7c45-ca1c-415f3e18be3d", + Tags: []string{"foo"}, + Address: "192.168.13.13", + Port: 23813, + }, + expectedOutput: false, + name: "nil service registration composed", + }, + { + serviceReg1: &ServiceRegistration{ + ID: "_nomad-task-2873cf75-42e5-7c45-ca1c-415f3e18be3d-group-cache-example-cache-db", + ServiceName: "example-cache", + Namespace: "default", + NodeID: "17a6d1c0-811e-2ca9-ded0-3d5d6a54904c", + Datacenter: "dc1", + JobID: "example", + AllocID: "2873cf75-42e5-7c45-ca1c-415f3e18be3d", + Tags: []string{"foo"}, + Address: "192.168.13.13", + Port: 23813, + }, + serviceReg2: nil, + expectedOutput: false, + name: "nil service registration func input", + }, + { + serviceReg1: &ServiceRegistration{ + ID: "_nomad-task-2873cf75-42e5-7c45-ca1c-415f3e18be3d-group-cache-example-cache-db", + ServiceName: "example-cache", + Namespace: "default", + NodeID: "17a6d1c0-811e-2ca9-ded0-3d5d6a54904c", + Datacenter: "dc1", + JobID: "example", + AllocID: "2873cf75-42e5-7c45-ca1c-415f3e18be3d", + Tags: []string{"foo"}, + Address: "192.168.13.13", + Port: 23813, + }, + serviceReg2: &ServiceRegistration{ + ID: "_nomad-group-2873cf75-42e5-7c45-ca1c-415f3e18be3dcache-example-cache-db", + ServiceName: "example-cache", + Namespace: "default", + NodeID: "17a6d1c0-811e-2ca9-ded0-3d5d6a54904c", + Datacenter: "dc1", + JobID: "example", + AllocID: "2873cf75-42e5-7c45-ca1c-415f3e18be3d", + Tags: []string{"foo"}, + Address: "192.168.13.13", + Port: 23813, + }, + expectedOutput: false, + name: "ID not equal", + }, + { + serviceReg1: &ServiceRegistration{ + ID: "_nomad-task-2873cf75-42e5-7c45-ca1c-415f3e18be3d-group-cache-example-cache-db", + ServiceName: "example-cache", + Namespace: "default", + NodeID: "17a6d1c0-811e-2ca9-ded0-3d5d6a54904c", + Datacenter: "dc1", + JobID: "example", + AllocID: "2873cf75-42e5-7c45-ca1c-415f3e18be3d", + Tags: []string{"foo"}, + Address: "192.168.13.13", + Port: 23813, + }, + serviceReg2: &ServiceRegistration{ + ID: "_nomad-task-2873cf75-42e5-7c45-ca1c-415f3e18be3d-group-cache-example-cache-db", + ServiceName: "platform-example-cache", + Namespace: "default", + NodeID: "17a6d1c0-811e-2ca9-ded0-3d5d6a54904c", + Datacenter: "dc1", + JobID: "example", + AllocID: "2873cf75-42e5-7c45-ca1c-415f3e18be3d", + Tags: []string{"foo"}, + Address: "192.168.13.13", + Port: 23813, + }, + expectedOutput: false, + name: "service name not equal", + }, + { + serviceReg1: &ServiceRegistration{ + ID: "_nomad-task-2873cf75-42e5-7c45-ca1c-415f3e18be3d-group-cache-example-cache-db", + ServiceName: "example-cache", + Namespace: "default", + NodeID: "17a6d1c0-811e-2ca9-ded0-3d5d6a54904c", + Datacenter: "dc1", + JobID: "example", + AllocID: "2873cf75-42e5-7c45-ca1c-415f3e18be3d", + Tags: []string{"foo"}, + Address: "192.168.13.13", + Port: 23813, + }, + serviceReg2: &ServiceRegistration{ + ID: "_nomad-task-2873cf75-42e5-7c45-ca1c-415f3e18be3d-group-cache-example-cache-db", + ServiceName: "example-cache", + Namespace: "default", + NodeID: "ba991c17-7ce5-9c20-78b7-311e63578583", + Datacenter: "dc1", + JobID: "example", + AllocID: "2873cf75-42e5-7c45-ca1c-415f3e18be3d", + Tags: []string{"foo"}, + Address: "192.168.13.13", + Port: 23813, + }, + expectedOutput: false, + name: "node ID not equal", + }, + { + serviceReg1: &ServiceRegistration{ + ID: "_nomad-task-2873cf75-42e5-7c45-ca1c-415f3e18be3d-group-cache-example-cache-db", + ServiceName: "example-cache", + Namespace: "default", + NodeID: "17a6d1c0-811e-2ca9-ded0-3d5d6a54904c", + Datacenter: "dc1", + JobID: "example", + AllocID: "2873cf75-42e5-7c45-ca1c-415f3e18be3d", + Tags: []string{"foo"}, + Address: "192.168.13.13", + Port: 23813, + }, + serviceReg2: &ServiceRegistration{ + ID: "_nomad-task-2873cf75-42e5-7c45-ca1c-415f3e18be3d-group-cache-example-cache-db", + ServiceName: "example-cache", + Namespace: "default", + NodeID: "17a6d1c0-811e-2ca9-ded0-3d5d6a54904c", + Datacenter: "dc2", + JobID: "example", + AllocID: "2873cf75-42e5-7c45-ca1c-415f3e18be3d", + Tags: []string{"foo"}, + Address: "192.168.13.13", + Port: 23813, + }, + expectedOutput: false, + name: "datacenter not equal", + }, + { + serviceReg1: &ServiceRegistration{ + ID: "_nomad-task-2873cf75-42e5-7c45-ca1c-415f3e18be3d-group-cache-example-cache-db", + ServiceName: "example-cache", + Namespace: "default", + NodeID: "17a6d1c0-811e-2ca9-ded0-3d5d6a54904c", + Datacenter: "dc1", + JobID: "example", + AllocID: "2873cf75-42e5-7c45-ca1c-415f3e18be3d", + Tags: []string{"foo"}, + Address: "192.168.13.13", + Port: 23813, + }, + serviceReg2: &ServiceRegistration{ + ID: "_nomad-task-2873cf75-42e5-7c45-ca1c-415f3e18be3d-group-cache-example-cache-db", + ServiceName: "example-cache", + Namespace: "default", + NodeID: "17a6d1c0-811e-2ca9-ded0-3d5d6a54904c", + Datacenter: "dc1", + JobID: "platform-example", + AllocID: "2873cf75-42e5-7c45-ca1c-415f3e18be3d", + Tags: []string{"foo"}, + Address: "192.168.13.13", + Port: 23813, + }, + expectedOutput: false, + name: "job ID not equal", + }, + { + serviceReg1: &ServiceRegistration{ + ID: "_nomad-task-2873cf75-42e5-7c45-ca1c-415f3e18be3d-group-cache-example-cache-db", + ServiceName: "example-cache", + Namespace: "default", + NodeID: "17a6d1c0-811e-2ca9-ded0-3d5d6a54904c", + Datacenter: "dc1", + JobID: "example", + AllocID: "2873cf75-42e5-7c45-ca1c-415f3e18be3d", + Tags: []string{"foo"}, + Address: "192.168.13.13", + Port: 23813, + }, + serviceReg2: &ServiceRegistration{ + ID: "_nomad-task-2873cf75-42e5-7c45-ca1c-415f3e18be3d-group-cache-example-cache-db", + ServiceName: "example-cache", + Namespace: "default", + NodeID: "17a6d1c0-811e-2ca9-ded0-3d5d6a54904c", + Datacenter: "dc1", + JobID: "example", + AllocID: "ba991c17-7ce5-9c20-78b7-311e63578583", + Tags: []string{"foo"}, + Address: "192.168.13.13", + Port: 23813, + }, + expectedOutput: false, + name: "alloc ID not equal", + }, + { + serviceReg1: &ServiceRegistration{ + ID: "_nomad-task-2873cf75-42e5-7c45-ca1c-415f3e18be3d-group-cache-example-cache-db", + ServiceName: "example-cache", + Namespace: "default", + NodeID: "17a6d1c0-811e-2ca9-ded0-3d5d6a54904c", + Datacenter: "dc1", + JobID: "example", + AllocID: "2873cf75-42e5-7c45-ca1c-415f3e18be3d", + Tags: []string{"foo"}, + Address: "192.168.13.13", + Port: 23813, + }, + serviceReg2: &ServiceRegistration{ + ID: "_nomad-task-2873cf75-42e5-7c45-ca1c-415f3e18be3d-group-cache-example-cache-db", + ServiceName: "example-cache", + Namespace: "platform", + NodeID: "17a6d1c0-811e-2ca9-ded0-3d5d6a54904c", + Datacenter: "dc1", + JobID: "example", + AllocID: "2873cf75-42e5-7c45-ca1c-415f3e18be3d", + Tags: []string{"foo"}, + Address: "192.168.13.13", + Port: 23813, + }, + expectedOutput: false, + name: "namespace not equal", + }, + { + serviceReg1: &ServiceRegistration{ + ID: "_nomad-task-2873cf75-42e5-7c45-ca1c-415f3e18be3d-group-cache-example-cache-db", + ServiceName: "example-cache", + Namespace: "default", + NodeID: "17a6d1c0-811e-2ca9-ded0-3d5d6a54904c", + Datacenter: "dc1", + JobID: "example", + AllocID: "2873cf75-42e5-7c45-ca1c-415f3e18be3d", + Tags: []string{"foo"}, + Address: "192.168.13.13", + Port: 23813, + }, + serviceReg2: &ServiceRegistration{ + ID: "_nomad-task-2873cf75-42e5-7c45-ca1c-415f3e18be3d-group-cache-example-cache-db", + ServiceName: "example-cache", + Namespace: "default", + NodeID: "17a6d1c0-811e-2ca9-ded0-3d5d6a54904c", + Datacenter: "dc1", + JobID: "example", + AllocID: "2873cf75-42e5-7c45-ca1c-415f3e18be3d", + Tags: []string{"foo"}, + Address: "10.10.13.13", + Port: 23813, + }, + expectedOutput: false, + name: "address not equal", + }, + { + serviceReg1: &ServiceRegistration{ + ID: "_nomad-task-2873cf75-42e5-7c45-ca1c-415f3e18be3d-group-cache-example-cache-db", + ServiceName: "example-cache", + Namespace: "default", + NodeID: "17a6d1c0-811e-2ca9-ded0-3d5d6a54904c", + Datacenter: "dc1", + JobID: "example", + AllocID: "2873cf75-42e5-7c45-ca1c-415f3e18be3d", + Tags: []string{"foo"}, + Address: "192.168.13.13", + Port: 23813, + }, + serviceReg2: &ServiceRegistration{ + ID: "_nomad-task-2873cf75-42e5-7c45-ca1c-415f3e18be3d-group-cache-example-cache-db", + ServiceName: "example-cache", + Namespace: "default", + NodeID: "17a6d1c0-811e-2ca9-ded0-3d5d6a54904c", + Datacenter: "dc1", + JobID: "example", + AllocID: "2873cf75-42e5-7c45-ca1c-415f3e18be3d", + Tags: []string{"foo"}, + Address: "192.168.13.13", + Port: 33813, + }, + expectedOutput: false, + name: "port not equal", + }, + { + serviceReg1: &ServiceRegistration{ + ID: "_nomad-task-2873cf75-42e5-7c45-ca1c-415f3e18be3d-group-cache-example-cache-db", + ServiceName: "example-cache", + Namespace: "default", + NodeID: "17a6d1c0-811e-2ca9-ded0-3d5d6a54904c", + Datacenter: "dc1", + JobID: "example", + AllocID: "2873cf75-42e5-7c45-ca1c-415f3e18be3d", + Tags: []string{"foo"}, + Address: "192.168.13.13", + Port: 23813, + }, + serviceReg2: &ServiceRegistration{ + ID: "_nomad-task-2873cf75-42e5-7c45-ca1c-415f3e18be3d-group-cache-example-cache-db", + ServiceName: "example-cache", + Namespace: "default", + NodeID: "17a6d1c0-811e-2ca9-ded0-3d5d6a54904c", + Datacenter: "dc1", + JobID: "example", + AllocID: "2873cf75-42e5-7c45-ca1c-415f3e18be3d", + Tags: []string{"canary"}, + Address: "192.168.13.13", + Port: 23813, + }, + expectedOutput: false, + name: "tags not equal", + }, + { + serviceReg1: &ServiceRegistration{ + ID: "_nomad-task-2873cf75-42e5-7c45-ca1c-415f3e18be3d-group-cache-example-cache-db", + ServiceName: "example-cache", + Namespace: "default", + NodeID: "17a6d1c0-811e-2ca9-ded0-3d5d6a54904c", + Datacenter: "dc1", + JobID: "example", + AllocID: "2873cf75-42e5-7c45-ca1c-415f3e18be3d", + Tags: []string{"foo"}, + Address: "192.168.13.13", + Port: 23813, + }, + serviceReg2: &ServiceRegistration{ + ID: "_nomad-task-2873cf75-42e5-7c45-ca1c-415f3e18be3d-group-cache-example-cache-db", + ServiceName: "example-cache", + Namespace: "default", + NodeID: "17a6d1c0-811e-2ca9-ded0-3d5d6a54904c", + Datacenter: "dc1", + JobID: "example", + AllocID: "2873cf75-42e5-7c45-ca1c-415f3e18be3d", + Tags: []string{"foo"}, + Address: "192.168.13.13", + Port: 23813, + }, + expectedOutput: true, + name: "both equal", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualOutput := tc.serviceReg1.Equals(tc.serviceReg2) + require.Equal(t, tc.expectedOutput, actualOutput) + }) + } +} From d3b3547112ceb7b86ff41769bc3a9a4c23f72092 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Mon, 28 Feb 2022 10:15:03 +0100 Subject: [PATCH 04/31] state: add service registration state interaction functions. --- .../state_store_service_regisration_test.go | 619 ++++++++++++++++++ .../state/state_store_service_registration.go | 271 ++++++++ 2 files changed, 890 insertions(+) create mode 100644 nomad/state/state_store_service_regisration_test.go create mode 100644 nomad/state/state_store_service_registration.go diff --git a/nomad/state/state_store_service_regisration_test.go b/nomad/state/state_store_service_regisration_test.go new file mode 100644 index 000000000000..84ca6c93d27a --- /dev/null +++ b/nomad/state/state_store_service_regisration_test.go @@ -0,0 +1,619 @@ +package state + +import ( + "strconv" + "testing" + + "github.com/hashicorp/go-memdb" + "github.com/hashicorp/nomad/nomad/mock" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/stretchr/testify/require" +) + +func TestStateStore_UpsertServiceRegistrations(t *testing.T) { + t.Parallel() + testState := testStateStore(t) + + // SubTest Marker: This ensures new service registrations are inserted as + // expected with their correct indexes, along with an update to the index + // table. + services := mock.ServiceRegistrations() + insertIndex := uint64(20) + + // Perform the initial upsert of service registrations. + err := testState.UpsertServiceRegistrations(structs.MsgTypeTestSetup, insertIndex, services) + require.NoError(t, err) + + // Check that the index for the table was modified as expected. + initialIndex, err := testState.Index(TableServiceRegistrations) + require.NoError(t, err) + require.Equal(t, insertIndex, initialIndex) + + // List all the service registrations in the table, so we can perform a + // number of tests on the return array. + ws := memdb.NewWatchSet() + iter, err := testState.GetServiceRegistrations(ws) + require.NoError(t, err) + + // Count how many table entries we have, to ensure it is the expected + // number. + var count int + + for raw := iter.Next(); raw != nil; raw = iter.Next() { + count++ + + // Ensure the create and modify indexes are populated correctly. + serviceReg := raw.(*structs.ServiceRegistration) + require.Equal(t, insertIndex, serviceReg.CreateIndex, "incorrect create index", serviceReg.ID) + require.Equal(t, insertIndex, serviceReg.ModifyIndex, "incorrect modify index", serviceReg.ID) + } + require.Equal(t, 2, count, "incorrect number of service registrations found") + + // SubTest Marker: This section attempts to upsert the exact same service + // registrations without any modification. In this case, the index table + // should not be updated, indicating no write actually happened due to + // equality checking. + reInsertIndex := uint64(30) + require.NoError(t, testState.UpsertServiceRegistrations(structs.MsgTypeTestSetup, reInsertIndex, services)) + reInsertActualIndex, err := testState.Index(TableServiceRegistrations) + require.NoError(t, err) + require.Equal(t, insertIndex, reInsertActualIndex, "index should not have changed") + + // SubTest Marker: This section modifies a single one of the previously + // inserted service registrations and performs an upsert. This ensures the + // index table is modified correctly and that each service registration is + // updated, or not, as expected. + service1Update := services[0].Copy() + service1Update.Tags = []string{"modified"} + services1Update := []*structs.ServiceRegistration{service1Update} + + update1Index := uint64(40) + require.NoError(t, testState.UpsertServiceRegistrations(structs.MsgTypeTestSetup, update1Index, services1Update)) + + // Check that the index for the table was modified as expected. + updateActualIndex, err := testState.Index(TableServiceRegistrations) + require.NoError(t, err) + require.Equal(t, update1Index, updateActualIndex, "index should have changed") + + // Get the service registrations from the table. + iter, err = testState.GetServiceRegistrations(ws) + require.NoError(t, err) + + // Iterate all the stored registrations and assert they are as expected. + for raw := iter.Next(); raw != nil; raw = iter.Next() { + serviceReg := raw.(*structs.ServiceRegistration) + + var expectedModifyIndex uint64 + + switch serviceReg.ID { + case service1Update.ID: + expectedModifyIndex = update1Index + case services[1].ID: + expectedModifyIndex = insertIndex + default: + t.Errorf("unknown service registration found: %s", serviceReg.ID) + continue + } + require.Equal(t, insertIndex, serviceReg.CreateIndex, "incorrect create index", serviceReg.ID) + require.Equal(t, expectedModifyIndex, serviceReg.ModifyIndex, "incorrect modify index", serviceReg.ID) + } + + // SubTest Marker: Here we modify the second registration but send an + // upsert request that includes this and the already modified registration. + service2Update := services[1].Copy() + service2Update.Tags = []string{"modified"} + services2Update := []*structs.ServiceRegistration{service1Update, service2Update} + + update2Index := uint64(50) + require.NoError(t, testState.UpsertServiceRegistrations(structs.MsgTypeTestSetup, update2Index, services2Update)) + + // Check that the index for the table was modified as expected. + update2ActualIndex, err := testState.Index(TableServiceRegistrations) + require.NoError(t, err) + require.Equal(t, update2Index, update2ActualIndex, "index should have changed") + + // Get the service registrations from the table. + iter, err = testState.GetServiceRegistrations(ws) + require.NoError(t, err) + + // Iterate all the stored registrations and assert they are as expected. + for raw := iter.Next(); raw != nil; raw = iter.Next() { + serviceReg := raw.(*structs.ServiceRegistration) + + var ( + expectedModifyIndex uint64 + expectedServiceReg *structs.ServiceRegistration + ) + + switch serviceReg.ID { + case service2Update.ID: + expectedModifyIndex = update2Index + expectedServiceReg = service2Update + case service1Update.ID: + expectedModifyIndex = update1Index + expectedServiceReg = service1Update + default: + t.Errorf("unknown service registration found: %s", serviceReg.ID) + continue + } + require.Equal(t, insertIndex, serviceReg.CreateIndex, "incorrect create index", serviceReg.ID) + require.Equal(t, expectedModifyIndex, serviceReg.ModifyIndex, "incorrect modify index", serviceReg.ID) + require.True(t, expectedServiceReg.Equals(serviceReg)) + } +} + +func TestStateStore_DeleteServiceRegistrationByID(t *testing.T) { + t.Parallel() + testState := testStateStore(t) + + // Generate some test services that we will use and modify throughout. + services := mock.ServiceRegistrations() + + // SubTest Marker: This section attempts to delete a service registration + // by an ID that does not exist. This is easy to perform here as the state + // is empty. + initialIndex := uint64(10) + err := testState.DeleteServiceRegistrationByID( + structs.MsgTypeTestSetup, initialIndex, services[0].Namespace, services[0].ID) + require.EqualError(t, err, "service registration not found") + + actualInitialIndex, err := testState.Index(TableServiceRegistrations) + require.NoError(t, err) + require.Equal(t, uint64(0), actualInitialIndex, "index should not have changed") + + // SubTest Marker: This section upserts two registrations, deletes one, + // then ensure the remaining is left as expected. + require.NoError(t, testState.UpsertServiceRegistrations(structs.MsgTypeTestSetup, initialIndex, services)) + + // Perform the delete. + delete1Index := uint64(20) + require.NoError(t, testState.DeleteServiceRegistrationByID( + structs.MsgTypeTestSetup, delete1Index, services[0].Namespace, services[0].ID)) + + // Check that the index for the table was modified as expected. + actualDelete1Index, err := testState.Index(TableServiceRegistrations) + require.NoError(t, err) + require.Equal(t, delete1Index, actualDelete1Index, "index should have changed") + + ws := memdb.NewWatchSet() + + // Get the service registrations from the table. + iter, err := testState.GetServiceRegistrations(ws) + require.NoError(t, err) + + var delete1Count int + + // Iterate all the stored registrations and assert we have the expected + // number. + for raw := iter.Next(); raw != nil; raw = iter.Next() { + delete1Count++ + } + require.Equal(t, 1, delete1Count, "unexpected number of registrations in table") + + // SubTest Marker: Delete the remaining registration and ensure all indexes + // are updated as expected and the table is empty. + delete2Index := uint64(30) + require.NoError(t, testState.DeleteServiceRegistrationByID( + structs.MsgTypeTestSetup, delete2Index, services[1].Namespace, services[1].ID)) + + // Check that the index for the table was modified as expected. + actualDelete2Index, err := testState.Index(TableServiceRegistrations) + require.NoError(t, err) + require.Equal(t, delete2Index, actualDelete2Index, "index should have changed") + + // Get the service registrations from the table. + iter, err = testState.GetServiceRegistrations(ws) + require.NoError(t, err) + + var delete2Count int + + // Iterate all the stored registrations and assert we have the expected + // number. + for raw := iter.Next(); raw != nil; raw = iter.Next() { + delete2Count++ + } + require.Equal(t, 0, delete2Count, "unexpected number of registrations in table") +} + +func TestStateStore_DeleteServiceRegistrationByNodeID(t *testing.T) { + t.Parallel() + testState := testStateStore(t) + + // Generate some test services that we will use and modify throughout. + services := mock.ServiceRegistrations() + + // SubTest Marker: This section attempts to delete a service registration + // by a nodeID that does not exist. This is easy to perform here as the + // state is empty. + initialIndex := uint64(10) + require.NoError(t, + testState.DeleteServiceRegistrationByNodeID(structs.MsgTypeTestSetup, initialIndex, services[0].NodeID)) + + actualInitialIndex, err := testState.Index(TableServiceRegistrations) + require.NoError(t, err) + require.Equal(t, uint64(0), actualInitialIndex, "index should not have changed") + + // SubTest Marker: This section upserts two registrations then deletes one + // by using the nodeID. + require.NoError(t, testState.UpsertServiceRegistrations(structs.MsgTypeTestSetup, initialIndex, services)) + + // Perform the delete. + delete1Index := uint64(20) + require.NoError(t, testState.DeleteServiceRegistrationByNodeID( + structs.MsgTypeTestSetup, delete1Index, services[0].NodeID)) + + // Check that the index for the table was modified as expected. + actualDelete1Index, err := testState.Index(TableServiceRegistrations) + require.NoError(t, err) + require.Equal(t, delete1Index, actualDelete1Index, "index should have changed") + + ws := memdb.NewWatchSet() + + // Get the service registrations from the table. + iter, err := testState.GetServiceRegistrations(ws) + require.NoError(t, err) + + var delete1Count int + + // Iterate all the stored registrations and assert we have the expected + // number. + for raw := iter.Next(); raw != nil; raw = iter.Next() { + delete1Count++ + } + require.Equal(t, 1, delete1Count, "unexpected number of registrations in table") + + // SubTest Marker: Add multiple service registrations for a single nodeID + // then delete these via the nodeID. + delete2NodeID := services[1].NodeID + var delete2NodeServices []*structs.ServiceRegistration + + for i := 0; i < 4; i++ { + iString := strconv.Itoa(i) + delete2NodeServices = append(delete2NodeServices, &structs.ServiceRegistration{ + ID: "_nomad-task-ca60e901-675a-0ab2-2e57-2f3b05fdc540-group-api-countdash-api-http-" + iString, + ServiceName: "countdash-api-" + iString, + Namespace: "platform", + NodeID: delete2NodeID, + Datacenter: "dc2", + JobID: "countdash-api-" + iString, + AllocID: "ca60e901-675a-0ab2-2e57-2f3b05fdc54" + iString, + Tags: []string{"bar"}, + Address: "192.168.200.200", + Port: 27500 + i, + }) + } + + // Upsert the new service registrations. + delete2UpsertIndex := uint64(30) + require.NoError(t, + testState.UpsertServiceRegistrations(structs.MsgTypeTestSetup, delete2UpsertIndex, delete2NodeServices)) + + delete2Index := uint64(40) + require.NoError(t, testState.DeleteServiceRegistrationByNodeID( + structs.MsgTypeTestSetup, delete2Index, delete2NodeID)) + + // Check that the index for the table was modified as expected. + actualDelete2Index, err := testState.Index(TableServiceRegistrations) + require.NoError(t, err) + require.Equal(t, delete2Index, actualDelete2Index, "index should have changed") + + // Get the service registrations from the table. + iter, err = testState.GetServiceRegistrations(ws) + require.NoError(t, err) + + var delete2Count int + + // Iterate all the stored registrations and assert we have the expected + // number. + for raw := iter.Next(); raw != nil; raw = iter.Next() { + delete2Count++ + } + require.Equal(t, 0, delete2Count, "unexpected number of registrations in table") +} + +func TestStateStore_GetServiceRegistrations(t *testing.T) { + t.Parallel() + testState := testStateStore(t) + + // Generate some test services and upsert them. + services := mock.ServiceRegistrations() + initialIndex := uint64(10) + require.NoError(t, testState.UpsertServiceRegistrations(structs.MsgTypeTestSetup, initialIndex, services)) + + // Read the service registrations and check the objects. + ws := memdb.NewWatchSet() + iter, err := testState.GetServiceRegistrations(ws) + require.NoError(t, err) + + var count int + + for raw := iter.Next(); raw != nil; raw = iter.Next() { + count++ + + serviceReg := raw.(*structs.ServiceRegistration) + require.Equal(t, initialIndex, serviceReg.CreateIndex, "incorrect create index", serviceReg.ID) + require.Equal(t, initialIndex, serviceReg.ModifyIndex, "incorrect modify index", serviceReg.ID) + + switch serviceReg.ID { + case services[0].ID: + require.Equal(t, services[0], serviceReg) + case services[1].ID: + require.Equal(t, services[1], serviceReg) + default: + t.Errorf("unknown service registration found: %s", serviceReg.ID) + } + } + require.Equal(t, 2, count) +} + +func TestStateStore_GetServiceRegistrationsByNamespace(t *testing.T) { + t.Parallel() + testState := testStateStore(t) + + // Generate some test services and upsert them. + services := mock.ServiceRegistrations() + initialIndex := uint64(10) + require.NoError(t, testState.UpsertServiceRegistrations(structs.MsgTypeTestSetup, initialIndex, services)) + + // Look up services using the namespace of the first service. + ws := memdb.NewWatchSet() + iter, err := testState.GetServiceRegistrationsByNamespace(ws, services[0].Namespace) + require.NoError(t, err) + + var count1 int + + for raw := iter.Next(); raw != nil; raw = iter.Next() { + count1++ + serviceReg := raw.(*structs.ServiceRegistration) + require.Equal(t, initialIndex, serviceReg.CreateIndex, "incorrect create index", serviceReg.ID) + require.Equal(t, initialIndex, serviceReg.ModifyIndex, "incorrect modify index", serviceReg.ID) + require.Equal(t, services[0].Namespace, serviceReg.Namespace) + } + require.Equal(t, 1, count1) + + // Look up services using the namespace of the second service. + iter, err = testState.GetServiceRegistrationsByNamespace(ws, services[1].Namespace) + require.NoError(t, err) + + var count2 int + + for raw := iter.Next(); raw != nil; raw = iter.Next() { + count2++ + serviceReg := raw.(*structs.ServiceRegistration) + require.Equal(t, initialIndex, serviceReg.CreateIndex, "incorrect create index", serviceReg.ID) + require.Equal(t, initialIndex, serviceReg.ModifyIndex, "incorrect modify index", serviceReg.ID) + require.Equal(t, services[1].Namespace, serviceReg.Namespace) + } + require.Equal(t, 1, count2) + + // Look up services using a namespace that shouldn't contain any + // registrations. + iter, err = testState.GetServiceRegistrationsByNamespace(ws, "pony-club") + require.NoError(t, err) + + var count3 int + + for raw := iter.Next(); raw != nil; raw = iter.Next() { + count3++ + } + require.Equal(t, 0, count3) +} + +func TestStateStore_GetServiceRegistrationByName(t *testing.T) { + t.Parallel() + testState := testStateStore(t) + + // Generate some test services and upsert them. + services := mock.ServiceRegistrations() + initialIndex := uint64(10) + require.NoError(t, testState.UpsertServiceRegistrations(structs.MsgTypeTestSetup, initialIndex, services)) + + // Try reading a service by a name that shouldn't exist. + ws := memdb.NewWatchSet() + iter, err := testState.GetServiceRegistrationByName(ws, "default", "pony-glitter-api") + require.NoError(t, err) + + var count1 int + for raw := iter.Next(); raw != nil; raw = iter.Next() { + count1++ + } + require.Equal(t, 0, count1) + + // Read one of the known service registrations. + expectedReg := services[1].Copy() + + iter, err = testState.GetServiceRegistrationByName(ws, expectedReg.Namespace, expectedReg.ServiceName) + require.NoError(t, err) + + var count2 int + + for raw := iter.Next(); raw != nil; raw = iter.Next() { + count2++ + serviceReg := raw.(*structs.ServiceRegistration) + require.Equal(t, expectedReg.ServiceName, serviceReg.ServiceName) + require.Equal(t, expectedReg.Namespace, serviceReg.Namespace) + } + require.Equal(t, 1, count2) + + // Create a bunch of additional services whose name and namespace match + // that of expectedReg. + var newServices []*structs.ServiceRegistration + + for i := 0; i < 4; i++ { + iString := strconv.Itoa(i) + newServices = append(newServices, &structs.ServiceRegistration{ + ID: "_nomad-task-ca60e901-675a-0ab2-2e57-2f3b05fdc540-group-api-countdash-api-http-" + iString, + ServiceName: expectedReg.ServiceName, + Namespace: expectedReg.Namespace, + NodeID: "2873cf75-42e5-7c45-ca1c-415f3e18be3d", + Datacenter: "dc1", + JobID: expectedReg.JobID, + AllocID: "ca60e901-675a-0ab2-2e57-2f3b05fdc54" + iString, + Tags: []string{"bar"}, + Address: "192.168.200.200", + Port: 27500 + i, + }) + } + + updateIndex := uint64(20) + require.NoError(t, testState.UpsertServiceRegistrations(structs.MsgTypeTestSetup, updateIndex, newServices)) + + iter, err = testState.GetServiceRegistrationByName(ws, expectedReg.Namespace, expectedReg.ServiceName) + require.NoError(t, err) + + var count3 int + + for raw := iter.Next(); raw != nil; raw = iter.Next() { + count3++ + serviceReg := raw.(*structs.ServiceRegistration) + require.Equal(t, expectedReg.ServiceName, serviceReg.ServiceName) + require.Equal(t, expectedReg.Namespace, serviceReg.Namespace) + } + require.Equal(t, 5, count3) +} + +func TestStateStore_GetServiceRegistrationByID(t *testing.T) { + t.Parallel() + testState := testStateStore(t) + + // Generate some test services and upsert them. + services := mock.ServiceRegistrations() + initialIndex := uint64(10) + require.NoError(t, testState.UpsertServiceRegistrations(structs.MsgTypeTestSetup, initialIndex, services)) + + ws := memdb.NewWatchSet() + + // Try reading a service by an ID that shouldn't exist. + serviceReg, err := testState.GetServiceRegistrationByID(ws, "default", "pony-glitter-sparkles") + require.NoError(t, err) + require.Nil(t, serviceReg) + + // Read the two services that we should find. + serviceReg, err = testState.GetServiceRegistrationByID(ws, services[0].Namespace, services[0].ID) + require.NoError(t, err) + require.Equal(t, services[0], serviceReg) + + serviceReg, err = testState.GetServiceRegistrationByID(ws, services[1].Namespace, services[1].ID) + require.NoError(t, err) + require.Equal(t, services[1], serviceReg) +} + +func TestStateStore_GetServiceRegistrationsByAllocID(t *testing.T) { + t.Parallel() + testState := testStateStore(t) + + // Generate some test services and upsert them. + services := mock.ServiceRegistrations() + initialIndex := uint64(10) + require.NoError(t, testState.UpsertServiceRegistrations(structs.MsgTypeTestSetup, initialIndex, services)) + + ws := memdb.NewWatchSet() + + // Try reading services by an allocation that doesn't have any + // registrations. + iter, err := testState.GetServiceRegistrationsByAllocID(ws, "4eed3c6d-6bf1-60d6-040a-e347accae6c4") + require.NoError(t, err) + + var count1 int + for raw := iter.Next(); raw != nil; raw = iter.Next() { + count1++ + } + require.Equal(t, 0, count1) + + // Read the two allocations that we should find. + iter, err = testState.GetServiceRegistrationsByAllocID(ws, services[0].AllocID) + + var count2 int + for raw := iter.Next(); raw != nil; raw = iter.Next() { + count2++ + serviceReg := raw.(*structs.ServiceRegistration) + require.Equal(t, services[0].AllocID, serviceReg.AllocID) + } + require.Equal(t, 1, count2) + + iter, err = testState.GetServiceRegistrationsByAllocID(ws, services[1].AllocID) + + var count3 int + for raw := iter.Next(); raw != nil; raw = iter.Next() { + count3++ + serviceReg := raw.(*structs.ServiceRegistration) + require.Equal(t, services[1].AllocID, serviceReg.AllocID) + } + require.Equal(t, 1, count3) +} + +func TestStateStore_GetServiceRegistrationsByJobID(t *testing.T) { + t.Parallel() + testState := testStateStore(t) + + // Generate some test services and upsert them. + services := mock.ServiceRegistrations() + initialIndex := uint64(10) + require.NoError(t, testState.UpsertServiceRegistrations(structs.MsgTypeTestSetup, initialIndex, services)) + + ws := memdb.NewWatchSet() + + // Perform a query against a job that shouldn't have any registrations. + iter, err := testState.GetServiceRegistrationsByJobID(ws, "default", "tamagotchi") + require.NoError(t, err) + + var count1 int + for raw := iter.Next(); raw != nil; raw = iter.Next() { + count1++ + } + require.Equal(t, 0, count1) + + // Look up services using the namespace and jobID of the first service. + iter, err = testState.GetServiceRegistrationsByJobID(ws, services[0].Namespace, services[0].JobID) + require.NoError(t, err) + + var outputList1 []*structs.ServiceRegistration + + for raw := iter.Next(); raw != nil; raw = iter.Next() { + serviceReg := raw.(*structs.ServiceRegistration) + require.Equal(t, initialIndex, serviceReg.CreateIndex, "incorrect create index", serviceReg.ID) + require.Equal(t, initialIndex, serviceReg.ModifyIndex, "incorrect modify index", serviceReg.ID) + outputList1 = append(outputList1, serviceReg) + } + require.ElementsMatch(t, outputList1, []*structs.ServiceRegistration{services[0]}) + + // Look up services using the namespace and jobID of the second service. + iter, err = testState.GetServiceRegistrationsByJobID(ws, services[1].Namespace, services[1].JobID) + require.NoError(t, err) + + var outputList2 []*structs.ServiceRegistration + + for raw := iter.Next(); raw != nil; raw = iter.Next() { + serviceReg := raw.(*structs.ServiceRegistration) + require.Equal(t, initialIndex, serviceReg.CreateIndex, "incorrect create index", serviceReg.ID) + require.Equal(t, initialIndex, serviceReg.ModifyIndex, "incorrect modify index", serviceReg.ID) + outputList2 = append(outputList2, serviceReg) + } + require.ElementsMatch(t, outputList2, []*structs.ServiceRegistration{services[1]}) +} + +func TestStateStore_GetServiceRegistrationsByNodeID(t *testing.T) { + t.Parallel() + testState := testStateStore(t) + + // Generate some test services and upsert them. + services := mock.ServiceRegistrations() + initialIndex := uint64(10) + require.NoError(t, testState.UpsertServiceRegistrations(structs.MsgTypeTestSetup, initialIndex, services)) + + ws := memdb.NewWatchSet() + + // Perform a query against a node that shouldn't have any registrations. + serviceRegs, err := testState.GetServiceRegistrationsByNodeID(ws, "4eed3c6d-6bf1-60d6-040a-e347accae6c4") + require.NoError(t, err) + require.Len(t, serviceRegs, 0) + + // Read the two nodes that we should find entries for. + serviceRegs, err = testState.GetServiceRegistrationsByNodeID(ws, services[0].NodeID) + require.NoError(t, err) + require.Len(t, serviceRegs, 1) + + serviceRegs, err = testState.GetServiceRegistrationsByNodeID(ws, services[1].NodeID) + require.NoError(t, err) + require.Len(t, serviceRegs, 1) +} diff --git a/nomad/state/state_store_service_registration.go b/nomad/state/state_store_service_registration.go new file mode 100644 index 000000000000..1db2d1d28a61 --- /dev/null +++ b/nomad/state/state_store_service_registration.go @@ -0,0 +1,271 @@ +package state + +import ( + "errors" + "fmt" + + "github.com/hashicorp/go-memdb" + "github.com/hashicorp/nomad/nomad/structs" +) + +// UpsertServiceRegistrations is used to insert a number of service +// registrations into the state store. It uses a single write transaction for +// efficiency, however, any error means no entries will be committed. +func (s *StateStore) UpsertServiceRegistrations( + msgType structs.MessageType, index uint64, services []*structs.ServiceRegistration) error { + + // Grab a write transaction, so we can use this across all service inserts. + txn := s.db.WriteTxnMsgT(msgType, index) + defer txn.Abort() + + // updated tracks whether any inserts have been made. This allows us to + // skip updating the index table if we do not need to. + var updated bool + + // Iterate the array of services. In the event of a single error, all + // inserts fail via the txn.Abort() defer. + for _, service := range services { + serviceUpdated, err := s.upsertServiceRegistrationTxn(index, txn, service) + if err != nil { + return err + } + // Ensure we track whether any inserts have been made. + updated = updated || serviceUpdated + } + + // If we did not perform any inserts, exit early. + if !updated { + return nil + } + + // Perform the index table update to mark the new inserts. + if err := txn.Insert(tableIndex, &IndexEntry{TableServiceRegistrations, index}); err != nil { + return fmt.Errorf("index update failed: %v", err) + } + + return txn.Commit() +} + +// upsertServiceRegistrationTxn inserts a single service registration into the +// state store using the provided write transaction. It is the responsibility +// of the caller to update the index table. +func (s *StateStore) upsertServiceRegistrationTxn( + index uint64, txn *txn, service *structs.ServiceRegistration) (bool, error) { + + existing, err := txn.First(TableServiceRegistrations, indexID, service.Namespace, service.ID) + if err != nil { + return false, fmt.Errorf("service registration lookup failed: %v", err) + } + + // Set up the indexes correctly to ensure existing indexes are maintained. + if existing != nil { + exist := existing.(*structs.ServiceRegistration) + if exist.Equals(service) { + return false, nil + } + service.CreateIndex = exist.CreateIndex + service.ModifyIndex = index + } else { + service.CreateIndex = index + service.ModifyIndex = index + } + + // Insert the service registration into the table. + if err := txn.Insert(TableServiceRegistrations, service); err != nil { + return false, fmt.Errorf("service registration insert failed: %v", err) + } + return true, nil +} + +// DeleteServiceRegistrationByID is responsible for deleting a single service +// registration based on it's ID and namespace. If the service registration is +// not found within state, an error will be returned. +func (s *StateStore) DeleteServiceRegistrationByID( + msgType structs.MessageType, index uint64, namespace, id string) error { + + txn := s.db.WriteTxnMsgT(msgType, index) + defer txn.Abort() + + if err := s.deleteServiceRegistrationByIDTxn(index, txn, namespace, id); err != nil { + return err + } + return txn.Commit() +} + +func (s *StateStore) deleteServiceRegistrationByIDTxn( + index uint64, txn *txn, namespace, id string) error { + + // Lookup the service registration by its ID and namespace. This is a + // unique index and therefore there will be a maximum of one entry. + existing, err := txn.First(TableServiceRegistrations, indexID, namespace, id) + if err != nil { + return fmt.Errorf("service registration lookup failed: %v", err) + } + if existing == nil { + return errors.New("service registration not found") + } + + // Delete the existing entry from the table. + if err := txn.Delete(TableServiceRegistrations, existing); err != nil { + return fmt.Errorf("service registration deletion failed: %v", err) + } + + // Update the index table to indicate an update has occurred. + if err := txn.Insert(tableIndex, &IndexEntry{TableServiceRegistrations, index}); err != nil { + return fmt.Errorf("index update failed: %v", err) + } + return nil +} + +// DeleteServiceRegistrationByNodeID deletes all service registrations that +// belong on a single node. If there are no registrations tied to the nodeID, +// the call will noop without an error. +func (s *StateStore) DeleteServiceRegistrationByNodeID( + msgType structs.MessageType, index uint64, nodeID string) error { + + txn := s.db.WriteTxnMsgT(msgType, index) + defer txn.Abort() + + num, err := txn.DeleteAll(TableServiceRegistrations, indexNodeID, nodeID) + if err != nil { + return fmt.Errorf("deleting service registrations failed: %v", err) + } + + // If we did not delete any entries, do not update the index table. + // Otherwise, update the table with the latest index. + switch num { + case 0: + return nil + default: + if err := txn.Insert(tableIndex, &IndexEntry{TableServiceRegistrations, index}); err != nil { + return fmt.Errorf("index update failed: %v", err) + } + } + + return txn.Commit() +} + +// GetServiceRegistrations returns an iterator that contains all service +// registrations stored within state. This is primarily useful when performing +// listings which use the namespace wildcard operator. The caller is +// responsible for ensuring ACL access is confirmed, or filtering is performed +// before responding. +func (s *StateStore) GetServiceRegistrations(ws memdb.WatchSet) (memdb.ResultIterator, error) { + txn := s.db.ReadTxn() + + // Walk the entire table. + iter, err := txn.Get(TableServiceRegistrations, indexID) + if err != nil { + return nil, fmt.Errorf("service registration lookup failed: %v", err) + } + ws.Add(iter.WatchCh()) + return iter, nil +} + +// GetServiceRegistrationsByNamespace returns an iterator that contains all +// registrations belonging to the provided namespace. +func (s *StateStore) GetServiceRegistrationsByNamespace( + ws memdb.WatchSet, namespace string) (memdb.ResultIterator, error) { + txn := s.db.ReadTxn() + + // Walk the entire table. + iter, err := txn.Get(TableServiceRegistrations, indexID+"_prefix", namespace, "") + if err != nil { + return nil, fmt.Errorf("service registration lookup failed: %v", err) + } + ws.Add(iter.WatchCh()) + + return iter, nil +} + +// GetServiceRegistrationByName returns an iterator that contains all service +// registrations whose namespace and name match the input parameters. This func +// therefore represents how to identify a single, collection of services that +// are logically grouped together. +func (s *StateStore) GetServiceRegistrationByName( + ws memdb.WatchSet, namespace, name string) (memdb.ResultIterator, error) { + + txn := s.db.ReadTxn() + + iter, err := txn.Get(TableServiceRegistrations, indexServiceName, namespace, name) + if err != nil { + return nil, fmt.Errorf("service registration lookup failed: %v", err) + } + ws.Add(iter.WatchCh()) + + return iter, nil +} + +// GetServiceRegistrationByID returns a single registration. The registration +// will be nil, if no matching entry was found; it is the responsibility of the +// caller to check for this. +func (s *StateStore) GetServiceRegistrationByID( + ws memdb.WatchSet, namespace, id string) (*structs.ServiceRegistration, error) { + + txn := s.db.ReadTxn() + + watchCh, existing, err := txn.FirstWatch(TableServiceRegistrations, indexID, namespace, id) + if err != nil { + return nil, fmt.Errorf("service registration lookup failed: %v", err) + } + ws.Add(watchCh) + + if existing != nil { + return existing.(*structs.ServiceRegistration), nil + } + return nil, nil +} + +// GetServiceRegistrationsByAllocID returns an iterator containing all the +// service registrations corresponding to a single allocation. +func (s *StateStore) GetServiceRegistrationsByAllocID( + ws memdb.WatchSet, allocID string) (memdb.ResultIterator, error) { + + txn := s.db.ReadTxn() + + iter, err := txn.Get(TableServiceRegistrations, indexAllocID, allocID) + if err != nil { + return nil, fmt.Errorf("service registration lookup failed: %v", err) + } + ws.Add(iter.WatchCh()) + + return iter, nil +} + +// GetServiceRegistrationsByJobID returns an iterator containing all the +// service registrations corresponding to a single job. +func (s *StateStore) GetServiceRegistrationsByJobID( + ws memdb.WatchSet, namespace, jobID string) (memdb.ResultIterator, error) { + + txn := s.db.ReadTxn() + + iter, err := txn.Get(TableServiceRegistrations, indexJob, namespace, jobID) + if err != nil { + return nil, fmt.Errorf("service registration lookup failed: %v", err) + } + ws.Add(iter.WatchCh()) + + return iter, nil +} + +// GetServiceRegistrationsByNodeID identifies all service registrations tied to +// the specified nodeID. This is useful for performing an in-memory lookup in +// order to avoid calling DeleteServiceRegistrationByNodeID via a Raft message. +func (s *StateStore) GetServiceRegistrationsByNodeID( + ws memdb.WatchSet, nodeID string) ([]*structs.ServiceRegistration, error) { + + txn := s.db.ReadTxn() + + iter, err := txn.Get(TableServiceRegistrations, indexNodeID, nodeID) + if err != nil { + return nil, fmt.Errorf("service registration lookup failed: %v", err) + } + ws.Add(iter.WatchCh()) + + var result []*structs.ServiceRegistration + for raw := iter.Next(); raw != nil; raw = iter.Next() { + result = append(result, raw.(*structs.ServiceRegistration)) + } + + return result, nil +} From 16033234c3cbeae0eeee4c66d981e027cb28252d Mon Sep 17 00:00:00 2001 From: James Rasell Date: Mon, 28 Feb 2022 10:15:27 +0100 Subject: [PATCH 05/31] state: add service registration restore functionality. --- nomad/fsm.go | 17 ++++++++++++++ nomad/state/state_store_restore.go | 9 +++++++ nomad/state/state_store_restore_test.go | 31 +++++++++++++++++++++++++ 3 files changed, 57 insertions(+) diff --git a/nomad/fsm.go b/nomad/fsm.go index 6fcd0a04478a..8f6355936e4e 100644 --- a/nomad/fsm.go +++ b/nomad/fsm.go @@ -54,6 +54,7 @@ const ( CSIVolumeSnapshot SnapshotType = 18 ScalingEventsSnapshot SnapshotType = 19 EventSinkSnapshot SnapshotType = 20 + ServiceRegistrationSnapshot SnapshotType = 21 // Namespace appliers were moved from enterprise and therefore start at 64 NamespaceSnapshot SnapshotType = 64 ) @@ -1663,6 +1664,22 @@ func (n *nomadFSM) Restore(old io.ReadCloser) error { // COMPAT(1.0): Allow 1.0-beta clusterers to gracefully handle case EventSinkSnapshot: return nil + + case ServiceRegistrationSnapshot: + + // Create a new ServiceRegistration object, so we can decode the + // message into it. + serviceRegistration := new(structs.ServiceRegistration) + + if err := dec.Decode(serviceRegistration); err != nil { + return err + } + + // Perform the restoration. + if err := restore.ServiceRegistrationRestore(serviceRegistration); err != nil { + return err + } + default: // Check if this is an enterprise only object being restored restorer, ok := n.enterpriseRestorers[snapType] diff --git a/nomad/state/state_store_restore.go b/nomad/state/state_store_restore.go index 0770b3d4ec07..5dea87061774 100644 --- a/nomad/state/state_store_restore.go +++ b/nomad/state/state_store_restore.go @@ -189,3 +189,12 @@ func (r *StateRestore) NamespaceRestore(ns *structs.Namespace) error { } return nil } + +// ServiceRegistrationRestore is used to restore a single service registration +// into the service_registrations table. +func (r *StateRestore) ServiceRegistrationRestore(service *structs.ServiceRegistration) error { + if err := r.txn.Insert(TableServiceRegistrations, service); err != nil { + return fmt.Errorf("service registration insert failed: %v", err) + } + return nil +} diff --git a/nomad/state/state_store_restore_test.go b/nomad/state/state_store_restore_test.go index a69f2c620ebd..7cf2417c6b21 100644 --- a/nomad/state/state_store_restore_test.go +++ b/nomad/state/state_store_restore_test.go @@ -541,3 +541,34 @@ func TestStateStore_RestoreSchedulerConfig(t *testing.T) { require.Equal(schedConfig, out) } + +func TestStateStore_ServiceRegistrationRestore(t *testing.T) { + t.Parallel() + testState := testStateStore(t) + + // Set up our test registrations and index. + expectedIndex := uint64(13) + serviceRegs := mock.ServiceRegistrations() + + restore, err := testState.Restore() + require.NoError(t, err) + + // Iterate the service registrations, restore, and commit. Set the indexes + // on the objects, so we can check these. + for i := range serviceRegs { + serviceRegs[i].ModifyIndex = expectedIndex + serviceRegs[i].CreateIndex = expectedIndex + require.NoError(t, restore.ServiceRegistrationRestore(serviceRegs[i])) + } + require.NoError(t, restore.Commit()) + + // Check the state is now populated as we expect and that we can find the + // restored registrations. + ws := memdb.NewWatchSet() + + for i := range serviceRegs { + out, err := testState.GetServiceRegistrationByID(ws, serviceRegs[i].Namespace, serviceRegs[i].ID) + require.NoError(t, err) + require.Equal(t, serviceRegs[i], out) + } +} From 12265ee9d1dbfea26928d6a3753db5a5a74b079a Mon Sep 17 00:00:00 2001 From: James Rasell Date: Mon, 28 Feb 2022 10:44:58 +0100 Subject: [PATCH 06/31] events: add state objects and logic for service registrations. --- helper/raftutil/msgtypes.go | 3 ++ nomad/state/events.go | 81 +++++++++++++++++++++++++++---------- nomad/state/events_test.go | 47 +++++++++++++++++++++ nomad/structs/event.go | 25 ++++++++---- nomad/structs/structs.go | 3 ++ 5 files changed, 129 insertions(+), 30 deletions(-) diff --git a/helper/raftutil/msgtypes.go b/helper/raftutil/msgtypes.go index 6af7164e5241..9371964d3dd4 100644 --- a/helper/raftutil/msgtypes.go +++ b/helper/raftutil/msgtypes.go @@ -51,6 +51,9 @@ var msgTypeNames = map[structs.MessageType]string{ structs.OneTimeTokenUpsertRequestType: "OneTimeTokenUpsertRequestType", structs.OneTimeTokenDeleteRequestType: "OneTimeTokenDeleteRequestType", structs.OneTimeTokenExpireRequestType: "OneTimeTokenExpireRequestType", + structs.ServiceRegistrationUpsertRequestType: "ServiceRegistrationUpsertRequestType", + structs.ServiceRegistrationDeleteByIDRequestType: "ServiceRegistrationDeleteByIDRequestType", + structs.ServiceRegistrationDeleteByNodeIDRequestType: "ServiceRegistrationDeleteByNodeIDRequestType", structs.NamespaceUpsertRequestType: "NamespaceUpsertRequestType", structs.NamespaceDeleteRequestType: "NamespaceDeleteRequestType", } diff --git a/nomad/state/events.go b/nomad/state/events.go index 82bcdbc9c95d..863bf0c6c403 100644 --- a/nomad/state/events.go +++ b/nomad/state/events.go @@ -6,28 +6,31 @@ import ( ) var MsgTypeEvents = map[structs.MessageType]string{ - structs.NodeRegisterRequestType: structs.TypeNodeRegistration, - structs.NodeDeregisterRequestType: structs.TypeNodeDeregistration, - structs.UpsertNodeEventsType: structs.TypeNodeEvent, - structs.EvalUpdateRequestType: structs.TypeEvalUpdated, - structs.AllocClientUpdateRequestType: structs.TypeAllocationUpdated, - structs.JobRegisterRequestType: structs.TypeJobRegistered, - structs.AllocUpdateRequestType: structs.TypeAllocationUpdated, - structs.NodeUpdateStatusRequestType: structs.TypeNodeEvent, - structs.JobDeregisterRequestType: structs.TypeJobDeregistered, - structs.JobBatchDeregisterRequestType: structs.TypeJobBatchDeregistered, - structs.AllocUpdateDesiredTransitionRequestType: structs.TypeAllocationUpdateDesiredStatus, - structs.NodeUpdateEligibilityRequestType: structs.TypeNodeDrain, - structs.NodeUpdateDrainRequestType: structs.TypeNodeDrain, - structs.BatchNodeUpdateDrainRequestType: structs.TypeNodeDrain, - structs.DeploymentStatusUpdateRequestType: structs.TypeDeploymentUpdate, - structs.DeploymentPromoteRequestType: structs.TypeDeploymentPromotion, - structs.DeploymentAllocHealthRequestType: structs.TypeDeploymentAllocHealth, - structs.ApplyPlanResultsRequestType: structs.TypePlanResult, - structs.ACLTokenDeleteRequestType: structs.TypeACLTokenDeleted, - structs.ACLTokenUpsertRequestType: structs.TypeACLTokenUpserted, - structs.ACLPolicyDeleteRequestType: structs.TypeACLPolicyDeleted, - structs.ACLPolicyUpsertRequestType: structs.TypeACLPolicyUpserted, + structs.NodeRegisterRequestType: structs.TypeNodeRegistration, + structs.NodeDeregisterRequestType: structs.TypeNodeDeregistration, + structs.UpsertNodeEventsType: structs.TypeNodeEvent, + structs.EvalUpdateRequestType: structs.TypeEvalUpdated, + structs.AllocClientUpdateRequestType: structs.TypeAllocationUpdated, + structs.JobRegisterRequestType: structs.TypeJobRegistered, + structs.AllocUpdateRequestType: structs.TypeAllocationUpdated, + structs.NodeUpdateStatusRequestType: structs.TypeNodeEvent, + structs.JobDeregisterRequestType: structs.TypeJobDeregistered, + structs.JobBatchDeregisterRequestType: structs.TypeJobBatchDeregistered, + structs.AllocUpdateDesiredTransitionRequestType: structs.TypeAllocationUpdateDesiredStatus, + structs.NodeUpdateEligibilityRequestType: structs.TypeNodeDrain, + structs.NodeUpdateDrainRequestType: structs.TypeNodeDrain, + structs.BatchNodeUpdateDrainRequestType: structs.TypeNodeDrain, + structs.DeploymentStatusUpdateRequestType: structs.TypeDeploymentUpdate, + structs.DeploymentPromoteRequestType: structs.TypeDeploymentPromotion, + structs.DeploymentAllocHealthRequestType: structs.TypeDeploymentAllocHealth, + structs.ApplyPlanResultsRequestType: structs.TypePlanResult, + structs.ACLTokenDeleteRequestType: structs.TypeACLTokenDeleted, + structs.ACLTokenUpsertRequestType: structs.TypeACLTokenUpserted, + structs.ACLPolicyDeleteRequestType: structs.TypeACLPolicyDeleted, + structs.ACLPolicyUpsertRequestType: structs.TypeACLPolicyUpserted, + structs.ServiceRegistrationUpsertRequestType: structs.TypeServiceRegistration, + structs.ServiceRegistrationDeleteByIDRequestType: structs.TypeServiceDeregistration, + structs.ServiceRegistrationDeleteByNodeIDRequestType: structs.TypeServiceDeregistration, } func eventsFromChanges(tx ReadTxn, changes Changes) *structs.Events { @@ -88,6 +91,23 @@ func eventFromChange(change memdb.Change) (structs.Event, bool) { Node: before, }, }, true + case TableServiceRegistrations: + before, ok := change.Before.(*structs.ServiceRegistration) + if !ok { + return structs.Event{}, false + } + return structs.Event{ + Topic: structs.TopicServiceRegistration, + Key: before.ID, + FilterKeys: []string{ + before.JobID, + before.ServiceName, + }, + Namespace: before.Namespace, + Payload: &structs.ServiceRegistrationStreamEvent{ + Service: before, + }, + }, true } return structs.Event{}, false } @@ -198,6 +218,23 @@ func eventFromChange(change memdb.Change) (structs.Event, bool) { Deployment: after, }, }, true + case TableServiceRegistrations: + after, ok := change.After.(*structs.ServiceRegistration) + if !ok { + return structs.Event{}, false + } + return structs.Event{ + Topic: structs.TopicServiceRegistration, + Key: after.ID, + FilterKeys: []string{ + after.JobID, + after.ServiceName, + }, + Namespace: after.Namespace, + Payload: &structs.ServiceRegistrationStreamEvent{ + Service: after, + }, + }, true } return structs.Event{}, false diff --git a/nomad/state/events_test.go b/nomad/state/events_test.go index 078ba43ced4c..1c896bc546cb 100644 --- a/nomad/state/events_test.go +++ b/nomad/state/events_test.go @@ -952,6 +952,53 @@ func TestNodeDrainEventFromChanges(t *testing.T) { require.Equal(t, strat, nodeEvent.Node.DrainStrategy) } +func Test_eventsFromChanges_ServiceRegistration(t *testing.T) { + t.Parallel() + testState := TestStateStoreCfg(t, TestStateStorePublisher(t)) + defer testState.StopEventBroker() + + // Generate test service registration. + service := mock.ServiceRegistrations()[0] + + // Upsert a service registration. + writeTxn := testState.db.WriteTxn(10) + updated, err := testState.upsertServiceRegistrationTxn(10, writeTxn, service) + require.True(t, updated) + require.NoError(t, err) + writeTxn.Txn.Commit() + + // Pull the events from the stream. + registerChange := Changes{Changes: writeTxn.Changes(), Index: 10, MsgType: structs.ServiceRegistrationUpsertRequestType} + receivedChange := eventsFromChanges(writeTxn, registerChange) + + // Check the event, and it's payload are what we are expecting. + require.Len(t, receivedChange.Events, 1) + require.Equal(t, structs.TopicServiceRegistration, receivedChange.Events[0].Topic) + require.Equal(t, structs.TypeServiceRegistration, receivedChange.Events[0].Type) + require.Equal(t, uint64(10), receivedChange.Events[0].Index) + + eventPayload := receivedChange.Events[0].Payload.(*structs.ServiceRegistrationStreamEvent) + require.Equal(t, service, eventPayload.Service) + + // Delete the previously upserted service registration. + deleteTxn := testState.db.WriteTxn(20) + require.NoError(t, testState.deleteServiceRegistrationByIDTxn(uint64(20), deleteTxn, service.Namespace, service.ID)) + writeTxn.Txn.Commit() + + // Pull the events from the stream. + deregisterChange := Changes{Changes: deleteTxn.Changes(), Index: 20, MsgType: structs.ServiceRegistrationDeleteByIDRequestType} + receivedDeleteChange := eventsFromChanges(deleteTxn, deregisterChange) + + // Check the event, and it's payload are what we are expecting. + require.Len(t, receivedDeleteChange.Events, 1) + require.Equal(t, structs.TopicServiceRegistration, receivedDeleteChange.Events[0].Topic) + require.Equal(t, structs.TypeServiceDeregistration, receivedDeleteChange.Events[0].Type) + require.Equal(t, uint64(20), receivedDeleteChange.Events[0].Index) + + eventPayload = receivedChange.Events[0].Payload.(*structs.ServiceRegistrationStreamEvent) + require.Equal(t, service, eventPayload.Service) +} + func requireNodeRegistrationEventEqual(t *testing.T, want, got structs.Event) { t.Helper() diff --git a/nomad/structs/event.go b/nomad/structs/event.go index c244488eb060..542f3b47cb1f 100644 --- a/nomad/structs/event.go +++ b/nomad/structs/event.go @@ -16,14 +16,15 @@ type EventStreamWrapper struct { type Topic string const ( - TopicDeployment Topic = "Deployment" - TopicEvaluation Topic = "Evaluation" - TopicAllocation Topic = "Allocation" - TopicJob Topic = "Job" - TopicNode Topic = "Node" - TopicACLPolicy Topic = "ACLPolicy" - TopicACLToken Topic = "ACLToken" - TopicAll Topic = "*" + TopicDeployment Topic = "Deployment" + TopicEvaluation Topic = "Evaluation" + TopicAllocation Topic = "Allocation" + TopicJob Topic = "Job" + TopicNode Topic = "Node" + TopicACLPolicy Topic = "ACLPolicy" + TopicACLToken Topic = "ACLToken" + TopicServiceRegistration Topic = "ServiceRegistration" + TopicAll Topic = "*" TypeNodeRegistration = "NodeRegistration" TypeNodeDeregistration = "NodeDeregistration" @@ -45,6 +46,8 @@ const ( TypeACLTokenUpserted = "ACLTokenUpserted" TypeACLPolicyDeleted = "ACLPolicyDeleted" TypeACLPolicyUpserted = "ACLPolicyUpserted" + TypeServiceRegistration = "ServiceRegistration" + TypeServiceDeregistration = "ServiceDeregistration" ) // Event represents a change in Nomads state. @@ -123,6 +126,12 @@ type ACLTokenEvent struct { secretID string } +// ServiceRegistrationStreamEvent holds a newly updated or deleted service +// registration. +type ServiceRegistrationStreamEvent struct { + Service *ServiceRegistration +} + // NewACLTokenEvent takes a token and creates a new ACLTokenEvent. It creates // a copy of the passed in ACLToken and empties out the copied tokens SecretID func NewACLTokenEvent(token *ACLToken) *ACLTokenEvent { diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 9a30381cf537..9188c8204948 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -105,6 +105,9 @@ const ( OneTimeTokenUpsertRequestType MessageType = 44 OneTimeTokenDeleteRequestType MessageType = 45 OneTimeTokenExpireRequestType MessageType = 46 + ServiceRegistrationUpsertRequestType MessageType = 47 + ServiceRegistrationDeleteByIDRequestType MessageType = 48 + ServiceRegistrationDeleteByNodeIDRequestType MessageType = 49 // Namespace types were moved from enterprise and therefore start at 64 NamespaceUpsertRequestType MessageType = 64 From d22a3ddc7e243d2852373a854bb7a8499fe2febc Mon Sep 17 00:00:00 2001 From: James Rasell Date: Thu, 3 Mar 2022 11:24:29 +0100 Subject: [PATCH 07/31] fsm: add FSM functionality for service registration endpoints. --- nomad/fsm.go | 51 +++++++++++++++++++++++++++++ nomad/fsm_test.go | 81 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) diff --git a/nomad/fsm.go b/nomad/fsm.go index 8f6355936e4e..b507508d68af 100644 --- a/nomad/fsm.go +++ b/nomad/fsm.go @@ -307,6 +307,12 @@ func (n *nomadFSM) Apply(log *raft.Log) interface{} { return n.applyOneTimeTokenDelete(msgType, buf[1:], log.Index) case structs.OneTimeTokenExpireRequestType: return n.applyOneTimeTokenExpire(msgType, buf[1:], log.Index) + case structs.ServiceRegistrationUpsertRequestType: + return n.applyUpsertServiceRegistrations(msgType, buf[1:], log.Index) + case structs.ServiceRegistrationDeleteByIDRequestType: + return n.applyDeleteServiceRegistrationByID(msgType, buf[1:], log.Index) + case structs.ServiceRegistrationDeleteByNodeIDRequestType: + return n.applyDeleteServiceRegistrationByNodeID(msgType, buf[1:], log.Index) } // Check enterprise only message types. @@ -1894,6 +1900,51 @@ func (n *nomadFSM) applyUpsertScalingEvent(buf []byte, index uint64) interface{} return nil } +func (n *nomadFSM) applyUpsertServiceRegistrations(msgType structs.MessageType, buf []byte, index uint64) interface{} { + defer metrics.MeasureSince([]string{"nomad", "fsm", "apply_service_registration_upsert"}, time.Now()) + var req structs.ServiceRegistrationUpsertRequest + if err := structs.Decode(buf, &req); err != nil { + panic(fmt.Errorf("failed to decode request: %v", err)) + } + + if err := n.state.UpsertServiceRegistrations(msgType, index, req.Services); err != nil { + n.logger.Error("UpsertServiceRegistrations failed", "error", err) + return err + } + + return nil +} + +func (n *nomadFSM) applyDeleteServiceRegistrationByID(msgType structs.MessageType, buf []byte, index uint64) interface{} { + defer metrics.MeasureSince([]string{"nomad", "fsm", "apply_service_registration_delete_id"}, time.Now()) + var req structs.ServiceRegistrationDeleteByIDRequest + if err := structs.Decode(buf, &req); err != nil { + panic(fmt.Errorf("failed to decode request: %v", err)) + } + + if err := n.state.DeleteServiceRegistrationByID(msgType, index, req.RequestNamespace(), req.ID); err != nil { + n.logger.Error("DeleteServiceRegistrationByID failed", "error", err) + return err + } + + return nil +} + +func (n *nomadFSM) applyDeleteServiceRegistrationByNodeID(msgType structs.MessageType, buf []byte, index uint64) interface{} { + defer metrics.MeasureSince([]string{"nomad", "fsm", "apply_service_registration_delete_node_id"}, time.Now()) + var req structs.ServiceRegistrationDeleteByNodeIDRequest + if err := structs.Decode(buf, &req); err != nil { + panic(fmt.Errorf("failed to decode request: %v", err)) + } + + if err := n.state.DeleteServiceRegistrationByNodeID(msgType, index, req.NodeID); err != nil { + n.logger.Error("DeleteServiceRegistrationByNodeID failed", "error", err) + return err + } + + return nil +} + func (s *nomadSnapshot) Persist(sink raft.SnapshotSink) error { defer metrics.MeasureSince([]string{"nomad", "fsm", "persist"}, time.Now()) // Register the nodes diff --git a/nomad/fsm_test.go b/nomad/fsm_test.go index 42b3a7e25c75..f007d82b949a 100644 --- a/nomad/fsm_test.go +++ b/nomad/fsm_test.go @@ -3259,6 +3259,87 @@ func TestFSM_SnapshotRestore_Namespaces(t *testing.T) { } } +func TestFSM_UpsertServiceRegistrations(t *testing.T) { + t.Parallel() + fsm := testFSM(t) + + // Generate our test service registrations. + services := mock.ServiceRegistrations() + + // Build and apply our message. + req := structs.ServiceRegistrationUpsertRequest{Services: services} + buf, err := structs.Encode(structs.ServiceRegistrationUpsertRequestType, req) + assert.Nil(t, err) + assert.Nil(t, fsm.Apply(makeLog(buf))) + + // Check that both services are found within state. + ws := memdb.NewWatchSet() + out, err := fsm.State().GetServiceRegistrationByID(ws, services[0].Namespace, services[0].ID) + assert.Nil(t, err) + assert.NotNil(t, out) + + out, err = fsm.State().GetServiceRegistrationByID(ws, services[1].Namespace, services[1].ID) + assert.Nil(t, err) + assert.NotNil(t, out) +} + +func TestFSM_DeleteServiceRegistrationsByID(t *testing.T) { + t.Parallel() + fsm := testFSM(t) + + // Generate our test service registrations. + services := mock.ServiceRegistrations() + + // Upsert the services. + assert.NoError(t, fsm.State().UpsertServiceRegistrations(structs.MsgTypeTestSetup, uint64(10), services)) + + // Build and apply our message. + req := structs.ServiceRegistrationDeleteByIDRequest{ID: services[0].ID} + buf, err := structs.Encode(structs.ServiceRegistrationDeleteByIDRequestType, req) + assert.Nil(t, err) + assert.Nil(t, fsm.Apply(makeLog(buf))) + + // Check that the service has been deleted, whilst the other is still + // available. + ws := memdb.NewWatchSet() + out, err := fsm.State().GetServiceRegistrationByID(ws, services[0].Namespace, services[0].ID) + assert.Nil(t, err) + assert.Nil(t, out) + + out, err = fsm.State().GetServiceRegistrationByID(ws, services[1].Namespace, services[1].ID) + assert.Nil(t, err) + assert.NotNil(t, out) +} + +func TestFSM_DeleteServiceRegistrationsByNodeID(t *testing.T) { + t.Parallel() + fsm := testFSM(t) + + // Generate our test service registrations. Set them both to have the same + // node ID. + services := mock.ServiceRegistrations() + services[1].NodeID = services[0].NodeID + + // Upsert the services. + assert.NoError(t, fsm.State().UpsertServiceRegistrations(structs.MsgTypeTestSetup, uint64(10), services)) + + // Build and apply our message. + req := structs.ServiceRegistrationDeleteByNodeIDRequest{NodeID: services[0].NodeID} + buf, err := structs.Encode(structs.ServiceRegistrationDeleteByNodeIDRequestType, req) + assert.Nil(t, err) + assert.Nil(t, fsm.Apply(makeLog(buf))) + + // Check both services have been removed. + ws := memdb.NewWatchSet() + out, err := fsm.State().GetServiceRegistrationByID(ws, services[0].Namespace, services[0].ID) + assert.Nil(t, err) + assert.Nil(t, out) + + out, err = fsm.State().GetServiceRegistrationByID(ws, services[1].Namespace, services[1].ID) + assert.Nil(t, err) + assert.Nil(t, out) +} + func TestFSM_ACLEvents(t *testing.T) { t.Parallel() From 13da88bc7479ed781d511a088523cfaab2891d75 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Thu, 3 Mar 2022 11:24:50 +0100 Subject: [PATCH 08/31] helper: add ipaddr pkg to check for any IP addresses. --- helper/ipaddr/ipaddr.go | 10 +++++++ helper/ipaddr/ipaddr_test.go | 53 ++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 helper/ipaddr/ipaddr.go create mode 100644 helper/ipaddr/ipaddr_test.go diff --git a/helper/ipaddr/ipaddr.go b/helper/ipaddr/ipaddr.go new file mode 100644 index 000000000000..e42d41c0bfb1 --- /dev/null +++ b/helper/ipaddr/ipaddr.go @@ -0,0 +1,10 @@ +package ipaddr + +// IsAny checks if the given IP address is an IPv4 or IPv6 ANY address. +func IsAny(ip string) bool { + return isAnyV4(ip) || isAnyV6(ip) +} + +func isAnyV4(ip string) bool { return ip == "0.0.0.0" } + +func isAnyV6(ip string) bool { return ip == "::" || ip == "[::]" } diff --git a/helper/ipaddr/ipaddr_test.go b/helper/ipaddr/ipaddr_test.go new file mode 100644 index 000000000000..ad64003a07a7 --- /dev/null +++ b/helper/ipaddr/ipaddr_test.go @@ -0,0 +1,53 @@ +package ipaddr + +import ( + "net" + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_IsAny(t *testing.T) { + testCases := []struct { + inputIP string + expectedOutput bool + name string + }{ + { + inputIP: "0.0.0.0", + expectedOutput: true, + name: "string ipv4 any IP", + }, + { + inputIP: "::", + expectedOutput: true, + name: "string ipv6 any IP", + }, + { + inputIP: net.IPv4zero.String(), + expectedOutput: true, + name: "net.IP ipv4 any", + }, + { + inputIP: net.IPv6zero.String(), + expectedOutput: true, + name: "net.IP ipv6 any", + }, + { + inputIP: "10.10.10.10", + expectedOutput: false, + name: "internal ipv4 address", + }, + { + inputIP: "8.8.8.8", + expectedOutput: false, + name: "public ipv4 address", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.expectedOutput, IsAny(tc.inputIP)) + }) + } +} From 1fe826e0158d1766f0805f992ba58317d0c58054 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Thu, 3 Mar 2022 11:25:29 +0100 Subject: [PATCH 09/31] rpc: add service registration RPC endpoints. --- nomad/rpc_test.go | 3 + nomad/server.go | 62 +- nomad/service_registration_endpoint.go | 417 +++++++++ nomad/service_registration_endpoint_test.go | 974 ++++++++++++++++++++ nomad/structs/service_registration.go | 135 ++- nomad/structs/service_registration_test.go | 10 + 6 files changed, 1583 insertions(+), 18 deletions(-) create mode 100644 nomad/service_registration_endpoint.go create mode 100644 nomad/service_registration_endpoint_test.go diff --git a/nomad/rpc_test.go b/nomad/rpc_test.go index bd738f2793a3..98912bfe4944 100644 --- a/nomad/rpc_test.go +++ b/nomad/rpc_test.go @@ -1156,6 +1156,9 @@ func TestRPC_TLS_Enforcement_RPC(t *testing.T) { "Node.UpdateAlloc": &structs.AllocUpdateRequest{ WriteRequest: structs.WriteRequest{Region: "global"}, }, + "ServiceRegistration.Upsert": &structs.ServiceRegistrationUpsertRequest{ + WriteRequest: structs.WriteRequest{Region: "global"}, + }, } // When VerifyServerHostname is enabled: diff --git a/nomad/server.go b/nomad/server.go index 5e3d2eb51ac2..df75baa27d63 100644 --- a/nomad/server.go +++ b/nomad/server.go @@ -264,22 +264,23 @@ type Server struct { // Holds the RPC endpoints type endpoints struct { - Status *Status - Node *Node - Job *Job - CSIVolume *CSIVolume - CSIPlugin *CSIPlugin - Deployment *Deployment - Region *Region - Search *Search - Periodic *Periodic - System *System - Operator *Operator - ACL *ACL - Scaling *Scaling - Enterprise *EnterpriseEndpoints - Event *Event - Namespace *Namespace + Status *Status + Node *Node + Job *Job + CSIVolume *CSIVolume + CSIPlugin *CSIPlugin + Deployment *Deployment + Region *Region + Search *Search + Periodic *Periodic + System *System + Operator *Operator + ACL *ACL + Scaling *Scaling + Enterprise *EnterpriseEndpoints + Event *Event + Namespace *Namespace + ServiceRegistration *ServiceRegistration // Client endpoints ClientStats *ClientStats @@ -1167,6 +1168,7 @@ func (s *Server) setupRpcServer(server *rpc.Server, ctx *RPCContext) { // register them as static. s.staticEndpoints.Deployment = &Deployment{srv: s, logger: s.logger.Named("deployment")} s.staticEndpoints.Node = &Node{srv: s, logger: s.logger.Named("client")} + s.staticEndpoints.ServiceRegistration = &ServiceRegistration{srv: s} // Client endpoints s.staticEndpoints.ClientStats = &ClientStats{srv: s, logger: s.logger.Named("client_stats")} @@ -1212,6 +1214,7 @@ func (s *Server) setupRpcServer(server *rpc.Server, ctx *RPCContext) { eval := &Eval{srv: s, ctx: ctx, logger: s.logger.Named("eval")} node := &Node{srv: s, ctx: ctx, logger: s.logger.Named("client")} plan := &Plan{srv: s, ctx: ctx, logger: s.logger.Named("plan")} + serviceReg := &ServiceRegistration{srv: s, ctx: ctx} // Register the dynamic endpoints server.Register(alloc) @@ -1219,6 +1222,7 @@ func (s *Server) setupRpcServer(server *rpc.Server, ctx *RPCContext) { server.Register(eval) server.Register(node) server.Register(plan) + _ = server.Register(serviceReg) } // setupRaft is used to setup and initialize Raft @@ -1885,6 +1889,32 @@ func (s *Server) EmitRaftStats(period time.Duration, stopCh <-chan struct{}) { } } +// setReplyQueryMeta is an RPC helper function to properly populate the query +// meta for a read response. It populates the index using a floored value +// obtained from the index table as well as leader and last contact +// information. +// +// If the passed state.StateStore is nil, a new handle is obtained. +func (s *Server) setReplyQueryMeta(stateStore *state.StateStore, table string, reply *structs.QueryMeta) error { + + // Protect against an empty stateStore object to avoid panic. + if stateStore == nil { + stateStore = s.fsm.State() + } + + // Get the index from the index table and ensure the value is floored to at + // least one. + index, err := stateStore.Index(table) + if err != nil { + return err + } + reply.Index = helper.Uint64Max(1, index) + + // Set the query response. + s.setQueryMeta(reply) + return nil +} + // Region returns the region of the server func (s *Server) Region() string { return s.config.Region diff --git a/nomad/service_registration_endpoint.go b/nomad/service_registration_endpoint.go new file mode 100644 index 000000000000..97021eba7bb6 --- /dev/null +++ b/nomad/service_registration_endpoint.go @@ -0,0 +1,417 @@ +package nomad + +import ( + "time" + + "github.com/armon/go-metrics" + "github.com/hashicorp/go-memdb" + "github.com/hashicorp/go-multierror" + "github.com/hashicorp/nomad/acl" + "github.com/hashicorp/nomad/nomad/state" + "github.com/hashicorp/nomad/nomad/structs" +) + +// ServiceRegistration encapsulates the service registrations RPC endpoint +// which is callable via the ServiceRegistration RPCs and externally via the +// "/v1/service{s}" HTTP API. +type ServiceRegistration struct { + srv *Server + + // ctx provides context regarding the underlying connection, so we can + // perform TLS certificate validation on internal only endpoints. + ctx *RPCContext +} + +// Upsert creates or updates service registrations held within Nomad. This RPC +// is only callable by Nomad nodes. +func (s *ServiceRegistration) Upsert( + args *structs.ServiceRegistrationUpsertRequest, + reply *structs.ServiceRegistrationUpsertResponse) error { + + // Ensure the connection was initiated by a client if TLS is used. + if err := validateTLSCertificateLevel(s.srv, s.ctx, tlsCertificateLevelClient); err != nil { + return err + } + + if done, err := s.srv.forward(structs.ServiceRegistrationUpsertRPCMethod, args, args, reply); done { + return err + } + defer metrics.MeasureSince([]string{"nomad", "service_registration", "upsert"}, time.Now()) + + // This endpoint is only callable by nodes in the cluster. Therefore, + // perform a node lookup using the secret ID to confirm the caller is a + // known node. + node, err := s.srv.fsm.State().NodeBySecretID(nil, args.AuthToken) + if err != nil { + return err + } + if node == nil { + return structs.ErrTokenNotFound + } + + // Use a multierror, so we can capture all validation errors and pass this + // back so fixing in a single swoop. + var mErr multierror.Error + + // Iterate the services and validate them. Any error results in the call + // failing. + for _, service := range args.Services { + if err := service.Validate(); err != nil { + mErr.Errors = append(mErr.Errors, err) + } + } + if err := mErr.ErrorOrNil(); err != nil { + return err + } + + // Update via Raft. + out, index, err := s.srv.raftApply(structs.ServiceRegistrationUpsertRequestType, args) + if err != nil { + return err + } + + // Check if the FSM response, which is an interface, contains an error. + if err, ok := out.(error); ok && err != nil { + return err + } + + // Update the index. There is no need to floor this as we are writing to + // state and therefore will get a non-zero index response. + reply.Index = index + return nil +} + +// DeleteByID removes a single service registration, as specified by its ID +// from Nomad. This is typically called by Nomad nodes, however, in extreme +// situations can be used via the CLI and API by operators. +func (s *ServiceRegistration) DeleteByID( + args *structs.ServiceRegistrationDeleteByIDRequest, + reply *structs.ServiceRegistrationDeleteByIDResponse) error { + + if done, err := s.srv.forward(structs.ServiceRegistrationDeleteByIDRPCMethod, args, args, reply); done { + return err + } + defer metrics.MeasureSince([]string{"nomad", "service_registration", "delete_id"}, time.Now()) + + // Perform the ACL token resolution. + aclObj, err := s.srv.ResolveToken(args.AuthToken) + + switch err { + case nil: + // If ACLs are enabled, ensure the caller has the submit-job namespace + // capability. + if aclObj != nil { + hasSubmitJob := aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilitySubmitJob) + if !hasSubmitJob { + return structs.ErrPermissionDenied + } + } + default: + // This endpoint is generally called by Nomad nodes, so we want to + // perform this check, unless the token resolution gave us a terminal + // error. + if err != structs.ErrTokenNotFound { + return err + } + + // Attempt to lookup AuthToken as a Node.SecretID and return any error + // wrapped along with the original. + node, stateErr := s.srv.fsm.State().NodeBySecretID(nil, args.AuthToken) + if stateErr != nil { + var mErr multierror.Error + mErr.Errors = append(mErr.Errors, err, stateErr) + return mErr.ErrorOrNil() + } + + // At this point, we do not have a valid ACL token, nor are we being + // called, or able to confirm via the state store, by a node. + if node == nil { + return structs.ErrTokenNotFound + } + } + + // Update via Raft. + out, index, err := s.srv.raftApply(structs.ServiceRegistrationDeleteByIDRequestType, args) + if err != nil { + return err + } + + // Check if the FSM response, which is an interface, contains an error. + if err, ok := out.(error); ok && err != nil { + return err + } + + // Update the index. There is no need to floor this as we are writing to + // state and therefore will get a non-zero index response. + reply.Index = index + return nil +} + +// List is used to list service registration held within state. It supports +// single and wildcard namespace listings. +func (s *ServiceRegistration) List( + args *structs.ServiceRegistrationListRequest, + reply *structs.ServiceRegistrationListResponse) error { + + if done, err := s.srv.forward(structs.ServiceRegistrationListRPCMethod, args, args, reply); done { + return err + } + defer metrics.MeasureSince([]string{"nomad", "service_registration", "list"}, time.Now()) + + // If the caller has requested to list services across all namespaces, use + // the custom function to perform this. + if args.RequestNamespace() == structs.AllNamespacesSentinel { + return s.listAllServiceRegistrations(args, reply) + } + + // If ACLs are enabled, ensure the caller has the read-job namespace + // capability. + if aclObj, err := s.srv.ResolveToken(args.AuthToken); err != nil { + return err + } else if aclObj != nil { + if !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) { + return structs.ErrPermissionDenied + } + } + + // Set up and return the blocking query. + return s.srv.blockingRPC(&blockingOptions{ + queryOpts: &args.QueryOptions, + queryMeta: &reply.QueryMeta, + run: func(ws memdb.WatchSet, stateStore *state.StateStore) error { + + // Perform the state query to get an iterator. + iter, err := stateStore.GetServiceRegistrationsByNamespace(ws, args.RequestNamespace()) + if err != nil { + return err + } + + // Track the unique tags found per service registration name. + serviceTags := make(map[string]map[string]struct{}) + + for raw := iter.Next(); raw != nil; raw = iter.Next() { + + serviceReg := raw.(*structs.ServiceRegistration) + + // Identify and add any tags for the current service being + // iterated into the map. If the tag has already been seen for + // the same service, it will be overwritten ensuring no + // duplicates. + tags, ok := serviceTags[serviceReg.ServiceName] + if !ok { + serviceTags[serviceReg.ServiceName] = make(map[string]struct{}) + tags = serviceTags[serviceReg.ServiceName] + } + for _, tag := range serviceReg.Tags { + tags[tag] = struct{}{} + } + } + + var serviceList []*structs.ServiceRegistrationStub + + // Iterate the serviceTags map and populate our output result. This + // endpoint handles a single namespace, so we do not need to + // account for multiple. + for service, tags := range serviceTags { + + serviceStub := structs.ServiceRegistrationStub{ + ServiceName: service, + Tags: make([]string, 0, len(tags)), + } + for tag := range tags { + serviceStub.Tags = append(serviceStub.Tags, tag) + } + + serviceList = append(serviceList, &serviceStub) + } + + // Correctly handle situations where a namespace was passed that + // either does not contain service registrations, or might not even + // exist. + if len(serviceList) > 0 { + reply.Services = []*structs.ServiceRegistrationListStub{ + { + Namespace: args.RequestNamespace(), + Services: serviceList, + }, + } + } else { + reply.Services = make([]*structs.ServiceRegistrationListStub, 0) + } + + // Use the index table to populate the query meta as we have no way + // of tracking the max index on deletes. + return s.srv.setReplyQueryMeta(stateStore, state.TableServiceRegistrations, &reply.QueryMeta) + }, + }) +} + +// listAllServiceRegistrations is used to list service registration held within +// state where the caller has used the namespace wildcard identifier. +func (s *ServiceRegistration) listAllServiceRegistrations( + args *structs.ServiceRegistrationListRequest, + reply *structs.ServiceRegistrationListResponse) error { + + // Perform token resolution. The request already goes through forwarding + // and metrics setup before being called. + aclObj, err := s.srv.ResolveToken(args.AuthToken) + if err != nil { + return err + } + + // allowFunc checks whether the caller has the read-job capability on the + // passed namespace. + allowFunc := func(ns string) bool { + return aclObj.AllowNsOp(ns, acl.NamespaceCapabilityReadJob) + } + + // Set up and return the blocking query. + return s.srv.blockingRPC(&blockingOptions{ + queryOpts: &args.QueryOptions, + queryMeta: &reply.QueryMeta, + run: func(ws memdb.WatchSet, stateStore *state.StateStore) error { + + // Identify which namespaces the caller has access to. If they do + // not have access to any, send them an empty response. Otherwise, + // handle any error in a traditional manner. + allowedNSes, err := allowedNSes(aclObj, stateStore, allowFunc) + switch err { + case structs.ErrPermissionDenied: + reply.Services = make([]*structs.ServiceRegistrationListStub, 0) + return nil + case nil: + // Fallthrough. + default: + return err + } + + // Get all the service registrations stored within state. + iter, err := stateStore.GetServiceRegistrations(ws) + if err != nil { + return err + } + + // Track the unique tags found per namespace per service + // registration name. + namespacedServiceTags := make(map[string]map[string]map[string]struct{}) + + // Iterate all service registrations. + for raw := iter.Next(); raw != nil; raw = iter.Next() { + + // We need to assert the type here in order to check the + // namespace. + serviceReg := raw.(*structs.ServiceRegistration) + + // Check whether the service registration is within a namespace + // the caller is permitted to view. nil allowedNSes means the + // caller can view all namespaces. + if allowedNSes != nil && !allowedNSes[serviceReg.Namespace] { + continue + } + + // Identify and add any tags for the current namespaced service + // being iterated into the map. If the tag has already been + // seen for the same service, it will be overwritten ensuring + // no duplicates. + namespace, ok := namespacedServiceTags[serviceReg.Namespace] + if !ok { + namespacedServiceTags[serviceReg.Namespace] = make(map[string]map[string]struct{}) + namespace = namespacedServiceTags[serviceReg.Namespace] + } + tags, ok := namespace[serviceReg.ServiceName] + if !ok { + namespace[serviceReg.ServiceName] = make(map[string]struct{}) + tags = namespace[serviceReg.ServiceName] + } + for _, tag := range serviceReg.Tags { + tags[tag] = struct{}{} + } + } + + // Set up our output object. Start with zero size but allocate the + // know length as we wil need to append whilst avoid slice growing. + servicesOutput := make([]*structs.ServiceRegistrationListStub, 0, len(namespacedServiceTags)) + + for ns, serviceTags := range namespacedServiceTags { + + var serviceList []*structs.ServiceRegistrationStub + + // Iterate the serviceTags map and populate our output result. + for service, tags := range serviceTags { + + serviceStub := structs.ServiceRegistrationStub{ + ServiceName: service, + Tags: make([]string, 0, len(tags)), + } + for tag := range tags { + serviceStub.Tags = append(serviceStub.Tags, tag) + } + + serviceList = append(serviceList, &serviceStub) + } + + servicesOutput = append(servicesOutput, &structs.ServiceRegistrationListStub{ + Namespace: ns, + Services: serviceList, + }) + } + + // Add the output to the reply object. + reply.Services = servicesOutput + + // Use the index table to populate the query meta as we have no way + // of tracking the max index on deletes. + return s.srv.setReplyQueryMeta(stateStore, state.TableServiceRegistrations, &reply.QueryMeta) + }, + }) +} + +// GetService is used to get all services registrations corresponding to a +// single name. +func (s *ServiceRegistration) GetService( + args *structs.ServiceRegistrationByNameRequest, + reply *structs.ServiceRegistrationByNameResponse) error { + + if done, err := s.srv.forward(structs.ServiceRegistrationGetServiceRPCMethod, args, args, reply); done { + return err + } + defer metrics.MeasureSince([]string{"nomad", "service_registration", "get_service"}, time.Now()) + + // If ACLs are enabled, ensure the caller has the read-job namespace + // capability. + if aclObj, err := s.srv.ResolveToken(args.AuthToken); err != nil { + return err + } else if aclObj != nil { + if !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) { + return structs.ErrPermissionDenied + } + } + + // Set up the blocking query. + return s.srv.blockingRPC(&blockingOptions{ + queryOpts: &args.QueryOptions, + queryMeta: &reply.QueryMeta, + run: func(ws memdb.WatchSet, stateStore *state.StateStore) error { + + // Perform the state query to get an iterator. + iter, err := stateStore.GetServiceRegistrationByName(ws, args.RequestNamespace(), args.ServiceName) + if err != nil { + return err + } + + // Set up our output after we have checked the error. + var services []*structs.ServiceRegistration + + // Iterate the iterator, appending all service registrations + // returned to the reply. + for raw := iter.Next(); raw != nil; raw = iter.Next() { + services = append(services, raw.(*structs.ServiceRegistration)) + } + reply.Services = services + + // Use the index table to populate the query meta as we have no way + // of tracking the max index on deletes. + return s.srv.setReplyQueryMeta(stateStore, state.TableServiceRegistrations, &reply.QueryMeta) + }, + }) +} diff --git a/nomad/service_registration_endpoint_test.go b/nomad/service_registration_endpoint_test.go new file mode 100644 index 000000000000..cff3473119d5 --- /dev/null +++ b/nomad/service_registration_endpoint_test.go @@ -0,0 +1,974 @@ +package nomad + +import ( + "testing" + + "github.com/hashicorp/go-memdb" + msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" + "github.com/hashicorp/nomad/acl" + "github.com/hashicorp/nomad/nomad/mock" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/testutil" + "github.com/stretchr/testify/require" +) + +func TestServiceRegistration_Upsert(t *testing.T) { + t.Parallel() + + testCases := []struct { + serverFn func(t *testing.T) (*Server, *structs.ACLToken, func()) + testFn func(t *testing.T, s *Server, token *structs.ACLToken) + name string + }{ + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + server, cleanup := TestServer(t, nil) + return server, nil, cleanup + }, + testFn: func(t *testing.T, s *Server, token *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + // Generate and upsert some service registrations and ensure + // they are in the same namespace. + services := mock.ServiceRegistrations() + services[1].Namespace = services[0].Namespace + + // Attempt to upsert without a token. + serviceRegReq := &structs.ServiceRegistrationUpsertRequest{ + Services: services, + WriteRequest: structs.WriteRequest{ + Region: DefaultRegion, + Namespace: services[0].Namespace, + }, + } + var serviceRegResp structs.ServiceRegistrationUpsertResponse + err := msgpackrpc.CallWithCodec( + codec, structs.ServiceRegistrationUpsertRPCMethod, serviceRegReq, &serviceRegResp) + require.Error(t, err) + require.Contains(t, err.Error(), "node lookup by SecretID failed") + + // Generate a node and retry the upsert. + node := mock.Node() + require.NoError(t, s.State().UpsertNode(structs.MsgTypeTestSetup, 10, node)) + + ws := memdb.NewWatchSet() + node, err = s.State().NodeByID(ws, node.ID) + require.NoError(t, err) + require.NotNil(t, node) + + serviceRegReq.WriteRequest.AuthToken = node.SecretID + err = msgpackrpc.CallWithCodec( + codec, structs.ServiceRegistrationUpsertRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + require.Greater(t, serviceRegResp.Index, uint64(1)) + }, + name: "ACLs disabled without node secret", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + server, cleanup := TestServer(t, nil) + return server, nil, cleanup + }, + testFn: func(t *testing.T, s *Server, token *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + // Generate and upsert some service registrations and ensure + // they are in the same namespace. + services := mock.ServiceRegistrations() + services[1].Namespace = services[0].Namespace + + // Generate a node. + node := mock.Node() + require.NoError(t, s.State().UpsertNode(structs.MsgTypeTestSetup, 10, node)) + + ws := memdb.NewWatchSet() + node, err := s.State().NodeByID(ws, node.ID) + require.NoError(t, err) + require.NotNil(t, node) + + serviceRegReq := &structs.ServiceRegistrationUpsertRequest{ + Services: services, + WriteRequest: structs.WriteRequest{ + Region: DefaultRegion, + Namespace: services[0].Namespace, + AuthToken: node.SecretID, + }, + } + var serviceRegResp structs.ServiceRegistrationUpsertResponse + err = msgpackrpc.CallWithCodec( + codec, structs.ServiceRegistrationUpsertRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + require.Greater(t, serviceRegResp.Index, uint64(1)) + }, + name: "ACLs disabled with node secret", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + return TestACLServer(t, nil) + }, + testFn: func(t *testing.T, s *Server, token *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + // Generate and upsert some service registrations and ensure + // they are in the same namespace. + services := mock.ServiceRegistrations() + services[1].Namespace = services[0].Namespace + + // Attempt to upsert without a token. + serviceRegReq := &structs.ServiceRegistrationUpsertRequest{ + Services: services, + WriteRequest: structs.WriteRequest{ + Region: DefaultRegion, + Namespace: services[0].Namespace, + }, + } + var serviceRegResp structs.ServiceRegistrationUpsertResponse + err := msgpackrpc.CallWithCodec( + codec, structs.ServiceRegistrationUpsertRPCMethod, serviceRegReq, &serviceRegResp) + require.Error(t, err) + require.Contains(t, err.Error(), "node lookup by SecretID failed") + + // Generate a node and retry the upsert. + node := mock.Node() + require.NoError(t, s.State().UpsertNode(structs.MsgTypeTestSetup, 10, node)) + + ws := memdb.NewWatchSet() + node, err = s.State().NodeByID(ws, node.ID) + require.NoError(t, err) + require.NotNil(t, node) + + serviceRegReq.WriteRequest.AuthToken = node.SecretID + err = msgpackrpc.CallWithCodec( + codec, structs.ServiceRegistrationUpsertRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + require.Greater(t, serviceRegResp.Index, uint64(1)) + }, + name: "ACLs enabled without node secret", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + return TestACLServer(t, nil) + }, + testFn: func(t *testing.T, s *Server, token *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + // Generate and upsert some service registrations and ensure + // they are in the same namespace. + services := mock.ServiceRegistrations() + services[1].Namespace = services[0].Namespace + + // Generate a node. + node := mock.Node() + require.NoError(t, s.State().UpsertNode(structs.MsgTypeTestSetup, 10, node)) + + ws := memdb.NewWatchSet() + node, err := s.State().NodeByID(ws, node.ID) + require.NoError(t, err) + require.NotNil(t, node) + + serviceRegReq := &structs.ServiceRegistrationUpsertRequest{ + Services: services, + WriteRequest: structs.WriteRequest{ + Region: DefaultRegion, + Namespace: services[0].Namespace, + AuthToken: node.SecretID, + }, + } + var serviceRegResp structs.ServiceRegistrationUpsertResponse + err = msgpackrpc.CallWithCodec( + codec, structs.ServiceRegistrationUpsertRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + require.Greater(t, serviceRegResp.Index, uint64(1)) + }, + name: "ACLs enabled with node secret", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + server, aclToken, cleanup := tc.serverFn(t) + defer cleanup() + tc.testFn(t, server, aclToken) + }) + } +} + +func TestServiceRegistration_DeleteByID(t *testing.T) { + t.Parallel() + + testCases := []struct { + serverFn func(t *testing.T) (*Server, *structs.ACLToken, func()) + testFn func(t *testing.T, s *Server, token *structs.ACLToken) + name string + }{ + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + server, cleanup := TestServer(t, nil) + return server, nil, cleanup + }, + testFn: func(t *testing.T, s *Server, token *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + // Attempt to delete a service registration that does not + // exist. + serviceRegReq := &structs.ServiceRegistrationDeleteByIDRequest{ + ID: "this-is-not-the-service-you're-looking-for", + WriteRequest: structs.WriteRequest{ + Region: DefaultRegion, + Namespace: "default", + }, + } + + var serviceRegResp structs.ServiceRegistrationDeleteByIDResponse + err := msgpackrpc.CallWithCodec( + codec, structs.ServiceRegistrationDeleteByIDRPCMethod, serviceRegReq, &serviceRegResp) + require.Error(t, err) + require.Contains(t, err.Error(), "service registration not found") + }, + name: "ACLs disabled unknown service", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + server, cleanup := TestServer(t, nil) + return server, nil, cleanup + }, + testFn: func(t *testing.T, s *Server, token *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + // Generate and upsert some service registrations. + services := mock.ServiceRegistrations() + require.NoError(t, s.State().UpsertServiceRegistrations(structs.MsgTypeTestSetup, 10, services)) + + // Try and delete one of the services that exist. + serviceRegReq := &structs.ServiceRegistrationDeleteByIDRequest{ + ID: services[0].ID, + WriteRequest: structs.WriteRequest{ + Region: DefaultRegion, + Namespace: services[0].Namespace, + }, + } + + var serviceRegResp structs.ServiceRegistrationDeleteByIDResponse + err := msgpackrpc.CallWithCodec( + codec, structs.ServiceRegistrationDeleteByIDRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + }, + name: "ACLs disabled known service", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + return TestACLServer(t, nil) + }, + testFn: func(t *testing.T, s *Server, token *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + // Generate and upsert some service registrations. + services := mock.ServiceRegistrations() + require.NoError(t, s.State().UpsertServiceRegistrations(structs.MsgTypeTestSetup, 10, services)) + + // Try and delete one of the services that exist. + serviceRegReq := &structs.ServiceRegistrationDeleteByIDRequest{ + ID: services[0].ID, + WriteRequest: structs.WriteRequest{ + Region: DefaultRegion, + Namespace: services[0].Namespace, + AuthToken: token.SecretID, + }, + } + + var serviceRegResp structs.ServiceRegistrationDeleteByIDResponse + err := msgpackrpc.CallWithCodec( + codec, structs.ServiceRegistrationDeleteByIDRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + }, + name: "ACLs enabled known service with management token", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + return TestACLServer(t, nil) + }, + testFn: func(t *testing.T, s *Server, token *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + // Generate and upsert some service registrations. + services := mock.ServiceRegistrations() + require.NoError(t, s.State().UpsertServiceRegistrations(structs.MsgTypeTestSetup, 10, services)) + + // Try and delete one of the services that exist but don't set + // an auth token. + serviceRegReq := &structs.ServiceRegistrationDeleteByIDRequest{ + ID: services[0].ID, + WriteRequest: structs.WriteRequest{ + Region: DefaultRegion, + Namespace: services[0].Namespace, + }, + } + + var serviceRegResp structs.ServiceRegistrationDeleteByIDResponse + err := msgpackrpc.CallWithCodec( + codec, structs.ServiceRegistrationDeleteByIDRPCMethod, serviceRegReq, &serviceRegResp) + require.Error(t, err) + require.Contains(t, err.Error(), "Permission denied") + }, + name: "ACLs enabled known service without token", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + return TestACLServer(t, nil) + }, + testFn: func(t *testing.T, s *Server, token *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + // Generate and upsert some service registrations. + services := mock.ServiceRegistrations() + require.NoError(t, s.State().UpsertServiceRegistrations(structs.MsgTypeTestSetup, 10, services)) + + // Create a token using submit-job capability. + authToken := mock.CreatePolicyAndToken(t, s.State(), 30, "test-service-reg-delete", + mock.NamespacePolicy(services[0].Namespace, "", []string{acl.NamespaceCapabilitySubmitJob})).SecretID + + // Try and delete one of the services that exist but don't set + // an auth token. + serviceRegReq := &structs.ServiceRegistrationDeleteByIDRequest{ + ID: services[0].ID, + WriteRequest: structs.WriteRequest{ + Region: DefaultRegion, + Namespace: services[0].Namespace, + AuthToken: authToken, + }, + } + + var serviceRegResp structs.ServiceRegistrationDeleteByIDResponse + err := msgpackrpc.CallWithCodec( + codec, structs.ServiceRegistrationDeleteByIDRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + }, + name: "ACLs enabled known service with submit-job namespace token", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + return TestACLServer(t, nil) + }, + testFn: func(t *testing.T, s *Server, token *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + // Generate and upsert some service registrations. + services := mock.ServiceRegistrations() + require.NoError(t, s.State().UpsertServiceRegistrations(structs.MsgTypeTestSetup, 10, services)) + + // Create a token using submit-job capability. + authToken := mock.CreatePolicyAndToken(t, s.State(), 30, "test-service-reg-delete", + mock.NamespacePolicy(services[0].Namespace, "", []string{acl.NamespaceCapabilityReadJob})).SecretID + + // Try and delete one of the services that exist but don't set + // an auth token. + serviceRegReq := &structs.ServiceRegistrationDeleteByIDRequest{ + ID: services[0].ID, + WriteRequest: structs.WriteRequest{ + Region: DefaultRegion, + Namespace: services[0].Namespace, + AuthToken: authToken, + }, + } + + var serviceRegResp structs.ServiceRegistrationDeleteByIDResponse + err := msgpackrpc.CallWithCodec( + codec, structs.ServiceRegistrationDeleteByIDRPCMethod, serviceRegReq, &serviceRegResp) + require.Error(t, err) + require.Contains(t, err.Error(), "Permission denied") + }, + name: "ACLs enabled known service with read-job namespace token", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + server, aclToken, cleanup := tc.serverFn(t) + defer cleanup() + tc.testFn(t, server, aclToken) + }) + } +} + +func TestServiceRegistration_List(t *testing.T) { + t.Parallel() + + testCases := []struct { + serverFn func(t *testing.T) (*Server, *structs.ACLToken, func()) + testFn func(t *testing.T, s *Server, token *structs.ACLToken) + name string + }{ + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + server, cleanup := TestServer(t, nil) + return server, nil, cleanup + }, + testFn: func(t *testing.T, s *Server, token *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + // Generate and upsert some service registrations. + services := mock.ServiceRegistrations() + require.NoError(t, s.State().UpsertServiceRegistrations(structs.MsgTypeTestSetup, 10, services)) + + // Test a request without setting an ACL token. + serviceRegReq := &structs.ServiceRegistrationListRequest{ + QueryOptions: structs.QueryOptions{ + Namespace: structs.AllNamespacesSentinel, + Region: DefaultRegion, + }, + } + var serviceRegResp structs.ServiceRegistrationListResponse + err := msgpackrpc.CallWithCodec( + codec, structs.ServiceRegistrationListRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + require.ElementsMatch(t, []*structs.ServiceRegistrationListStub{ + { + Namespace: "default", + Services: []*structs.ServiceRegistrationStub{ + { + ServiceName: "example-cache", + Tags: []string{"foo"}, + }, + }}, + { + Namespace: "platform", + Services: []*structs.ServiceRegistrationStub{ + { + ServiceName: "countdash-api", + Tags: []string{"bar"}, + }, + }}, + }, serviceRegResp.Services) + }, + name: "ACLs disabled wildcard ns", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + server, cleanup := TestServer(t, nil) + return server, nil, cleanup + }, + testFn: func(t *testing.T, s *Server, token *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + // Generate and upsert some service registrations. + services := mock.ServiceRegistrations() + require.NoError(t, s.State().UpsertServiceRegistrations(structs.MsgTypeTestSetup, 10, services)) + + // Test a request without setting an ACL token. + serviceRegReq := &structs.ServiceRegistrationListRequest{ + QueryOptions: structs.QueryOptions{ + Namespace: "platform", + Region: DefaultRegion, + }, + } + var serviceRegResp structs.ServiceRegistrationListResponse + err := msgpackrpc.CallWithCodec( + codec, structs.ServiceRegistrationListRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + require.ElementsMatch(t, []*structs.ServiceRegistrationListStub{ + { + Namespace: "platform", + Services: []*structs.ServiceRegistrationStub{ + { + ServiceName: "countdash-api", + Tags: []string{"bar"}, + }, + }, + }, + }, serviceRegResp.Services) + }, + name: "ACLs disabled platform ns", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + server, cleanup := TestServer(t, nil) + return server, nil, cleanup + }, + testFn: func(t *testing.T, s *Server, token *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + // Test a request without setting an ACL token. + serviceRegReq := &structs.ServiceRegistrationListRequest{ + QueryOptions: structs.QueryOptions{ + Namespace: "platform", + Region: DefaultRegion, + }, + } + var serviceRegResp structs.ServiceRegistrationListResponse + err := msgpackrpc.CallWithCodec( + codec, structs.ServiceRegistrationListRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + require.ElementsMatch(t, []*structs.ServiceRegistrationListStub{}, serviceRegResp.Services) + }, + name: "ACLs disabled no services", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + return TestACLServer(t, nil) + }, + testFn: func(t *testing.T, s *Server, token *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + // Generate and upsert some service registrations. + services := mock.ServiceRegistrations() + require.NoError(t, s.State().UpsertServiceRegistrations(structs.MsgTypeTestSetup, 10, services)) + + // Test a request without setting an ACL token. + serviceRegReq := &structs.ServiceRegistrationListRequest{ + QueryOptions: structs.QueryOptions{ + Namespace: structs.AllNamespacesSentinel, + Region: DefaultRegion, + }, + } + var serviceRegResp structs.ServiceRegistrationListResponse + err := msgpackrpc.CallWithCodec( + codec, structs.ServiceRegistrationListRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + require.ElementsMatch(t, []*structs.ServiceRegistrationListStub{}, serviceRegResp.Services) + }, + name: "ACLs enabled wildcard ns without token", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + return TestACLServer(t, nil) + }, + testFn: func(t *testing.T, s *Server, token *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + // Generate and upsert some service registrations. + services := mock.ServiceRegistrations() + require.NoError(t, s.State().UpsertServiceRegistrations(structs.MsgTypeTestSetup, 10, services)) + + // Test a request without setting an ACL token. + serviceRegReq := &structs.ServiceRegistrationListRequest{ + QueryOptions: structs.QueryOptions{ + Namespace: "default", + Region: DefaultRegion, + }, + } + var serviceRegResp structs.ServiceRegistrationListResponse + err := msgpackrpc.CallWithCodec( + codec, structs.ServiceRegistrationListRPCMethod, serviceRegReq, &serviceRegResp) + require.Error(t, err) + require.Contains(t, err.Error(), "Permission denied") + }, + name: "ACLs enabled default ns without token", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + return TestACLServer(t, nil) + }, + testFn: func(t *testing.T, s *Server, token *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + // Generate and upsert some service registrations. + services := mock.ServiceRegistrations() + require.NoError(t, s.State().UpsertServiceRegistrations(structs.MsgTypeTestSetup, 10, services)) + + // Test a request without setting an ACL token. + serviceRegReq := &structs.ServiceRegistrationListRequest{ + QueryOptions: structs.QueryOptions{ + Namespace: structs.AllNamespacesSentinel, + Region: DefaultRegion, + AuthToken: token.SecretID, + }, + } + var serviceRegResp structs.ServiceRegistrationListResponse + err := msgpackrpc.CallWithCodec( + codec, structs.ServiceRegistrationListRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + require.ElementsMatch(t, []*structs.ServiceRegistrationListStub{ + { + Namespace: "default", + Services: []*structs.ServiceRegistrationStub{ + { + ServiceName: "example-cache", + Tags: []string{"foo"}, + }, + }}, + { + Namespace: "platform", + Services: []*structs.ServiceRegistrationStub{ + { + ServiceName: "countdash-api", + Tags: []string{"bar"}, + }, + }}, + }, serviceRegResp.Services) + }, + name: "ACLs enabled wildcard with management token", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + return TestACLServer(t, nil) + }, + testFn: func(t *testing.T, s *Server, token *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + // Generate and upsert some service registrations. + services := mock.ServiceRegistrations() + require.NoError(t, s.State().UpsertServiceRegistrations(structs.MsgTypeTestSetup, 10, services)) + + // Test a request without setting an ACL token. + serviceRegReq := &structs.ServiceRegistrationListRequest{ + QueryOptions: structs.QueryOptions{ + Namespace: "default", + Region: DefaultRegion, + AuthToken: token.SecretID, + }, + } + var serviceRegResp structs.ServiceRegistrationListResponse + err := msgpackrpc.CallWithCodec( + codec, structs.ServiceRegistrationListRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + require.ElementsMatch(t, []*structs.ServiceRegistrationListStub{ + { + Namespace: "default", + Services: []*structs.ServiceRegistrationStub{ + { + ServiceName: "example-cache", + Tags: []string{"foo"}, + }, + }}, + }, serviceRegResp.Services) + }, + name: "ACLs enabled default ns with management token", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + return TestACLServer(t, nil) + }, + testFn: func(t *testing.T, s *Server, token *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + // Create a policy and grab the token which has the read-job + // capability on the platform namespace. + customToken := mock.CreatePolicyAndToken(t, s.State(), 5, "test-valid-autoscaler", + mock.NamespacePolicy("platform", "", []string{acl.NamespaceCapabilityReadJob})).SecretID + + // Generate and upsert some service registrations. + services := mock.ServiceRegistrations() + require.NoError(t, s.State().UpsertServiceRegistrations(structs.MsgTypeTestSetup, 10, services)) + + // Test a request without setting an ACL token. + serviceRegReq := &structs.ServiceRegistrationListRequest{ + QueryOptions: structs.QueryOptions{ + Namespace: "platform", + Region: DefaultRegion, + AuthToken: customToken, + }, + } + var serviceRegResp structs.ServiceRegistrationListResponse + err := msgpackrpc.CallWithCodec( + codec, structs.ServiceRegistrationListRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + require.ElementsMatch(t, []*structs.ServiceRegistrationListStub{ + { + Namespace: "platform", + Services: []*structs.ServiceRegistrationStub{ + { + ServiceName: "countdash-api", + Tags: []string{"bar"}, + }, + }}, + }, serviceRegResp.Services) + }, + name: "ACLs enabled with read-job policy token", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + return TestACLServer(t, nil) + }, + testFn: func(t *testing.T, s *Server, token *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + // Create a namespace as this is needed when using an ACL like + // we do in this test. + ns := &structs.Namespace{ + Name: "platform", + Description: "test namespace", + CreateIndex: 5, + ModifyIndex: 5, + } + ns.SetHash() + require.NoError(t, s.State().UpsertNamespaces(5, []*structs.Namespace{ns})) + + // Create a policy and grab the token which has the read-job + // capability on the platform namespace. + customToken := mock.CreatePolicyAndToken(t, s.State(), 10, "test-valid-autoscaler", + mock.NamespacePolicy("platform", "", []string{acl.NamespaceCapabilityReadJob})).SecretID + + // Generate and upsert some service registrations. + services := mock.ServiceRegistrations() + require.NoError(t, s.State().UpsertServiceRegistrations(structs.MsgTypeTestSetup, 20, services)) + + // Test a request without setting an ACL token. + serviceRegReq := &structs.ServiceRegistrationListRequest{ + QueryOptions: structs.QueryOptions{ + Namespace: structs.AllNamespacesSentinel, + Region: DefaultRegion, + AuthToken: customToken, + }, + } + var serviceRegResp structs.ServiceRegistrationListResponse + err := msgpackrpc.CallWithCodec( + codec, structs.ServiceRegistrationListRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + require.ElementsMatch(t, []*structs.ServiceRegistrationListStub{ + { + Namespace: "platform", + Services: []*structs.ServiceRegistrationStub{ + { + ServiceName: "countdash-api", + Tags: []string{"bar"}, + }, + }}, + }, serviceRegResp.Services) + }, + name: "ACLs enabled wildcard ns with restricted token", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + return TestACLServer(t, nil) + }, + testFn: func(t *testing.T, s *Server, token *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + // Create a namespace as this is needed when using an ACL like + // we do in this test. + ns := &structs.Namespace{ + Name: "platform", + Description: "test namespace", + CreateIndex: 5, + ModifyIndex: 5, + } + ns.SetHash() + require.NoError(t, s.State().UpsertNamespaces(5, []*structs.Namespace{ns})) + + // Create a policy and grab the token which has the read policy + // on the platform namespace. + customToken := mock.CreatePolicyAndToken(t, s.State(), 10, "test-valid-autoscaler", + mock.NamespacePolicy("platform", "read", nil)).SecretID + + // Generate and upsert some service registrations. + services := mock.ServiceRegistrations() + require.NoError(t, s.State().UpsertServiceRegistrations(structs.MsgTypeTestSetup, 20, services)) + + // Test a request without setting an ACL token. + serviceRegReq := &structs.ServiceRegistrationListRequest{ + QueryOptions: structs.QueryOptions{ + Namespace: structs.AllNamespacesSentinel, + Region: DefaultRegion, + AuthToken: customToken, + }, + } + var serviceRegResp structs.ServiceRegistrationListResponse + err := msgpackrpc.CallWithCodec( + codec, structs.ServiceRegistrationListRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + require.ElementsMatch(t, []*structs.ServiceRegistrationListStub{ + { + Namespace: "platform", + Services: []*structs.ServiceRegistrationStub{ + { + ServiceName: "countdash-api", + Tags: []string{"bar"}, + }, + }}, + }, serviceRegResp.Services) + }, + name: "ACLs enabled with read namespace policy token", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + server, aclToken, cleanup := tc.serverFn(t) + defer cleanup() + tc.testFn(t, server, aclToken) + }) + } +} + +func TestServiceRegistration_GetService(t *testing.T) { + t.Parallel() + + testCases := []struct { + serverFn func(t *testing.T) (*Server, *structs.ACLToken, func()) + testFn func(t *testing.T, s *Server, token *structs.ACLToken) + name string + }{ + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + server, cleanup := TestServer(t, nil) + return server, nil, cleanup + }, + testFn: func(t *testing.T, s *Server, _ *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + // Generate mock services then upsert them individually using different indexes. + services := mock.ServiceRegistrations() + + require.NoError(t, s.fsm.State().UpsertServiceRegistrations( + structs.MsgTypeTestSetup, 10, []*structs.ServiceRegistration{services[0]})) + + require.NoError(t, s.fsm.State().UpsertServiceRegistrations( + structs.MsgTypeTestSetup, 20, []*structs.ServiceRegistration{services[1]})) + + // Lookup the first registration. + serviceRegReq := &structs.ServiceRegistrationByNameRequest{ + ServiceName: services[0].ServiceName, + QueryOptions: structs.QueryOptions{ + Namespace: services[0].Namespace, + Region: s.Region(), + }, + } + var serviceRegResp structs.ServiceRegistrationByNameResponse + err := msgpackrpc.CallWithCodec(codec, structs.ServiceRegistrationGetServiceRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + require.Equal(t, uint64(10), serviceRegResp.Services[0].CreateIndex) + require.Equal(t, uint64(20), serviceRegResp.Index) + require.Len(t, serviceRegResp.Services, 1) + + // Lookup the second registration. + serviceRegReq2 := &structs.ServiceRegistrationByNameRequest{ + ServiceName: services[1].ServiceName, + QueryOptions: structs.QueryOptions{ + Namespace: services[1].Namespace, + Region: s.Region(), + }, + } + var serviceRegResp2 structs.ServiceRegistrationByNameResponse + err = msgpackrpc.CallWithCodec(codec, structs.ServiceRegistrationGetServiceRPCMethod, serviceRegReq2, &serviceRegResp2) + require.NoError(t, err) + require.Equal(t, uint64(20), serviceRegResp2.Services[0].CreateIndex) + require.Equal(t, uint64(20), serviceRegResp.Index) + require.Len(t, serviceRegResp2.Services, 1) + + // Perform a lookup with namespace and service name that shouldn't produce + // results. + serviceRegReq3 := &structs.ServiceRegistrationByNameRequest{ + ServiceName: services[0].ServiceName, + QueryOptions: structs.QueryOptions{ + Namespace: services[1].Namespace, + Region: s.Region(), + }, + } + var serviceRegResp3 structs.ServiceRegistrationByNameResponse + err = msgpackrpc.CallWithCodec(codec, structs.ServiceRegistrationGetServiceRPCMethod, serviceRegReq3, &serviceRegResp3) + require.NoError(t, err) + require.Len(t, serviceRegResp3.Services, 0) + }, + name: "ACLs disabled", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + return TestACLServer(t, nil) + }, + testFn: func(t *testing.T, s *Server, token *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + // Generate mock services then upsert them individually using different indexes. + services := mock.ServiceRegistrations() + + require.NoError(t, s.fsm.State().UpsertServiceRegistrations( + structs.MsgTypeTestSetup, 10, []*structs.ServiceRegistration{services[0]})) + + require.NoError(t, s.fsm.State().UpsertServiceRegistrations( + structs.MsgTypeTestSetup, 20, []*structs.ServiceRegistration{services[1]})) + + // Lookup the first registration without using an ACL token + // which should fail. + serviceRegReq := &structs.ServiceRegistrationByNameRequest{ + ServiceName: services[0].ServiceName, + QueryOptions: structs.QueryOptions{ + Namespace: services[0].Namespace, + Region: s.Region(), + }, + } + var serviceRegResp structs.ServiceRegistrationByNameResponse + err := msgpackrpc.CallWithCodec(codec, structs.ServiceRegistrationGetServiceRPCMethod, serviceRegReq, &serviceRegResp) + require.Error(t, err) + require.Contains(t, err.Error(), "Permission denied") + + // Lookup the first registration using the management token. + serviceRegReq2 := &structs.ServiceRegistrationByNameRequest{ + ServiceName: services[0].ServiceName, + QueryOptions: structs.QueryOptions{ + Namespace: services[0].Namespace, + Region: s.Region(), + AuthToken: token.SecretID, + }, + } + var serviceRegResp2 structs.ServiceRegistrationByNameResponse + err = msgpackrpc.CallWithCodec(codec, structs.ServiceRegistrationGetServiceRPCMethod, serviceRegReq2, &serviceRegResp2) + require.Nil(t, err) + require.Len(t, serviceRegResp2.Services, 1) + require.EqualValues(t, 20, serviceRegResp2.Index) + + // Create a read policy for the default namespace and test this + // can correctly read the first service. + authToken1 := mock.CreatePolicyAndToken(t, s.State(), 30, "test-service-reg-get", + mock.NamespacePolicy(structs.DefaultNamespace, "read", nil)).SecretID + serviceRegReq3 := &structs.ServiceRegistrationByNameRequest{ + ServiceName: services[0].ServiceName, + QueryOptions: structs.QueryOptions{ + Namespace: services[0].Namespace, + Region: s.Region(), + AuthToken: authToken1, + }, + } + var serviceRegResp3 structs.ServiceRegistrationByNameResponse + err = msgpackrpc.CallWithCodec(codec, structs.ServiceRegistrationGetServiceRPCMethod, serviceRegReq3, &serviceRegResp3) + require.Nil(t, err) + require.Len(t, serviceRegResp3.Services, 1) + require.EqualValues(t, 20, serviceRegResp2.Index) + + // Attempting to lookup services in a different namespace should fail. + serviceRegReq4 := &structs.ServiceRegistrationByNameRequest{ + ServiceName: services[1].ServiceName, + QueryOptions: structs.QueryOptions{ + Namespace: services[1].Namespace, + Region: s.Region(), + AuthToken: authToken1, + }, + } + var serviceRegResp4 structs.ServiceRegistrationByNameResponse + err = msgpackrpc.CallWithCodec(codec, structs.ServiceRegistrationGetServiceRPCMethod, serviceRegReq4, &serviceRegResp4) + require.Error(t, err) + require.Contains(t, err.Error(), "Permission denied") + }, + name: "ACLs enabled", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + server, aclToken, cleanup := tc.serverFn(t) + defer cleanup() + tc.testFn(t, server, aclToken) + }) + } +} diff --git a/nomad/structs/service_registration.go b/nomad/structs/service_registration.go index 9ebbe873b4d0..88c03c4d8838 100644 --- a/nomad/structs/service_registration.go +++ b/nomad/structs/service_registration.go @@ -1,6 +1,41 @@ package structs -import "github.com/hashicorp/nomad/helper" +import ( + "fmt" + + "github.com/hashicorp/nomad/helper" + "github.com/hashicorp/nomad/helper/ipaddr" +) + +const ( + // ServiceRegistrationUpsertRPCMethod is the RPC method for upserting + // service registrations into Nomad state. + // + // Args: ServiceRegistrationUpsertRequest + // Reply: ServiceRegistrationUpsertResponse + ServiceRegistrationUpsertRPCMethod = "ServiceRegistration.Upsert" + + // ServiceRegistrationDeleteByIDRPCMethod is the RPC method for deleting + // a service registration by its ID. + // + // Args: ServiceRegistrationDeleteByIDRequest + // Reply: ServiceRegistrationDeleteByIDResponse + ServiceRegistrationDeleteByIDRPCMethod = "ServiceRegistration.DeleteByID" + + // ServiceRegistrationListRPCMethod is the RPC method for listing service + // registrations within Nomad. + // + // Args: ServiceRegistrationListRequest + // Reply: ServiceRegistrationListResponse + ServiceRegistrationListRPCMethod = "ServiceRegistration.List" + + // ServiceRegistrationGetServiceRPCMethod is the RPC method for detailing a + // service and its registrations according to its name. + // + // Args: ServiceRegistrationByNameRequest + // Reply: ServiceRegistrationByNameResponse + ServiceRegistrationGetServiceRPCMethod = "ServiceRegistration.GetService" +) // ServiceRegistration is the internal representation of a Nomad service // registration. @@ -24,7 +59,8 @@ type ServiceRegistration struct { NodeID string // Datacenter is the DC identifier of the node as identified by - // Node.Datacenter. It is denormalized here to allow filtering services by datacenter without looking up every node. + // Node.Datacenter. It is denormalized here to allow filtering services by + // datacenter without looking up every node. Datacenter string // JobID is Job.ID and represents the job which contained the service block @@ -105,3 +141,98 @@ func (s *ServiceRegistration) Equals(o *ServiceRegistration) bool { } return true } + +// Validate ensures the upserted service registration contains valid +// information and routing capabilities. Objects should never fail here as +// Nomad controls the entire registration process; but it's possible +// configuration problems could cause failures. +func (s *ServiceRegistration) Validate() error { + if ipaddr.IsAny(s.Address) { + return fmt.Errorf("invalid service registration address") + } + return nil +} + +// ServiceRegistrationUpsertRequest is the request object used to upsert one or +// more service registrations. +type ServiceRegistrationUpsertRequest struct { + Services []*ServiceRegistration + WriteRequest +} + +// ServiceRegistrationUpsertResponse is the response object when one or more +// service registrations have been successfully upserted into state. +type ServiceRegistrationUpsertResponse struct { + WriteMeta +} + +// ServiceRegistrationDeleteByIDRequest is the request object to delete a +// service registration as specified by the ID parameter. +type ServiceRegistrationDeleteByIDRequest struct { + ID string + WriteRequest +} + +// ServiceRegistrationDeleteByIDResponse is the response object when performing a +// deletion of an individual service registration. +type ServiceRegistrationDeleteByIDResponse struct { + WriteMeta +} + +// ServiceRegistrationDeleteByNodeIDRequest is the request object to delete all +// service registrations assigned to a particular node. +type ServiceRegistrationDeleteByNodeIDRequest struct { + NodeID string + WriteRequest +} + +// ServiceRegistrationDeleteByNodeIDResponse is the response object when +// performing a deletion of all service registrations assigned to a particular +// node. +type ServiceRegistrationDeleteByNodeIDResponse struct { + WriteMeta +} + +// ServiceRegistrationListRequest is the request object when performing service +// registration listings. +type ServiceRegistrationListRequest struct { + QueryOptions +} + +// ServiceRegistrationListResponse is the response object when performing a +// list of services. This is specifically concise to reduce the serialisation +// and network costs endpoint incur, particularly when performing blocking list +// queries. +type ServiceRegistrationListResponse struct { + Services []*ServiceRegistrationListStub + QueryMeta +} + +// ServiceRegistrationListStub is the object which contains a list of namespace +// service registrations and their tags. +type ServiceRegistrationListStub struct { + Namespace string + Services []*ServiceRegistrationStub +} + +// ServiceRegistrationStub is the stub object describing an individual +// namespaced service. The object is built in a manner which would allow us to +// add additional fields in the future, if we wanted. +type ServiceRegistrationStub struct { + ServiceName string + Tags []string +} + +// ServiceRegistrationByNameRequest is the request object to perform a lookup +// of services matching a specific name. +type ServiceRegistrationByNameRequest struct { + ServiceName string + QueryOptions +} + +// ServiceRegistrationByNameResponse is the response object when performing a +// lookup of services matching a specific name. +type ServiceRegistrationByNameResponse struct { + Services []*ServiceRegistration + QueryMeta +} diff --git a/nomad/structs/service_registration_test.go b/nomad/structs/service_registration_test.go index 2d362b821aa5..d8fe06b27537 100644 --- a/nomad/structs/service_registration_test.go +++ b/nomad/structs/service_registration_test.go @@ -381,3 +381,13 @@ func TestServiceRegistration_Equal(t *testing.T) { }) } } + +func TestServiceRegistrationListRequest_StaleReadSupport(t *testing.T) { + req := &ServiceRegistrationListRequest{} + require.True(t, req.IsRead()) +} + +func TestServiceRegistrationByNameRequest_StaleReadSupport(t *testing.T) { + req := &ServiceRegistrationByNameRequest{} + require.True(t, req.IsRead()) +} From 487ec37d7ac387c7ef0b5e97a4b337bd2fffaffa Mon Sep 17 00:00:00 2001 From: James Rasell Date: Thu, 3 Mar 2022 11:25:55 +0100 Subject: [PATCH 10/31] rpc: add alloc service registration list RPC endpoint. --- nomad/alloc_endpoint.go | 64 ++++++++ nomad/alloc_endpoint_test.go | 309 +++++++++++++++++++++++++++++++++++ nomad/structs/alloc.go | 24 +++ nomad/structs/alloc_test.go | 12 ++ 4 files changed, 409 insertions(+) create mode 100644 nomad/structs/alloc.go create mode 100644 nomad/structs/alloc_test.go diff --git a/nomad/alloc_endpoint.go b/nomad/alloc_endpoint.go index 92abee62f4a5..8ed1490a615d 100644 --- a/nomad/alloc_endpoint.go +++ b/nomad/alloc_endpoint.go @@ -380,3 +380,67 @@ func (a *Alloc) UpdateDesiredTransition(args *structs.AllocUpdateDesiredTransiti reply.Index = index return nil } + +// GetServiceRegistrations returns a list of service registrations which belong +// to the passed allocation ID. +func (a *Alloc) GetServiceRegistrations( + args *structs.AllocServiceRegistrationsRequest, + reply *structs.AllocServiceRegistrationsResponse) error { + + if done, err := a.srv.forward(structs.AllocServiceRegistrationsRPCMethod, args, args, reply); done { + return err + } + defer metrics.MeasureSince([]string{"nomad", "alloc", "get_service_registrations"}, time.Now()) + + // If ACLs are enabled, ensure the caller has the read-job namespace + // capability. + aclObj, err := a.srv.ResolveToken(args.AuthToken) + if err != nil { + return err + } else if aclObj != nil { + if !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) { + return structs.ErrPermissionDenied + } + } + + // Set up the blocking query. + return a.srv.blockingRPC(&blockingOptions{ + queryOpts: &args.QueryOptions, + queryMeta: &reply.QueryMeta, + run: func(ws memdb.WatchSet, stateStore *state.StateStore) error { + + // Read the allocation to ensure its namespace matches the request + // args. + alloc, err := stateStore.AllocByID(ws, args.AllocID) + if err != nil { + return err + } + + // Guard against the alloc not-existing or that the namespace does + // not match the request arguments. + if alloc == nil || alloc.Namespace != args.RequestNamespace() { + return nil + } + + // Perform the state query to get an iterator. + iter, err := stateStore.GetServiceRegistrationsByAllocID(ws, args.AllocID) + if err != nil { + return err + } + + // Set up our output after we have checked the error. + services := make([]*structs.ServiceRegistration, 0) + + // Iterate the iterator, appending all service registrations + // returned to the reply. + for raw := iter.Next(); raw != nil; raw = iter.Next() { + services = append(services, raw.(*structs.ServiceRegistration)) + } + reply.Services = services + + // Use the index table to populate the query meta as we have no way + // of tracking the max index on deletes. + return a.srv.setReplyQueryMeta(stateStore, state.TableServiceRegistrations, &reply.QueryMeta) + }, + }) +} diff --git a/nomad/alloc_endpoint_test.go b/nomad/alloc_endpoint_test.go index d34abfbbf2dc..ceb7aee1270f 100644 --- a/nomad/alloc_endpoint_test.go +++ b/nomad/alloc_endpoint_test.go @@ -1034,3 +1034,312 @@ func TestAllocEndpoint_List_AllNamespaces_ACL_OSS(t *testing.T) { } } + +func TestAlloc_GetServiceRegistrations(t *testing.T) { + t.Parallel() + + // This function is a helper function to set up an allocation and service + // which can be queried. + correctSetupFn := func(s *Server) (error, string, *structs.ServiceRegistration) { + // Generate an upsert an allocation. + alloc := mock.Alloc() + err := s.State().UpsertAllocs(structs.MsgTypeTestSetup, 10, []*structs.Allocation{alloc}) + if err != nil { + return nil, "", nil + } + + // Generate services. Set the allocation ID to the first, so it + // matches the allocation. The alloc and first service both + // reside in the default namespace. + services := mock.ServiceRegistrations() + services[0].AllocID = alloc.ID + err = s.State().UpsertServiceRegistrations(structs.MsgTypeTestSetup, 20, services) + + return err, alloc.ID, services[0] + } + + testCases := []struct { + serverFn func(t *testing.T) (*Server, *structs.ACLToken, func()) + testFn func(t *testing.T, s *Server, token *structs.ACLToken) + name string + }{ + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + server, cleanup := TestServer(t, nil) + return server, nil, cleanup + }, + testFn: func(t *testing.T, s *Server, _ *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + err, allocID, service := correctSetupFn(s) + require.NoError(t, err) + + // Perform a lookup on the first service. + serviceRegReq := &structs.AllocServiceRegistrationsRequest{ + AllocID: allocID, + QueryOptions: structs.QueryOptions{ + Namespace: service.Namespace, + Region: s.Region(), + }, + } + var serviceRegResp structs.AllocServiceRegistrationsResponse + err = msgpackrpc.CallWithCodec(codec, structs.AllocServiceRegistrationsRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + require.EqualValues(t, uint64(20), serviceRegResp.Index) + require.ElementsMatch(t, serviceRegResp.Services, []*structs.ServiceRegistration{service}) + }, + name: "ACLs disabled alloc found with regs", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + server, cleanup := TestServer(t, nil) + return server, nil, cleanup + }, + testFn: func(t *testing.T, s *Server, _ *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + // Generate and upsert our services. + services := mock.ServiceRegistrations() + require.NoError(t, s.State().UpsertServiceRegistrations(structs.MsgTypeTestSetup, 20, services)) + + // Perform a lookup on the first service using the allocation + // ID. This allocation does not exist within the Nomad state + // meaning the service is orphaned or the caller used an + // incorrect allocation ID. + serviceRegReq := &structs.AllocServiceRegistrationsRequest{ + AllocID: services[0].AllocID, + QueryOptions: structs.QueryOptions{ + Namespace: services[0].Namespace, + Region: s.Region(), + }, + } + var serviceRegResp structs.AllocServiceRegistrationsResponse + err := msgpackrpc.CallWithCodec(codec, structs.AllocServiceRegistrationsRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + require.Nil(t, serviceRegResp.Services) + }, + name: "ACLs disabled alloc not found", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + server, cleanup := TestServer(t, nil) + return server, nil, cleanup + }, + testFn: func(t *testing.T, s *Server, _ *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + err, allocID, _ := correctSetupFn(s) + require.NoError(t, err) + + // Perform a lookup on the first service using the allocation + // ID but a random namespace. The namespace on the allocation + // does therefore not match the request args. + serviceRegReq := &structs.AllocServiceRegistrationsRequest{ + AllocID: allocID, + QueryOptions: structs.QueryOptions{ + Namespace: "platform", + Region: s.Region(), + }, + } + var serviceRegResp structs.AllocServiceRegistrationsResponse + err = msgpackrpc.CallWithCodec(codec, structs.AllocServiceRegistrationsRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + require.ElementsMatch(t, serviceRegResp.Services, []*structs.ServiceRegistration{}) + }, + name: "ACLs disabled alloc found in different namespace than request", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + server, cleanup := TestServer(t, nil) + return server, nil, cleanup + }, + testFn: func(t *testing.T, s *Server, _ *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + // Generate an upsert an allocation. + alloc := mock.Alloc() + require.NoError(t, s.State().UpsertAllocs( + structs.MsgTypeTestSetup, 10, []*structs.Allocation{alloc})) + + // Perform a lookup using the allocation information. + serviceRegReq := &structs.AllocServiceRegistrationsRequest{ + AllocID: alloc.ID, + QueryOptions: structs.QueryOptions{ + Namespace: alloc.Namespace, + Region: s.Region(), + }, + } + var serviceRegResp structs.AllocServiceRegistrationsResponse + err := msgpackrpc.CallWithCodec(codec, structs.AllocServiceRegistrationsRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + require.ElementsMatch(t, serviceRegResp.Services, []*structs.ServiceRegistration{}) + }, + name: "ACLs disabled alloc found without regs", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + return TestACLServer(t, nil) + }, + testFn: func(t *testing.T, s *Server, token *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + err, allocID, service := correctSetupFn(s) + require.NoError(t, err) + + // Perform a lookup using the allocation information. + serviceRegReq := &structs.AllocServiceRegistrationsRequest{ + AllocID: allocID, + QueryOptions: structs.QueryOptions{ + Namespace: service.Namespace, + Region: s.Region(), + AuthToken: token.SecretID, + }, + } + var serviceRegResp structs.AllocServiceRegistrationsResponse + err = msgpackrpc.CallWithCodec(codec, structs.AllocServiceRegistrationsRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + require.ElementsMatch(t, serviceRegResp.Services, []*structs.ServiceRegistration{service}) + }, + name: "ACLs enabled use management token", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + return TestACLServer(t, nil) + }, + testFn: func(t *testing.T, s *Server, _ *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + err, allocID, service := correctSetupFn(s) + require.NoError(t, err) + + // Create and policy and grab the auth token. + authToken := mock.CreatePolicyAndToken(t, s.State(), 30, "test-node-get-service-reg", + mock.NamespacePolicy(service.Namespace, "", []string{acl.NamespaceCapabilityReadJob})).SecretID + + // Perform a lookup using the allocation information. + serviceRegReq := &structs.AllocServiceRegistrationsRequest{ + AllocID: allocID, + QueryOptions: structs.QueryOptions{ + Namespace: service.Namespace, + Region: s.Region(), + AuthToken: authToken, + }, + } + var serviceRegResp structs.AllocServiceRegistrationsResponse + err = msgpackrpc.CallWithCodec(codec, structs.AllocServiceRegistrationsRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + require.ElementsMatch(t, serviceRegResp.Services, []*structs.ServiceRegistration{service}) + }, + name: "ACLs enabled use read-job namespace capability token", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + return TestACLServer(t, nil) + }, + testFn: func(t *testing.T, s *Server, _ *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + err, allocID, service := correctSetupFn(s) + require.NoError(t, err) + + // Create and policy and grab the auth token. + authToken := mock.CreatePolicyAndToken(t, s.State(), 30, "test-node-get-service-reg", + mock.NamespacePolicy(service.Namespace, "read", nil)).SecretID + + // Perform a lookup using the allocation information. + serviceRegReq := &structs.AllocServiceRegistrationsRequest{ + AllocID: allocID, + QueryOptions: structs.QueryOptions{ + Namespace: service.Namespace, + Region: s.Region(), + AuthToken: authToken, + }, + } + var serviceRegResp structs.AllocServiceRegistrationsResponse + err = msgpackrpc.CallWithCodec(codec, structs.AllocServiceRegistrationsRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + require.ElementsMatch(t, serviceRegResp.Services, []*structs.ServiceRegistration{service}) + }, + name: "ACLs enabled use read namespace policy token", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + return TestACLServer(t, nil) + }, + testFn: func(t *testing.T, s *Server, _ *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + err, allocID, service := correctSetupFn(s) + require.NoError(t, err) + + // Create and policy and grab the auth token. + authToken := mock.CreatePolicyAndToken(t, s.State(), 30, "test-node-get-service-reg", + mock.NamespacePolicy("ohno", "read", nil)).SecretID + + // Perform a lookup using the allocation information. + serviceRegReq := &structs.AllocServiceRegistrationsRequest{ + AllocID: allocID, + QueryOptions: structs.QueryOptions{ + Namespace: service.Namespace, + Region: s.Region(), + AuthToken: authToken, + }, + } + var serviceRegResp structs.AllocServiceRegistrationsResponse + err = msgpackrpc.CallWithCodec(codec, structs.AllocServiceRegistrationsRPCMethod, serviceRegReq, &serviceRegResp) + require.Error(t, err) + require.Contains(t, err.Error(), "Permission denied") + require.Empty(t, serviceRegResp.Services) + }, + name: "ACLs enabled use read incorrect namespace policy token", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + return TestACLServer(t, nil) + }, + testFn: func(t *testing.T, s *Server, _ *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + err, allocID, service := correctSetupFn(s) + require.NoError(t, err) + + // Create and policy and grab the auth token. + authToken := mock.CreatePolicyAndToken(t, s.State(), 30, "test-node-get-service-reg", + mock.NamespacePolicy(service.Namespace, "", []string{acl.NamespaceCapabilityReadScalingPolicy})).SecretID + + // Perform a lookup using the allocation information. + serviceRegReq := &structs.AllocServiceRegistrationsRequest{ + AllocID: allocID, + QueryOptions: structs.QueryOptions{ + Namespace: service.Namespace, + Region: s.Region(), + AuthToken: authToken, + }, + } + var serviceRegResp structs.AllocServiceRegistrationsResponse + err = msgpackrpc.CallWithCodec(codec, structs.AllocServiceRegistrationsRPCMethod, serviceRegReq, &serviceRegResp) + require.Error(t, err) + require.Contains(t, err.Error(), "Permission denied") + require.Empty(t, serviceRegResp.Services) + }, + name: "ACLs enabled use incorrect capability", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + server, aclToken, cleanup := tc.serverFn(t) + defer cleanup() + tc.testFn(t, server, aclToken) + }) + } +} diff --git a/nomad/structs/alloc.go b/nomad/structs/alloc.go new file mode 100644 index 000000000000..9f1dee8d079c --- /dev/null +++ b/nomad/structs/alloc.go @@ -0,0 +1,24 @@ +package structs + +const ( + // AllocServiceRegistrationsRPCMethod is the RPC method for listing all + // service registrations assigned to a specific allocation. + // + // Args: AllocServiceRegistrationsRequest + // Reply: AllocServiceRegistrationsResponse + AllocServiceRegistrationsRPCMethod = "Alloc.GetServiceRegistrations" +) + +// AllocServiceRegistrationsRequest is the request object used to list all +// service registrations belonging to the specified Allocation.ID. +type AllocServiceRegistrationsRequest struct { + AllocID string + QueryOptions +} + +// AllocServiceRegistrationsResponse is the response object when performing a +// listing of services belonging to an allocation. +type AllocServiceRegistrationsResponse struct { + Services []*ServiceRegistration + QueryMeta +} diff --git a/nomad/structs/alloc_test.go b/nomad/structs/alloc_test.go new file mode 100644 index 000000000000..ce2ce52dab8f --- /dev/null +++ b/nomad/structs/alloc_test.go @@ -0,0 +1,12 @@ +package structs + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestAllocServiceRegistrationsRequest_StaleReadSupport(t *testing.T) { + req := &AllocServiceRegistrationsRequest{} + require.True(t, req.IsRead()) +} From a674fb388badf1ff25ba30473bd92a92cd9822b0 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Thu, 3 Mar 2022 11:26:14 +0100 Subject: [PATCH 11/31] rpc: add job service registration list RPC endpoint. --- nomad/job_endpoint.go | 62 +++++++++ nomad/job_endpoint_test.go | 277 +++++++++++++++++++++++++++++++++++++ nomad/structs/job.go | 24 ++++ nomad/structs/job_test.go | 12 ++ 4 files changed, 375 insertions(+) create mode 100644 nomad/structs/job.go create mode 100644 nomad/structs/job_test.go diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index 418cbe1afa3f..a26600290bc8 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -2217,3 +2217,65 @@ func (j *Job) ScaleStatus(args *structs.JobScaleStatusRequest, }} return j.srv.blockingRPC(&opts) } + +// GetServiceRegistrations returns a list of service registrations which belong +// to the passed job ID. +func (j *Job) GetServiceRegistrations( + args *structs.JobServiceRegistrationsRequest, + reply *structs.JobServiceRegistrationsResponse) error { + + if done, err := j.srv.forward(structs.JobServiceRegistrationsRPCMethod, args, args, reply); done { + return err + } + defer metrics.MeasureSince([]string{"nomad", "job", "get_service_registrations"}, time.Now()) + + // If ACLs are enabled, ensure the caller has the read-job namespace + // capability. + aclObj, err := j.srv.ResolveToken(args.AuthToken) + if err != nil { + return err + } else if aclObj != nil { + if !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) { + return structs.ErrPermissionDenied + } + } + + // Set up the blocking query. + return j.srv.blockingRPC(&blockingOptions{ + queryOpts: &args.QueryOptions, + queryMeta: &reply.QueryMeta, + run: func(ws memdb.WatchSet, stateStore *state.StateStore) error { + + job, err := stateStore.JobByID(ws, args.RequestNamespace(), args.JobID) + if err != nil { + return err + } + + // Guard against the job not-existing. Do not create an empty list + // to allow the API to determine whether the job was found or not. + if job == nil { + return nil + } + + // Perform the state query to get an iterator. + iter, err := stateStore.GetServiceRegistrationsByJobID(ws, args.RequestNamespace(), args.JobID) + if err != nil { + return err + } + + // Set up our output after we have checked the error. + services := make([]*structs.ServiceRegistration, 0) + + // Iterate the iterator, appending all service registrations + // returned to the reply. + for raw := iter.Next(); raw != nil; raw = iter.Next() { + services = append(services, raw.(*structs.ServiceRegistration)) + } + reply.Services = services + + // Use the index table to populate the query meta as we have no way + // of tracking the max index on deletes. + return j.srv.setReplyQueryMeta(stateStore, state.TableServiceRegistrations, &reply.QueryMeta) + }, + }) +} diff --git a/nomad/job_endpoint_test.go b/nomad/job_endpoint_test.go index 3850290fa4f7..e844578faa23 100644 --- a/nomad/job_endpoint_test.go +++ b/nomad/job_endpoint_test.go @@ -7740,3 +7740,280 @@ func TestJobEndpoint_GetScaleStatus_ACL(t *testing.T) { require.NotNil(validResp.JobScaleStatus) } } + +func TestJob_GetServiceRegistrations(t *testing.T) { + t.Parallel() + + // This function is a helper function to set up job and service which can + // be queried. + correctSetupFn := func(s *Server) (error, string, *structs.ServiceRegistration) { + // Generate an upsert a job. + job := mock.Job() + err := s.State().UpsertJob(structs.MsgTypeTestSetup, 10, job) + if err != nil { + return nil, "", nil + } + + // Generate services. Set the jobID on the first service so this + // matches the job now held in state. + services := mock.ServiceRegistrations() + services[0].JobID = job.ID + err = s.State().UpsertServiceRegistrations(structs.MsgTypeTestSetup, 20, services) + + return err, job.ID, services[0] + } + + testCases := []struct { + serverFn func(t *testing.T) (*Server, *structs.ACLToken, func()) + testFn func(t *testing.T, s *Server, token *structs.ACLToken) + name string + }{ + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + server, cleanup := TestServer(t, nil) + return server, nil, cleanup + }, + testFn: func(t *testing.T, s *Server, _ *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + err, jobID, service := correctSetupFn(s) + require.NoError(t, err) + + // Perform a lookup and test the response. + serviceRegReq := &structs.JobServiceRegistrationsRequest{ + JobID: jobID, + QueryOptions: structs.QueryOptions{ + Namespace: service.Namespace, + Region: s.Region(), + }, + } + var serviceRegResp structs.JobServiceRegistrationsResponse + err = msgpackrpc.CallWithCodec(codec, structs.JobServiceRegistrationsRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + require.EqualValues(t, uint64(20), serviceRegResp.Index) + require.ElementsMatch(t, serviceRegResp.Services, []*structs.ServiceRegistration{service}) + }, + name: "ACLs disabled job found with regs", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + server, cleanup := TestServer(t, nil) + return server, nil, cleanup + }, + testFn: func(t *testing.T, s *Server, _ *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + // Generate and upsert our services. + services := mock.ServiceRegistrations() + require.NoError(t, s.State().UpsertServiceRegistrations(structs.MsgTypeTestSetup, 20, services)) + + // Perform a lookup on the first service using the job ID. This + // job does not exist within the Nomad state meaning the + // service is orphaned or the caller used an incorrect job ID. + serviceRegReq := &structs.JobServiceRegistrationsRequest{ + JobID: services[0].JobID, + QueryOptions: structs.QueryOptions{ + Namespace: services[0].Namespace, + Region: s.Region(), + }, + } + var serviceRegResp structs.JobServiceRegistrationsResponse + err := msgpackrpc.CallWithCodec(codec, structs.JobServiceRegistrationsRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + require.Nil(t, serviceRegResp.Services) + }, + name: "ACLs disabled job not found", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + server, cleanup := TestServer(t, nil) + return server, nil, cleanup + }, + testFn: func(t *testing.T, s *Server, _ *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + // Generate an upsert a job. + job := mock.Job() + require.NoError(t, s.State().UpsertJob(structs.MsgTypeTestSetup, 10, job)) + + // Perform a lookup and test the response. + serviceRegReq := &structs.JobServiceRegistrationsRequest{ + JobID: job.ID, + QueryOptions: structs.QueryOptions{ + Namespace: job.Namespace, + Region: s.Region(), + }, + } + var serviceRegResp structs.JobServiceRegistrationsResponse + err := msgpackrpc.CallWithCodec(codec, structs.JobServiceRegistrationsRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + require.ElementsMatch(t, serviceRegResp.Services, []*structs.ServiceRegistration{}) + }, + name: "ACLs disabled job found without regs", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + return TestACLServer(t, nil) + }, + testFn: func(t *testing.T, s *Server, token *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + err, jobID, service := correctSetupFn(s) + require.NoError(t, err) + + // Perform a lookup and test the response. + serviceRegReq := &structs.JobServiceRegistrationsRequest{ + JobID: jobID, + QueryOptions: structs.QueryOptions{ + Namespace: service.Namespace, + Region: s.Region(), + AuthToken: token.SecretID, + }, + } + var serviceRegResp structs.JobServiceRegistrationsResponse + err = msgpackrpc.CallWithCodec(codec, structs.JobServiceRegistrationsRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + require.ElementsMatch(t, serviceRegResp.Services, []*structs.ServiceRegistration{service}) + }, + name: "ACLs enabled use management token", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + return TestACLServer(t, nil) + }, + testFn: func(t *testing.T, s *Server, _ *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + err, jobID, service := correctSetupFn(s) + require.NoError(t, err) + + // Create and policy and grab the auth token. + authToken := mock.CreatePolicyAndToken(t, s.State(), 30, "test-node-get-service-reg", + mock.NamespacePolicy(service.Namespace, "", []string{acl.NamespaceCapabilityReadJob})).SecretID + + // Perform a lookup and test the response. + serviceRegReq := &structs.JobServiceRegistrationsRequest{ + JobID: jobID, + QueryOptions: structs.QueryOptions{ + Namespace: service.Namespace, + Region: s.Region(), + AuthToken: authToken, + }, + } + var serviceRegResp structs.JobServiceRegistrationsResponse + err = msgpackrpc.CallWithCodec(codec, structs.JobServiceRegistrationsRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + require.ElementsMatch(t, serviceRegResp.Services, []*structs.ServiceRegistration{service}) + }, + name: "ACLs enabled use read-job namespace capability token", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + return TestACLServer(t, nil) + }, + testFn: func(t *testing.T, s *Server, _ *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + err, jobID, service := correctSetupFn(s) + require.NoError(t, err) + + // Create and policy and grab the auth token. + authToken := mock.CreatePolicyAndToken(t, s.State(), 30, "test-node-get-service-reg", + mock.NamespacePolicy(service.Namespace, "read", nil)).SecretID + + // Perform a lookup and test the response. + serviceRegReq := &structs.JobServiceRegistrationsRequest{ + JobID: jobID, + QueryOptions: structs.QueryOptions{ + Namespace: service.Namespace, + Region: s.Region(), + AuthToken: authToken, + }, + } + var serviceRegResp structs.JobServiceRegistrationsResponse + err = msgpackrpc.CallWithCodec(codec, structs.JobServiceRegistrationsRPCMethod, serviceRegReq, &serviceRegResp) + require.NoError(t, err) + require.ElementsMatch(t, serviceRegResp.Services, []*structs.ServiceRegistration{service}) + }, + name: "ACLs enabled use read namespace policy token", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + return TestACLServer(t, nil) + }, + testFn: func(t *testing.T, s *Server, _ *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + err, jobID, service := correctSetupFn(s) + require.NoError(t, err) + + // Create and policy and grab the auth token. + authToken := mock.CreatePolicyAndToken(t, s.State(), 30, "test-node-get-service-reg", + mock.NamespacePolicy("ohno", "read", nil)).SecretID + + // Perform a lookup and test the response. + serviceRegReq := &structs.JobServiceRegistrationsRequest{ + JobID: jobID, + QueryOptions: structs.QueryOptions{ + Namespace: service.Namespace, + Region: s.Region(), + AuthToken: authToken, + }, + } + var serviceRegResp structs.JobServiceRegistrationsResponse + err = msgpackrpc.CallWithCodec(codec, structs.JobServiceRegistrationsRPCMethod, serviceRegReq, &serviceRegResp) + require.Error(t, err) + require.Contains(t, err.Error(), "Permission denied") + require.Empty(t, serviceRegResp.Services) + }, + name: "ACLs enabled use read incorrect namespace policy token", + }, + { + serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { + return TestACLServer(t, nil) + }, + testFn: func(t *testing.T, s *Server, _ *structs.ACLToken) { + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + + err, jobID, service := correctSetupFn(s) + require.NoError(t, err) + + // Create and policy and grab the auth token. + authToken := mock.CreatePolicyAndToken(t, s.State(), 30, "test-node-get-service-reg", + mock.NamespacePolicy(service.Namespace, "", []string{acl.NamespaceCapabilityReadScalingPolicy})).SecretID + + // Perform a lookup and test the response. + serviceRegReq := &structs.JobServiceRegistrationsRequest{ + JobID: jobID, + QueryOptions: structs.QueryOptions{ + Namespace: service.Namespace, + Region: s.Region(), + AuthToken: authToken, + }, + } + var serviceRegResp structs.JobServiceRegistrationsResponse + err = msgpackrpc.CallWithCodec(codec, structs.JobServiceRegistrationsRPCMethod, serviceRegReq, &serviceRegResp) + require.Error(t, err) + require.Contains(t, err.Error(), "Permission denied") + require.Empty(t, serviceRegResp.Services) + }, + name: "ACLs enabled use incorrect capability", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + server, aclToken, cleanup := tc.serverFn(t) + defer cleanup() + tc.testFn(t, server, aclToken) + }) + } +} diff --git a/nomad/structs/job.go b/nomad/structs/job.go new file mode 100644 index 000000000000..07c79023af79 --- /dev/null +++ b/nomad/structs/job.go @@ -0,0 +1,24 @@ +package structs + +const ( + // JobServiceRegistrationsRPCMethod is the RPC method for listing all + // service registrations assigned to a specific namespaced job. + // + // Args: JobServiceRegistrationsRequest + // Reply: JobServiceRegistrationsResponse + JobServiceRegistrationsRPCMethod = "Job.GetServiceRegistrations" +) + +// JobServiceRegistrationsRequest is the request object used to list all +// service registrations belonging to the specified Job.ID. +type JobServiceRegistrationsRequest struct { + JobID string + QueryOptions +} + +// JobServiceRegistrationsResponse is the response object when performing a +// listing of services belonging to a namespaced job. +type JobServiceRegistrationsResponse struct { + Services []*ServiceRegistration + QueryMeta +} diff --git a/nomad/structs/job_test.go b/nomad/structs/job_test.go new file mode 100644 index 000000000000..e64339510ce4 --- /dev/null +++ b/nomad/structs/job_test.go @@ -0,0 +1,12 @@ +package structs + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestServiceRegistrationsRequest_StaleReadSupport(t *testing.T) { + req := &AllocServiceRegistrationsRequest{} + require.True(t, req.IsRead()) +} From 5093be70f21cac7210cebb931cac50a7917db5b7 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Thu, 3 Mar 2022 12:13:00 +0100 Subject: [PATCH 12/31] http: add agent service registration HTTP endpoint. --- command/agent/http.go | 4 + .../agent/service_registration_endpoint.go | 124 +++++++ .../service_registration_endpoint_test.go | 305 ++++++++++++++++++ 3 files changed, 433 insertions(+) create mode 100644 command/agent/service_registration_endpoint.go create mode 100644 command/agent/service_registration_endpoint_test.go diff --git a/command/agent/http.go b/command/agent/http.go index 8568a0b0e9d9..ba48f268e144 100644 --- a/command/agent/http.go +++ b/command/agent/http.go @@ -341,6 +341,10 @@ func (s HTTPServer) registerHandlers(enableDebug bool) { s.mux.HandleFunc("/v1/agent/health", s.wrap(s.HealthRequest)) s.mux.HandleFunc("/v1/agent/host", s.wrap(s.AgentHostRequest)) + // Register our service registration handlers. + s.mux.HandleFunc("/v1/services", s.wrap(s.ServiceRegistrationListRequest)) + s.mux.HandleFunc("/v1/service/", s.wrap(s.ServiceRegistrationRequest)) + // Monitor is *not* an untrusted endpoint despite the log contents // potentially containing unsanitized user input. Monitor, like // "/v1/client/fs/logs", explicitly sets a "text/plain" or diff --git a/command/agent/service_registration_endpoint.go b/command/agent/service_registration_endpoint.go new file mode 100644 index 000000000000..bf89e62c80c0 --- /dev/null +++ b/command/agent/service_registration_endpoint.go @@ -0,0 +1,124 @@ +package agent + +import ( + "net/http" + "strings" + + "github.com/hashicorp/nomad/nomad/structs" +) + +// ServiceRegistrationListRequest performs a listing of service registrations +// using the structs.ServiceRegistrationListRPCMethod RPC endpoint and is +// callable via the /v1/services HTTP API. +func (s *HTTPServer) ServiceRegistrationListRequest( + resp http.ResponseWriter, req *http.Request) (interface{}, error) { + + // The endpoint only supports GET requests. + if req.Method != http.MethodGet { + return nil, CodedError(http.StatusMethodNotAllowed, ErrInvalidMethod) + } + + // Set up the request args and parse this to ensure the query options are + // set. + args := structs.ServiceRegistrationListRequest{} + + if s.parse(resp, req, &args.Region, &args.QueryOptions) { + return nil, nil + } + + // Perform the RPC request. + var reply structs.ServiceRegistrationListResponse + if err := s.agent.RPC(structs.ServiceRegistrationListRPCMethod, &args, &reply); err != nil { + return nil, err + } + + setMeta(resp, &reply.QueryMeta) + + if reply.Services == nil { + reply.Services = make([]*structs.ServiceRegistrationListStub, 0) + } + return reply.Services, nil +} + +// ServiceRegistrationRequest is callable via the /v1/service/ HTTP API and +// handles service reads and individual service registration deletions. +func (s *HTTPServer) ServiceRegistrationRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + + // Grab the suffix of the request, so we can further understand it. + reqSuffix := strings.TrimPrefix(req.URL.Path, "/v1/service/") + + // Split the request suffix in order to identify whether this is a lookup + // of a service, or whether this includes a service and service identifier. + suffixParts := strings.Split(reqSuffix, "/") + + switch len(suffixParts) { + case 1: + // This endpoint only supports GET. + if req.Method != http.MethodGet { + return nil, CodedError(http.StatusMethodNotAllowed, ErrInvalidMethod) + } + + // Ensure the service ID is not an empty string which is possible if + // the caller requested "/v1/service/service-name/" + if suffixParts[0] == "" { + return nil, CodedError(http.StatusBadRequest, "missing service name") + } + + return s.serviceGetRequest(resp, req, suffixParts[0]) + + case 2: + // This endpoint only supports DELETE. + if req.Method != http.MethodDelete { + return nil, CodedError(http.StatusMethodNotAllowed, ErrInvalidMethod) + } + + // Ensure the service ID is not an empty string which is possible if + // the caller requested "/v1/service/service-name/" + if suffixParts[1] == "" { + return nil, CodedError(http.StatusBadRequest, "missing service id") + } + + return s.serviceDeleteRequest(resp, req, suffixParts[1]) + + default: + return nil, CodedError(http.StatusBadRequest, "invalid URI") + } +} + +// serviceGetRequest performs a reading of service registrations by name using +// the structs.ServiceRegistrationGetServiceRPCMethod RPC endpoint. +func (s *HTTPServer) serviceGetRequest( + resp http.ResponseWriter, req *http.Request, serviceName string) (interface{}, error) { + + args := structs.ServiceRegistrationByNameRequest{ServiceName: serviceName} + if s.parse(resp, req, &args.Region, &args.QueryOptions) { + return nil, nil + } + + var reply structs.ServiceRegistrationByNameResponse + if err := s.agent.RPC(structs.ServiceRegistrationGetServiceRPCMethod, &args, &reply); err != nil { + return nil, err + } + setIndex(resp, reply.Index) + + if reply.Services == nil { + reply.Services = make([]*structs.ServiceRegistration, 0) + } + return reply.Services, nil +} + +// serviceDeleteRequest performs a reading of service registrations by name using +// the structs.ServiceRegistrationDeleteByIDRPCMethod RPC endpoint. +func (s *HTTPServer) serviceDeleteRequest( + resp http.ResponseWriter, req *http.Request, serviceID string) (interface{}, error) { + + args := structs.ServiceRegistrationDeleteByIDRequest{ID: serviceID} + s.parseWriteRequest(req, &args.WriteRequest) + + var reply structs.ServiceRegistrationDeleteByIDResponse + if err := s.agent.RPC(structs.ServiceRegistrationDeleteByIDRPCMethod, &args, &reply); err != nil { + return nil, err + } + setIndex(resp, reply.Index) + return nil, nil +} diff --git a/command/agent/service_registration_endpoint_test.go b/command/agent/service_registration_endpoint_test.go new file mode 100644 index 000000000000..e02edc0607a3 --- /dev/null +++ b/command/agent/service_registration_endpoint_test.go @@ -0,0 +1,305 @@ +package agent + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/hashicorp/go-memdb" + "github.com/hashicorp/nomad/nomad/mock" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/stretchr/testify/require" +) + +func TestHTTPServer_ServiceRegistrationListRequest(t *testing.T) { + t.Parallel() + + testCases := []struct { + testFn func(srv *TestAgent) + name string + }{ + { + testFn: func(s *TestAgent) { + + // Grab the state, so we can manipulate it and test against it. + testState := s.Agent.server.State() + + // Generate service registrations and upsert. + serviceRegs := mock.ServiceRegistrations() + require.NoError(t, testState.UpsertServiceRegistrations( + structs.MsgTypeTestSetup, 10, serviceRegs)) + + // Build the HTTP request. + req, err := http.NewRequest(http.MethodGet, "/v1/services", nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Send the HTTP request. + obj, err := s.Server.ServiceRegistrationListRequest(respW, req) + require.NoError(t, err) + require.NotNil(t, obj) + + // Check the index is not zero. + require.EqualValues(t, "10", respW.Header().Get("X-Nomad-Index")) + require.ElementsMatch(t, []*structs.ServiceRegistrationListStub{ + { + Namespace: "default", + Services: []*structs.ServiceRegistrationStub{ + { + ServiceName: "example-cache", + Tags: []string{"foo"}, + }, + }, + }, + }, obj.([]*structs.ServiceRegistrationListStub)) + }, + name: "list default namespace", + }, + { + testFn: func(s *TestAgent) { + + // Grab the state, so we can manipulate it and test against it. + testState := s.Agent.server.State() + + // Generate service registrations and upsert. + serviceRegs := mock.ServiceRegistrations() + require.NoError(t, testState.UpsertServiceRegistrations( + structs.MsgTypeTestSetup, 10, serviceRegs)) + + // Build the HTTP request. + req, err := http.NewRequest(http.MethodGet, "/v1/services?namespace=platform", nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Send the HTTP request. + obj, err := s.Server.ServiceRegistrationListRequest(respW, req) + require.NoError(t, err) + require.NotNil(t, obj) + + // Check the index is not zero. + require.EqualValues(t, "10", respW.Header().Get("X-Nomad-Index")) + require.ElementsMatch(t, []*structs.ServiceRegistrationListStub{ + { + Namespace: "platform", + Services: []*structs.ServiceRegistrationStub{ + { + ServiceName: "countdash-api", + Tags: []string{"bar"}, + }, + }, + }, + }, obj.([]*structs.ServiceRegistrationListStub)) + }, + name: "list platform namespace", + }, + { + testFn: func(s *TestAgent) { + + // Grab the state, so we can manipulate it and test against it. + testState := s.Agent.server.State() + + // Generate service registrations and upsert. + serviceRegs := mock.ServiceRegistrations() + require.NoError(t, testState.UpsertServiceRegistrations( + structs.MsgTypeTestSetup, 10, serviceRegs)) + + // Build the HTTP request. + req, err := http.NewRequest(http.MethodGet, "/v1/services?namespace=*", nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Send the HTTP request. + obj, err := s.Server.ServiceRegistrationListRequest(respW, req) + require.NoError(t, err) + require.NotNil(t, obj) + + // Check the index is not zero. + require.EqualValues(t, "10", respW.Header().Get("X-Nomad-Index")) + require.ElementsMatch(t, []*structs.ServiceRegistrationListStub{ + { + Namespace: "default", + Services: []*structs.ServiceRegistrationStub{ + { + ServiceName: "example-cache", + Tags: []string{"foo"}, + }, + }, + }, + { + Namespace: "platform", + Services: []*structs.ServiceRegistrationStub{ + { + ServiceName: "countdash-api", + Tags: []string{"bar"}, + }, + }, + }, + }, obj.([]*structs.ServiceRegistrationListStub)) + }, + name: "list wildcard namespace", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + httpTest(t, nil, tc.testFn) + }) + } +} + +func TestHTTPServer_ServiceRegistrationRequest(t *testing.T) { + t.Parallel() + + testCases := []struct { + testFn func(srv *TestAgent) + name string + }{ + { + testFn: func(s *TestAgent) { + + // Grab the state, so we can manipulate it and test against it. + testState := s.Agent.server.State() + + // Generate a service registration and upsert this. + serviceReg := mock.ServiceRegistrations()[0] + require.NoError(t, testState.UpsertServiceRegistrations( + structs.MsgTypeTestSetup, 10, []*structs.ServiceRegistration{serviceReg})) + + // Build the HTTP request. + path := fmt.Sprintf("/v1/service/%s/%s", serviceReg.ServiceName, serviceReg.ID) + req, err := http.NewRequest(http.MethodDelete, path, nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Send the HTTP request. + obj, err := s.Server.ServiceRegistrationRequest(respW, req) + require.NoError(t, err) + require.Nil(t, obj) + + // Check the index is not zero. + require.NotZero(t, respW.Header().Get("X-Nomad-Index")) + + // Check that the service is not found within state. + out, err := testState.GetServiceRegistrationByID(memdb.NewWatchSet(), serviceReg.Namespace, serviceReg.ID) + require.Nil(t, out) + require.NoError(t, err) + }, + name: "delete by ID", + }, + { + testFn: func(s *TestAgent) { + + // Grab the state, so we can manipulate it and test against it. + testState := s.Agent.server.State() + + // Generate a service registration and upsert this. + serviceReg := mock.ServiceRegistrations()[0] + require.NoError(t, testState.UpsertServiceRegistrations( + structs.MsgTypeTestSetup, 10, []*structs.ServiceRegistration{serviceReg})) + + // Build the HTTP request. + path := fmt.Sprintf("/v1/service/%s", serviceReg.ServiceName) + req, err := http.NewRequest(http.MethodGet, path, nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Send the HTTP request. + obj, err := s.Server.ServiceRegistrationRequest(respW, req) + require.NoError(t, err) + + // Check the index is not zero and that we see the service + // registration. + require.NotZero(t, respW.Header().Get("X-Nomad-Index")) + require.Equal(t, serviceReg, obj.([]*structs.ServiceRegistration)[0]) + }, + name: "get service by name", + }, + { + testFn: func(s *TestAgent) { + + // Build the HTTP request. + req, err := http.NewRequest(http.MethodGet, "/v1/service/foo/bar/baz/bonkers", nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Send the HTTP request. + obj, err := s.Server.ServiceRegistrationRequest(respW, req) + require.Error(t, err) + require.Contains(t, err.Error(), "invalid URI") + require.Nil(t, obj) + }, + name: "incorrect URI format", + }, + { + testFn: func(s *TestAgent) { + + // Build the HTTP request. + req, err := http.NewRequest(http.MethodGet, "/v1/service/", nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Send the HTTP request. + obj, err := s.Server.ServiceRegistrationRequest(respW, req) + require.Error(t, err) + require.Contains(t, err.Error(), "missing service name") + require.Nil(t, obj) + }, + name: "get service empty name", + }, + { + testFn: func(s *TestAgent) { + + // Build the HTTP request. + req, err := http.NewRequest(http.MethodHead, "/v1/service/foo", nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Send the HTTP request. + obj, err := s.Server.ServiceRegistrationRequest(respW, req) + require.Error(t, err) + require.Contains(t, err.Error(), "Invalid method") + require.Nil(t, obj) + }, + name: "get service incorrect method", + }, + { + testFn: func(s *TestAgent) { + + // Build the HTTP request. + req, err := http.NewRequest(http.MethodDelete, "/v1/service/foo/", nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Send the HTTP request. + obj, err := s.Server.ServiceRegistrationRequest(respW, req) + require.Error(t, err) + require.Contains(t, err.Error(), "missing service id") + require.Nil(t, obj) + }, + name: "delete service empty id", + }, + { + testFn: func(s *TestAgent) { + + // Build the HTTP request. + req, err := http.NewRequest(http.MethodHead, "/v1/service/foo/bar", nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Send the HTTP request. + obj, err := s.Server.ServiceRegistrationRequest(respW, req) + require.Error(t, err) + require.Contains(t, err.Error(), "Invalid method") + require.Nil(t, obj) + }, + name: "delete service incorrect method", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + httpTest(t, nil, tc.testFn) + }) + } +} From d3f634329ccf0b15790ac6d1d5d1e9b91578c0b4 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Thu, 3 Mar 2022 12:13:13 +0100 Subject: [PATCH 13/31] http: add job service registration agent HTTP endpoint. --- command/agent/job_endpoint.go | 40 +++++++++++ command/agent/job_endpoint_test.go | 109 +++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+) diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 6576cefbee6d..46a0a60d63db 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -15,6 +15,10 @@ import ( "github.com/hashicorp/nomad/nomad/structs" ) +// jobNotFoundErr is an error string which can be used as the return string +// alongside a 404 when a job is not found. +const jobNotFoundErr = "job not found" + func (s *HTTPServer) JobsRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { switch req.Method { case "GET": @@ -86,6 +90,9 @@ func (s *HTTPServer) JobSpecificRequest(resp http.ResponseWriter, req *http.Requ case strings.HasSuffix(path, "/scale"): jobName := strings.TrimSuffix(path, "/scale") return s.jobScale(resp, req, jobName) + case strings.HasSuffix(path, "/services"): + jobName := strings.TrimSuffix(path, "/services") + return s.jobServiceRegistrations(resp, req, jobName) default: return s.jobCRUD(resp, req, path) } @@ -751,6 +758,39 @@ func (s *HTTPServer) JobsParseRequest(resp http.ResponseWriter, req *http.Reques return jobStruct, nil } +// jobServiceRegistrations returns a list of all service registrations assigned +// to the job identifier. It is callable via the +// /v1/job/:jobID/services HTTP API and uses the +// structs.JobServiceRegistrationsRPCMethod RPC method. +func (s *HTTPServer) jobServiceRegistrations( + resp http.ResponseWriter, req *http.Request, jobID string) (interface{}, error) { + + // The endpoint only supports GET requests. + if req.Method != http.MethodGet { + return nil, CodedError(http.StatusMethodNotAllowed, ErrInvalidMethod) + } + + // Set up the request args and parse this to ensure the query options are + // set. + args := structs.JobServiceRegistrationsRequest{JobID: jobID} + if s.parse(resp, req, &args.Region, &args.QueryOptions) { + return nil, nil + } + + // Perform the RPC request. + var reply structs.JobServiceRegistrationsResponse + if err := s.agent.RPC(structs.JobServiceRegistrationsRPCMethod, &args, &reply); err != nil { + return nil, err + } + + setMeta(resp, &reply.QueryMeta) + + if reply.Services == nil { + return nil, CodedError(http.StatusNotFound, jobNotFoundErr) + } + return reply.Services, nil +} + // apiJobAndRequestToStructs parses the query params from the incoming // request and converts to a structs.Job and WriteRequest with the func (s *HTTPServer) apiJobAndRequestToStructs(job *api.Job, req *http.Request, apiReq api.WriteRequest) (*structs.Job, *structs.WriteRequest) { diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index ff4d615a6d70..bbdd6479f80d 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -1,6 +1,7 @@ package agent import ( + "fmt" "net/http" "net/http/httptest" "reflect" @@ -2246,6 +2247,114 @@ func TestJobs_NamespaceForJob(t *testing.T) { } } +func TestHTTPServer_jobServiceRegistrations(t *testing.T) { + t.Parallel() + + testCases := []struct { + testFn func(srv *TestAgent) + name string + }{ + { + testFn: func(s *TestAgent) { + + // Grab the state, so we can manipulate it and test against it. + testState := s.Agent.server.State() + + // Generate a job and upsert this. + job := mock.Job() + require.NoError(t, testState.UpsertJob(structs.MsgTypeTestSetup, 10, job)) + + // Generate a service registration, assigned the jobID to the + // mocked jobID, and upsert this. + serviceReg := mock.ServiceRegistrations()[0] + serviceReg.JobID = job.ID + require.NoError(t, testState.UpsertServiceRegistrations( + structs.MsgTypeTestSetup, 20, []*structs.ServiceRegistration{serviceReg})) + + // Build the HTTP request. + path := fmt.Sprintf("/v1/job/%s/services", job.ID) + req, err := http.NewRequest(http.MethodGet, path, nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Send the HTTP request. + obj, err := s.Server.JobSpecificRequest(respW, req) + require.NoError(t, err) + + // Check the response. + require.Equal(t, "20", respW.Header().Get("X-Nomad-Index")) + require.ElementsMatch(t, []*structs.ServiceRegistration{serviceReg}, + obj.([]*structs.ServiceRegistration)) + }, + name: "job has registrations", + }, + { + testFn: func(s *TestAgent) { + + // Grab the state, so we can manipulate it and test against it. + testState := s.Agent.server.State() + + // Generate a job and upsert this. + job := mock.Job() + require.NoError(t, testState.UpsertJob(structs.MsgTypeTestSetup, 10, job)) + + // Build the HTTP request. + path := fmt.Sprintf("/v1/job/%s/services", job.ID) + req, err := http.NewRequest(http.MethodGet, path, nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Send the HTTP request. + obj, err := s.Server.JobSpecificRequest(respW, req) + require.NoError(t, err) + + // Check the response. + require.Equal(t, "1", respW.Header().Get("X-Nomad-Index")) + require.ElementsMatch(t, []*structs.ServiceRegistration{}, obj.([]*structs.ServiceRegistration)) + }, + name: "job without registrations", + }, + { + testFn: func(s *TestAgent) { + + // Build the HTTP request. + req, err := http.NewRequest(http.MethodGet, "/v1/job/example/services", nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Send the HTTP request. + obj, err := s.Server.JobSpecificRequest(respW, req) + require.Error(t, err) + require.Contains(t, err.Error(), "job not found") + require.Nil(t, obj) + }, + name: "job not found", + }, + { + testFn: func(s *TestAgent) { + + // Build the HTTP request. + req, err := http.NewRequest(http.MethodHead, "/v1/job/example/services", nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Send the HTTP request. + obj, err := s.Server.JobSpecificRequest(respW, req) + require.Error(t, err) + require.Contains(t, err.Error(), "Invalid method") + require.Nil(t, obj) + }, + name: "incorrect method", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + httpTest(t, nil, tc.testFn) + }) + } +} + func TestJobs_ApiJobToStructsJob(t *testing.T) { apiJob := &api.Job{ Stop: helper.BoolToPtr(true), From ab52b02def56a60c2dc3f33e011dd97c3eff4721 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Thu, 3 Mar 2022 12:13:32 +0100 Subject: [PATCH 14/31] http: add alloc service registration agent HTTP endpoint. --- command/agent/alloc_endpoint.go | 35 +++++++++ command/agent/alloc_endpoint_test.go | 113 +++++++++++++++++++++++++++ 2 files changed, 148 insertions(+) diff --git a/command/agent/alloc_endpoint.go b/command/agent/alloc_endpoint.go index f6f724001bae..d5bfdf151ccb 100644 --- a/command/agent/alloc_endpoint.go +++ b/command/agent/alloc_endpoint.go @@ -86,6 +86,8 @@ func (s *HTTPServer) AllocSpecificRequest(resp http.ResponseWriter, req *http.Re switch tokens[1] { case "stop": return s.allocStop(allocID, resp, req) + case "services": + return s.allocServiceRegistrations(resp, req, allocID) } return nil, CodedError(404, resourceNotFoundErr) @@ -167,6 +169,39 @@ func (s *HTTPServer) allocStop(allocID string, resp http.ResponseWriter, req *ht return &out, nil } +// allocServiceRegistrations returns a list of all service registrations +// assigned to the job identifier. It is callable via the +// /v1/allocation/:alloc_id/services HTTP API and uses the +// structs.AllocServiceRegistrationsRPCMethod RPC method. +func (s *HTTPServer) allocServiceRegistrations( + resp http.ResponseWriter, req *http.Request, allocID string) (interface{}, error) { + + // The endpoint only supports GET requests. + if req.Method != http.MethodGet { + return nil, CodedError(http.StatusMethodNotAllowed, ErrInvalidMethod) + } + + // Set up the request args and parse this to ensure the query options are + // set. + args := structs.AllocServiceRegistrationsRequest{AllocID: allocID} + if s.parse(resp, req, &args.Region, &args.QueryOptions) { + return nil, nil + } + + // Perform the RPC request. + var reply structs.AllocServiceRegistrationsResponse + if err := s.agent.RPC(structs.AllocServiceRegistrationsRPCMethod, &args, &reply); err != nil { + return nil, err + } + + setMeta(resp, &reply.QueryMeta) + + if reply.Services == nil { + return nil, CodedError(http.StatusNotFound, allocNotFoundErr) + } + return reply.Services, nil +} + func (s *HTTPServer) ClientAllocRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { reqSuffix := strings.TrimPrefix(req.URL.Path, "/v1/client/allocation/") diff --git a/command/agent/alloc_endpoint_test.go b/command/agent/alloc_endpoint_test.go index ce1d994bc389..8d8b293f49cc 100644 --- a/command/agent/alloc_endpoint_test.go +++ b/command/agent/alloc_endpoint_test.go @@ -433,6 +433,119 @@ func TestHTTP_AllocStop(t *testing.T) { }) } +func TestHTTP_allocServiceRegistrations(t *testing.T) { + t.Parallel() + + testCases := []struct { + testFn func(srv *TestAgent) + name string + }{ + { + testFn: func(s *TestAgent) { + + // Grab the state, so we can manipulate it and test against it. + testState := s.Agent.server.State() + + // Generate an alloc and upsert this. + alloc := mock.Alloc() + require.NoError(t, testState.UpsertAllocs( + structs.MsgTypeTestSetup, 10, []*structs.Allocation{alloc})) + + // Generate a service registration, assigned the allocID to the + // mocked allocation ID, and upsert this. + serviceReg := mock.ServiceRegistrations()[0] + serviceReg.AllocID = alloc.ID + require.NoError(t, testState.UpsertServiceRegistrations( + structs.MsgTypeTestSetup, 20, []*structs.ServiceRegistration{serviceReg})) + + // Build the HTTP request. + path := fmt.Sprintf("/v1/allocation/%s/services", alloc.ID) + req, err := http.NewRequest(http.MethodGet, path, nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Send the HTTP request. + obj, err := s.Server.AllocSpecificRequest(respW, req) + require.NoError(t, err) + + // Check the response. + require.Equal(t, "20", respW.Header().Get("X-Nomad-Index")) + require.ElementsMatch(t, []*structs.ServiceRegistration{serviceReg}, + obj.([]*structs.ServiceRegistration)) + }, + name: "alloc has registrations", + }, + { + testFn: func(s *TestAgent) { + + // Grab the state, so we can manipulate it and test against it. + testState := s.Agent.server.State() + + // Generate an alloc and upsert this. + alloc := mock.Alloc() + require.NoError(t, testState.UpsertAllocs( + structs.MsgTypeTestSetup, 10, []*structs.Allocation{alloc})) + + // Build the HTTP request. + path := fmt.Sprintf("/v1/allocation/%s/services", alloc.ID) + req, err := http.NewRequest(http.MethodGet, path, nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Send the HTTP request. + obj, err := s.Server.AllocSpecificRequest(respW, req) + require.NoError(t, err) + + // Check the response. + require.Equal(t, "1", respW.Header().Get("X-Nomad-Index")) + require.ElementsMatch(t, []*structs.ServiceRegistration{}, + obj.([]*structs.ServiceRegistration)) + }, + name: "alloc without registrations", + }, + { + testFn: func(s *TestAgent) { + + // Build the HTTP request. + path := fmt.Sprintf("/v1/allocation/%s/services", uuid.Generate()) + req, err := http.NewRequest(http.MethodGet, path, nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Send the HTTP request. + obj, err := s.Server.AllocSpecificRequest(respW, req) + require.Error(t, err) + require.Contains(t, err.Error(), "allocation not found") + require.Nil(t, obj) + }, + name: "alloc not found", + }, + { + testFn: func(s *TestAgent) { + + // Build the HTTP request. + path := fmt.Sprintf("/v1/allocation/%s/services", uuid.Generate()) + req, err := http.NewRequest(http.MethodHead, path, nil) + require.NoError(t, err) + respW := httptest.NewRecorder() + + // Send the HTTP request. + obj, err := s.Server.AllocSpecificRequest(respW, req) + require.Error(t, err) + require.Contains(t, err.Error(), "Invalid method") + require.Nil(t, obj) + }, + name: "alloc not found", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + httpTest(t, nil, tc.testFn) + }) + } +} + func TestHTTP_AllocStats(t *testing.T) { t.Parallel() require := require.New(t) From 16cb776f9c91f2c26f50bcd91043b3109ef1b2cd Mon Sep 17 00:00:00 2001 From: James Rasell Date: Thu, 3 Mar 2022 12:14:00 +0100 Subject: [PATCH 15/31] api: add service registration HTTP API wrapper. --- api/allocations.go | 8 ++ api/allocations_test.go | 4 + api/jobs.go | 8 ++ api/jobs_test.go | 4 + api/service_registrations.go | 129 ++++++++++++++++++++++++++++++ api/service_registrations_test.go | 17 ++++ 6 files changed, 170 insertions(+) create mode 100644 api/service_registrations.go create mode 100644 api/service_registrations_test.go diff --git a/api/allocations.go b/api/allocations.go index 128541ca370b..67fc6ca29897 100644 --- a/api/allocations.go +++ b/api/allocations.go @@ -147,6 +147,14 @@ func (a *Allocations) Signal(alloc *Allocation, q *QueryOptions, task, signal st return err } +// Services is used to return a list of service registrations associated to the +// specified allocID. +func (a *Allocations) Services(allocID string, q *QueryOptions) ([]*ServiceRegistration, *QueryMeta, error) { + var resp []*ServiceRegistration + qm, err := a.client.query("/v1/allocation/"+allocID+"/services", &resp, q) + return resp, qm, err +} + // Allocation is used for serialization of allocations. type Allocation struct { ID string diff --git a/api/allocations_test.go b/api/allocations_test.go index cc306ec88c09..31fcb7d74e7d 100644 --- a/api/allocations_test.go +++ b/api/allocations_test.go @@ -396,3 +396,7 @@ func TestAllocations_ShouldMigrate(t *testing.T) { require.False(t, DesiredTransition{}.ShouldMigrate()) require.False(t, DesiredTransition{Migrate: boolToPtr(false)}.ShouldMigrate()) } + +func TestAllocations_Services(t *testing.T) { + // TODO(jrasell) add tests once registration process is in place. +} diff --git a/api/jobs.go b/api/jobs.go index 61d0c2892ced..aa934bfb8595 100644 --- a/api/jobs.go +++ b/api/jobs.go @@ -459,6 +459,14 @@ func (j *Jobs) Stable(jobID string, version uint64, stable bool, return &resp, wm, nil } +// Services is used to return a list of service registrations associated to the +// specified jobID. +func (j *Jobs) Services(jobID string, q *QueryOptions) ([]*ServiceRegistration, *QueryMeta, error) { + var resp []*ServiceRegistration + qm, err := j.client.query("/v1/job/"+jobID+"/services", &resp, q) + return resp, qm, err +} + // periodicForceResponse is used to deserialize a force response type periodicForceResponse struct { EvalID string diff --git a/api/jobs_test.go b/api/jobs_test.go index f21f7cab2484..ed5909c28bd2 100644 --- a/api/jobs_test.go +++ b/api/jobs_test.go @@ -2356,3 +2356,7 @@ func TestJobs_ScaleStatus(t *testing.T) { // Check that the result is what we expect require.Equal(groupCount, result.TaskGroups[groupName].Desired) } + +func TestJobs_Services(t *testing.T) { + // TODO(jrasell) add tests once registration process is in place. +} diff --git a/api/service_registrations.go b/api/service_registrations.go new file mode 100644 index 000000000000..ebf1418b41fe --- /dev/null +++ b/api/service_registrations.go @@ -0,0 +1,129 @@ +package api + +import ( + "fmt" + "net/url" +) + +// ServiceRegistrations is used to query the service endpoints. +type ServiceRegistrations struct { + client *Client +} + +// ServiceRegistration is an instance of a single allocation advertising itself +// as a named service with a specific address. Each registration is constructed +// from the job specification Service block. Whether the service is registered +// within Nomad, and therefore generates a ServiceRegistration is controlled by +// the Service.Provider parameter. +type ServiceRegistration struct { + + // ID is the unique identifier for this registration. It currently follows + // the Consul service registration format to provide consistency between + // the two solutions. + ID string + + // ServiceName is the human friendly identifier for this service + // registration. + ServiceName string + + // Namespace represents the namespace within which this service is + // registered. + Namespace string + + // NodeID is Node.ID on which this service registration is currently + // running. + NodeID string + + // Datacenter is the DC identifier of the node as identified by + // Node.Datacenter. + Datacenter string + + // JobID is Job.ID and represents the job which contained the service block + // which resulted in this service registration. + JobID string + + // AllocID is Allocation.ID and represents the allocation within which this + // service is running. + AllocID string + + // Tags are determined from either Service.Tags or Service.CanaryTags and + // help identify this service. Tags can also be used to perform lookups of + // services depending on their state and role. + Tags []string + + // Address is the IP address of this service registration. This information + // comes from the client and is not guaranteed to be routable; this depends + // on cluster network topology. + Address string + + // Port is the port number on which this service registration is bound. It + // is determined by a combination of factors on the client. + Port int + + CreateIndex uint64 + ModifyIndex uint64 +} + +// ServiceRegistrationListStub represents all service registrations held within a +// single namespace. +type ServiceRegistrationListStub struct { + + // Namespace details the namespace in which these services have been + // registered. + Namespace string + + // Services is a list of services found within the namespace. + Services []*ServiceRegistrationStub +} + +// ServiceRegistrationStub is the stub object describing an individual +// namespaced service. The object is built in a manner which would allow us to +// add additional fields in the future, if we wanted. +type ServiceRegistrationStub struct { + + // ServiceName is the human friendly name for this service as specified + // within Service.Name. + ServiceName string + + // Tags is a list of unique tags found for this service. The list is + // de-duplicated automatically by Nomad. + Tags []string +} + +// ServiceRegistrations returns a new handle on the services endpoints. +func (c *Client) ServiceRegistrations() *ServiceRegistrations { + return &ServiceRegistrations{client: c} +} + +// List can be used to list all service registrations currently stored within +// the target namespace. It returns a stub response object. +func (s *ServiceRegistrations) List(q *QueryOptions) ([]*ServiceRegistrationListStub, *QueryMeta, error) { + var resp []*ServiceRegistrationListStub + qm, err := s.client.query("/v1/services", &resp, q) + if err != nil { + return nil, qm, err + } + return resp, qm, nil +} + +// Get is used to return a list of service registrations whose name matches the +// specified parameter. +func (s *ServiceRegistrations) Get(serviceName string, q *QueryOptions) ([]*ServiceRegistration, *QueryMeta, error) { + var resp []*ServiceRegistration + qm, err := s.client.query("/v1/service/"+url.PathEscape(serviceName), &resp, q) + if err != nil { + return nil, qm, err + } + return resp, qm, nil +} + +// Delete can be used to delete an individual service registration as defined +// by its service name and service ID. +func (s *ServiceRegistrations) Delete(serviceName, serviceID string, q *WriteOptions) (*WriteMeta, error) { + path := fmt.Sprintf("/v1/service/%s/%s", url.PathEscape(serviceName), url.PathEscape(serviceID)) + wm, err := s.client.delete(path, nil, q) + if err != nil { + return nil, err + } + return wm, nil +} diff --git a/api/service_registrations_test.go b/api/service_registrations_test.go new file mode 100644 index 000000000000..b957e194b432 --- /dev/null +++ b/api/service_registrations_test.go @@ -0,0 +1,17 @@ +package api + +import ( + "testing" +) + +func TestServiceRegistrations_List(t *testing.T) { + // TODO(jrasell) add tests once registration process is in place. +} + +func TestServiceRegistrations_Get(t *testing.T) { + // TODO(jrasell) add tests once registration process is in place. +} + +func TestServiceRegistrations_Delete(t *testing.T) { + // TODO(jrasell) add tests once registration process is in place. +} From 463f93813116caa0bacc775b0efaec2afa5cec5d Mon Sep 17 00:00:00 2001 From: James Rasell Date: Mon, 14 Mar 2022 09:21:20 +0100 Subject: [PATCH 16/31] jobspec: add service block provider parameter and validation. --- command/agent/job_endpoint.go | 1 + command/agent/job_endpoint_test.go | 2 + nomad/structs/consul.go | 8 +- nomad/structs/diff_test.go | 103 ++++++++++ nomad/structs/services.go | 82 +++++++- nomad/structs/services_test.go | 80 ++++++++ nomad/structs/structs.go | 30 ++- nomad/structs/structs_test.go | 319 ++++++++++++++++++++++++----- 8 files changed, 560 insertions(+), 65 deletions(-) diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 46a0a60d63db..e29fe751dfb4 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -1372,6 +1372,7 @@ func ApiServicesToStructs(in []*api.Service, group bool) []*structs.Service { Meta: helper.CopyMapStringString(s.Meta), CanaryMeta: helper.CopyMapStringString(s.CanaryMeta), OnUpdate: s.OnUpdate, + Provider: s.Provider, } if l := len(s.Checks); l != 0 { diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index bbdd6479f80d..dfe1c8253f25 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -2902,6 +2902,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { Services: []*structs.Service{ { Name: "groupserviceA", + Provider: "consul", Tags: []string{"a", "b"}, CanaryTags: []string{"d", "e"}, EnableTagOverride: true, @@ -2993,6 +2994,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { Services: []*structs.Service{ { Name: "serviceA", + Provider: "consul", Tags: []string{"1", "2"}, CanaryTags: []string{"3", "4"}, EnableTagOverride: true, diff --git a/nomad/structs/consul.go b/nomad/structs/consul.go index aa934a1a2a56..2c988c30d46a 100644 --- a/nomad/structs/consul.go +++ b/nomad/structs/consul.go @@ -70,13 +70,17 @@ func (j *Job) ConsulUsages() map[string]*ConsulUsage { // Gather group services for _, service := range tg.Services { - m[namespace].Services = append(m[namespace].Services, service.Name) + if service.Provider == ServiceProviderConsul { + m[namespace].Services = append(m[namespace].Services, service.Name) + } } // Gather task services and KV usage for _, task := range tg.Tasks { for _, service := range task.Services { - m[namespace].Services = append(m[namespace].Services, service.Name) + if service.Provider == ServiceProviderConsul { + m[namespace].Services = append(m[namespace].Services, service.Name) + } } if len(task.Templates) > 0 { m[namespace].KV = true diff --git a/nomad/structs/diff_test.go b/nomad/structs/diff_test.go index ef950b90d421..53b158262843 100644 --- a/nomad/structs/diff_test.go +++ b/nomad/structs/diff_test.go @@ -2872,6 +2872,12 @@ func TestTaskGroupDiff(t *testing.T) { Old: "", New: "", }, + { + Type: DiffTypeNone, + Name: "Provider", + Old: "", + New: "", + }, { Type: DiffTypeEdited, Name: "TaskName", @@ -5601,6 +5607,10 @@ func TestTaskDiff(t *testing.T) { Old: "foo", New: "bar", }, + { + Type: DiffTypeNone, + Name: "Provider", + }, { Type: DiffTypeAdded, Name: "TaskName", @@ -5745,6 +5755,10 @@ func TestTaskDiff(t *testing.T) { Type: DiffTypeNone, Name: "PortLabel", }, + { + Type: DiffTypeNone, + Name: "Provider", + }, { Type: DiffTypeNone, Name: "TaskName", @@ -6264,6 +6278,10 @@ func TestTaskDiff(t *testing.T) { Old: "", New: "", }, + { + Type: DiffTypeNone, + Name: "Provider", + }, { Type: DiffTypeNone, Name: "TaskName", @@ -7347,6 +7365,10 @@ func TestServicesDiff(t *testing.T) { Old: "http", New: "https", }, + { + Type: DiffTypeNone, + Name: "Provider", + }, { Type: DiffTypeNone, Name: "TaskName", @@ -7432,6 +7454,10 @@ func TestServicesDiff(t *testing.T) { Name: "PortLabel", New: "http", }, + { + Type: DiffTypeNone, + Name: "Provider", + }, { Type: DiffTypeNone, Name: "TaskName", @@ -7491,6 +7517,10 @@ func TestServicesDiff(t *testing.T) { Name: "PortLabel", New: "https", }, + { + Type: DiffTypeNone, + Name: "Provider", + }, { Type: DiffTypeNone, Name: "TaskName", @@ -7556,6 +7586,9 @@ func TestServicesDiff(t *testing.T) { Name: "PortLabel", Old: "http", New: "https-redirect", + }, { + Type: DiffTypeNone, + Name: "Provider", }, { Type: DiffTypeNone, @@ -7627,6 +7660,10 @@ func TestServicesDiff(t *testing.T) { Old: "http", New: "http", }, + { + Type: DiffTypeNone, + Name: "Provider", + }, { Type: DiffTypeNone, Name: "TaskName", @@ -7654,6 +7691,72 @@ func TestServicesDiff(t *testing.T) { }, }, }, + { + Name: "Service with different provider", + Contextual: true, + Old: []*Service{ + { + Name: "webapp", + Provider: "nomad", + PortLabel: "http", + }, + }, + New: []*Service{ + { + Name: "webapp", + Provider: "consul", + PortLabel: "http", + }, + }, + Expected: []*ObjectDiff{ + { + Type: DiffTypeEdited, + Name: "Service", + Fields: []*FieldDiff{ + { + Type: DiffTypeNone, + Name: "AddressMode", + }, + { + Type: DiffTypeNone, + Name: "EnableTagOverride", + Old: "false", + New: "false", + }, + { + Type: DiffTypeNone, + Name: "Name", + Old: "webapp", + New: "webapp", + }, + { + Type: DiffTypeNone, + Name: "Namespace", + }, + { + Type: DiffTypeNone, + Name: "OnUpdate", + }, + { + Type: DiffTypeNone, + Name: "PortLabel", + Old: "http", + New: "http", + }, + { + Type: DiffTypeEdited, + Name: "Provider", + Old: "nomad", + New: "consul", + }, + { + Type: DiffTypeNone, + Name: "TaskName", + }, + }, + }, + }, + }, } for _, c := range cases { diff --git a/nomad/structs/services.go b/nomad/structs/services.go index 7c6a1c4e5371..dc9209220036 100644 --- a/nomad/structs/services.go +++ b/nomad/structs/services.go @@ -420,6 +420,15 @@ const ( AddressModeHost = "host" AddressModeDriver = "driver" AddressModeAlloc = "alloc" + + // ServiceProviderConsul is the default service provider and the way Nomad + // worked before native service discovery. + ServiceProviderConsul = "consul" + + // ServiceProviderNomad is the native service discovery provider. At the + // time of writing, there are a number of restrictions around its + // functionality and use. + ServiceProviderNomad = "nomad" ) // Service represents a Consul service definition @@ -468,6 +477,11 @@ type Service struct { // OnUpdate Specifies how the service and its checks should be evaluated // during an update OnUpdate string + + // Provider dictates which service discovery provider to use. This can be + // either ServiceProviderConsul or ServiceProviderNomad and defaults to the former when + // left empty by the operator. + Provider string } const ( @@ -504,7 +518,7 @@ func (s *Service) Copy() *Service { // Canonicalize interpolates values of Job, Task Group and Task in the Service // Name. This also generates check names, service id and check ids. -func (s *Service) Canonicalize(job string, taskGroup string, task string) { +func (s *Service) Canonicalize(job, taskGroup, task, jobNamespace string) { // Ensure empty lists are treated as null to avoid scheduler issues when // using DeepEquals if len(s.Tags) == 0 { @@ -528,10 +542,23 @@ func (s *Service) Canonicalize(job string, taskGroup string, task string) { check.Canonicalize(s.Name) } + // Set the provider to its default value. The value of consul ensures this + // new feature and parameter behaves in the same manner a previous versions + // which did not include this. + if s.Provider == "" { + s.Provider = ServiceProviderConsul + } + // Consul API returns "default" whether the namespace is empty or set as - // such, so we coerce our copy of the service to be the same. - if s.Namespace == "" { + // such, so we coerce our copy of the service to be the same if using the + // consul provider. + // + // When using ServiceProviderNomad, set the namespace to that of the job. This + // makes modifications and diffs on the service correct. + if s.Namespace == "" && s.Provider == ServiceProviderConsul { s.Namespace = "default" + } else if s.Provider == ServiceProviderNomad { + s.Namespace = jobNamespace } } @@ -563,6 +590,26 @@ func (s *Service) Validate() error { mErr.Errors = append(mErr.Errors, fmt.Errorf("Service on_update must be %q, %q, or %q; not %q", OnUpdateRequireHealthy, OnUpdateIgnoreWarn, OnUpdateIgnore, s.OnUpdate)) } + // Up until this point, all service validation has been independent of the + // provider. From this point on, we have different validation paths. We can + // also catch an incorrect provider parameter. + switch s.Provider { + case ServiceProviderConsul: + s.validateConsulService(&mErr) + case ServiceProviderNomad: + s.validateNomadService(&mErr) + default: + mErr.Errors = append(mErr.Errors, fmt.Errorf("Service provider must be %q, or %q; not %q", + ServiceProviderConsul, ServiceProviderNomad, s.Provider)) + } + + return mErr.ErrorOrNil() +} + +// validateConsulService performs validation on a service which is using the +// consul provider. +func (s *Service) validateConsulService(mErr *multierror.Error) { + // check checks for _, c := range s.Checks { if s.PortLabel == "" && c.PortLabel == "" && c.RequiresPort() { @@ -595,8 +642,23 @@ func (s *Service) Validate() error { mErr.Errors = append(mErr.Errors, fmt.Errorf("Service %s is Connect Native and requires setting the task", s.Name)) } } +} - return mErr.ErrorOrNil() +// validateNomadService performs validation on a service which is using the +// nomad provider. +func (s *Service) validateNomadService(mErr *multierror.Error) { + + // Service blocks for the Nomad provider do not support checks. We perform + // a nil check, as an empty check list is nilled within the service + // canonicalize function. + if s.Checks != nil { + mErr.Errors = append(mErr.Errors, errors.New("Service with provider nomad cannot include Check blocks")) + } + + // Services using the Nomad provider do not support Consul connect. + if s.Connect != nil { + mErr.Errors = append(mErr.Errors, errors.New("Service with provider nomad cannot include Connect blocks")) + } } // ValidateName checks if the service Name is valid and should be called after @@ -614,7 +676,8 @@ func (s *Service) ValidateName(name string) error { } // Hash returns a base32 encoded hash of a Service's contents excluding checks -// as they're hashed independently. +// as they're hashed independently and the provider in order to not cause churn +// during cluster upgrades. func (s *Service) Hash(allocID, taskName string, canary bool) string { h := sha1.New() hashString(h, allocID) @@ -632,6 +695,11 @@ func (s *Service) Hash(allocID, taskName string, canary bool) string { hashString(h, s.OnUpdate) hashString(h, s.Namespace) + // Don't hash the provider parameter, so we don't cause churn of all + // registered services when upgrading Nomad versions. The provider is not + // used at the level the hash is and therefore is not needed to tell + // whether the service has changed. + // Base32 is used for encoding the hash as sha1 hashes can always be // encoded without padding, only 4 bytes larger than base64, and saves // 8 bytes vs hex. Since these hashes are used in Consul URLs it's nice @@ -687,6 +755,10 @@ func (s *Service) Equals(o *Service) bool { return s == o } + if s.Provider != o.Provider { + return false + } + if s.Namespace != o.Namespace { return false } diff --git a/nomad/structs/services_test.go b/nomad/structs/services_test.go index 9375366d7e5f..12c98ecaa52a 100644 --- a/nomad/structs/services_test.go +++ b/nomad/structs/services_test.go @@ -1,6 +1,8 @@ package structs import ( + "errors" + "github.com/hashicorp/go-multierror" "testing" "time" @@ -1472,3 +1474,81 @@ func TestConsulMeshGateway_Validate(t *testing.T) { require.NoError(t, err) }) } + +func TestService_validateNomadService(t *testing.T) { + t.Parallel() + + testCases := []struct { + inputService *Service + inputErr *multierror.Error + expectedOutputErrors []error + name string + }{ + { + inputService: &Service{ + Name: "webapp", + PortLabel: "http", + Namespace: "default", + Provider: "nomad", + }, + inputErr: &multierror.Error{}, + expectedOutputErrors: []error{}, + name: "valid service", + }, + { + inputService: &Service{ + Name: "webapp", + PortLabel: "http", + Namespace: "default", + Provider: "nomad", + Checks: []*ServiceCheck{ + {Name: "some-check"}, + }, + }, + inputErr: &multierror.Error{}, + expectedOutputErrors: []error{errors.New("Service with provider nomad cannot include Check blocks")}, + name: "invalid service due to checks", + }, + { + inputService: &Service{ + Name: "webapp", + PortLabel: "http", + Namespace: "default", + Provider: "nomad", + Connect: &ConsulConnect{ + Native: true, + }, + }, + inputErr: &multierror.Error{}, + expectedOutputErrors: []error{errors.New("Service with provider nomad cannot include Connect blocks")}, + name: "invalid service due to connect", + }, + { + inputService: &Service{ + Name: "webapp", + PortLabel: "http", + Namespace: "default", + Provider: "nomad", + Connect: &ConsulConnect{ + Native: true, + }, + Checks: []*ServiceCheck{ + {Name: "some-check"}, + }, + }, + inputErr: &multierror.Error{}, + expectedOutputErrors: []error{ + errors.New("Service with provider nomad cannot include Check blocks"), + errors.New("Service with provider nomad cannot include Connect blocks"), + }, + name: "invalid service due to checks and connect", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tc.inputService.validateNomadService(tc.inputErr) + require.ElementsMatch(t, tc.expectedOutputErrors, tc.inputErr.Errors) + }) + } +} diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 9188c8204948..797a5c5119b0 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -6195,7 +6195,7 @@ func (tg *TaskGroup) Canonicalize(job *Job) { } for _, service := range tg.Services { - service.Canonicalize(job.Name, tg.Name, "group") + service.Canonicalize(job.Name, tg.Name, "group", job.Namespace) } for _, network := range tg.Networks { @@ -6491,6 +6491,10 @@ func (tg *TaskGroup) validateServices() error { var mErr multierror.Error knownTasks := make(map[string]struct{}) + // Track the providers used for this task group. Currently, Nomad only + // allows the use of a single service provider within a task group. + configuredProviders := make(map[string]struct{}) + // Create a map of known tasks and their services so we can compare // vs the group-level services and checks for _, task := range tg.Tasks { @@ -6504,9 +6508,22 @@ func (tg *TaskGroup) validateServices() error { mErr.Errors = append(mErr.Errors, fmt.Errorf("Check %s is invalid: only task group service checks can be assigned tasks", check.Name)) } } + + // Add the service provider to the tracking, if it has not already + // been seen. + if _, ok := configuredProviders[service.Provider]; !ok { + configuredProviders[service.Provider] = struct{}{} + } } } for i, service := range tg.Services { + + // Add the service provider to the tracking, if it has not already been + // seen. + if _, ok := configuredProviders[service.Provider]; !ok { + configuredProviders[service.Provider] = struct{}{} + } + if err := service.Validate(); err != nil { outer := fmt.Errorf("Service[%d] %s validation failed: %s", i, service.Name, err) mErr.Errors = append(mErr.Errors, outer) @@ -6535,6 +6552,15 @@ func (tg *TaskGroup) validateServices() error { } } } + + // The initial feature release of native service discovery only allows for + // a single service provider to be used across all services in a task + // group. + if len(configuredProviders) > 1 { + mErr.Errors = append(mErr.Errors, + errors.New("Multiple service providers used: task group services must use the same provider")) + } + return mErr.ErrorOrNil() } @@ -6946,7 +6972,7 @@ func (t *Task) Canonicalize(job *Job, tg *TaskGroup) { } for _, service := range t.Services { - service.Canonicalize(job.Name, tg.Name, t.Name) + service.Canonicalize(job.Name, tg.Name, t.Name, job.Namespace) } // If Resources are nil initialize them to defaults, otherwise canonicalize diff --git a/nomad/structs/structs_test.go b/nomad/structs/structs_test.go index 40b343910b8f..ddfc242fe36d 100644 --- a/nomad/structs/structs_test.go +++ b/nomad/structs/structs_test.go @@ -371,6 +371,7 @@ func testJob() *Job { { Name: "${TASK}-frontend", PortLabel: "http", + Provider: "consul", }, }, Tasks: []*Task{ @@ -1203,7 +1204,8 @@ func TestTaskGroup_Validate(t *testing.T) { Name: "group-a", Services: []*Service{ { - Name: "service-a", + Name: "service-a", + Provider: "consul", Checks: []*ServiceCheck{ { Name: "check-a", @@ -1225,6 +1227,47 @@ func TestTaskGroup_Validate(t *testing.T) { expected = `Check check-a invalid: only script and gRPC checks should have tasks` require.Contains(t, err.Error(), expected) + tg = &TaskGroup{ + Name: "group-a", + Services: []*Service{ + { + Name: "service-a", + Provider: "nomad", + }, + { + Name: "service-b", + Provider: "consul", + }, + }, + Tasks: []*Task{{Name: "task-a"}}, + } + err = tg.Validate(&Job{}) + expected = "Multiple service providers used: task group services must use the same provider" + require.Contains(t, err.Error(), expected) + + tg = &TaskGroup{ + Name: "group-a", + Services: []*Service{ + { + Name: "service-a", + Provider: "nomad", + }, + }, + Tasks: []*Task{ + { + Name: "task-a", + Services: []*Service{ + { + Name: "service-b", + Provider: "consul", + }, + }, + }, + }, + } + err = tg.Validate(&Job{}) + expected = "Multiple service providers used: task group services must use the same provider" + require.Contains(t, err.Error(), expected) } func TestTaskGroupNetwork_Validate(t *testing.T) { @@ -1689,6 +1732,7 @@ func TestNetworkResource_Copy(t *testing.T) { func TestTask_Validate_Services(t *testing.T) { s1 := &Service{ Name: "service-name", + Provider: "consul", PortLabel: "bar", Checks: []*ServiceCheck{ { @@ -1711,15 +1755,18 @@ func TestTask_Validate_Services(t *testing.T) { s2 := &Service{ Name: "service-name", + Provider: "consul", PortLabel: "bar", } s3 := &Service{ Name: "service-A", + Provider: "consul", PortLabel: "a", } s4 := &Service{ Name: "service-A", + Provider: "consul", PortLabel: "b", } @@ -1812,25 +1859,30 @@ func TestTask_Validate_Service_AddressMode_Ok(t *testing.T) { { // https://github.com/hashicorp/nomad/issues/3681#issuecomment-357274177 Name: "DriverModeWithLabel", + Provider: "consul", PortLabel: "http", AddressMode: AddressModeDriver, }, { Name: "DriverModeWithPort", + Provider: "consul", PortLabel: "80", AddressMode: AddressModeDriver, }, { Name: "HostModeWithLabel", + Provider: "consul", PortLabel: "http", AddressMode: AddressModeHost, }, { Name: "HostModeWithoutLabel", + Provider: "consul", AddressMode: AddressModeHost, }, { Name: "DriverModeWithoutLabel", + Provider: "consul", AddressMode: AddressModeDriver, }, } @@ -2030,6 +2082,7 @@ func TestTask_Validate_Service_Check_AddressMode(t *testing.T) { { Service: &Service{ Name: "invalid-driver", + Provider: "consul", PortLabel: "80", AddressMode: "host", }, @@ -2054,6 +2107,7 @@ func TestTask_Validate_Service_Check_AddressMode(t *testing.T) { { Service: &Service{ Name: "http-driver-fail-2", + Provider: "consul", PortLabel: "80", AddressMode: "driver", Checks: []*ServiceCheck{ @@ -2071,6 +2125,7 @@ func TestTask_Validate_Service_Check_AddressMode(t *testing.T) { { Service: &Service{ Name: "http-driver-fail-3", + Provider: "consul", PortLabel: "80", AddressMode: "driver", Checks: []*ServiceCheck{ @@ -2088,6 +2143,7 @@ func TestTask_Validate_Service_Check_AddressMode(t *testing.T) { { Service: &Service{ Name: "http-driver-passes", + Provider: "consul", PortLabel: "80", AddressMode: "driver", Checks: []*ServiceCheck{ @@ -2117,7 +2173,8 @@ func TestTask_Validate_Service_Check_AddressMode(t *testing.T) { }, { Service: &Service{ - Name: "empty-address-3673-passes-1", + Name: "empty-address-3673-passes-1", + Provider: "consul", Checks: []*ServiceCheck{ { Name: "valid-port-label", @@ -2143,7 +2200,8 @@ func TestTask_Validate_Service_Check_AddressMode(t *testing.T) { }, { Service: &Service{ - Name: "empty-address-3673-fails", + Name: "empty-address-3673-fails", + Provider: "consul", Checks: []*ServiceCheck{ { Name: "empty-is-not-ok", @@ -2192,8 +2250,9 @@ func TestTask_Validate_Service_Check_GRPC(t *testing.T) { Timeout: time.Second, } service := &Service{ - Name: "test", - Checks: []*ServiceCheck{invalidGRPC}, + Name: "test", + Provider: "consul", + Checks: []*ServiceCheck{invalidGRPC}, } assert.Error(t, service.Validate()) @@ -3119,6 +3178,7 @@ func BenchmarkEncodeDecode(b *testing.B) { func TestInvalidServiceCheck(t *testing.T) { s := Service{ Name: "service-name", + Provider: "consul", PortLabel: "bar", Checks: []*ServiceCheck{ { @@ -3133,6 +3193,7 @@ func TestInvalidServiceCheck(t *testing.T) { s = Service{ Name: "service.name", + Provider: "consul", PortLabel: "bar", } if err := s.ValidateName(s.Name); err == nil { @@ -3141,6 +3202,7 @@ func TestInvalidServiceCheck(t *testing.T) { s = Service{ Name: "-my-service", + Provider: "consul", PortLabel: "bar", } if err := s.Validate(); err == nil { @@ -3149,6 +3211,7 @@ func TestInvalidServiceCheck(t *testing.T) { s = Service{ Name: "my-service-${NOMAD_META_FOO}", + Provider: "consul", PortLabel: "bar", } if err := s.Validate(); err != nil { @@ -3157,6 +3220,7 @@ func TestInvalidServiceCheck(t *testing.T) { s = Service{ Name: "my_service-${NOMAD_META_FOO}", + Provider: "consul", PortLabel: "bar", } if err := s.Validate(); err == nil { @@ -3165,6 +3229,7 @@ func TestInvalidServiceCheck(t *testing.T) { s = Service{ Name: "abcdef0123456789-abcdef0123456789-abcdef0123456789-abcdef0123456", + Provider: "consul", PortLabel: "bar", } if err := s.ValidateName(s.Name); err == nil { @@ -3172,7 +3237,8 @@ func TestInvalidServiceCheck(t *testing.T) { } s = Service{ - Name: "service-name", + Name: "service-name", + Provider: "consul", Checks: []*ServiceCheck{ { Name: "check-tcp", @@ -3194,7 +3260,8 @@ func TestInvalidServiceCheck(t *testing.T) { } s = Service{ - Name: "service-name", + Name: "service-name", + Provider: "consul", Checks: []*ServiceCheck{ { Name: "check-script", @@ -3210,7 +3277,8 @@ func TestInvalidServiceCheck(t *testing.T) { } s = Service{ - Name: "service-name", + Name: "service-name", + Provider: "consul", Checks: []*ServiceCheck{ { Name: "tcp-check", @@ -3261,62 +3329,198 @@ func TestDistinctCheckID(t *testing.T) { } func TestService_Canonicalize(t *testing.T) { - job := "example" - taskGroup := "cache" - task := "redis" - - s := Service{ - Name: "${TASK}-db", - } - - s.Canonicalize(job, taskGroup, task) - if s.Name != "redis-db" { - t.Fatalf("Expected name: %v, Actual: %v", "redis-db", s.Name) - } - - s.Name = "db" - s.Canonicalize(job, taskGroup, task) - if s.Name != "db" { - t.Fatalf("Expected name: %v, Actual: %v", "redis-db", s.Name) - } - - s.Name = "${JOB}-${TASKGROUP}-${TASK}-db" - s.Canonicalize(job, taskGroup, task) - if s.Name != "example-cache-redis-db" { - t.Fatalf("Expected name: %v, Actual: %v", "example-cache-redis-db", s.Name) + testCases := []struct { + inputService *Service + inputJob string + inputTaskGroup string + inputTask string + inputJobNamespace string + expectedOutputService *Service + name string + }{ + { + inputService: &Service{ + Name: "${TASK}-db", + }, + inputJob: "example", + inputTaskGroup: "cache", + inputTask: "redis", + inputJobNamespace: "platform", + expectedOutputService: &Service{ + Name: "redis-db", + Provider: "consul", + Namespace: "default", + }, + name: "interpolate task in name", + }, + { + inputService: &Service{ + Name: "db", + }, + inputJob: "example", + inputTaskGroup: "cache", + inputTask: "redis", + inputJobNamespace: "platform", + expectedOutputService: &Service{ + Name: "db", + Provider: "consul", + Namespace: "default", + }, + name: "no interpolation in name", + }, + { + inputService: &Service{ + Name: "${JOB}-${TASKGROUP}-${TASK}-db", + }, + inputJob: "example", + inputTaskGroup: "cache", + inputTask: "redis", + inputJobNamespace: "platform", + expectedOutputService: &Service{ + Name: "example-cache-redis-db", + Provider: "consul", + Namespace: "default", + }, + name: "interpolate job, taskgroup and task in name", + }, + { + inputService: &Service{ + Name: "${BASE}-db", + }, + inputJob: "example", + inputTaskGroup: "cache", + inputTask: "redis", + inputJobNamespace: "platform", + expectedOutputService: &Service{ + Name: "example-cache-redis-db", + Provider: "consul", + Namespace: "default", + }, + name: "interpolate base in name", + }, + { + inputService: &Service{ + Name: "db", + Provider: "nomad", + }, + inputJob: "example", + inputTaskGroup: "cache", + inputTask: "redis", + inputJobNamespace: "platform", + expectedOutputService: &Service{ + Name: "db", + Provider: "nomad", + Namespace: "platform", + }, + name: "nomad provider", + }, } - s.Name = "${BASE}-db" - s.Canonicalize(job, taskGroup, task) - if s.Name != "example-cache-redis-db" { - t.Fatalf("Expected name: %v, Actual: %v", "example-cache-redis-db", s.Name) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tc.inputService.Canonicalize(tc.inputJob, tc.inputTaskGroup, tc.inputTask, tc.inputJobNamespace) + assert.Equal(t, tc.expectedOutputService, tc.inputService) + }) } - } func TestService_Validate(t *testing.T) { - s := Service{ - Name: "testservice", + testCases := []struct { + inputService *Service + expectedError bool + expectedErrorContains string + name string + }{ + { + inputService: &Service{ + Name: "testservice", + }, + expectedError: false, + name: "base service", + }, + { + inputService: &Service{ + Name: "testservice", + Connect: &ConsulConnect{ + Native: true, + }, + }, + expectedError: true, + expectedErrorContains: "Connect Native and requires setting the task", + name: "Native Connect without task name", + }, + { + inputService: &Service{ + Name: "testservice", + TaskName: "testtask", + Connect: &ConsulConnect{ + Native: true, + }, + }, + expectedError: false, + name: "Native Connect with task name", + }, + { + inputService: &Service{ + Name: "testservice", + TaskName: "testtask", + Connect: &ConsulConnect{ + Native: true, + SidecarService: &ConsulSidecarService{}, + }, + }, + expectedError: true, + expectedErrorContains: "Consul Connect must be exclusively native", + name: "Native Connect with Sidecar", + }, + { + inputService: &Service{ + Name: "testservice", + Provider: "nomad", + Checks: []*ServiceCheck{ + { + Name: "servicecheck", + }, + }, + }, + expectedError: true, + expectedErrorContains: "Service with provider nomad cannot include Check blocks", + name: "provider nomad with checks", + }, + { + inputService: &Service{ + Name: "testservice", + Provider: "nomad", + Connect: &ConsulConnect{ + Native: true, + }, + }, + expectedError: true, + expectedErrorContains: "Service with provider nomad cannot include Connect blocks", + name: "provider nomad with connect", + }, + { + inputService: &Service{ + Name: "testservice", + Provider: "nomad", + }, + expectedError: false, + name: "provider nomad valid", + }, } - s.Canonicalize("testjob", "testgroup", "testtask") - - // Base service should be valid - require.NoError(t, s.Validate()) - - // Native Connect requires task name on service - s.Connect = &ConsulConnect{ - Native: true, + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tc.inputService.Canonicalize("testjob", "testgroup", "testtask", "testnamespace") + err := tc.inputService.Validate() + if tc.expectedError { + assert.Error(t, err) + assert.Contains(t, err.Error(), tc.expectedErrorContains) + } else { + assert.NoError(t, err) + } + }) } - require.Error(t, s.Validate()) - - // Native Connect should work with task name on service set - s.TaskName = "testtask" - require.NoError(t, s.Validate()) - - // Native Connect + Sidecar should be invalid - s.Connect.SidecarService = &ConsulSidecarService{} - require.Error(t, s.Validate()) } func TestService_Equals(t *testing.T) { @@ -3324,7 +3528,7 @@ func TestService_Equals(t *testing.T) { Name: "testservice", } - s.Canonicalize("testjob", "testgroup", "testtask") + s.Canonicalize("testjob", "testgroup", "testtask", "default") o := s.Copy() @@ -3362,6 +3566,9 @@ func TestService_Equals(t *testing.T) { o.EnableTagOverride = true assertDiff() + + o.Provider = "nomad" + assertDiff() } func TestJob_ExpandServiceNames(t *testing.T) { From 86f02a385ace32a5a0baa2b0276b7590e5afed90 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Mon, 14 Mar 2022 10:00:53 +0100 Subject: [PATCH 17/31] hcl1: add service block provider parameter. --- jobspec/parse_service.go | 1 + jobspec/parse_test.go | 26 ++++++++++++++++++++++ jobspec/test-fixtures/service-provider.hcl | 14 ++++++++++++ 3 files changed, 41 insertions(+) create mode 100644 jobspec/test-fixtures/service-provider.hcl diff --git a/jobspec/parse_service.go b/jobspec/parse_service.go index fe679805bd32..9ca32e352ae6 100644 --- a/jobspec/parse_service.go +++ b/jobspec/parse_service.go @@ -51,6 +51,7 @@ func parseService(o *ast.ObjectItem) (*api.Service, error) { "meta", "canary_meta", "on_update", + "provider", } if err := checkHCLKeys(o.Val, valid); err != nil { return nil, err diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index aa219c1172eb..c774cb33508e 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -1763,6 +1763,32 @@ func TestParse(t *testing.T) { }, false, }, + { + "service-provider.hcl", + &api.Job{ + ID: stringToPtr("service-provider"), + Name: stringToPtr("service-provider"), + TaskGroups: []*api.TaskGroup{ + { + Count: intToPtr(5), + Name: stringToPtr("group"), + Tasks: []*api.Task{ + { + Name: "task", + Driver: "docker", + Services: []*api.Service{ + { + Name: "service-provider", + Provider: "nomad", + }, + }, + }, + }, + }, + }, + }, + false, + }, } for _, tc := range cases { diff --git a/jobspec/test-fixtures/service-provider.hcl b/jobspec/test-fixtures/service-provider.hcl new file mode 100644 index 000000000000..6a31599d1160 --- /dev/null +++ b/jobspec/test-fixtures/service-provider.hcl @@ -0,0 +1,14 @@ +job "service-provider" { + group "group" { + count = 5 + + task "task" { + driver = "docker" + + service { + name = "service-provider" + provider = "nomad" + } + } + } +} From 9215d0c897f0365522fb7706e0c3009750f48289 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Mon, 14 Mar 2022 10:01:20 +0100 Subject: [PATCH 18/31] api: add service block provider parameter. --- api/jobs_test.go | 1 + api/services.go | 14 ++++++++++++++ api/services_test.go | 1 + 3 files changed, 16 insertions(+) diff --git a/api/jobs_test.go b/api/jobs_test.go index ed5909c28bd2..0f9ba8855b93 100644 --- a/api/jobs_test.go +++ b/api/jobs_test.go @@ -742,6 +742,7 @@ func TestJobs_Canonicalize(t *testing.T) { PortLabel: "db", AddressMode: "auto", OnUpdate: "require_healthy", + Provider: "consul", Checks: []ServiceCheck{ { Name: "alive", diff --git a/api/services.go b/api/services.go index 9017b1396033..d927c514ec2e 100644 --- a/api/services.go +++ b/api/services.go @@ -116,12 +116,21 @@ type Service struct { CanaryMeta map[string]string `hcl:"canary_meta,block"` TaskName string `mapstructure:"task" hcl:"task,optional"` OnUpdate string `mapstructure:"on_update" hcl:"on_update,optional"` + + // Provider defines which backend system provides the service registration + // mechanism for this service. This supports either structs.ProviderConsul + // or structs.ProviderNomad and defaults for the former. + Provider string `hcl:"provider,optional"` } const ( OnUpdateRequireHealthy = "require_healthy" OnUpdateIgnoreWarn = "ignore_warnings" OnUpdateIgnore = "ignore" + + // ServiceProviderConsul is the default provider for services when no + // parameter is set. + ServiceProviderConsul = "consul" ) // Canonicalize the Service by ensuring its name and address mode are set. Task @@ -145,6 +154,11 @@ func (s *Service) Canonicalize(t *Task, tg *TaskGroup, job *Job) { s.OnUpdate = OnUpdateRequireHealthy } + // Default the service provider. + if s.Provider == "" { + s.Provider = ServiceProviderConsul + } + s.Connect.Canonicalize() // Canonicalize CheckRestart on Checks and merge Service.CheckRestart diff --git a/api/services_test.go b/api/services_test.go index 648c975d80c5..28dd5d96320f 100644 --- a/api/services_test.go +++ b/api/services_test.go @@ -21,6 +21,7 @@ func TestService_Canonicalize(t *testing.T) { require.Equal(t, fmt.Sprintf("%s-%s-%s", *j.Name, *tg.Name, task.Name), s.Name) require.Equal(t, "auto", s.AddressMode) require.Equal(t, OnUpdateRequireHealthy, s.OnUpdate) + require.Equal(t, ServiceProviderConsul, s.Provider) } func TestServiceCheck_Canonicalize(t *testing.T) { From b78e436f10d4652dcb0422041e491483b634a615 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Mon, 14 Mar 2022 11:48:13 +0100 Subject: [PATCH 19/31] config: add native service discovery admin boolean parameter. --- client/config/config.go | 4 ++++ command/agent/agent.go | 4 ++++ command/agent/agent_test.go | 8 ++++++++ command/agent/config.go | 15 +++++++++++++++ command/agent/config_test.go | 2 ++ 5 files changed, 33 insertions(+) diff --git a/client/config/config.go b/client/config/config.go index 9ec256eabb6d..2ce0568e9f1b 100644 --- a/client/config/config.go +++ b/client/config/config.go @@ -279,6 +279,10 @@ type Config struct { // ReservableCores if set overrides the set of reservable cores reported in fingerprinting. ReservableCores []uint16 + + // NomadServiceDiscovery determines whether the Nomad native service + // discovery client functionality is enabled. + NomadServiceDiscovery bool } // ClientTemplateConfig is configuration on the client specific to template diff --git a/command/agent/agent.go b/command/agent/agent.go index 61688bf13cf6..b65e101ef658 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -704,6 +704,10 @@ func convertClientConfig(agentConfig *Config) (*clientconfig.Config, error) { conf.ReservableCores = cores.ToSlice() } + if agentConfig.Client.NomadServiceDiscovery != nil { + conf.NomadServiceDiscovery = *agentConfig.Client.NomadServiceDiscovery + } + return conf, nil } diff --git a/command/agent/agent_test.go b/command/agent/agent_test.go index f61e47c049c4..a0d1d891f845 100644 --- a/command/agent/agent_test.go +++ b/command/agent/agent_test.go @@ -519,6 +519,14 @@ func TestAgent_ClientConfig(t *testing.T) { if c.Node.HTTPAddr != expectedHttpAddr { t.Fatalf("Expected http addr: %v, got: %v", expectedHttpAddr, c.Node.HTTPAddr) } + + // Test the default, and then custom setting of the client service + // discovery boolean. + require.True(t, c.NomadServiceDiscovery) + conf.Client.NomadServiceDiscovery = helper.BoolToPtr(false) + c, err = a.clientConfig() + require.NoError(t, err) + require.False(t, c.NomadServiceDiscovery) } func TestAgent_ClientConfig_ReservedCores(t *testing.T) { diff --git a/command/agent/config.go b/command/agent/config.go index 7ed47aa73d2e..2821f70f550e 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -318,6 +318,12 @@ type ClientConfig struct { // doest not exist Nomad will attempt to create it during startup. Defaults to '/nomad' CgroupParent string `hcl:"cgroup_parent"` + // NomadServiceDiscovery is a boolean parameter which allows operators to + // enable/disable to Nomad native service discovery feature on the client. + // This parameter is exposed via the Nomad fingerprinter and used to ensure + // correct scheduling decisions on allocations which require this. + NomadServiceDiscovery *bool `hcl:"nomad_service_discovery"` + // ExtraKeysHCL is used by hcl to surface unexpected keys ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` } @@ -915,6 +921,7 @@ func DevConfig(mode *devModeConfig) *Config { DisableSandbox: false, } conf.Client.BindWildcardDefaultHostNetwork = true + conf.Client.NomadServiceDiscovery = helper.BoolToPtr(true) conf.Telemetry.PrometheusMetrics = true conf.Telemetry.PublishAllocationMetrics = true conf.Telemetry.PublishNodeMetrics = true @@ -966,6 +973,7 @@ func DefaultConfig() *Config { BindWildcardDefaultHostNetwork: true, CNIPath: "/opt/cni/bin", CNIConfigDir: "/opt/cni/config", + NomadServiceDiscovery: helper.BoolToPtr(true), }, Server: &ServerConfig{ Enabled: false, @@ -1763,6 +1771,13 @@ func (a *ClientConfig) Merge(b *ClientConfig) *ClientConfig { if b.BindWildcardDefaultHostNetwork { result.BindWildcardDefaultHostNetwork = true } + + // This value is a pointer, therefore if it is not nil the user has + // supplied an override value. + if b.NomadServiceDiscovery != nil { + result.NomadServiceDiscovery = b.NomadServiceDiscovery + } + return &result } diff --git a/command/agent/config_test.go b/command/agent/config_test.go index b795f8fad9e8..aaf65ff268f2 100644 --- a/command/agent/config_test.go +++ b/command/agent/config_test.go @@ -126,6 +126,7 @@ func TestConfig_Merge(t *testing.T) { DiskMB: 10, ReservedPorts: "1,10-30,55", }, + NomadServiceDiscovery: helper.BoolToPtr(false), }, Server: &ServerConfig{ Enabled: false, @@ -314,6 +315,7 @@ func TestConfig_Merge(t *testing.T) { GCParallelDestroys: 6, GCDiskUsageThreshold: 71, GCInodeUsageThreshold: 86, + NomadServiceDiscovery: helper.BoolToPtr(false), }, Server: &ServerConfig{ Enabled: true, From 8e2c71e76b39597056714cf65537cfd601b3d022 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Mon, 14 Mar 2022 12:42:01 +0100 Subject: [PATCH 20/31] client: add service discovery feature enabled attribute. --- client/fingerprint/nomad.go | 3 +++ client/fingerprint/nomad_test.go | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/client/fingerprint/nomad.go b/client/fingerprint/nomad.go index bee62758ec05..89525dbfdc4d 100644 --- a/client/fingerprint/nomad.go +++ b/client/fingerprint/nomad.go @@ -1,6 +1,8 @@ package fingerprint import ( + "strconv" + log "github.com/hashicorp/go-hclog" ) @@ -20,6 +22,7 @@ func (f *NomadFingerprint) Fingerprint(req *FingerprintRequest, resp *Fingerprin resp.AddAttribute("nomad.advertise.address", req.Node.HTTPAddr) resp.AddAttribute("nomad.version", req.Config.Version.VersionNumber()) resp.AddAttribute("nomad.revision", req.Config.Version.Revision) + resp.AddAttribute("nomad.service_discovery", strconv.FormatBool(req.Config.NomadServiceDiscovery)) resp.Detected = true return nil } diff --git a/client/fingerprint/nomad_test.go b/client/fingerprint/nomad_test.go index c2dba204fc7f..8ce3b6a2f415 100644 --- a/client/fingerprint/nomad_test.go +++ b/client/fingerprint/nomad_test.go @@ -7,6 +7,7 @@ import ( "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/version" + "github.com/stretchr/testify/require" ) func TestNomadFingerprint(t *testing.T) { @@ -20,6 +21,7 @@ func TestNomadFingerprint(t *testing.T) { Revision: r, Version: v, }, + NomadServiceDiscovery: true, } node := &structs.Node{ Attributes: make(map[string]string), @@ -52,4 +54,7 @@ func TestNomadFingerprint(t *testing.T) { if response.Attributes["nomad.advertise.address"] != h { t.Fatalf("incorrect advertise address") } + + serviceDisco := response.Attributes["nomad.service_discovery"] + require.Equal(t, "true", serviceDisco, "service_discovery attr incorrect") } From f5be44806b54c8e98289148c1bfda5239fb78ea8 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Mon, 14 Mar 2022 12:42:12 +0100 Subject: [PATCH 21/31] job: add native service discovery job constraint mutator. --- nomad/job_endpoint_hooks.go | 35 ++++++- nomad/job_endpoint_hooks_test.go | 169 +++++++++++++++++++++++++++++++ nomad/structs/job.go | 38 +++++++ nomad/structs/job_test.go | 144 ++++++++++++++++++++++++++ 4 files changed, 385 insertions(+), 1 deletion(-) create mode 100644 nomad/job_endpoint_hooks_test.go diff --git a/nomad/job_endpoint_hooks.go b/nomad/job_endpoint_hooks.go index c1df901af232..61bb42b20952 100644 --- a/nomad/job_endpoint_hooks.go +++ b/nomad/job_endpoint_hooks.go @@ -23,6 +23,17 @@ var ( RTarget: ">= 0.6.1", Operand: structs.ConstraintSemver, } + + // nativeServiceDiscoveryConstraint is the constraint injected into task + // groups that utilise Nomad's native service discovery feature. This is + // needed, as operators can disable the client functionality, and therefore + // we need to ensure task groups are placed where they can run + // successfully. + nativeServiceDiscoveryConstraint = &structs.Constraint{ + LTarget: "${attr.nomad.service_discovery}", + RTarget: "true", + Operand: "=", + } ) type admissionController interface { @@ -120,8 +131,11 @@ func (jobImpliedConstraints) Mutate(j *structs.Job) (*structs.Job, []error, erro // Get the required signals signals := j.RequiredSignals() + // Identify which task groups are utilising Nomad native service discovery. + nativeServiceDisco := j.RequiredNativeServiceDiscovery() + // Hot path - if len(signals) == 0 && len(policies) == 0 { + if len(signals) == 0 && len(policies) == 0 && len(nativeServiceDisco) == 0 { return j, nil, nil } @@ -171,6 +185,25 @@ func (jobImpliedConstraints) Mutate(j *structs.Job) (*structs.Job, []error, erro } } + // Add the Nomad service discovery constraints. + for _, tg := range j.TaskGroups { + if ok := nativeServiceDisco[tg.Name]; !ok { + continue + } + + found := false + for _, c := range tg.Constraints { + if c.Equals(nativeServiceDiscoveryConstraint) { + found = true + break + } + } + + if !found { + tg.Constraints = append(tg.Constraints, nativeServiceDiscoveryConstraint) + } + } + return j, nil, nil } diff --git a/nomad/job_endpoint_hooks_test.go b/nomad/job_endpoint_hooks_test.go new file mode 100644 index 000000000000..8e146972a277 --- /dev/null +++ b/nomad/job_endpoint_hooks_test.go @@ -0,0 +1,169 @@ +package nomad + +import ( + "testing" + + "github.com/hashicorp/nomad/nomad/structs" + "github.com/stretchr/testify/require" +) + +func Test_jobImpliedConstraints_Mutate(t *testing.T) { + t.Parallel() + + testCases := []struct { + inputJob *structs.Job + expectedOutputJob *structs.Job + expectedOutputWarnings []error + expectedOutputError error + name string + }{ + { + inputJob: &structs.Job{ + Name: "example", + TaskGroups: []*structs.TaskGroup{ + { + Name: "example-group-1", + }, + }, + }, + expectedOutputJob: &structs.Job{ + Name: "example", + TaskGroups: []*structs.TaskGroup{ + { + Name: "example-group-1", + }, + }, + }, + expectedOutputWarnings: nil, + expectedOutputError: nil, + name: "no needed constraints", + }, + { + inputJob: &structs.Job{ + Name: "example", + TaskGroups: []*structs.TaskGroup{ + { + Name: "example-group-1", + Services: []*structs.Service{ + { + Name: "example-group-service-1", + Provider: structs.ServiceProviderNomad, + }, + }, + }, + }, + }, + expectedOutputJob: &structs.Job{ + Name: "example", + TaskGroups: []*structs.TaskGroup{ + { + Name: "example-group-1", + Services: []*structs.Service{ + { + Name: "example-group-service-1", + Provider: structs.ServiceProviderNomad, + }, + }, + Constraints: []*structs.Constraint{nativeServiceDiscoveryConstraint}, + }, + }, + }, + expectedOutputWarnings: nil, + expectedOutputError: nil, + name: "task group nomad discovery", + }, + { + inputJob: &structs.Job{ + Name: "example", + TaskGroups: []*structs.TaskGroup{ + { + Name: "example-group-1", + Services: []*structs.Service{ + { + Name: "example-group-service-1", + Provider: structs.ServiceProviderNomad, + }, + }, + Constraints: []*structs.Constraint{nativeServiceDiscoveryConstraint}, + }, + }, + }, + expectedOutputJob: &structs.Job{ + Name: "example", + TaskGroups: []*structs.TaskGroup{ + { + Name: "example-group-1", + Services: []*structs.Service{ + { + Name: "example-group-service-1", + Provider: structs.ServiceProviderNomad, + }, + }, + Constraints: []*structs.Constraint{nativeServiceDiscoveryConstraint}, + }, + }, + }, + expectedOutputWarnings: nil, + expectedOutputError: nil, + name: "task group nomad discovery constraint found", + }, + { + inputJob: &structs.Job{ + Name: "example", + TaskGroups: []*structs.TaskGroup{ + { + Name: "example-group-1", + Services: []*structs.Service{ + { + Name: "example-group-service-1", + Provider: structs.ServiceProviderNomad, + }, + }, + Constraints: []*structs.Constraint{ + { + LTarget: "${node.class}", + RTarget: "high-memory", + Operand: "=", + }, + }, + }, + }, + }, + expectedOutputJob: &structs.Job{ + Name: "example", + TaskGroups: []*structs.TaskGroup{ + { + Name: "example-group-1", + Services: []*structs.Service{ + { + Name: "example-group-service-1", + Provider: structs.ServiceProviderNomad, + }, + }, + Constraints: []*structs.Constraint{ + { + LTarget: "${node.class}", + RTarget: "high-memory", + Operand: "=", + }, + nativeServiceDiscoveryConstraint, + }, + }, + }, + }, + expectedOutputWarnings: nil, + expectedOutputError: nil, + name: "task group nomad discovery other constraints", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + impl := jobImpliedConstraints{} + actualJob, actualWarnings, actualError := impl.Mutate(tc.inputJob) + require.Equal(t, tc.expectedOutputJob, actualJob) + require.ElementsMatch(t, tc.expectedOutputWarnings, actualWarnings) + require.Equal(t, tc.expectedOutputError, actualError) + }) + } +} diff --git a/nomad/structs/job.go b/nomad/structs/job.go index 07c79023af79..7d638f51dd9f 100644 --- a/nomad/structs/job.go +++ b/nomad/structs/job.go @@ -22,3 +22,41 @@ type JobServiceRegistrationsResponse struct { Services []*ServiceRegistration QueryMeta } + +// RequiredNativeServiceDiscovery identifies which task groups, if any, within +// the job are utilising Nomad native service discovery. +func (j *Job) RequiredNativeServiceDiscovery() map[string]bool { + groups := make(map[string]bool) + + for _, tg := range j.TaskGroups { + + // It is possible for services using the Nomad provider to be + // configured at the task group level, so check here first. + if requiresNativeServiceDiscovery(tg.Services) { + groups[tg.Name] = true + continue + } + + // Iterate the tasks within the task group to check the services + // configured at this more traditional level. + for _, task := range tg.Tasks { + if requiresNativeServiceDiscovery(task.Services) { + groups[tg.Name] = true + continue + } + } + } + + return groups +} + +// requiresNativeServiceDiscovery identifies whether any of the services passed +// to the function are utilising Nomad native service discovery. +func requiresNativeServiceDiscovery(services []*Service) bool { + for _, tgService := range services { + if tgService.Provider == ServiceProviderNomad { + return true + } + } + return false +} diff --git a/nomad/structs/job_test.go b/nomad/structs/job_test.go index e64339510ce4..254c315b1f96 100644 --- a/nomad/structs/job_test.go +++ b/nomad/structs/job_test.go @@ -10,3 +10,147 @@ func TestServiceRegistrationsRequest_StaleReadSupport(t *testing.T) { req := &AllocServiceRegistrationsRequest{} require.True(t, req.IsRead()) } + +func TestJob_RequiresNativeServiceDiscovery(t *testing.T) { + testCases := []struct { + inputJob *Job + expectedOutput map[string]bool + name string + }{ + { + inputJob: &Job{ + TaskGroups: []*TaskGroup{ + { + Name: "group1", + Services: []*Service{ + {Provider: "nomad"}, + {Provider: "nomad"}, + }, + }, + { + Name: "group2", + Services: []*Service{ + {Provider: "nomad"}, + {Provider: "nomad"}, + }, + }, + }, + }, + expectedOutput: map[string]bool{"group1": true, "group2": true}, + name: "multiple group services with Nomad provider", + }, + { + inputJob: &Job{ + TaskGroups: []*TaskGroup{ + { + Name: "group1", + Tasks: []*Task{ + { + Services: []*Service{ + {Provider: "nomad"}, + {Provider: "nomad"}, + }, + }, + { + Services: []*Service{ + {Provider: "nomad"}, + {Provider: "nomad"}, + }, + }, + }, + }, + { + Name: "group2", + Tasks: []*Task{ + { + Services: []*Service{ + {Provider: "nomad"}, + {Provider: "nomad"}, + }, + }, + { + Services: []*Service{ + {Provider: "nomad"}, + {Provider: "nomad"}, + }, + }, + }, + }, + }, + }, + expectedOutput: map[string]bool{"group1": true, "group2": true}, + name: "multiple task services with Nomad provider", + }, + { + inputJob: &Job{ + TaskGroups: []*TaskGroup{ + { + Name: "group1", + Services: []*Service{ + {Provider: "consul"}, + {Provider: "consul"}, + }, + }, + { + Name: "group2", + Services: []*Service{ + {Provider: "consul"}, + {Provider: "consul"}, + }, + }, + }, + }, + expectedOutput: map[string]bool{}, + name: "multiple group services with Consul provider", + }, + { + inputJob: &Job{ + TaskGroups: []*TaskGroup{ + { + Name: "group1", + Tasks: []*Task{ + { + Services: []*Service{ + {Provider: "consul"}, + {Provider: "consul"}, + }, + }, + { + Services: []*Service{ + {Provider: "consul"}, + {Provider: "consul"}, + }, + }, + }, + }, + { + Name: "group2", + Tasks: []*Task{ + { + Services: []*Service{ + {Provider: "consul"}, + {Provider: "consul"}, + }, + }, + { + Services: []*Service{ + {Provider: "consul"}, + {Provider: "consul"}, + }, + }, + }, + }, + }, + }, + expectedOutput: map[string]bool{}, + name: "multiple task services with Consul provider", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualOutput := tc.inputJob.RequiredNativeServiceDiscovery() + require.Equal(t, tc.expectedOutput, actualOutput) + }) + } +} From 6e8f32a290f36f74c2b0580526c4d1a2d79f64db Mon Sep 17 00:00:00 2001 From: James Rasell Date: Tue, 15 Mar 2022 09:38:30 +0100 Subject: [PATCH 22/31] client: refactor common service registration objects from Consul. This commit performs refactoring to pull out common service registration objects into a new `client/serviceregistration` package. This new package will form the base point for all client specific service registration functionality. The Consul specific implementation is not moved as it also includes non-service registration implementations; this reduces the blast radius of the changes as well. --- client/allochealth/tracker.go | 11 +- client/allochealth/tracker_test.go | 48 +-- client/allocrunner/alloc_runner.go | 3 +- client/allocrunner/alloc_runner_test.go | 28 +- client/allocrunner/alloc_runner_unix_test.go | 4 +- client/allocrunner/config.go | 3 +- client/allocrunner/groupservice_hook.go | 28 +- client/allocrunner/groupservice_hook_test.go | 12 +- client/allocrunner/health_hook.go | 6 +- client/allocrunner/health_hook_test.go | 32 +- .../taskrunner/envoy_bootstrap_hook.go | 8 +- .../taskrunner/script_check_hook.go | 10 +- .../taskrunner/script_check_hook_test.go | 9 +- client/allocrunner/taskrunner/service_hook.go | 30 +- .../taskrunner/service_hook_test.go | 6 +- client/allocrunner/taskrunner/task_runner.go | 5 +- .../taskrunner/task_runner_test.go | 9 +- client/allocrunner/testing.go | 3 +- client/client.go | 5 +- client/client_test.go | 7 +- client/consul/consul.go | 24 -- client/consul/consul_testing.go | 113 ------ client/serviceregistration/address.go | 136 +++++++ client/serviceregistration/address_test.go | 361 +++++++++++++++++ client/serviceregistration/id.go | 27 ++ client/serviceregistration/id_test.go | 36 ++ client/serviceregistration/mock/mock.go | 125 ++++++ .../service_registration.go | 157 ++++++++ .../service_registration_test.go | 53 +++ client/serviceregistration/workload.go | 95 +++++ client/serviceregistration/workload_test.go | 49 +++ client/state/upgrade_int_test.go | 4 +- client/testing.go | 4 +- command/agent/consul/group_test.go | 3 +- command/agent/consul/service_client.go | 323 +++------------ command/agent/consul/service_client_test.go | 5 +- command/agent/consul/structs.go | 71 +--- command/agent/consul/unit_test.go | 379 +----------------- 38 files changed, 1245 insertions(+), 987 deletions(-) delete mode 100644 client/consul/consul_testing.go create mode 100644 client/serviceregistration/address.go create mode 100644 client/serviceregistration/address_test.go create mode 100644 client/serviceregistration/id.go create mode 100644 client/serviceregistration/id_test.go create mode 100644 client/serviceregistration/mock/mock.go create mode 100644 client/serviceregistration/service_registration.go create mode 100644 client/serviceregistration/service_registration_test.go create mode 100644 client/serviceregistration/workload.go create mode 100644 client/serviceregistration/workload_test.go diff --git a/client/allochealth/tracker.go b/client/allochealth/tracker.go index 7c17b7e364de..2bd727180f75 100644 --- a/client/allochealth/tracker.go +++ b/client/allochealth/tracker.go @@ -9,9 +9,8 @@ import ( "github.com/hashicorp/consul/api" hclog "github.com/hashicorp/go-hclog" - cconsul "github.com/hashicorp/nomad/client/consul" + "github.com/hashicorp/nomad/client/serviceregistration" cstructs "github.com/hashicorp/nomad/client/structs" - "github.com/hashicorp/nomad/command/agent/consul" "github.com/hashicorp/nomad/nomad/structs" ) @@ -56,7 +55,7 @@ type Tracker struct { allocUpdates *cstructs.AllocListener // consulClient is used to look up the state of the task's checks - consulClient cconsul.ConsulServiceAPI + consulClient serviceregistration.Handler // healthy is used to signal whether we have determined the allocation to be // healthy or unhealthy @@ -93,7 +92,7 @@ type Tracker struct { // listener and consul API object are given so that the watcher can detect // health changes. func NewTracker(parentCtx context.Context, logger hclog.Logger, alloc *structs.Allocation, - allocUpdates *cstructs.AllocListener, consulClient cconsul.ConsulServiceAPI, + allocUpdates *cstructs.AllocListener, consulClient serviceregistration.Handler, minHealthyTime time.Duration, useChecks bool) *Tracker { // Do not create a named sub-logger as the hook controlling @@ -377,7 +376,7 @@ func (t *Tracker) watchConsulEvents() { consulChecksErr := false // allocReg are the registered objects in Consul for the allocation - var allocReg *consul.AllocRegistration + var allocReg *serviceregistration.AllocRegistration OUTER: for { @@ -482,7 +481,7 @@ OUTER: type taskHealthState struct { task *structs.Task state *structs.TaskState - taskRegistrations *consul.ServiceRegistrations + taskRegistrations *serviceregistration.ServiceRegistrations } // event takes the deadline time for the allocation to be healthy and the update diff --git a/client/allochealth/tracker_test.go b/client/allochealth/tracker_test.go index f4aec166d9e2..f40d016d5958 100644 --- a/client/allochealth/tracker_test.go +++ b/client/allochealth/tracker_test.go @@ -8,9 +8,9 @@ import ( "time" consulapi "github.com/hashicorp/consul/api" - "github.com/hashicorp/nomad/client/consul" + "github.com/hashicorp/nomad/client/serviceregistration" + regMock "github.com/hashicorp/nomad/client/serviceregistration/mock" cstructs "github.com/hashicorp/nomad/client/structs" - agentconsul "github.com/hashicorp/nomad/command/agent/consul" "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" @@ -39,9 +39,9 @@ func TestTracker_Checks_Healthy(t *testing.T) { Name: task.Services[0].Checks[0].Name, Status: consulapi.HealthPassing, } - taskRegs := map[string]*agentconsul.ServiceRegistrations{ + taskRegs := map[string]*serviceregistration.ServiceRegistrations{ task.Name: { - Services: map[string]*agentconsul.ServiceRegistration{ + Services: map[string]*serviceregistration.ServiceRegistration{ task.Services[0].Name: { Service: &consulapi.AgentService{ ID: "foo", @@ -59,13 +59,13 @@ func TestTracker_Checks_Healthy(t *testing.T) { // Don't reply on the first call var called uint64 - consul := consul.NewMockConsulServiceClient(t, logger) - consul.AllocRegistrationsFn = func(string) (*agentconsul.AllocRegistration, error) { + consul := regMock.NewServiceRegistrationHandler(logger) + consul.AllocRegistrationsFn = func(string) (*serviceregistration.AllocRegistration, error) { if atomic.AddUint64(&called, 1) == 1 { return nil, nil } - reg := &agentconsul.AllocRegistration{ + reg := &serviceregistration.AllocRegistration{ Tasks: taskRegs, } @@ -111,7 +111,7 @@ func TestTracker_Checks_PendingPostStop_Healthy(t *testing.T) { b := cstructs.NewAllocBroadcaster(logger) defer b.Close() - consul := consul.NewMockConsulServiceClient(t, logger) + consul := regMock.NewServiceRegistrationHandler(logger) ctx, cancelFn := context.WithCancel(context.Background()) defer cancelFn() @@ -152,7 +152,7 @@ func TestTracker_Succeeded_PostStart_Healthy(t *testing.T) { b := cstructs.NewAllocBroadcaster(logger) defer b.Close() - consul := consul.NewMockConsulServiceClient(t, logger) + consul := regMock.NewServiceRegistrationHandler(logger) ctx, cancelFn := context.WithCancel(context.Background()) defer cancelFn() @@ -199,9 +199,9 @@ func TestTracker_Checks_Unhealthy(t *testing.T) { Name: task.Services[0].Checks[1].Name, Status: consulapi.HealthCritical, } - taskRegs := map[string]*agentconsul.ServiceRegistrations{ + taskRegs := map[string]*serviceregistration.ServiceRegistrations{ task.Name: { - Services: map[string]*agentconsul.ServiceRegistration{ + Services: map[string]*serviceregistration.ServiceRegistration{ task.Services[0].Name: { Service: &consulapi.AgentService{ ID: "foo", @@ -219,13 +219,13 @@ func TestTracker_Checks_Unhealthy(t *testing.T) { // Don't reply on the first call var called uint64 - consul := consul.NewMockConsulServiceClient(t, logger) - consul.AllocRegistrationsFn = func(string) (*agentconsul.AllocRegistration, error) { + consul := regMock.NewServiceRegistrationHandler(logger) + consul.AllocRegistrationsFn = func(string) (*serviceregistration.AllocRegistration, error) { if atomic.AddUint64(&called, 1) == 1 { return nil, nil } - reg := &agentconsul.AllocRegistration{ + reg := &serviceregistration.AllocRegistration{ Tasks: taskRegs, } @@ -341,9 +341,9 @@ func TestTracker_Checks_Healthy_Before_TaskHealth(t *testing.T) { Name: task.Services[0].Checks[0].Name, Status: consulapi.HealthPassing, } - taskRegs := map[string]*agentconsul.ServiceRegistrations{ + taskRegs := map[string]*serviceregistration.ServiceRegistrations{ task.Name: { - Services: map[string]*agentconsul.ServiceRegistration{ + Services: map[string]*serviceregistration.ServiceRegistration{ task.Services[0].Name: { Service: &consulapi.AgentService{ ID: "foo", @@ -361,13 +361,13 @@ func TestTracker_Checks_Healthy_Before_TaskHealth(t *testing.T) { // Don't reply on the first call var called uint64 - consul := consul.NewMockConsulServiceClient(t, logger) - consul.AllocRegistrationsFn = func(string) (*agentconsul.AllocRegistration, error) { + consul := regMock.NewServiceRegistrationHandler(logger) + consul.AllocRegistrationsFn = func(string) (*serviceregistration.AllocRegistration, error) { if atomic.AddUint64(&called, 1) == 1 { return nil, nil } - reg := &agentconsul.AllocRegistration{ + reg := &serviceregistration.AllocRegistration{ Tasks: taskRegs, } @@ -480,9 +480,9 @@ func TestTracker_Checks_OnUpdate(t *testing.T) { Name: task.Services[0].Checks[0].Name, Status: tc.consulResp, } - taskRegs := map[string]*agentconsul.ServiceRegistrations{ + taskRegs := map[string]*serviceregistration.ServiceRegistrations{ task.Name: { - Services: map[string]*agentconsul.ServiceRegistration{ + Services: map[string]*serviceregistration.ServiceRegistration{ task.Services[0].Name: { Service: &consulapi.AgentService{ ID: "foo", @@ -503,13 +503,13 @@ func TestTracker_Checks_OnUpdate(t *testing.T) { // Don't reply on the first call var called uint64 - consul := consul.NewMockConsulServiceClient(t, logger) - consul.AllocRegistrationsFn = func(string) (*agentconsul.AllocRegistration, error) { + consul := regMock.NewServiceRegistrationHandler(logger) + consul.AllocRegistrationsFn = func(string) (*serviceregistration.AllocRegistration, error) { if atomic.AddUint64(&called, 1) == 1 { return nil, nil } - reg := &agentconsul.AllocRegistration{ + reg := &serviceregistration.AllocRegistration{ Tasks: taskRegs, } diff --git a/client/allocrunner/alloc_runner.go b/client/allocrunner/alloc_runner.go index c846b0916931..17971c72bb9b 100644 --- a/client/allocrunner/alloc_runner.go +++ b/client/allocrunner/alloc_runner.go @@ -22,6 +22,7 @@ import ( cinterfaces "github.com/hashicorp/nomad/client/interfaces" "github.com/hashicorp/nomad/client/pluginmanager/csimanager" "github.com/hashicorp/nomad/client/pluginmanager/drivermanager" + "github.com/hashicorp/nomad/client/serviceregistration" cstate "github.com/hashicorp/nomad/client/state" cstructs "github.com/hashicorp/nomad/client/structs" "github.com/hashicorp/nomad/client/vaultclient" @@ -63,7 +64,7 @@ type allocRunner struct { // consulClient is the client used by the consul service hook for // registering services and checks - consulClient consul.ConsulServiceAPI + consulClient serviceregistration.Handler // consulProxiesClient is the client used by the envoy version hook for // looking up supported envoy versions of the consul agent. diff --git a/client/allocrunner/alloc_runner_test.go b/client/allocrunner/alloc_runner_test.go index 749f7207898f..23d59dc6915b 100644 --- a/client/allocrunner/alloc_runner_test.go +++ b/client/allocrunner/alloc_runner_test.go @@ -11,9 +11,9 @@ import ( "github.com/hashicorp/consul/api" "github.com/hashicorp/nomad/client/allochealth" "github.com/hashicorp/nomad/client/allocwatcher" - cconsul "github.com/hashicorp/nomad/client/consul" + "github.com/hashicorp/nomad/client/serviceregistration" + regMock "github.com/hashicorp/nomad/client/serviceregistration/mock" "github.com/hashicorp/nomad/client/state" - "github.com/hashicorp/nomad/command/agent/consul" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" @@ -577,9 +577,9 @@ func TestAllocRunner_TaskGroup_ShutdownDelay(t *testing.T) { }) // Get consul client operations - consulClient := conf.Consul.(*cconsul.MockConsulServiceClient) + consulClient := conf.Consul.(*regMock.ServiceRegistrationHandler) consulOpts := consulClient.GetOps() - var groupRemoveOp cconsul.MockConsulOp + var groupRemoveOp regMock.Operation for _, op := range consulOpts { // Grab the first deregistration request if op.Op == "remove" && op.Name == "group-web" { @@ -1030,12 +1030,12 @@ func TestAllocRunner_DeploymentHealth_Unhealthy_Checks(t *testing.T) { defer cleanup() // Only return the check as healthy after a duration - consulClient := conf.Consul.(*cconsul.MockConsulServiceClient) - consulClient.AllocRegistrationsFn = func(allocID string) (*consul.AllocRegistration, error) { - return &consul.AllocRegistration{ - Tasks: map[string]*consul.ServiceRegistrations{ + consulClient := conf.Consul.(*regMock.ServiceRegistrationHandler) + consulClient.AllocRegistrationsFn = func(allocID string) (*serviceregistration.AllocRegistration, error) { + return &serviceregistration.AllocRegistration{ + Tasks: map[string]*serviceregistration.ServiceRegistrations{ task.Name: { - Services: map[string]*consul.ServiceRegistration{ + Services: map[string]*serviceregistration.ServiceRegistration{ "123": { Service: &api.AgentService{Service: "fakeservice"}, Checks: []*api.AgentCheck{checkUnhealthy}, @@ -1352,12 +1352,12 @@ func TestAllocRunner_TaskFailed_KillTG(t *testing.T) { conf, cleanup := testAllocRunnerConfig(t, alloc) defer cleanup() - consulClient := conf.Consul.(*cconsul.MockConsulServiceClient) - consulClient.AllocRegistrationsFn = func(allocID string) (*consul.AllocRegistration, error) { - return &consul.AllocRegistration{ - Tasks: map[string]*consul.ServiceRegistrations{ + consulClient := conf.Consul.(*regMock.ServiceRegistrationHandler) + consulClient.AllocRegistrationsFn = func(allocID string) (*serviceregistration.AllocRegistration, error) { + return &serviceregistration.AllocRegistration{ + Tasks: map[string]*serviceregistration.ServiceRegistrations{ task.Name: { - Services: map[string]*consul.ServiceRegistration{ + Services: map[string]*serviceregistration.ServiceRegistration{ "123": { Service: &api.AgentService{Service: "fakeservice"}, Checks: []*api.AgentCheck{checkHealthy}, diff --git a/client/allocrunner/alloc_runner_unix_test.go b/client/allocrunner/alloc_runner_unix_test.go index 41b5dabc8db0..63d06fd3fea5 100644 --- a/client/allocrunner/alloc_runner_unix_test.go +++ b/client/allocrunner/alloc_runner_unix_test.go @@ -11,7 +11,7 @@ import ( "testing" "time" - "github.com/hashicorp/nomad/client/consul" + regMock "github.com/hashicorp/nomad/client/serviceregistration/mock" "github.com/hashicorp/nomad/client/state" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" @@ -125,7 +125,7 @@ func TestAllocRunner_Restore_RunningTerminal(t *testing.T) { // - removal during exited is de-duped due to prekill // - removal during stop is de-duped due to prekill // 1 removal group during stop - consulOps := conf2.Consul.(*consul.MockConsulServiceClient).GetOps() + consulOps := conf2.Consul.(*regMock.ServiceRegistrationHandler).GetOps() require.Len(t, consulOps, 2) for _, op := range consulOps { require.Equal(t, "remove", op.Op) diff --git a/client/allocrunner/config.go b/client/allocrunner/config.go index 343d4eec5527..d1500c906744 100644 --- a/client/allocrunner/config.go +++ b/client/allocrunner/config.go @@ -11,6 +11,7 @@ import ( "github.com/hashicorp/nomad/client/lib/cgutil" "github.com/hashicorp/nomad/client/pluginmanager/csimanager" "github.com/hashicorp/nomad/client/pluginmanager/drivermanager" + "github.com/hashicorp/nomad/client/serviceregistration" cstate "github.com/hashicorp/nomad/client/state" "github.com/hashicorp/nomad/client/vaultclient" "github.com/hashicorp/nomad/nomad/structs" @@ -31,7 +32,7 @@ type Config struct { StateDB cstate.StateDB // Consul is the Consul client used to register task services and checks - Consul consul.ConsulServiceAPI + Consul serviceregistration.Handler // ConsulProxies is the Consul client used to lookup supported envoy versions // of the Consul agent. diff --git a/client/allocrunner/groupservice_hook.go b/client/allocrunner/groupservice_hook.go index 69eae41e8bf7..1d5a6205373f 100644 --- a/client/allocrunner/groupservice_hook.go +++ b/client/allocrunner/groupservice_hook.go @@ -7,7 +7,7 @@ import ( log "github.com/hashicorp/go-hclog" "github.com/hashicorp/nomad/client/allocrunner/interfaces" - "github.com/hashicorp/nomad/client/consul" + "github.com/hashicorp/nomad/client/serviceregistration" "github.com/hashicorp/nomad/client/taskenv" agentconsul "github.com/hashicorp/nomad/command/agent/consul" "github.com/hashicorp/nomad/nomad/structs" @@ -27,7 +27,7 @@ type groupServiceHook struct { allocID string group string restarter agentconsul.WorkloadRestarter - consulClient consul.ConsulServiceAPI + consulClient serviceregistration.Handler consulNamespace string prerun bool deregistered bool @@ -51,7 +51,7 @@ type groupServiceHook struct { type groupServiceHookConfig struct { alloc *structs.Allocation - consul consul.ConsulServiceAPI + consul serviceregistration.Handler consulNamespace string restarter agentconsul.WorkloadRestarter taskEnvBuilder *taskenv.Builder @@ -217,7 +217,7 @@ func (h *groupServiceHook) deregister() { } } -func (h *groupServiceHook) getWorkloadServices() *agentconsul.WorkloadServices { +func (h *groupServiceHook) getWorkloadServices() *serviceregistration.WorkloadServices { // Interpolate with the task's environment interpolatedServices := taskenv.InterpolateServices(h.taskEnvBuilder.Build(), h.services) @@ -227,15 +227,15 @@ func (h *groupServiceHook) getWorkloadServices() *agentconsul.WorkloadServices { } // Create task services struct with request's driver metadata - return &agentconsul.WorkloadServices{ - AllocID: h.allocID, - Group: h.group, - ConsulNamespace: h.consulNamespace, - Restarter: h.restarter, - Services: interpolatedServices, - Networks: h.networks, - NetworkStatus: netStatus, - Ports: h.ports, - Canary: h.canary, + return &serviceregistration.WorkloadServices{ + AllocID: h.allocID, + Group: h.group, + Namespace: h.consulNamespace, + Restarter: h.restarter, + Services: interpolatedServices, + Networks: h.networks, + NetworkStatus: netStatus, + Ports: h.ports, + Canary: h.canary, } } diff --git a/client/allocrunner/groupservice_hook_test.go b/client/allocrunner/groupservice_hook_test.go index 61d9a38b49ad..4fae46a300af 100644 --- a/client/allocrunner/groupservice_hook_test.go +++ b/client/allocrunner/groupservice_hook_test.go @@ -8,7 +8,7 @@ import ( consulapi "github.com/hashicorp/consul/api" ctestutil "github.com/hashicorp/consul/sdk/testutil" "github.com/hashicorp/nomad/client/allocrunner/interfaces" - "github.com/hashicorp/nomad/client/consul" + regMock "github.com/hashicorp/nomad/client/serviceregistration/mock" "github.com/hashicorp/nomad/client/taskenv" agentconsul "github.com/hashicorp/nomad/command/agent/consul" "github.com/hashicorp/nomad/helper" @@ -35,7 +35,7 @@ func TestGroupServiceHook_NoGroupServices(t *testing.T) { PortLabel: "9999", }} logger := testlog.HCLogger(t) - consulClient := consul.NewMockConsulServiceClient(t, logger) + consulClient := regMock.NewServiceRegistrationHandler(logger) h := newGroupServiceHook(groupServiceHookConfig{ alloc: alloc, @@ -71,7 +71,7 @@ func TestGroupServiceHook_ShutdownDelayUpdate(t *testing.T) { alloc.Job.TaskGroups[0].ShutdownDelay = helper.TimeToPtr(10 * time.Second) logger := testlog.HCLogger(t) - consulClient := consul.NewMockConsulServiceClient(t, logger) + consulClient := regMock.NewServiceRegistrationHandler(logger) h := newGroupServiceHook(groupServiceHookConfig{ alloc: alloc, @@ -106,7 +106,7 @@ func TestGroupServiceHook_GroupServices(t *testing.T) { alloc := mock.ConnectAlloc() logger := testlog.HCLogger(t) - consulClient := consul.NewMockConsulServiceClient(t, logger) + consulClient := regMock.NewServiceRegistrationHandler(logger) h := newGroupServiceHook(groupServiceHookConfig{ alloc: alloc, @@ -152,7 +152,7 @@ func TestGroupServiceHook_NoNetwork(t *testing.T) { } logger := testlog.HCLogger(t) - consulClient := consul.NewMockConsulServiceClient(t, logger) + consulClient := regMock.NewServiceRegistrationHandler(logger) h := newGroupServiceHook(groupServiceHookConfig{ alloc: alloc, @@ -196,7 +196,7 @@ func TestGroupServiceHook_getWorkloadServices(t *testing.T) { } logger := testlog.HCLogger(t) - consulClient := consul.NewMockConsulServiceClient(t, logger) + consulClient := regMock.NewServiceRegistrationHandler(logger) h := newGroupServiceHook(groupServiceHookConfig{ alloc: alloc, diff --git a/client/allocrunner/health_hook.go b/client/allocrunner/health_hook.go index feabfba2c466..4c2bef43ab90 100644 --- a/client/allocrunner/health_hook.go +++ b/client/allocrunner/health_hook.go @@ -9,7 +9,7 @@ import ( log "github.com/hashicorp/go-hclog" "github.com/hashicorp/nomad/client/allochealth" "github.com/hashicorp/nomad/client/allocrunner/interfaces" - "github.com/hashicorp/nomad/client/consul" + "github.com/hashicorp/nomad/client/serviceregistration" cstructs "github.com/hashicorp/nomad/client/structs" "github.com/hashicorp/nomad/nomad/structs" ) @@ -33,7 +33,7 @@ type allocHealthWatcherHook struct { healthSetter healthSetter // consul client used to monitor health checks - consul consul.ConsulServiceAPI + consul serviceregistration.Handler // listener is given to trackers to listen for alloc updates and closed // when the alloc is destroyed. @@ -68,7 +68,7 @@ type allocHealthWatcherHook struct { } func newAllocHealthWatcherHook(logger log.Logger, alloc *structs.Allocation, hs healthSetter, - listener *cstructs.AllocListener, consul consul.ConsulServiceAPI) interfaces.RunnerHook { + listener *cstructs.AllocListener, consul serviceregistration.Handler) interfaces.RunnerHook { // Neither deployments nor migrations care about the health of // non-service jobs so never watch their health diff --git a/client/allocrunner/health_hook_test.go b/client/allocrunner/health_hook_test.go index 62c14892560f..0f11e072c739 100644 --- a/client/allocrunner/health_hook_test.go +++ b/client/allocrunner/health_hook_test.go @@ -7,9 +7,9 @@ import ( consulapi "github.com/hashicorp/consul/api" "github.com/hashicorp/nomad/client/allocrunner/interfaces" - "github.com/hashicorp/nomad/client/consul" + "github.com/hashicorp/nomad/client/serviceregistration" + regMock "github.com/hashicorp/nomad/client/serviceregistration/mock" cstructs "github.com/hashicorp/nomad/client/structs" - agentconsul "github.com/hashicorp/nomad/command/agent/consul" "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" @@ -92,7 +92,7 @@ func TestHealthHook_PrerunPostrun(t *testing.T) { b := cstructs.NewAllocBroadcaster(logger) defer b.Close() - consul := consul.NewMockConsulServiceClient(t, logger) + consul := regMock.NewServiceRegistrationHandler(logger) hs := &mockHealthSetter{} h := newAllocHealthWatcherHook(logger, mock.Alloc(), hs, b.Listen(), consul) @@ -130,7 +130,7 @@ func TestHealthHook_PrerunUpdatePostrun(t *testing.T) { b := cstructs.NewAllocBroadcaster(logger) defer b.Close() - consul := consul.NewMockConsulServiceClient(t, logger) + consul := regMock.NewServiceRegistrationHandler(logger) hs := &mockHealthSetter{} h := newAllocHealthWatcherHook(logger, alloc.Copy(), hs, b.Listen(), consul).(*allocHealthWatcherHook) @@ -169,7 +169,7 @@ func TestHealthHook_UpdatePrerunPostrun(t *testing.T) { b := cstructs.NewAllocBroadcaster(logger) defer b.Close() - consul := consul.NewMockConsulServiceClient(t, logger) + consul := regMock.NewServiceRegistrationHandler(logger) hs := &mockHealthSetter{} h := newAllocHealthWatcherHook(logger, alloc.Copy(), hs, b.Listen(), consul).(*allocHealthWatcherHook) @@ -210,7 +210,7 @@ func TestHealthHook_Postrun(t *testing.T) { b := cstructs.NewAllocBroadcaster(logger) defer b.Close() - consul := consul.NewMockConsulServiceClient(t, logger) + consul := regMock.NewServiceRegistrationHandler(logger) hs := &mockHealthSetter{} h := newAllocHealthWatcherHook(logger, mock.Alloc(), hs, b.Listen(), consul).(*allocHealthWatcherHook) @@ -243,9 +243,9 @@ func TestHealthHook_SetHealth_healthy(t *testing.T) { Name: task.Services[0].Checks[0].Name, Status: consulapi.HealthPassing, } - taskRegs := map[string]*agentconsul.ServiceRegistrations{ + taskRegs := map[string]*serviceregistration.ServiceRegistrations{ task.Name: { - Services: map[string]*agentconsul.ServiceRegistration{ + Services: map[string]*serviceregistration.ServiceRegistration{ task.Services[0].Name: { Service: &consulapi.AgentService{ ID: "foo", @@ -263,14 +263,14 @@ func TestHealthHook_SetHealth_healthy(t *testing.T) { // Don't reply on the first call called := false - consul := consul.NewMockConsulServiceClient(t, logger) - consul.AllocRegistrationsFn = func(string) (*agentconsul.AllocRegistration, error) { + consul := regMock.NewServiceRegistrationHandler(logger) + consul.AllocRegistrationsFn = func(string) (*serviceregistration.AllocRegistration, error) { if !called { called = true return nil, nil } - reg := &agentconsul.AllocRegistration{ + reg := &serviceregistration.AllocRegistration{ Tasks: taskRegs, } @@ -331,9 +331,9 @@ func TestHealthHook_SetHealth_unhealthy(t *testing.T) { Name: task.Services[0].Checks[1].Name, Status: consulapi.HealthCritical, } - taskRegs := map[string]*agentconsul.ServiceRegistrations{ + taskRegs := map[string]*serviceregistration.ServiceRegistrations{ task.Name: { - Services: map[string]*agentconsul.ServiceRegistration{ + Services: map[string]*serviceregistration.ServiceRegistration{ task.Services[0].Name: { Service: &consulapi.AgentService{ ID: "foo", @@ -351,14 +351,14 @@ func TestHealthHook_SetHealth_unhealthy(t *testing.T) { // Don't reply on the first call called := false - consul := consul.NewMockConsulServiceClient(t, logger) - consul.AllocRegistrationsFn = func(string) (*agentconsul.AllocRegistration, error) { + consul := regMock.NewServiceRegistrationHandler(logger) + consul.AllocRegistrationsFn = func(string) (*serviceregistration.AllocRegistration, error) { if !called { called = true return nil, nil } - reg := &agentconsul.AllocRegistration{ + reg := &serviceregistration.AllocRegistration{ Tasks: taskRegs, } diff --git a/client/allocrunner/taskrunner/envoy_bootstrap_hook.go b/client/allocrunner/taskrunner/envoy_bootstrap_hook.go index a3025927be1b..712b5c2365dd 100644 --- a/client/allocrunner/taskrunner/envoy_bootstrap_hook.go +++ b/client/allocrunner/taskrunner/envoy_bootstrap_hook.go @@ -17,8 +17,8 @@ import ( "github.com/hashicorp/go-hclog" "github.com/hashicorp/nomad/client/allocdir" ifs "github.com/hashicorp/nomad/client/allocrunner/interfaces" + "github.com/hashicorp/nomad/client/serviceregistration" "github.com/hashicorp/nomad/client/taskenv" - agentconsul "github.com/hashicorp/nomad/command/agent/consul" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/nomad/structs/config" @@ -451,9 +451,9 @@ func (h *envoyBootstrapHook) grpcAddress(env map[string]string) string { } func (h *envoyBootstrapHook) proxyServiceID(group string, service *structs.Service) string { - // Note, it is critical the ID here matches what is actually registered in Consul. - // See: WorkloadServices.Name in structs.go - return agentconsul.MakeAllocServiceID(h.alloc.ID, "group-"+group, service) + // Note, it is critical the ID here matches what is actually registered in + // Consul. See: WorkloadServices.Name in serviceregistration/workload.go. + return serviceregistration.MakeAllocServiceID(h.alloc.ID, "group-"+group, service) } // newEnvoyBootstrapArgs is used to prepare for the invocation of the diff --git a/client/allocrunner/taskrunner/script_check_hook.go b/client/allocrunner/taskrunner/script_check_hook.go index 332ec6673d46..e556b6f72002 100644 --- a/client/allocrunner/taskrunner/script_check_hook.go +++ b/client/allocrunner/taskrunner/script_check_hook.go @@ -10,7 +10,7 @@ import ( log "github.com/hashicorp/go-hclog" "github.com/hashicorp/nomad/client/allocrunner/interfaces" tinterfaces "github.com/hashicorp/nomad/client/allocrunner/taskrunner/interfaces" - "github.com/hashicorp/nomad/client/consul" + "github.com/hashicorp/nomad/client/serviceregistration" "github.com/hashicorp/nomad/client/taskenv" agentconsul "github.com/hashicorp/nomad/command/agent/consul" "github.com/hashicorp/nomad/nomad/structs" @@ -26,7 +26,7 @@ const defaultShutdownWait = time.Minute type scriptCheckHookConfig struct { alloc *structs.Allocation task *structs.Task - consul consul.ConsulServiceAPI + consul serviceregistration.Handler logger log.Logger shutdownWait time.Duration } @@ -34,7 +34,7 @@ type scriptCheckHookConfig struct { // scriptCheckHook implements a task runner hook for running script // checks in the context of a task type scriptCheckHook struct { - consul consul.ConsulServiceAPI + consul serviceregistration.Handler consulNamespace string alloc *structs.Allocation task *structs.Task @@ -182,7 +182,7 @@ func (h *scriptCheckHook) newScriptChecks() map[string]*scriptCheck { if check.Type != structs.ServiceCheckScript { continue } - serviceID := agentconsul.MakeAllocServiceID( + serviceID := serviceregistration.MakeAllocServiceID( h.alloc.ID, h.task.Name, service) sc := newScriptCheck(&scriptCheckConfig{ consulNamespace: h.consulNamespace, @@ -222,7 +222,7 @@ func (h *scriptCheckHook) newScriptChecks() map[string]*scriptCheck { continue } groupTaskName := "group-" + tg.Name - serviceID := agentconsul.MakeAllocServiceID( + serviceID := serviceregistration.MakeAllocServiceID( h.alloc.ID, groupTaskName, service) sc := newScriptCheck(&scriptCheckConfig{ consulNamespace: h.consulNamespace, diff --git a/client/allocrunner/taskrunner/script_check_hook_test.go b/client/allocrunner/taskrunner/script_check_hook_test.go index 0d50c4fc0c4f..5463a0b2e1ae 100644 --- a/client/allocrunner/taskrunner/script_check_hook_test.go +++ b/client/allocrunner/taskrunner/script_check_hook_test.go @@ -10,7 +10,8 @@ import ( "github.com/hashicorp/consul/api" hclog "github.com/hashicorp/go-hclog" "github.com/hashicorp/nomad/client/allocrunner/taskrunner/interfaces" - "github.com/hashicorp/nomad/client/consul" + "github.com/hashicorp/nomad/client/serviceregistration" + regMock "github.com/hashicorp/nomad/client/serviceregistration/mock" "github.com/hashicorp/nomad/client/taskenv" agentconsul "github.com/hashicorp/nomad/command/agent/consul" "github.com/hashicorp/nomad/helper/testlog" @@ -226,7 +227,7 @@ func TestScript_Exec_Codes(t *testing.T) { func TestScript_TaskEnvInterpolation(t *testing.T) { logger := testlog.HCLogger(t) - consulClient := consul.NewMockConsulServiceClient(t, logger) + consulClient := regMock.NewServiceRegistrationHandler(logger) exec, cancel := newBlockingScriptExec() defer cancel() @@ -262,7 +263,7 @@ func TestScript_TaskEnvInterpolation(t *testing.T) { scHook.driverExec = exec expectedSvc := svcHook.getWorkloadServices().Services[0] - expected := agentconsul.MakeCheckID(agentconsul.MakeAllocServiceID( + expected := agentconsul.MakeCheckID(serviceregistration.MakeAllocServiceID( alloc.ID, task.Name, expectedSvc), expectedSvc.Checks[0]) actual := scHook.newScriptChecks() @@ -278,7 +279,7 @@ func TestScript_TaskEnvInterpolation(t *testing.T) { svcHook.taskEnv = env expectedSvc = svcHook.getWorkloadServices().Services[0] - expected = agentconsul.MakeCheckID(agentconsul.MakeAllocServiceID( + expected = agentconsul.MakeCheckID(serviceregistration.MakeAllocServiceID( alloc.ID, task.Name, expectedSvc), expectedSvc.Checks[0]) actual = scHook.newScriptChecks() diff --git a/client/allocrunner/taskrunner/service_hook.go b/client/allocrunner/taskrunner/service_hook.go index ecc022c40f80..9684317e5411 100644 --- a/client/allocrunner/taskrunner/service_hook.go +++ b/client/allocrunner/taskrunner/service_hook.go @@ -8,7 +8,7 @@ import ( log "github.com/hashicorp/go-hclog" "github.com/hashicorp/nomad/client/allocrunner/interfaces" tinterfaces "github.com/hashicorp/nomad/client/allocrunner/taskrunner/interfaces" - "github.com/hashicorp/nomad/client/consul" + "github.com/hashicorp/nomad/client/serviceregistration" "github.com/hashicorp/nomad/client/taskenv" agentconsul "github.com/hashicorp/nomad/command/agent/consul" "github.com/hashicorp/nomad/nomad/structs" @@ -24,7 +24,7 @@ var _ interfaces.TaskUpdateHook = &serviceHook{} type serviceHookConfig struct { alloc *structs.Allocation task *structs.Task - consulServices consul.ConsulServiceAPI + consulServices serviceregistration.Handler consulNamespace string // Restarter is a subset of the TaskLifecycle interface @@ -37,7 +37,7 @@ type serviceHook struct { allocID string taskName string consulNamespace string - consulServices consul.ConsulServiceAPI + consulServices serviceregistration.Handler restarter agentconsul.WorkloadRestarter logger log.Logger @@ -193,21 +193,21 @@ func (h *serviceHook) Stop(ctx context.Context, req *interfaces.TaskStopRequest, return nil } -func (h *serviceHook) getWorkloadServices() *agentconsul.WorkloadServices { +func (h *serviceHook) getWorkloadServices() *serviceregistration.WorkloadServices { // Interpolate with the task's environment interpolatedServices := taskenv.InterpolateServices(h.taskEnv, h.services) // Create task services struct with request's driver metadata - return &agentconsul.WorkloadServices{ - AllocID: h.allocID, - Task: h.taskName, - ConsulNamespace: h.consulNamespace, - Restarter: h.restarter, - Services: interpolatedServices, - DriverExec: h.driverExec, - DriverNetwork: h.driverNet, - Networks: h.networks, - Canary: h.canary, - Ports: h.ports, + return &serviceregistration.WorkloadServices{ + AllocID: h.allocID, + Task: h.taskName, + Namespace: h.consulNamespace, + Restarter: h.restarter, + Services: interpolatedServices, + DriverExec: h.driverExec, + DriverNetwork: h.driverNet, + Networks: h.networks, + Canary: h.canary, + Ports: h.ports, } } diff --git a/client/allocrunner/taskrunner/service_hook_test.go b/client/allocrunner/taskrunner/service_hook_test.go index bdae6bfd0c96..4489e722008c 100644 --- a/client/allocrunner/taskrunner/service_hook_test.go +++ b/client/allocrunner/taskrunner/service_hook_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/hashicorp/nomad/client/allocrunner/interfaces" - "github.com/hashicorp/nomad/client/consul" + regMock "github.com/hashicorp/nomad/client/serviceregistration/mock" "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/nomad/mock" "github.com/stretchr/testify/require" @@ -20,7 +20,7 @@ var _ interfaces.TaskUpdateHook = (*serviceHook)(nil) func TestUpdate_beforePoststart(t *testing.T) { alloc := mock.Alloc() logger := testlog.HCLogger(t) - c := consul.NewMockConsulServiceClient(t, logger) + c := regMock.NewServiceRegistrationHandler(logger) hook := newServiceHook(serviceHookConfig{ alloc: alloc, @@ -56,7 +56,7 @@ func Test_serviceHook_multipleDeRegisterCall(t *testing.T) { alloc := mock.Alloc() logger := testlog.HCLogger(t) - c := consul.NewMockConsulServiceClient(t, logger) + c := regMock.NewServiceRegistrationHandler(logger) hook := newServiceHook(serviceHookConfig{ alloc: alloc, diff --git a/client/allocrunner/taskrunner/task_runner.go b/client/allocrunner/taskrunner/task_runner.go index 3cf751cf12b4..41938aea33fd 100644 --- a/client/allocrunner/taskrunner/task_runner.go +++ b/client/allocrunner/taskrunner/task_runner.go @@ -25,6 +25,7 @@ import ( cinterfaces "github.com/hashicorp/nomad/client/interfaces" "github.com/hashicorp/nomad/client/pluginmanager/csimanager" "github.com/hashicorp/nomad/client/pluginmanager/drivermanager" + "github.com/hashicorp/nomad/client/serviceregistration" cstate "github.com/hashicorp/nomad/client/state" cstructs "github.com/hashicorp/nomad/client/structs" "github.com/hashicorp/nomad/client/taskenv" @@ -166,7 +167,7 @@ type TaskRunner struct { // consulClient is the client used by the consul service hook for // registering services and checks - consulServiceClient consul.ConsulServiceAPI + consulServiceClient serviceregistration.Handler // consulProxiesClient is the client used by the envoy version hook for // asking consul what version of envoy nomad should inject into the connect @@ -248,7 +249,7 @@ type Config struct { Logger log.Logger // Consul is the client to use for managing Consul service registrations - Consul consul.ConsulServiceAPI + Consul serviceregistration.Handler // ConsulProxies is the client to use for looking up supported envoy versions // from Consul. diff --git a/client/allocrunner/taskrunner/task_runner_test.go b/client/allocrunner/taskrunner/task_runner_test.go index 77741c8024e8..139808d913e4 100644 --- a/client/allocrunner/taskrunner/task_runner_test.go +++ b/client/allocrunner/taskrunner/task_runner_test.go @@ -24,6 +24,7 @@ import ( consulapi "github.com/hashicorp/nomad/client/consul" "github.com/hashicorp/nomad/client/devicemanager" "github.com/hashicorp/nomad/client/pluginmanager/drivermanager" + regMock "github.com/hashicorp/nomad/client/serviceregistration/mock" cstate "github.com/hashicorp/nomad/client/state" ctestutil "github.com/hashicorp/nomad/client/testutil" "github.com/hashicorp/nomad/client/vaultclient" @@ -109,7 +110,7 @@ func testTaskRunnerConfig(t *testing.T, alloc *structs.Allocation, taskName stri Task: thisTask, TaskDir: taskDir, Logger: clientConf.Logger, - Consul: consulapi.NewMockConsulServiceClient(t, logger), + Consul: regMock.NewServiceRegistrationHandler(logger), ConsulSI: consulapi.NewMockServiceIdentitiesClient(), Vault: vaultclient.NewMockVaultClient(), StateDB: cstate.NoopDB{}, @@ -939,7 +940,7 @@ func TestTaskRunner_ShutdownDelay(t *testing.T) { tr, conf, cleanup := runTestTaskRunner(t, alloc, task.Name) defer cleanup() - mockConsul := conf.Consul.(*consulapi.MockConsulServiceClient) + mockConsul := conf.Consul.(*regMock.ServiceRegistrationHandler) // Wait for the task to start testWaitForTaskToStart(t, tr) @@ -1027,7 +1028,7 @@ func TestTaskRunner_NoShutdownDelay(t *testing.T) { tr, conf, cleanup := runTestTaskRunner(t, alloc, task.Name) defer cleanup() - mockConsul := conf.Consul.(*consulapi.MockConsulServiceClient) + mockConsul := conf.Consul.(*regMock.ServiceRegistrationHandler) testWaitForTaskToStart(t, tr) @@ -2479,7 +2480,7 @@ func TestTaskRunner_UnregisterConsul_Retries(t *testing.T) { state := tr.TaskState() require.Equal(t, structs.TaskStateDead, state.State) - consul := conf.Consul.(*consulapi.MockConsulServiceClient) + consul := conf.Consul.(*regMock.ServiceRegistrationHandler) consulOps := consul.GetOps() require.Len(t, consulOps, 4) diff --git a/client/allocrunner/testing.go b/client/allocrunner/testing.go index da850719ef4e..4fde2cd02348 100644 --- a/client/allocrunner/testing.go +++ b/client/allocrunner/testing.go @@ -14,6 +14,7 @@ import ( "github.com/hashicorp/nomad/client/consul" "github.com/hashicorp/nomad/client/devicemanager" "github.com/hashicorp/nomad/client/pluginmanager/drivermanager" + "github.com/hashicorp/nomad/client/serviceregistration/mock" "github.com/hashicorp/nomad/client/state" "github.com/hashicorp/nomad/client/vaultclient" "github.com/hashicorp/nomad/nomad/structs" @@ -62,7 +63,7 @@ func testAllocRunnerConfig(t *testing.T, alloc *structs.Allocation) (*Config, fu Logger: clientConf.Logger, ClientConfig: clientConf, StateDB: state.NoopDB{}, - Consul: consul.NewMockConsulServiceClient(t, clientConf.Logger), + Consul: mock.NewServiceRegistrationHandler(clientConf.Logger), ConsulSI: consul.NewMockServiceIdentitiesClient(), Vault: vaultclient.NewMockVaultClient(), StateUpdater: &MockStateUpdater{}, diff --git a/client/client.go b/client/client.go index 826453170d8c..c70a1547daac 100644 --- a/client/client.go +++ b/client/client.go @@ -39,6 +39,7 @@ import ( "github.com/hashicorp/nomad/client/pluginmanager/csimanager" "github.com/hashicorp/nomad/client/pluginmanager/drivermanager" "github.com/hashicorp/nomad/client/servers" + "github.com/hashicorp/nomad/client/serviceregistration" "github.com/hashicorp/nomad/client/state" "github.com/hashicorp/nomad/client/stats" cstructs "github.com/hashicorp/nomad/client/structs" @@ -227,7 +228,7 @@ type Client struct { // consulService is Nomad's custom Consul client for managing services // and checks. - consulService consulApi.ConsulServiceAPI + consulService serviceregistration.Handler // consulProxies is Nomad's custom Consul client for looking up supported // envoy versions @@ -322,7 +323,7 @@ var ( // registered via https://golang.org/pkg/net/rpc/#Server.RegisterName in place // of the client's normal RPC handlers. This allows server tests to override // the behavior of the client. -func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulProxies consulApi.SupportedProxiesAPI, consulService consulApi.ConsulServiceAPI, rpcs map[string]interface{}) (*Client, error) { +func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulProxies consulApi.SupportedProxiesAPI, consulService serviceregistration.Handler, rpcs map[string]interface{}) (*Client, error) { // Create the tls wrapper var tlsWrap tlsutil.RegionWrapper if cfg.TLSConfig.EnableRPC { diff --git a/client/client_test.go b/client/client_test.go index d27ab0ed0619..30fbf384bae1 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -14,8 +14,9 @@ import ( memdb "github.com/hashicorp/go-memdb" trstate "github.com/hashicorp/nomad/client/allocrunner/taskrunner/state" "github.com/hashicorp/nomad/client/config" - consulApi "github.com/hashicorp/nomad/client/consul" "github.com/hashicorp/nomad/client/fingerprint" + regMock "github.com/hashicorp/nomad/client/serviceregistration/mock" + cstate "github.com/hashicorp/nomad/client/state" "github.com/hashicorp/nomad/command/agent/consul" "github.com/hashicorp/nomad/helper/pluginutils/catalog" "github.com/hashicorp/nomad/helper/pluginutils/singleton" @@ -29,8 +30,6 @@ import ( psstructs "github.com/hashicorp/nomad/plugins/shared/structs" "github.com/hashicorp/nomad/testutil" "github.com/stretchr/testify/assert" - - cstate "github.com/hashicorp/nomad/client/state" "github.com/stretchr/testify/require" ) @@ -615,7 +614,7 @@ func TestClient_SaveRestoreState(t *testing.T) { logger := testlog.HCLogger(t) c1.config.Logger = logger consulCatalog := consul.NewMockCatalog(logger) - mockService := consulApi.NewMockConsulServiceClient(t, logger) + mockService := regMock.NewServiceRegistrationHandler(logger) // ensure we use non-shutdown driver instances c1.config.PluginLoader = catalog.TestPluginLoaderWithOptions(t, "", c1.config.Options, nil) diff --git a/client/consul/consul.go b/client/consul/consul.go index 9459e375d710..05f77418c8f7 100644 --- a/client/consul/consul.go +++ b/client/consul/consul.go @@ -1,33 +1,9 @@ package consul import ( - "github.com/hashicorp/nomad/command/agent/consul" "github.com/hashicorp/nomad/nomad/structs" ) -// ConsulServiceAPI is the interface the Nomad Client uses to register and -// remove services and checks from Consul. -// -// ACL requirements -// - service:write -type ConsulServiceAPI interface { - // RegisterWorkload with Consul. Adds all service entries and checks to Consul. - RegisterWorkload(*consul.WorkloadServices) error - - // RemoveWorkload from Consul. Removes all service entries and checks. - RemoveWorkload(*consul.WorkloadServices) - - // UpdateWorkload in Consul. Does not alter the service if only checks have - // changed. - UpdateWorkload(old, newTask *consul.WorkloadServices) error - - // AllocRegistrations returns the registrations for the given allocation. - AllocRegistrations(allocID string) (*consul.AllocRegistration, error) - - // UpdateTTL is used to update the TTL of a check. - UpdateTTL(id, namespace, output, status string) error -} - // TokenDeriverFunc takes an allocation and a set of tasks and derives a // service identity token for each. Requests go through nomad server. type TokenDeriverFunc func(*structs.Allocation, []string) (map[string]string, error) diff --git a/client/consul/consul_testing.go b/client/consul/consul_testing.go deleted file mode 100644 index 605bf96cfc8d..000000000000 --- a/client/consul/consul_testing.go +++ /dev/null @@ -1,113 +0,0 @@ -package consul - -import ( - "fmt" - "sync" - "time" - - log "github.com/hashicorp/go-hclog" - "github.com/hashicorp/nomad/command/agent/consul" - testing "github.com/mitchellh/go-testing-interface" -) - -// MockConsulOp represents the register/deregister operations. -type MockConsulOp struct { - Op string // add, remove, or update - AllocID string - Name string // task or group name - OccurredAt time.Time -} - -func NewMockConsulOp(op, allocID, name string) MockConsulOp { - switch op { - case "add", "remove", "update", "alloc_registrations", - "add_group", "remove_group", "update_group", "update_ttl": - default: - panic(fmt.Errorf("invalid consul op: %s", op)) - } - return MockConsulOp{ - Op: op, - AllocID: allocID, - Name: name, - OccurredAt: time.Now(), - } -} - -// MockConsulServiceClient implements the ConsulServiceAPI interface to record -// and log task registration/deregistration. -type MockConsulServiceClient struct { - ops []MockConsulOp - mu sync.Mutex - - logger log.Logger - - // AllocRegistrationsFn allows injecting return values for the - // AllocRegistrations function. - AllocRegistrationsFn func(allocID string) (*consul.AllocRegistration, error) -} - -func NewMockConsulServiceClient(t testing.T, logger log.Logger) *MockConsulServiceClient { - logger = logger.Named("mock_consul") - m := MockConsulServiceClient{ - ops: make([]MockConsulOp, 0, 20), - logger: logger, - } - return &m -} - -func (m *MockConsulServiceClient) UpdateWorkload(old, newSvcs *consul.WorkloadServices) error { - m.mu.Lock() - defer m.mu.Unlock() - m.logger.Trace("UpdateWorkload", "alloc_id", newSvcs.AllocID, "name", newSvcs.Name(), - "old_services", len(old.Services), "new_services", len(newSvcs.Services), - ) - m.ops = append(m.ops, NewMockConsulOp("update", newSvcs.AllocID, newSvcs.Name())) - return nil -} - -func (m *MockConsulServiceClient) RegisterWorkload(svcs *consul.WorkloadServices) error { - m.mu.Lock() - defer m.mu.Unlock() - m.logger.Trace("RegisterWorkload", "alloc_id", svcs.AllocID, "name", svcs.Name(), - "services", len(svcs.Services), - ) - m.ops = append(m.ops, NewMockConsulOp("add", svcs.AllocID, svcs.Name())) - return nil -} - -func (m *MockConsulServiceClient) RemoveWorkload(svcs *consul.WorkloadServices) { - m.mu.Lock() - defer m.mu.Unlock() - m.logger.Trace("RemoveWorkload", "alloc_id", svcs.AllocID, "name", svcs.Name(), - "services", len(svcs.Services), - ) - m.ops = append(m.ops, NewMockConsulOp("remove", svcs.AllocID, svcs.Name())) -} - -func (m *MockConsulServiceClient) AllocRegistrations(allocID string) (*consul.AllocRegistration, error) { - m.mu.Lock() - defer m.mu.Unlock() - m.logger.Trace("AllocRegistrations", "alloc_id", allocID) - m.ops = append(m.ops, NewMockConsulOp("alloc_registrations", allocID, "")) - - if m.AllocRegistrationsFn != nil { - return m.AllocRegistrationsFn(allocID) - } - - return nil, nil -} - -func (m *MockConsulServiceClient) UpdateTTL(checkID, namespace, output, status string) error { - // TODO(tgross): this method is here so we can implement the - // interface but the locking we need for testing creates a lot - // of opportunities for deadlocks in testing that will never - // appear in live code. - m.logger.Trace("UpdateTTL", "check_id", checkID, "namespace", namespace, "status", status) - return nil -} - -func (m *MockConsulServiceClient) GetOps() []MockConsulOp { - m.mu.Lock() - defer m.mu.Unlock() - return m.ops -} diff --git a/client/serviceregistration/address.go b/client/serviceregistration/address.go new file mode 100644 index 000000000000..4a67c94ab957 --- /dev/null +++ b/client/serviceregistration/address.go @@ -0,0 +1,136 @@ +package serviceregistration + +import ( + "fmt" + "strconv" + + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/plugins/drivers" +) + +// GetAddress returns the IP and port to use for a service or check. If no port +// label is specified (an empty value), zero values are returned because no +// address could be resolved. +func GetAddress( + addrMode, portLabel string, networks structs.Networks, driverNet *drivers.DriverNetwork, + ports structs.AllocatedPorts, netStatus *structs.AllocNetworkStatus) (string, int, error) { + + switch addrMode { + case structs.AddressModeAuto: + if driverNet.Advertise() { + addrMode = structs.AddressModeDriver + } else { + addrMode = structs.AddressModeHost + } + return GetAddress(addrMode, portLabel, networks, driverNet, ports, netStatus) + case structs.AddressModeHost: + if portLabel == "" { + if len(networks) != 1 { + // If no networks are specified return zero + // values. Consul will advertise the host IP + // with no port. This is the pre-0.7.1 behavior + // some people rely on. + return "", 0, nil + } + + return networks[0].IP, 0, nil + } + + // Default path: use host ip:port + // Try finding port in the AllocatedPorts struct first + // Check in Networks struct for backwards compatibility if not found + mapping, ok := ports.Get(portLabel) + if !ok { + mapping = networks.Port(portLabel) + if mapping.Value > 0 { + return mapping.HostIP, mapping.Value, nil + } + + // If port isn't a label, try to parse it as a literal port number + port, err := strconv.Atoi(portLabel) + if err != nil { + // Don't include Atoi error message as user likely + // never intended it to be a numeric and it creates a + // confusing error message + return "", 0, fmt.Errorf("invalid port %q: port label not found", portLabel) + } + if port <= 0 { + return "", 0, fmt.Errorf("invalid port: %q: port must be >0", portLabel) + } + + // A number was given which will use the Consul agent's address and the given port + // Returning a blank string as an address will use the Consul agent's address + return "", port, nil + } + return mapping.HostIP, mapping.Value, nil + + case structs.AddressModeDriver: + // Require a driver network if driver address mode is used + if driverNet == nil { + return "", 0, fmt.Errorf(`cannot use address_mode="driver": no driver network exists`) + } + + // If no port label is specified just return the IP + if portLabel == "" { + return driverNet.IP, 0, nil + } + + // If the port is a label, use the driver's port (not the host's) + if port, ok := ports.Get(portLabel); ok { + return driverNet.IP, port.To, nil + } + + // Check if old style driver portmap is used + if port, ok := driverNet.PortMap[portLabel]; ok { + return driverNet.IP, port, nil + } + + // If port isn't a label, try to parse it as a literal port number + port, err := strconv.Atoi(portLabel) + if err != nil { + // Don't include Atoi error message as user likely + // never intended it to be a numeric and it creates a + // confusing error message + return "", 0, fmt.Errorf("invalid port label %q: port labels in driver address_mode must be numeric or in the driver's port map", portLabel) + } + if port <= 0 { + return "", 0, fmt.Errorf("invalid port: %q: port must be >0", portLabel) + } + + return driverNet.IP, port, nil + + case structs.AddressModeAlloc: + if netStatus == nil { + return "", 0, fmt.Errorf(`cannot use address_mode="alloc": no allocation network status reported`) + } + + // If no port label is specified just return the IP + if portLabel == "" { + return netStatus.Address, 0, nil + } + + // If port is a label and is found then return it + if port, ok := ports.Get(portLabel); ok { + // Use port.To value unless not set + if port.To > 0 { + return netStatus.Address, port.To, nil + } + return netStatus.Address, port.Value, nil + } + + // Check if port is a literal number + port, err := strconv.Atoi(portLabel) + if err != nil { + // User likely specified wrong port label here + return "", 0, fmt.Errorf("invalid port %q: port label not found or is not numeric", portLabel) + } + if port <= 0 { + return "", 0, fmt.Errorf("invalid port: %q: port must be >0", portLabel) + } + return netStatus.Address, port, nil + + default: + // Shouldn't happen due to validation, but enforce invariants + return "", 0, fmt.Errorf("invalid address mode %q", addrMode) + } +} diff --git a/client/serviceregistration/address_test.go b/client/serviceregistration/address_test.go new file mode 100644 index 000000000000..fbad6acd1022 --- /dev/null +++ b/client/serviceregistration/address_test.go @@ -0,0 +1,361 @@ +package serviceregistration + +import ( + "testing" + + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/plugins/drivers" + "github.com/stretchr/testify/require" +) + +func Test_GetAddress(t *testing.T) { + const HostIP = "127.0.0.1" + + testCases := []struct { + name string + + // Parameters + mode string + portLabel string + host map[string]int // will be converted to structs.Networks + driver *drivers.DriverNetwork + ports structs.AllocatedPorts + status *structs.AllocNetworkStatus + + // Results + expectedIP string + expectedPort int + expectedErr string + }{ + // Valid Configurations + { + name: "ExampleService", + mode: structs.AddressModeAuto, + portLabel: "db", + host: map[string]int{"db": 12435}, + driver: &drivers.DriverNetwork{ + PortMap: map[string]int{"db": 6379}, + IP: "10.1.2.3", + }, + expectedIP: HostIP, + expectedPort: 12435, + }, + { + name: "host", + mode: structs.AddressModeHost, + portLabel: "db", + host: map[string]int{"db": 12345}, + driver: &drivers.DriverNetwork{ + PortMap: map[string]int{"db": 6379}, + IP: "10.1.2.3", + }, + expectedIP: HostIP, + expectedPort: 12345, + }, + { + name: "driver", + mode: structs.AddressModeDriver, + portLabel: "db", + host: map[string]int{"db": 12345}, + driver: &drivers.DriverNetwork{ + PortMap: map[string]int{"db": 6379}, + IP: "10.1.2.3", + }, + expectedIP: "10.1.2.3", + expectedPort: 6379, + }, + { + name: "AutoDriver", + mode: structs.AddressModeAuto, + portLabel: "db", + host: map[string]int{"db": 12345}, + driver: &drivers.DriverNetwork{ + PortMap: map[string]int{"db": 6379}, + IP: "10.1.2.3", + AutoAdvertise: true, + }, + expectedIP: "10.1.2.3", + expectedPort: 6379, + }, + { + name: "DriverCustomPort", + mode: structs.AddressModeDriver, + portLabel: "7890", + host: map[string]int{"db": 12345}, + driver: &drivers.DriverNetwork{ + PortMap: map[string]int{"db": 6379}, + IP: "10.1.2.3", + }, + expectedIP: "10.1.2.3", + expectedPort: 7890, + }, + + // Invalid Configurations + { + name: "DriverWithoutNetwork", + mode: structs.AddressModeDriver, + portLabel: "db", + host: map[string]int{"db": 12345}, + driver: nil, + expectedErr: "no driver network exists", + }, + { + name: "DriverBadPort", + mode: structs.AddressModeDriver, + portLabel: "bad-port-label", + host: map[string]int{"db": 12345}, + driver: &drivers.DriverNetwork{ + PortMap: map[string]int{"db": 6379}, + IP: "10.1.2.3", + }, + expectedErr: "invalid port", + }, + { + name: "DriverZeroPort", + mode: structs.AddressModeDriver, + portLabel: "0", + driver: &drivers.DriverNetwork{ + IP: "10.1.2.3", + }, + expectedErr: "invalid port", + }, + { + name: "HostBadPort", + mode: structs.AddressModeHost, + portLabel: "bad-port-label", + expectedErr: "invalid port", + }, + { + name: "InvalidMode", + mode: "invalid-mode", + portLabel: "80", + expectedErr: "invalid address mode", + }, + { + name: "NoPort_AutoMode", + mode: structs.AddressModeAuto, + expectedIP: HostIP, + }, + { + name: "NoPort_HostMode", + mode: structs.AddressModeHost, + expectedIP: HostIP, + }, + { + name: "NoPort_DriverMode", + mode: structs.AddressModeDriver, + driver: &drivers.DriverNetwork{ + IP: "10.1.2.3", + }, + expectedIP: "10.1.2.3", + }, + + // Scenarios using port 0.12 networking fields (NetworkStatus, AllocatedPortMapping) + { + name: "ExampleServer_withAllocatedPorts", + mode: structs.AddressModeAuto, + portLabel: "db", + ports: []structs.AllocatedPortMapping{ + { + Label: "db", + Value: 12435, + To: 6379, + HostIP: HostIP, + }, + }, + status: &structs.AllocNetworkStatus{ + InterfaceName: "eth0", + Address: "172.26.0.1", + }, + expectedIP: HostIP, + expectedPort: 12435, + }, + { + name: "Host_withAllocatedPorts", + mode: structs.AddressModeHost, + portLabel: "db", + ports: []structs.AllocatedPortMapping{ + { + Label: "db", + Value: 12345, + To: 6379, + HostIP: HostIP, + }, + }, + status: &structs.AllocNetworkStatus{ + InterfaceName: "eth0", + Address: "172.26.0.1", + }, + expectedIP: HostIP, + expectedPort: 12345, + }, + { + name: "Driver_withAllocatedPorts", + mode: structs.AddressModeDriver, + portLabel: "db", + ports: []structs.AllocatedPortMapping{ + { + Label: "db", + Value: 12345, + To: 6379, + HostIP: HostIP, + }, + }, + driver: &drivers.DriverNetwork{ + IP: "10.1.2.3", + }, + status: &structs.AllocNetworkStatus{ + InterfaceName: "eth0", + Address: "172.26.0.1", + }, + expectedIP: "10.1.2.3", + expectedPort: 6379, + }, + { + name: "AutoDriver_withAllocatedPorts", + mode: structs.AddressModeAuto, + portLabel: "db", + ports: []structs.AllocatedPortMapping{ + { + Label: "db", + Value: 12345, + To: 6379, + HostIP: HostIP, + }, + }, + driver: &drivers.DriverNetwork{ + IP: "10.1.2.3", + AutoAdvertise: true, + }, + status: &structs.AllocNetworkStatus{ + InterfaceName: "eth0", + Address: "172.26.0.1", + }, + expectedIP: "10.1.2.3", + expectedPort: 6379, + }, + { + name: "DriverCustomPort_withAllocatedPorts", + mode: structs.AddressModeDriver, + portLabel: "7890", + ports: []structs.AllocatedPortMapping{ + { + Label: "db", + Value: 12345, + To: 6379, + HostIP: HostIP, + }, + }, + driver: &drivers.DriverNetwork{ + IP: "10.1.2.3", + }, + status: &structs.AllocNetworkStatus{ + InterfaceName: "eth0", + Address: "172.26.0.1", + }, + expectedIP: "10.1.2.3", + expectedPort: 7890, + }, + { + name: "Host_MultiHostInterface", + mode: structs.AddressModeAuto, + portLabel: "db", + ports: []structs.AllocatedPortMapping{ + { + Label: "db", + Value: 12345, + To: 6379, + HostIP: "127.0.0.100", + }, + }, + status: &structs.AllocNetworkStatus{ + InterfaceName: "eth0", + Address: "172.26.0.1", + }, + expectedIP: "127.0.0.100", + expectedPort: 12345, + }, + { + name: "Alloc", + mode: structs.AddressModeAlloc, + portLabel: "db", + ports: []structs.AllocatedPortMapping{ + { + Label: "db", + Value: 12345, + To: 6379, + HostIP: HostIP, + }, + }, + status: &structs.AllocNetworkStatus{ + InterfaceName: "eth0", + Address: "172.26.0.1", + }, + expectedIP: "172.26.0.1", + expectedPort: 6379, + }, + { + name: "Alloc no to value", + mode: structs.AddressModeAlloc, + portLabel: "db", + ports: []structs.AllocatedPortMapping{ + { + Label: "db", + Value: 12345, + HostIP: HostIP, + }, + }, + status: &structs.AllocNetworkStatus{ + InterfaceName: "eth0", + Address: "172.26.0.1", + }, + expectedIP: "172.26.0.1", + expectedPort: 12345, + }, + { + name: "AllocCustomPort", + mode: structs.AddressModeAlloc, + portLabel: "6379", + status: &structs.AllocNetworkStatus{ + InterfaceName: "eth0", + Address: "172.26.0.1", + }, + expectedIP: "172.26.0.1", + expectedPort: 6379, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + + // Convert host port map into a structs.Networks. + networks := []*structs.NetworkResource{ + { + IP: HostIP, + ReservedPorts: make([]structs.Port, len(tc.host)), + }, + } + + i := 0 + for label, port := range tc.host { + networks[0].ReservedPorts[i].Label = label + networks[0].ReservedPorts[i].Value = port + i++ + } + + // Run the GetAddress function. + actualIP, actualPort, actualErr := GetAddress( + tc.mode, tc.portLabel, networks, tc.driver, tc.ports, tc.status) + + // Assert the results + require.Equal(t, tc.expectedIP, actualIP, "IP mismatch") + require.Equal(t, tc.expectedPort, actualPort, "Port mismatch") + if tc.expectedErr == "" { + require.Nil(t, actualErr) + } else { + require.Error(t, actualErr) + require.Contains(t, actualErr.Error(), tc.expectedErr) + } + }) + } +} diff --git a/client/serviceregistration/id.go b/client/serviceregistration/id.go new file mode 100644 index 000000000000..edcabd4c6f69 --- /dev/null +++ b/client/serviceregistration/id.go @@ -0,0 +1,27 @@ +package serviceregistration + +import ( + "fmt" + + "github.com/hashicorp/nomad/nomad/structs" +) + +const ( + // nomadServicePrefix is the prefix that scopes all Nomad registered + // services (both agent and task entries). + nomadServicePrefix = "_nomad" + + // nomadTaskPrefix is the prefix that scopes Nomad registered services + // for tasks. + nomadTaskPrefix = nomadServicePrefix + "-task-" +) + +// MakeAllocServiceID creates a unique ID for identifying an alloc service in +// a service registration provider. Both Nomad and Consul solutions use the +// same ID format to provide consistency. +// +// Example Service ID: _nomad-task-b4e61df9-b095-d64e-f241-23860da1375f-redis-http-http +func MakeAllocServiceID(allocID, taskName string, service *structs.Service) string { + return fmt.Sprintf("%s%s-%s-%s-%s", + nomadTaskPrefix, allocID, taskName, service.Name, service.PortLabel) +} diff --git a/client/serviceregistration/id_test.go b/client/serviceregistration/id_test.go new file mode 100644 index 000000000000..b41085a9fd8e --- /dev/null +++ b/client/serviceregistration/id_test.go @@ -0,0 +1,36 @@ +package serviceregistration + +import ( + "testing" + + "github.com/hashicorp/nomad/nomad/structs" + "github.com/stretchr/testify/require" +) + +func Test_MakeAllocServiceID(t *testing.T) { + testCases := []struct { + inputAllocID string + inputTaskName string + inputService *structs.Service + expectedOutput string + name string + }{ + { + inputAllocID: "7ac7c672-1824-6f06-644c-4c249e1578b9", + inputTaskName: "cache", + inputService: &structs.Service{ + Name: "redis", + PortLabel: "db", + }, + expectedOutput: "_nomad-task-7ac7c672-1824-6f06-644c-4c249e1578b9-cache-redis-db", + name: "generic 1", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualOutput := MakeAllocServiceID(tc.inputAllocID, tc.inputTaskName, tc.inputService) + require.Equal(t, tc.expectedOutput, actualOutput) + }) + } +} diff --git a/client/serviceregistration/mock/mock.go b/client/serviceregistration/mock/mock.go new file mode 100644 index 000000000000..899c4f8246fd --- /dev/null +++ b/client/serviceregistration/mock/mock.go @@ -0,0 +1,125 @@ +package mock + +import ( + "fmt" + "sync" + "time" + + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/nomad/client/serviceregistration" +) + +// Ensure that the mock handler implements the service registration handler +// interface. +var _ serviceregistration.Handler = (*ServiceRegistrationHandler)(nil) + +// ServiceRegistrationHandler is the mock implementation of the +// serviceregistration.Handler interface and can be used for testing. +type ServiceRegistrationHandler struct { + log hclog.Logger + + // ops tracks the requested operations by the caller during the entire + // lifecycle of the ServiceRegistrationHandler. The mutex should be used + // whenever interacting with this. + mu sync.Mutex + ops []Operation + + // AllocRegistrationsFn allows injecting return values for the + // AllocRegistrations function. + AllocRegistrationsFn func(allocID string) (*serviceregistration.AllocRegistration, error) +} + +// NewServiceRegistrationHandler returns a ready to use +// ServiceRegistrationHandler for testing. +func NewServiceRegistrationHandler(log hclog.Logger) *ServiceRegistrationHandler { + return &ServiceRegistrationHandler{ + ops: make([]Operation, 0, 20), + log: log.Named("mock_service_registration"), + } +} + +func (h *ServiceRegistrationHandler) RegisterWorkload(services *serviceregistration.WorkloadServices) error { + h.mu.Lock() + defer h.mu.Unlock() + + h.log.Trace("RegisterWorkload", "alloc_id", services.AllocID, + "name", services.Name(), "services", len(services.Services)) + + h.ops = append(h.ops, newOperation("add", services.AllocID, services.Name())) + return nil +} + +func (h *ServiceRegistrationHandler) RemoveWorkload(services *serviceregistration.WorkloadServices) { + h.mu.Lock() + defer h.mu.Unlock() + + h.log.Trace("RemoveWorkload", "alloc_id", services.AllocID, + "name", services.Name(), "services", len(services.Services)) + + h.ops = append(h.ops, newOperation("remove", services.AllocID, services.Name())) +} + +func (h *ServiceRegistrationHandler) UpdateWorkload(old, newServices *serviceregistration.WorkloadServices) error { + h.mu.Lock() + defer h.mu.Unlock() + + h.log.Trace("UpdateWorkload", "alloc_id", newServices.AllocID, "name", newServices.Name(), + "old_services", len(old.Services), "new_services", len(newServices.Services)) + + h.ops = append(h.ops, newOperation("update", newServices.AllocID, newServices.Name())) + return nil +} + +func (h *ServiceRegistrationHandler) AllocRegistrations(allocID string) (*serviceregistration.AllocRegistration, error) { + h.mu.Lock() + defer h.mu.Unlock() + + h.log.Trace("AllocRegistrations", "alloc_id", allocID) + h.ops = append(h.ops, newOperation("alloc_registrations", allocID, "")) + + if h.AllocRegistrationsFn != nil { + return h.AllocRegistrationsFn(allocID) + } + return nil, nil +} + +func (h *ServiceRegistrationHandler) UpdateTTL(checkID, namespace, output, status string) error { + // TODO(tgross): this method is here so we can implement the + // interface but the locking we need for testing creates a lot + // of opportunities for deadlocks in testing that will never + // appear in live code. + h.log.Trace("UpdateTTL", "check_id", checkID, "namespace", namespace, "status", status) + return nil +} + +// GetOps returns all stored operations within the handler. +func (h *ServiceRegistrationHandler) GetOps() []Operation { + h.mu.Lock() + defer h.mu.Unlock() + + return h.ops +} + +// Operation represents the register/deregister operations. +type Operation struct { + Op string // add, remove, or update + AllocID string + Name string // task or group name + OccurredAt time.Time +} + +// newOperation generates a new Operation for the given parameters. +func newOperation(op, allocID, name string) Operation { + switch op { + case "add", "remove", "update", "alloc_registrations", + "add_group", "remove_group", "update_group", "update_ttl": + default: + panic(fmt.Errorf("invalid consul op: %s", op)) + } + return Operation{ + Op: op, + AllocID: allocID, + Name: name, + OccurredAt: time.Now(), + } +} diff --git a/client/serviceregistration/service_registration.go b/client/serviceregistration/service_registration.go new file mode 100644 index 000000000000..4981ff1cfebb --- /dev/null +++ b/client/serviceregistration/service_registration.go @@ -0,0 +1,157 @@ +package serviceregistration + +import ( + "context" + + "github.com/hashicorp/consul/api" + "github.com/hashicorp/nomad/helper" + "github.com/hashicorp/nomad/nomad/structs" +) + +// Handler is the interface the Nomad Client uses to register, update and +// remove services and checks from service registration providers. Currently, +// Consul and Nomad are supported providers. +// +// When utilising Consul, the ACL "service:write" is required. It supports all +// functionality and is the OG/GOAT. +// +// When utilising Nomad, the client secret ID is used for authorisation. It +// currently supports service registrations only. +type Handler interface { + + // RegisterWorkload adds all service entries and checks to the backend + // provider. Whilst callers attempt to ensure WorkloadServices.Services is + // not empty before calling this function, implementations should also + // perform this. + RegisterWorkload(workload *WorkloadServices) error + + // RemoveWorkload all service entries and checks from the backend provider + // that are found within the passed WorkloadServices object. Whilst callers + // attempt to ensure WorkloadServices.Services is not empty before calling + // this function, implementations should also perform this. + RemoveWorkload(workload *WorkloadServices) + + // UpdateWorkload removes workload as specified by the old parameter, and + // adds workload as specified by the new parameter. Callers do not perform + // any deduplication on both objects, it is therefore the responsibility of + // the implementation. + UpdateWorkload(old, newTask *WorkloadServices) error + + // AllocRegistrations returns the registrations for the given allocation. + AllocRegistrations(allocID string) (*AllocRegistration, error) + + // UpdateTTL is used to update the TTL of an individual service + // registration check. + UpdateTTL(id, namespace, output, status string) error +} + +// WorkloadRestarter allows the checkWatcher to restart tasks or entire task +// groups. +type WorkloadRestarter interface { + Restart(ctx context.Context, event *structs.TaskEvent, failure bool) error +} + +// AllocRegistration holds the status of services registered for a particular +// allocations by task. +type AllocRegistration struct { + // Tasks maps the name of a task to its registered services and checks. + Tasks map[string]*ServiceRegistrations +} + +// Copy performs a deep copy of the AllocRegistration object. +func (a *AllocRegistration) Copy() *AllocRegistration { + c := &AllocRegistration{ + Tasks: make(map[string]*ServiceRegistrations, len(a.Tasks)), + } + + for k, v := range a.Tasks { + c.Tasks[k] = v.copy() + } + + return c +} + +// NumServices returns the number of registered services. +func (a *AllocRegistration) NumServices() int { + if a == nil { + return 0 + } + + total := 0 + for _, treg := range a.Tasks { + for _, sreg := range treg.Services { + if sreg.Service != nil { + total++ + } + } + } + + return total +} + +// NumChecks returns the number of registered checks. +func (a *AllocRegistration) NumChecks() int { + if a == nil { + return 0 + } + + total := 0 + for _, treg := range a.Tasks { + for _, sreg := range treg.Services { + total += len(sreg.Checks) + } + } + + return total +} + +// ServiceRegistrations holds the status of services registered for a +// particular task or task group. +type ServiceRegistrations struct { + Services map[string]*ServiceRegistration +} + +func (t *ServiceRegistrations) copy() *ServiceRegistrations { + c := &ServiceRegistrations{ + Services: make(map[string]*ServiceRegistration, len(t.Services)), + } + + for k, v := range t.Services { + c.Services[k] = v.copy() + } + + return c +} + +// ServiceRegistration holds the status of a registered Consul Service and its +// Checks. +type ServiceRegistration struct { + // serviceID and checkIDs are internal fields that track just the IDs of the + // services/checks registered in Consul. It is used to materialize the other + // fields when queried. + ServiceID string + CheckIDs map[string]struct{} + + // CheckOnUpdate is a map of checkIDs and the associated OnUpdate value + // from the ServiceCheck It is used to determine how a reported checks + // status should be evaluated. + CheckOnUpdate map[string]string + + // Service is the AgentService registered in Consul. + Service *api.AgentService + + // Checks is the status of the registered checks. + Checks []*api.AgentCheck +} + +func (s *ServiceRegistration) copy() *ServiceRegistration { + // Copy does not copy the external fields but only the internal fields. + // This is so that the caller of AllocRegistrations can not access the + // internal fields and that method uses these fields to populate the + // external fields. + return &ServiceRegistration{ + ServiceID: s.ServiceID, + CheckIDs: helper.CopyMapStringStruct(s.CheckIDs), + CheckOnUpdate: helper.CopyMapStringString(s.CheckOnUpdate), + } +} diff --git a/client/serviceregistration/service_registration_test.go b/client/serviceregistration/service_registration_test.go new file mode 100644 index 000000000000..dc6589886dc6 --- /dev/null +++ b/client/serviceregistration/service_registration_test.go @@ -0,0 +1,53 @@ +package serviceregistration + +import ( + "testing" + + "github.com/hashicorp/nomad/nomad/structs" + "github.com/stretchr/testify/require" +) + +func TestAllocRegistration_Copy(t *testing.T) { + testCases := []struct { + inputAllocRegistration *AllocRegistration + name string + }{ + { + inputAllocRegistration: &AllocRegistration{ + Tasks: map[string]*ServiceRegistrations{}, + }, + name: "empty tasks map", + }, + { + inputAllocRegistration: &AllocRegistration{ + Tasks: map[string]*ServiceRegistrations{ + "cache": { + Services: map[string]*ServiceRegistration{ + "redis-db": { + ServiceID: "service-id-1", + CheckIDs: map[string]struct{}{ + "check-id-1": {}, + "check-id-2": {}, + "check-id-3": {}, + }, + CheckOnUpdate: map[string]string{ + "check-id-1": structs.OnUpdateIgnore, + "check-id-2": structs.OnUpdateRequireHealthy, + "check-id-3": structs.OnUpdateIgnoreWarn, + }, + }, + }, + }, + }, + }, + name: "non-empty tasks map", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualOutput := tc.inputAllocRegistration.Copy() + require.Equal(t, tc.inputAllocRegistration, actualOutput) + }) + } +} diff --git a/client/serviceregistration/workload.go b/client/serviceregistration/workload.go new file mode 100644 index 000000000000..064f4fa06304 --- /dev/null +++ b/client/serviceregistration/workload.go @@ -0,0 +1,95 @@ +package serviceregistration + +import ( + "github.com/hashicorp/nomad/client/allocrunner/taskrunner/interfaces" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/plugins/drivers" +) + +// WorkloadServices describes services defined in either a Task or TaskGroup +// that need to be syncronized with a service registration provider. +type WorkloadServices struct { + AllocID string + + // Group in which the service belongs for a group-level service, or the + // group in which task belongs for a task-level service. + Group string + + // Task in which the service belongs for task-level service. Will be empty + // for a group-level service. + Task string + + // JobID provides additional context for providers regarding which job + // caused this registration. + JobID string + + // Canary indicates whether, or not the allocation is a canary. This is + // used to build the correct tags mapping. + Canary bool + + // Namespace is the provider namespace in which services will be + // registered, if the provider supports this functionality. + Namespace string + + // Restarter allows restarting the task or task group depending on the + // check_restart stanzas. + Restarter WorkloadRestarter + + // Services and checks to register for the task. + Services []*structs.Service + + // Networks from the task's resources stanza. + // TODO: remove and use Ports + Networks structs.Networks + + // NetworkStatus from alloc if network namespace is created. + // Can be nil. + NetworkStatus *structs.AllocNetworkStatus + + // AllocatedPorts is the list of port mappings. + Ports structs.AllocatedPorts + + // DriverExec is the script executor for the task's driver. For group + // services this is nil and script execution is managed by a tasklet in the + // taskrunner script_check_hook. + DriverExec interfaces.ScriptExecutor + + // DriverNetwork is the network specified by the driver and may be nil. + DriverNetwork *drivers.DriverNetwork +} + +// RegistrationProvider identifies the service registration provider for the +// WorkloadServices. +func (ws *WorkloadServices) RegistrationProvider() string { + + // Protect against an empty array; it would be embarrassing to panic here. + if len(ws.Services) == 0 { + return "" + } + + // Note(jrasell): a Nomad task group can only currently utilise a single + // service provider for all services included within it. In the event we + // remove this restriction, this will need to change along which a lot of + // other logic. + return ws.Services[0].Provider +} + +// Copy method for easing tests. +func (ws *WorkloadServices) Copy() *WorkloadServices { + newTS := new(WorkloadServices) + *newTS = *ws + + // Deep copy Services + newTS.Services = make([]*structs.Service, len(ws.Services)) + for i := range ws.Services { + newTS.Services[i] = ws.Services[i].Copy() + } + return newTS +} + +func (ws *WorkloadServices) Name() string { + if ws.Task != "" { + return ws.Task + } + return "group-" + ws.Group +} diff --git a/client/serviceregistration/workload_test.go b/client/serviceregistration/workload_test.go new file mode 100644 index 000000000000..3152e7492bee --- /dev/null +++ b/client/serviceregistration/workload_test.go @@ -0,0 +1,49 @@ +package serviceregistration + +import ( + "testing" + + "github.com/hashicorp/nomad/nomad/structs" + "github.com/stretchr/testify/require" +) + +func TestWorkloadServices_RegistrationProvider(t *testing.T) { + testCases := []struct { + inputWorkloadServices *WorkloadServices + expectedOutput string + name string + }{ + { + inputWorkloadServices: &WorkloadServices{ + Services: nil, + }, + expectedOutput: "", + name: "nil panic check", + }, + { + inputWorkloadServices: &WorkloadServices{ + Services: []*structs.Service{ + {Provider: structs.ServiceProviderNomad}, + }, + }, + expectedOutput: "nomad", + name: "nomad provider", + }, + { + inputWorkloadServices: &WorkloadServices{ + Services: []*structs.Service{ + {Provider: structs.ServiceProviderConsul}, + }, + }, + expectedOutput: "consul", + name: "consul provider", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualOutput := tc.inputWorkloadServices.RegistrationProvider() + require.Equal(t, tc.expectedOutput, actualOutput) + }) + } +} diff --git a/client/state/upgrade_int_test.go b/client/state/upgrade_int_test.go index 96df3fbadce7..22c70581b196 100644 --- a/client/state/upgrade_int_test.go +++ b/client/state/upgrade_int_test.go @@ -14,10 +14,10 @@ import ( "github.com/hashicorp/nomad/client/allocrunner" "github.com/hashicorp/nomad/client/allocwatcher" clientconfig "github.com/hashicorp/nomad/client/config" - "github.com/hashicorp/nomad/client/consul" "github.com/hashicorp/nomad/client/devicemanager" dmstate "github.com/hashicorp/nomad/client/devicemanager/state" "github.com/hashicorp/nomad/client/pluginmanager/drivermanager" + regMock "github.com/hashicorp/nomad/client/serviceregistration/mock" . "github.com/hashicorp/nomad/client/state" "github.com/hashicorp/nomad/client/vaultclient" "github.com/hashicorp/nomad/helper/boltdd" @@ -206,7 +206,7 @@ func checkUpgradedAlloc(t *testing.T, path string, db StateDB, alloc *structs.Al Logger: clientConf.Logger, ClientConfig: clientConf, StateDB: db, - Consul: consul.NewMockConsulServiceClient(t, clientConf.Logger), + Consul: regMock.NewServiceRegistrationHandler(clientConf.Logger), Vault: vaultclient.NewMockVaultClient(), StateUpdater: &allocrunner.MockStateUpdater{}, PrevAllocWatcher: allocwatcher.NoopPrevAlloc{}, diff --git a/client/testing.go b/client/testing.go index 94681f76e18e..564f4273d1f8 100644 --- a/client/testing.go +++ b/client/testing.go @@ -7,9 +7,9 @@ import ( "time" "github.com/hashicorp/nomad/client/config" - consulapi "github.com/hashicorp/nomad/client/consul" "github.com/hashicorp/nomad/client/fingerprint" "github.com/hashicorp/nomad/client/servers" + "github.com/hashicorp/nomad/client/serviceregistration/mock" agentconsul "github.com/hashicorp/nomad/command/agent/consul" "github.com/hashicorp/nomad/helper/pluginutils/catalog" "github.com/hashicorp/nomad/helper/pluginutils/singleton" @@ -53,7 +53,7 @@ func TestClientWithRPCs(t testing.T, cb func(c *config.Config), rpcs map[string] conf.PluginSingletonLoader = singleton.NewSingletonLoader(logger, conf.PluginLoader) } mockCatalog := agentconsul.NewMockCatalog(logger) - mockService := consulapi.NewMockConsulServiceClient(t, logger) + mockService := mock.NewServiceRegistrationHandler(logger) client, err := NewClient(conf, mockCatalog, nil, mockService, rpcs) if err != nil { cleanup() diff --git a/command/agent/consul/group_test.go b/command/agent/consul/group_test.go index a76aac73e1f8..dd7431f7f70f 100644 --- a/command/agent/consul/group_test.go +++ b/command/agent/consul/group_test.go @@ -7,6 +7,7 @@ import ( consulapi "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/sdk/testutil" + "github.com/hashicorp/nomad/client/serviceregistration" "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" @@ -92,7 +93,7 @@ func TestConsul_Connect(t *testing.T) { require.NoError(t, err) require.Len(t, services, 2) - serviceID := MakeAllocServiceID(alloc.ID, "group-"+alloc.TaskGroup, tg.Services[0]) + serviceID := serviceregistration.MakeAllocServiceID(alloc.ID, "group-"+alloc.TaskGroup, tg.Services[0]) connectID := serviceID + "-sidecar-proxy" require.Contains(t, services, serviceID) diff --git a/command/agent/consul/service_client.go b/command/agent/consul/service_client.go index afd06d92edd3..1d888b81e9ca 100644 --- a/command/agent/consul/service_client.go +++ b/command/agent/consul/service_client.go @@ -14,14 +14,13 @@ import ( "time" "github.com/armon/go-metrics" - log "github.com/hashicorp/go-hclog" - "github.com/hashicorp/nomad/helper/envoy" - "github.com/pkg/errors" - "github.com/hashicorp/consul/api" + log "github.com/hashicorp/go-hclog" + "github.com/hashicorp/nomad/client/serviceregistration" "github.com/hashicorp/nomad/helper" + "github.com/hashicorp/nomad/helper/envoy" "github.com/hashicorp/nomad/nomad/structs" - "github.com/hashicorp/nomad/plugins/drivers" + "github.com/pkg/errors" ) const ( @@ -370,109 +369,6 @@ func (o operations) String() string { return fmt.Sprintf("<%d, %d, %d, %d>", len(o.regServices), len(o.regChecks), len(o.deregServices), len(o.deregChecks)) } -// AllocRegistration holds the status of services registered for a particular -// allocations by task. -type AllocRegistration struct { - // Tasks maps the name of a task to its registered services and checks - Tasks map[string]*ServiceRegistrations -} - -func (a *AllocRegistration) copy() *AllocRegistration { - c := &AllocRegistration{ - Tasks: make(map[string]*ServiceRegistrations, len(a.Tasks)), - } - - for k, v := range a.Tasks { - c.Tasks[k] = v.copy() - } - - return c -} - -// NumServices returns the number of registered services -func (a *AllocRegistration) NumServices() int { - if a == nil { - return 0 - } - - total := 0 - for _, treg := range a.Tasks { - for _, sreg := range treg.Services { - if sreg.Service != nil { - total++ - } - } - } - - return total -} - -// NumChecks returns the number of registered checks -func (a *AllocRegistration) NumChecks() int { - if a == nil { - return 0 - } - - total := 0 - for _, treg := range a.Tasks { - for _, sreg := range treg.Services { - total += len(sreg.Checks) - } - } - - return total -} - -// ServiceRegistrations holds the status of services registered for a particular -// task or task group. -type ServiceRegistrations struct { - Services map[string]*ServiceRegistration -} - -func (t *ServiceRegistrations) copy() *ServiceRegistrations { - c := &ServiceRegistrations{ - Services: make(map[string]*ServiceRegistration, len(t.Services)), - } - - for k, v := range t.Services { - c.Services[k] = v.copy() - } - - return c -} - -// ServiceRegistration holds the status of a registered Consul Service and its -// Checks. -type ServiceRegistration struct { - // serviceID and checkIDs are internal fields that track just the IDs of the - // services/checks registered in Consul. It is used to materialize the other - // fields when queried. - serviceID string - checkIDs map[string]struct{} - - // CheckOnUpdate is a map of checkIDs and the associated OnUpdate value - // from the ServiceCheck It is used to determine how a reported checks - // status should be evaluated. - CheckOnUpdate map[string]string - - // Service is the AgentService registered in Consul. - Service *api.AgentService - - // Checks is the status of the registered checks. - Checks []*api.AgentCheck -} - -func (s *ServiceRegistration) copy() *ServiceRegistration { - // Copy does not copy the external fields but only the internal fields. This - // is so that the caller of AllocRegistrations can not access the internal - // fields and that method uses these fields to populate the external fields. - return &ServiceRegistration{ - serviceID: s.serviceID, - checkIDs: helper.CopyMapStringStruct(s.checkIDs), - CheckOnUpdate: helper.CopyMapStringString(s.CheckOnUpdate), - } -} - // ServiceClient handles task and agent service registration with Consul. type ServiceClient struct { agentAPI AgentAPI @@ -503,7 +399,7 @@ type ServiceClient struct { // allocRegistrations stores the services and checks that are registered // with Consul by allocation ID. - allocRegistrations map[string]*AllocRegistration + allocRegistrations map[string]*serviceregistration.AllocRegistration allocRegistrationsLock sync.RWMutex // Nomad agent services and checks that are recorded so they can be removed @@ -550,7 +446,7 @@ func NewServiceClient(agentAPI AgentAPI, namespacesClient *NamespacesClient, log checks: make(map[string]*api.AgentCheckRegistration), explicitlyDeregisteredServices: make(map[string]bool), explicitlyDeregisteredChecks: make(map[string]bool), - allocRegistrations: make(map[string]*AllocRegistration), + allocRegistrations: make(map[string]*serviceregistration.AllocRegistration), agentServices: make(map[string]struct{}), agentChecks: make(map[string]struct{}), checkWatcher: newCheckWatcher(logger, agentAPI, namespacesClient), @@ -1033,14 +929,15 @@ func (c *ServiceClient) RegisterAgent(role string, services []*structs.Service) // serviceRegs creates service registrations, check registrations, and script // checks from a service. It returns a service registration object with the // service and check IDs populated. -func (c *ServiceClient) serviceRegs(ops *operations, service *structs.Service, workload *WorkloadServices) ( - *ServiceRegistration, error) { +func (c *ServiceClient) serviceRegs( + ops *operations, service *structs.Service, workload *serviceregistration.WorkloadServices) ( + *serviceregistration.ServiceRegistration, error) { // Get the services ID - id := MakeAllocServiceID(workload.AllocID, workload.Name(), service) - sreg := &ServiceRegistration{ - serviceID: id, - checkIDs: make(map[string]struct{}, len(service.Checks)), + id := serviceregistration.MakeAllocServiceID(workload.AllocID, workload.Name(), service) + sreg := &serviceregistration.ServiceRegistration{ + ServiceID: id, + CheckIDs: make(map[string]struct{}, len(service.Checks)), CheckOnUpdate: make(map[string]string, len(service.Checks)), } @@ -1051,7 +948,8 @@ func (c *ServiceClient) serviceRegs(ops *operations, service *structs.Service, w } // Determine the address to advertise based on the mode - ip, port, err := getAddress(addrMode, service.PortLabel, workload.Networks, workload.DriverNetwork, workload.Ports, workload.NetworkStatus) + ip, port, err := serviceregistration.GetAddress( + addrMode, service.PortLabel, workload.Networks, workload.DriverNetwork, workload.Ports, workload.NetworkStatus) if err != nil { return nil, fmt.Errorf("unable to get address for service %q: %v", service.Name, err) } @@ -1135,7 +1033,7 @@ func (c *ServiceClient) serviceRegs(ops *operations, service *structs.Service, w Kind: kind, ID: id, Name: service.Name, - Namespace: workload.ConsulNamespace, + Namespace: workload.Namespace, Tags: tags, EnableTagOverride: service.EnableTagOverride, Address: ip, @@ -1152,7 +1050,7 @@ func (c *ServiceClient) serviceRegs(ops *operations, service *structs.Service, w return nil, err } for _, registration := range checkRegs { - sreg.checkIDs[registration.ID] = struct{}{} + sreg.CheckIDs[registration.ID] = struct{}{} ops.regChecks = append(ops.regChecks, registration) } @@ -1161,7 +1059,7 @@ func (c *ServiceClient) serviceRegs(ops *operations, service *structs.Service, w // checkRegs creates check registrations for the given service func (c *ServiceClient) checkRegs(serviceID string, service *structs.Service, - workload *WorkloadServices, sreg *ServiceRegistration) ([]*api.AgentCheckRegistration, error) { + workload *serviceregistration.WorkloadServices, sreg *serviceregistration.ServiceRegistration) ([]*api.AgentCheckRegistration, error) { registrations := make([]*api.AgentCheckRegistration, 0, len(service.Checks)) for _, check := range service.Checks { @@ -1181,14 +1079,15 @@ func (c *ServiceClient) checkRegs(serviceID string, service *structs.Service, } var err error - ip, port, err = getAddress(addrMode, portLabel, workload.Networks, workload.DriverNetwork, workload.Ports, workload.NetworkStatus) + ip, port, err = serviceregistration.GetAddress( + addrMode, portLabel, workload.Networks, workload.DriverNetwork, workload.Ports, workload.NetworkStatus) if err != nil { return nil, fmt.Errorf("error getting address for check %q: %v", check.Name, err) } } checkID := MakeCheckID(serviceID, check) - registration, err := createCheckReg(serviceID, checkID, check, ip, port, workload.ConsulNamespace) + registration, err := createCheckReg(serviceID, checkID, check, ip, port, workload.Namespace) if err != nil { return nil, fmt.Errorf("failed to add check %q: %v", check.Name, err) } @@ -1205,15 +1104,15 @@ func (c *ServiceClient) checkRegs(serviceID string, service *structs.Service, // Checks will always use the IP from the Task struct (host's IP). // // Actual communication with Consul is done asynchronously (see Run). -func (c *ServiceClient) RegisterWorkload(workload *WorkloadServices) error { +func (c *ServiceClient) RegisterWorkload(workload *serviceregistration.WorkloadServices) error { // Fast path numServices := len(workload.Services) if numServices == 0 { return nil } - t := new(ServiceRegistrations) - t.Services = make(map[string]*ServiceRegistration, numServices) + t := new(serviceregistration.ServiceRegistrations) + t.Services = make(map[string]*serviceregistration.ServiceRegistration, numServices) ops := &operations{} for _, service := range workload.Services { @@ -1221,7 +1120,7 @@ func (c *ServiceClient) RegisterWorkload(workload *WorkloadServices) error { if err != nil { return err } - t.Services[sreg.serviceID] = sreg + t.Services[sreg.ServiceID] = sreg } // Add the workload to the allocation's registration @@ -1232,7 +1131,7 @@ func (c *ServiceClient) RegisterWorkload(workload *WorkloadServices) error { // Start watching checks. Done after service registrations are built // since an error building them could leak watches. for _, service := range workload.Services { - serviceID := MakeAllocServiceID(workload.AllocID, workload.Name(), service) + serviceID := serviceregistration.MakeAllocServiceID(workload.AllocID, workload.Name(), service) for _, check := range service.Checks { if check.TriggersRestarts() { checkID := MakeCheckID(serviceID, check) @@ -1247,19 +1146,19 @@ func (c *ServiceClient) RegisterWorkload(workload *WorkloadServices) error { // changed. // // DriverNetwork must not change between invocations for the same allocation. -func (c *ServiceClient) UpdateWorkload(old, newWorkload *WorkloadServices) error { +func (c *ServiceClient) UpdateWorkload(old, newWorkload *serviceregistration.WorkloadServices) error { ops := new(operations) - regs := new(ServiceRegistrations) - regs.Services = make(map[string]*ServiceRegistration, len(newWorkload.Services)) + regs := new(serviceregistration.ServiceRegistrations) + regs.Services = make(map[string]*serviceregistration.ServiceRegistration, len(newWorkload.Services)) newIDs := make(map[string]*structs.Service, len(newWorkload.Services)) for _, s := range newWorkload.Services { - newIDs[MakeAllocServiceID(newWorkload.AllocID, newWorkload.Name(), s)] = s + newIDs[serviceregistration.MakeAllocServiceID(newWorkload.AllocID, newWorkload.Name(), s)] = s } // Loop over existing Services to see if they have been removed for _, existingSvc := range old.Services { - existingID := MakeAllocServiceID(old.AllocID, old.Name(), existingSvc) + existingID := serviceregistration.MakeAllocServiceID(old.AllocID, old.Name(), existingSvc) newSvc, ok := newIDs[existingID] if !ok { @@ -1285,9 +1184,9 @@ func (c *ServiceClient) UpdateWorkload(old, newWorkload *WorkloadServices) error } // Service still exists so add it to the task's registration - sreg := &ServiceRegistration{ - serviceID: existingID, - checkIDs: make(map[string]struct{}, len(newSvc.Checks)), + sreg := &serviceregistration.ServiceRegistration{ + ServiceID: existingID, + CheckIDs: make(map[string]struct{}, len(newSvc.Checks)), CheckOnUpdate: make(map[string]string, len(newSvc.Checks)), } regs.Services[existingID] = sreg @@ -1305,7 +1204,7 @@ func (c *ServiceClient) UpdateWorkload(old, newWorkload *WorkloadServices) error // Check is still required. Remove it from the map so it doesn't get // deleted later. delete(existingChecks, checkID) - sreg.checkIDs[checkID] = struct{}{} + sreg.CheckIDs[checkID] = struct{}{} sreg.CheckOnUpdate[checkID] = check.OnUpdate } @@ -1316,7 +1215,7 @@ func (c *ServiceClient) UpdateWorkload(old, newWorkload *WorkloadServices) error } for _, registration := range checkRegs { - sreg.checkIDs[registration.ID] = struct{}{} + sreg.CheckIDs[registration.ID] = struct{}{} sreg.CheckOnUpdate[registration.ID] = check.OnUpdate ops.regChecks = append(ops.regChecks, registration) } @@ -1345,7 +1244,7 @@ func (c *ServiceClient) UpdateWorkload(old, newWorkload *WorkloadServices) error return err } - regs.Services[sreg.serviceID] = sreg + regs.Services[sreg.ServiceID] = sreg } // Add the task to the allocation's registration @@ -1370,11 +1269,11 @@ func (c *ServiceClient) UpdateWorkload(old, newWorkload *WorkloadServices) error // RemoveWorkload from Consul. Removes all service entries and checks. // // Actual communication with Consul is done asynchronously (see Run). -func (c *ServiceClient) RemoveWorkload(workload *WorkloadServices) { +func (c *ServiceClient) RemoveWorkload(workload *serviceregistration.WorkloadServices) { ops := operations{} for _, service := range workload.Services { - id := MakeAllocServiceID(workload.AllocID, workload.Name(), service) + id := serviceregistration.MakeAllocServiceID(workload.AllocID, workload.Name(), service) ops.deregServices = append(ops.deregServices, id) for _, check := range service.Checks { @@ -1406,7 +1305,7 @@ func normalizeNamespace(namespace string) string { // AllocRegistrations returns the registrations for the given allocation. If the // allocation has no registrations, the response is a nil object. -func (c *ServiceClient) AllocRegistrations(allocID string) (*AllocRegistration, error) { +func (c *ServiceClient) AllocRegistrations(allocID string) (*serviceregistration.AllocRegistration, error) { // Get the internal struct using the lock c.allocRegistrationsLock.RLock() regInternal, ok := c.allocRegistrations[allocID] @@ -1416,7 +1315,7 @@ func (c *ServiceClient) AllocRegistrations(allocID string) (*AllocRegistration, } // Copy so we don't expose internal structs - reg := regInternal.copy() + reg := regInternal.Copy() c.allocRegistrationsLock.RUnlock() // Get the list of all namespaces created so we can iterate them. @@ -1451,7 +1350,7 @@ func (c *ServiceClient) AllocRegistrations(allocID string) (*AllocRegistration, for _, treg := range reg.Tasks { for serviceID, sreg := range treg.Services { sreg.Service = services[serviceID] - for checkID := range sreg.checkIDs { + for checkID := range sreg.CheckIDs { if check, ok := checks[checkID]; ok { sreg.Checks = append(sreg.Checks, check) } @@ -1547,14 +1446,14 @@ func (c *ServiceClient) Shutdown() error { } // addRegistration adds the service registrations for the given allocation. -func (c *ServiceClient) addRegistrations(allocID, taskName string, reg *ServiceRegistrations) { +func (c *ServiceClient) addRegistrations(allocID, taskName string, reg *serviceregistration.ServiceRegistrations) { c.allocRegistrationsLock.Lock() defer c.allocRegistrationsLock.Unlock() alloc, ok := c.allocRegistrations[allocID] if !ok { - alloc = &AllocRegistration{ - Tasks: make(map[string]*ServiceRegistrations), + alloc = &serviceregistration.AllocRegistration{ + Tasks: make(map[string]*serviceregistration.ServiceRegistrations), } c.allocRegistrations[allocID] = alloc } @@ -1592,14 +1491,6 @@ func makeAgentServiceID(role string, service *structs.Service) string { return fmt.Sprintf("%s-%s-%s", nomadServicePrefix, role, service.Hash(role, "", false)) } -// MakeAllocServiceID creates a unique ID for identifying an alloc service in -// Consul. -// -// Example Service ID: _nomad-task-b4e61df9-b095-d64e-f241-23860da1375f-redis-http-http -func MakeAllocServiceID(allocID, taskName string, service *structs.Service) string { - return fmt.Sprintf("%s%s-%s-%s-%s", nomadTaskPrefix, allocID, taskName, service.Name, service.PortLabel) -} - // MakeCheckID creates a unique ID for a check. // // Example Check ID: _nomad-check-434ae42f9a57c5705344974ac38de2aee0ee089d @@ -1768,127 +1659,3 @@ func getNomadSidecar(id string, services map[string]*api.AgentService) *api.Agen sidecarID := id + sidecarSuffix return services[sidecarID] } - -// getAddress returns the IP and port to use for a service or check. If no port -// label is specified (an empty value), zero values are returned because no -// address could be resolved. -func getAddress(addrMode, portLabel string, networks structs.Networks, driverNet *drivers.DriverNetwork, ports structs.AllocatedPorts, netStatus *structs.AllocNetworkStatus) (string, int, error) { - switch addrMode { - case structs.AddressModeAuto: - if driverNet.Advertise() { - addrMode = structs.AddressModeDriver - } else { - addrMode = structs.AddressModeHost - } - return getAddress(addrMode, portLabel, networks, driverNet, ports, netStatus) - case structs.AddressModeHost: - if portLabel == "" { - if len(networks) != 1 { - // If no networks are specified return zero - // values. Consul will advertise the host IP - // with no port. This is the pre-0.7.1 behavior - // some people rely on. - return "", 0, nil - } - - return networks[0].IP, 0, nil - } - - // Default path: use host ip:port - // Try finding port in the AllocatedPorts struct first - // Check in Networks struct for backwards compatibility if not found - mapping, ok := ports.Get(portLabel) - if !ok { - mapping = networks.Port(portLabel) - if mapping.Value > 0 { - return mapping.HostIP, mapping.Value, nil - } - - // If port isn't a label, try to parse it as a literal port number - port, err := strconv.Atoi(portLabel) - if err != nil { - // Don't include Atoi error message as user likely - // never intended it to be a numeric and it creates a - // confusing error message - return "", 0, fmt.Errorf("invalid port %q: port label not found", portLabel) - } - if port <= 0 { - return "", 0, fmt.Errorf("invalid port: %q: port must be >0", portLabel) - } - - // A number was given which will use the Consul agent's address and the given port - // Returning a blank string as an address will use the Consul agent's address - return "", port, nil - } - return mapping.HostIP, mapping.Value, nil - - case structs.AddressModeDriver: - // Require a driver network if driver address mode is used - if driverNet == nil { - return "", 0, fmt.Errorf(`cannot use address_mode="driver": no driver network exists`) - } - - // If no port label is specified just return the IP - if portLabel == "" { - return driverNet.IP, 0, nil - } - - // If the port is a label, use the driver's port (not the host's) - if port, ok := ports.Get(portLabel); ok { - return driverNet.IP, port.To, nil - } - - // Check if old style driver portmap is used - if port, ok := driverNet.PortMap[portLabel]; ok { - return driverNet.IP, port, nil - } - - // If port isn't a label, try to parse it as a literal port number - port, err := strconv.Atoi(portLabel) - if err != nil { - // Don't include Atoi error message as user likely - // never intended it to be a numeric and it creates a - // confusing error message - return "", 0, fmt.Errorf("invalid port label %q: port labels in driver address_mode must be numeric or in the driver's port map", portLabel) - } - if port <= 0 { - return "", 0, fmt.Errorf("invalid port: %q: port must be >0", portLabel) - } - - return driverNet.IP, port, nil - - case structs.AddressModeAlloc: - if netStatus == nil { - return "", 0, fmt.Errorf(`cannot use address_mode="alloc": no allocation network status reported`) - } - - // If no port label is specified just return the IP - if portLabel == "" { - return netStatus.Address, 0, nil - } - - // If port is a label and is found then return it - if port, ok := ports.Get(portLabel); ok { - // Use port.To value unless not set - if port.To > 0 { - return netStatus.Address, port.To, nil - } - return netStatus.Address, port.Value, nil - } - - // Check if port is a literal number - port, err := strconv.Atoi(portLabel) - if err != nil { - // User likely specified wrong port label here - return "", 0, fmt.Errorf("invalid port %q: port label not found or is not numeric", portLabel) - } - if port <= 0 { - return "", 0, fmt.Errorf("invalid port: %q: port must be >0", portLabel) - } - return netStatus.Address, port, nil - - default: - // Shouldn't happen due to validation, but enforce invariants - return "", 0, fmt.Errorf("invalid address mode %q", addrMode) - } -} diff --git a/command/agent/consul/service_client_test.go b/command/agent/consul/service_client_test.go index 9cacaa38ddce..e1dcc7a915af 100644 --- a/command/agent/consul/service_client_test.go +++ b/command/agent/consul/service_client_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/hashicorp/consul/api" + "github.com/hashicorp/nomad/client/serviceregistration" "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/structs" @@ -393,7 +394,7 @@ func TestServiceRegistration_CheckOnUpdate(t *testing.T) { sc := NewServiceClient(mockAgent, namespacesClient, logger, true) allocID := uuid.Generate() - ws := &WorkloadServices{ + ws := &serviceregistration.WorkloadServices{ AllocID: allocID, Task: "taskname", Restarter: &restartRecorder{}, @@ -444,7 +445,7 @@ func TestServiceRegistration_CheckOnUpdate(t *testing.T) { } // Update - wsUpdate := new(WorkloadServices) + wsUpdate := new(serviceregistration.WorkloadServices) *wsUpdate = *ws wsUpdate.Services[0].Checks[0].OnUpdate = structs.OnUpdateRequireHealthy diff --git a/command/agent/consul/structs.go b/command/agent/consul/structs.go index 1163d8c592f7..d8e9210ce13a 100644 --- a/command/agent/consul/structs.go +++ b/command/agent/consul/structs.go @@ -1,64 +1,22 @@ package consul import ( - "github.com/hashicorp/nomad/client/allocrunner/taskrunner/interfaces" + "github.com/hashicorp/nomad/client/serviceregistration" "github.com/hashicorp/nomad/client/taskenv" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/plugins/drivers" ) -// WorkloadServices describes services defined in either a Task or TaskGroup -// that need to be syncronized with Consul. -type WorkloadServices struct { - AllocID string - - // Name of the task and task group the services are defined for. For - // group based services, Task will be empty. - Task string - Group string - - // Canary indicates whether or not the allocation is a canary. - Canary bool - - // ConsulNamespace is the consul namespace in which services will be registered. - ConsulNamespace string - - // Restarter allows restarting the task or task group depending on the - // check_restart stanzas. - Restarter WorkloadRestarter - - // Services and checks to register for the task. - Services []*structs.Service - - // Networks from the task's resources stanza. - // TODO: remove and use Ports - Networks structs.Networks - - // NetworkStatus from alloc if network namespace is created. - // Can be nil. - NetworkStatus *structs.AllocNetworkStatus - - // AllocatedPorts is the list of port mappings. - Ports structs.AllocatedPorts - - // DriverExec is the script executor for the task's driver. - // For group services this is nil and script execution is managed by - // a tasklet in the taskrunner script_check_hook. - DriverExec interfaces.ScriptExecutor - - // DriverNetwork is the network specified by the driver and may be nil. - DriverNetwork *drivers.DriverNetwork -} - -func BuildAllocServices(node *structs.Node, alloc *structs.Allocation, restarter WorkloadRestarter) *WorkloadServices { +func BuildAllocServices( + node *structs.Node, alloc *structs.Allocation, restarter WorkloadRestarter) *serviceregistration.WorkloadServices { //TODO(schmichael) only support one network for now net := alloc.AllocatedResources.Shared.Networks[0] tg := alloc.Job.LookupTaskGroup(alloc.TaskGroup) - ws := &WorkloadServices{ + ws := &serviceregistration.WorkloadServices{ AllocID: alloc.ID, Group: alloc.TaskGroup, Services: taskenv.InterpolateServices(taskenv.NewBuilder(mock.Node(), alloc, nil, alloc.Job.Region).Build(), tg.Services), @@ -82,24 +40,3 @@ func BuildAllocServices(node *structs.Node, alloc *structs.Allocation, restarter return ws } - -// Copy method for easing tests -func (ws *WorkloadServices) Copy() *WorkloadServices { - newTS := new(WorkloadServices) - *newTS = *ws - - // Deep copy Services - newTS.Services = make([]*structs.Service, len(ws.Services)) - for i := range ws.Services { - newTS.Services[i] = ws.Services[i].Copy() - } - return newTS -} - -func (ws *WorkloadServices) Name() string { - if ws.Task != "" { - return ws.Task - } - - return "group-" + ws.Group -} diff --git a/command/agent/consul/unit_test.go b/command/agent/consul/unit_test.go index b3f035ad373e..3a5d29c11e73 100644 --- a/command/agent/consul/unit_test.go +++ b/command/agent/consul/unit_test.go @@ -11,12 +11,12 @@ import ( "time" "github.com/hashicorp/consul/api" + "github.com/hashicorp/nomad/client/serviceregistration" "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/plugins/drivers" "github.com/kr/pretty" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -26,8 +26,8 @@ const ( yPort = 1235 ) -func testWorkload() *WorkloadServices { - return &WorkloadServices{ +func testWorkload() *serviceregistration.WorkloadServices { + return &serviceregistration.WorkloadServices{ AllocID: uuid.Generate(), Task: "taskname", Restarter: &restartRecorder{}, @@ -65,7 +65,7 @@ func (r *restartRecorder) Restart(ctx context.Context, event *structs.TaskEvent, type testFakeCtx struct { ServiceClient *ServiceClient FakeConsul *MockAgent - Workload *WorkloadServices + Workload *serviceregistration.WorkloadServices } var errNoOps = fmt.Errorf("testing error: no pending operations") @@ -502,8 +502,8 @@ func TestConsul_ChangeChecks(t *testing.T) { t.Fatalf("service ID changed") } - for newID := range sreg.checkIDs { - if _, ok := otherServiceReg.checkIDs[newID]; ok { + for newID := range sreg.CheckIDs { + if _, ok := otherServiceReg.CheckIDs[newID]; ok { t.Fatalf("check IDs should change") } } @@ -1349,361 +1349,6 @@ func TestCreateCheckReg_GRPC(t *testing.T) { require.Equal(t, expected, actual) } -// TestGetAddress asserts Nomad uses the correct ip and port for services and -// checks depending on port labels, driver networks, and address mode. -func TestGetAddress(t *testing.T) { - const HostIP = "127.0.0.1" - - cases := []struct { - Name string - - // Parameters - Mode string - PortLabel string - Host map[string]int // will be converted to structs.Networks - Driver *drivers.DriverNetwork - Ports structs.AllocatedPorts - Status *structs.AllocNetworkStatus - - // Results - ExpectedIP string - ExpectedPort int - ExpectedErr string - }{ - // Valid Configurations - { - Name: "ExampleService", - Mode: structs.AddressModeAuto, - PortLabel: "db", - Host: map[string]int{"db": 12435}, - Driver: &drivers.DriverNetwork{ - PortMap: map[string]int{"db": 6379}, - IP: "10.1.2.3", - }, - ExpectedIP: HostIP, - ExpectedPort: 12435, - }, - { - Name: "Host", - Mode: structs.AddressModeHost, - PortLabel: "db", - Host: map[string]int{"db": 12345}, - Driver: &drivers.DriverNetwork{ - PortMap: map[string]int{"db": 6379}, - IP: "10.1.2.3", - }, - ExpectedIP: HostIP, - ExpectedPort: 12345, - }, - { - Name: "Driver", - Mode: structs.AddressModeDriver, - PortLabel: "db", - Host: map[string]int{"db": 12345}, - Driver: &drivers.DriverNetwork{ - PortMap: map[string]int{"db": 6379}, - IP: "10.1.2.3", - }, - ExpectedIP: "10.1.2.3", - ExpectedPort: 6379, - }, - { - Name: "AutoDriver", - Mode: structs.AddressModeAuto, - PortLabel: "db", - Host: map[string]int{"db": 12345}, - Driver: &drivers.DriverNetwork{ - PortMap: map[string]int{"db": 6379}, - IP: "10.1.2.3", - AutoAdvertise: true, - }, - ExpectedIP: "10.1.2.3", - ExpectedPort: 6379, - }, - { - Name: "DriverCustomPort", - Mode: structs.AddressModeDriver, - PortLabel: "7890", - Host: map[string]int{"db": 12345}, - Driver: &drivers.DriverNetwork{ - PortMap: map[string]int{"db": 6379}, - IP: "10.1.2.3", - }, - ExpectedIP: "10.1.2.3", - ExpectedPort: 7890, - }, - - // Invalid Configurations - { - Name: "DriverWithoutNetwork", - Mode: structs.AddressModeDriver, - PortLabel: "db", - Host: map[string]int{"db": 12345}, - Driver: nil, - ExpectedErr: "no driver network exists", - }, - { - Name: "DriverBadPort", - Mode: structs.AddressModeDriver, - PortLabel: "bad-port-label", - Host: map[string]int{"db": 12345}, - Driver: &drivers.DriverNetwork{ - PortMap: map[string]int{"db": 6379}, - IP: "10.1.2.3", - }, - ExpectedErr: "invalid port", - }, - { - Name: "DriverZeroPort", - Mode: structs.AddressModeDriver, - PortLabel: "0", - Driver: &drivers.DriverNetwork{ - IP: "10.1.2.3", - }, - ExpectedErr: "invalid port", - }, - { - Name: "HostBadPort", - Mode: structs.AddressModeHost, - PortLabel: "bad-port-label", - ExpectedErr: "invalid port", - }, - { - Name: "InvalidMode", - Mode: "invalid-mode", - PortLabel: "80", - ExpectedErr: "invalid address mode", - }, - { - Name: "NoPort_AutoMode", - Mode: structs.AddressModeAuto, - ExpectedIP: HostIP, - }, - { - Name: "NoPort_HostMode", - Mode: structs.AddressModeHost, - ExpectedIP: HostIP, - }, - { - Name: "NoPort_DriverMode", - Mode: structs.AddressModeDriver, - Driver: &drivers.DriverNetwork{ - IP: "10.1.2.3", - }, - ExpectedIP: "10.1.2.3", - }, - - // Scenarios using port 0.12 networking fields (NetworkStatus, AllocatedPortMapping) - { - Name: "ExampleServer_withAllocatedPorts", - Mode: structs.AddressModeAuto, - PortLabel: "db", - Ports: []structs.AllocatedPortMapping{ - { - Label: "db", - Value: 12435, - To: 6379, - HostIP: HostIP, - }, - }, - Status: &structs.AllocNetworkStatus{ - InterfaceName: "eth0", - Address: "172.26.0.1", - }, - ExpectedIP: HostIP, - ExpectedPort: 12435, - }, - { - Name: "Host_withAllocatedPorts", - Mode: structs.AddressModeHost, - PortLabel: "db", - Ports: []structs.AllocatedPortMapping{ - { - Label: "db", - Value: 12345, - To: 6379, - HostIP: HostIP, - }, - }, - Status: &structs.AllocNetworkStatus{ - InterfaceName: "eth0", - Address: "172.26.0.1", - }, - ExpectedIP: HostIP, - ExpectedPort: 12345, - }, - { - Name: "Driver_withAllocatedPorts", - Mode: structs.AddressModeDriver, - PortLabel: "db", - Ports: []structs.AllocatedPortMapping{ - { - Label: "db", - Value: 12345, - To: 6379, - HostIP: HostIP, - }, - }, - Driver: &drivers.DriverNetwork{ - IP: "10.1.2.3", - }, - Status: &structs.AllocNetworkStatus{ - InterfaceName: "eth0", - Address: "172.26.0.1", - }, - ExpectedIP: "10.1.2.3", - ExpectedPort: 6379, - }, - { - Name: "AutoDriver_withAllocatedPorts", - Mode: structs.AddressModeAuto, - PortLabel: "db", - Ports: []structs.AllocatedPortMapping{ - { - Label: "db", - Value: 12345, - To: 6379, - HostIP: HostIP, - }, - }, - Driver: &drivers.DriverNetwork{ - IP: "10.1.2.3", - AutoAdvertise: true, - }, - Status: &structs.AllocNetworkStatus{ - InterfaceName: "eth0", - Address: "172.26.0.1", - }, - ExpectedIP: "10.1.2.3", - ExpectedPort: 6379, - }, - { - Name: "DriverCustomPort_withAllocatedPorts", - Mode: structs.AddressModeDriver, - PortLabel: "7890", - Ports: []structs.AllocatedPortMapping{ - { - Label: "db", - Value: 12345, - To: 6379, - HostIP: HostIP, - }, - }, - Driver: &drivers.DriverNetwork{ - IP: "10.1.2.3", - }, - Status: &structs.AllocNetworkStatus{ - InterfaceName: "eth0", - Address: "172.26.0.1", - }, - ExpectedIP: "10.1.2.3", - ExpectedPort: 7890, - }, - { - Name: "Host_MultiHostInterface", - Mode: structs.AddressModeAuto, - PortLabel: "db", - Ports: []structs.AllocatedPortMapping{ - { - Label: "db", - Value: 12345, - To: 6379, - HostIP: "127.0.0.100", - }, - }, - Status: &structs.AllocNetworkStatus{ - InterfaceName: "eth0", - Address: "172.26.0.1", - }, - ExpectedIP: "127.0.0.100", - ExpectedPort: 12345, - }, - { - Name: "Alloc", - Mode: structs.AddressModeAlloc, - PortLabel: "db", - Ports: []structs.AllocatedPortMapping{ - { - Label: "db", - Value: 12345, - To: 6379, - HostIP: HostIP, - }, - }, - Status: &structs.AllocNetworkStatus{ - InterfaceName: "eth0", - Address: "172.26.0.1", - }, - ExpectedIP: "172.26.0.1", - ExpectedPort: 6379, - }, - { - Name: "Alloc no to value", - Mode: structs.AddressModeAlloc, - PortLabel: "db", - Ports: []structs.AllocatedPortMapping{ - { - Label: "db", - Value: 12345, - HostIP: HostIP, - }, - }, - Status: &structs.AllocNetworkStatus{ - InterfaceName: "eth0", - Address: "172.26.0.1", - }, - ExpectedIP: "172.26.0.1", - ExpectedPort: 12345, - }, - { - Name: "AllocCustomPort", - Mode: structs.AddressModeAlloc, - PortLabel: "6379", - Status: &structs.AllocNetworkStatus{ - InterfaceName: "eth0", - Address: "172.26.0.1", - }, - ExpectedIP: "172.26.0.1", - ExpectedPort: 6379, - }, - } - - for _, tc := range cases { - t.Run(tc.Name, func(t *testing.T) { - // convert host port map into a structs.Networks - networks := []*structs.NetworkResource{ - { - IP: HostIP, - ReservedPorts: make([]structs.Port, len(tc.Host)), - }, - } - - i := 0 - for label, port := range tc.Host { - networks[0].ReservedPorts[i].Label = label - networks[0].ReservedPorts[i].Value = port - i++ - } - - // Run getAddress - ip, port, err := getAddress(tc.Mode, tc.PortLabel, networks, tc.Driver, tc.Ports, tc.Status) - - // Assert the results - assert.Equal(t, tc.ExpectedIP, ip, "IP mismatch") - assert.Equal(t, tc.ExpectedPort, port, "Port mismatch") - if tc.ExpectedErr == "" { - assert.Nil(t, err) - } else { - if err == nil { - t.Fatalf("expected error containing %q but err=nil", tc.ExpectedErr) - } else { - assert.Contains(t, err.Error(), tc.ExpectedErr) - } - } - }) - } -} - func TestConsul_ServiceName_Duplicates(t *testing.T) { t.Parallel() ctx := setupFake(t) @@ -1789,7 +1434,7 @@ func TestConsul_ServiceDeregistration_OutProbation(t *testing.T) { }, }, } - remainingWorkloadServiceID := MakeAllocServiceID(remainingWorkload.AllocID, + remainingWorkloadServiceID := serviceregistration.MakeAllocServiceID(remainingWorkload.AllocID, remainingWorkload.Name(), remainingWorkload.Services[0]) require.NoError(ctx.ServiceClient.RegisterWorkload(remainingWorkload)) @@ -1812,7 +1457,7 @@ func TestConsul_ServiceDeregistration_OutProbation(t *testing.T) { }, }, } - explicitlyRemovedWorkloadServiceID := MakeAllocServiceID(explicitlyRemovedWorkload.AllocID, + explicitlyRemovedWorkloadServiceID := serviceregistration.MakeAllocServiceID(explicitlyRemovedWorkload.AllocID, explicitlyRemovedWorkload.Name(), explicitlyRemovedWorkload.Services[0]) require.NoError(ctx.ServiceClient.RegisterWorkload(explicitlyRemovedWorkload)) @@ -1837,7 +1482,7 @@ func TestConsul_ServiceDeregistration_OutProbation(t *testing.T) { }, }, } - outofbandWorkloadServiceID := MakeAllocServiceID(outofbandWorkload.AllocID, + outofbandWorkloadServiceID := serviceregistration.MakeAllocServiceID(outofbandWorkload.AllocID, outofbandWorkload.Name(), outofbandWorkload.Services[0]) require.NoError(ctx.ServiceClient.RegisterWorkload(outofbandWorkload)) @@ -1898,7 +1543,7 @@ func TestConsul_ServiceDeregistration_InProbation(t *testing.T) { }, }, } - remainingWorkloadServiceID := MakeAllocServiceID(remainingWorkload.AllocID, + remainingWorkloadServiceID := serviceregistration.MakeAllocServiceID(remainingWorkload.AllocID, remainingWorkload.Name(), remainingWorkload.Services[0]) require.NoError(ctx.ServiceClient.RegisterWorkload(remainingWorkload)) @@ -1921,7 +1566,7 @@ func TestConsul_ServiceDeregistration_InProbation(t *testing.T) { }, }, } - explicitlyRemovedWorkloadServiceID := MakeAllocServiceID(explicitlyRemovedWorkload.AllocID, + explicitlyRemovedWorkloadServiceID := serviceregistration.MakeAllocServiceID(explicitlyRemovedWorkload.AllocID, explicitlyRemovedWorkload.Name(), explicitlyRemovedWorkload.Services[0]) require.NoError(ctx.ServiceClient.RegisterWorkload(explicitlyRemovedWorkload)) @@ -1946,7 +1591,7 @@ func TestConsul_ServiceDeregistration_InProbation(t *testing.T) { }, }, } - outofbandWorkloadServiceID := MakeAllocServiceID(outofbandWorkload.AllocID, + outofbandWorkloadServiceID := serviceregistration.MakeAllocServiceID(outofbandWorkload.AllocID, outofbandWorkload.Name(), outofbandWorkload.Services[0]) require.NoError(ctx.ServiceClient.RegisterWorkload(outofbandWorkload)) From 47229473b9cbadd2662d556f1d033f90a3cbdce4 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Tue, 15 Mar 2022 12:43:52 +0100 Subject: [PATCH 23/31] client: add service registration wrapper to handle providers. The service registration wrapper handles sending requests to backend providers without the caller needing to know this information. This will be used within the task and alloc runner service hooks when performing service registration activities. --- client/serviceregistration/wrapper/wrapper.go | 131 ++++++ .../wrapper/wrapper_test.go | 384 ++++++++++++++++++ 2 files changed, 515 insertions(+) create mode 100644 client/serviceregistration/wrapper/wrapper.go create mode 100644 client/serviceregistration/wrapper/wrapper_test.go diff --git a/client/serviceregistration/wrapper/wrapper.go b/client/serviceregistration/wrapper/wrapper.go new file mode 100644 index 000000000000..37b5e4f614be --- /dev/null +++ b/client/serviceregistration/wrapper/wrapper.go @@ -0,0 +1,131 @@ +package wrapper + +import ( + "fmt" + + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/nomad/client/serviceregistration" + "github.com/hashicorp/nomad/nomad/structs" +) + +// HandlerWrapper is used to wrap service registration implementations of the +// Handler interface. We do not use a map or similar to store the handlers, so +// we can avoid having to use a lock. This may need to be updated if we ever +// support additional registration providers. +type HandlerWrapper struct { + log hclog.Logger + + // consulServiceProvider is the handler for services where Consul is the + // provider. This provider is always created and available. + consulServiceProvider serviceregistration.Handler + + // nomadServiceProvider is the handler for services where Nomad is the + // provider. + nomadServiceProvider serviceregistration.Handler +} + +// NewHandlerWrapper configures and returns a HandlerWrapper for use within +// client hooks that need to interact with service and check registrations. It +// mimics the serviceregistration.Handler interface, but returns the +// implementation to allow future flexibility and is initially only intended +// for use with the alloc and task runner service hooks. +func NewHandlerWrapper( + log hclog.Logger, consulProvider, nomadProvider serviceregistration.Handler) *HandlerWrapper { + return &HandlerWrapper{ + log: log, + nomadServiceProvider: nomadProvider, + consulServiceProvider: consulProvider, + } +} + +// RegisterWorkload wraps the serviceregistration.Handler RegisterWorkload +// function. It determines which backend provider to call and passes the +// workload unless the provider is unknown, in which case an error will be +// returned. +func (h *HandlerWrapper) RegisterWorkload(workload *serviceregistration.WorkloadServices) error { + + // Don't rely on callers to check there are no services to register. + if len(workload.Services) == 0 { + return nil + } + + provider := workload.RegistrationProvider() + + switch provider { + case structs.ServiceProviderNomad: + return h.nomadServiceProvider.RegisterWorkload(workload) + case structs.ServiceProviderConsul: + return h.consulServiceProvider.RegisterWorkload(workload) + default: + return fmt.Errorf("unknown service registration provider: %q", provider) + } +} + +// RemoveWorkload wraps the serviceregistration.Handler RemoveWorkload +// function. It determines which backend provider to call and passes the +// workload unless the provider is unknown. +func (h *HandlerWrapper) RemoveWorkload(services *serviceregistration.WorkloadServices) { + + // Don't rely on callers to check there are no services to remove. + if len(services.Services) == 0 { + return + } + + provider := services.RegistrationProvider() + + // Call the correct provider. In the case it is not supported, we can't do + // much apart from log, although we should never reach this point because + // of job validation. + switch provider { + case structs.ServiceProviderNomad: + h.nomadServiceProvider.RemoveWorkload(services) + case structs.ServiceProviderConsul: + h.consulServiceProvider.RemoveWorkload(services) + default: + h.log.Error("unknown service registration provider", "provider", provider) + } +} + +// UpdateWorkload identifies which provider to call for the new and old +// workloads provided. In the event both use the same provider, the +// UpdateWorkload function will be called, otherwise the register and remove +// functions will be called. +func (h *HandlerWrapper) UpdateWorkload(old, new *serviceregistration.WorkloadServices) error { + + // Hot path to exit if there is nothing to do. + if len(old.Services) == 0 && len(new.Services) == 0 { + return nil + } + + newProvider := new.RegistrationProvider() + oldProvider := old.RegistrationProvider() + + // If the new and old services use the same provider, call the + // UpdateWorkload and leave it at that. + if newProvider == oldProvider { + switch newProvider { + case structs.ServiceProviderNomad: + return h.nomadServiceProvider.UpdateWorkload(old, new) + case structs.ServiceProviderConsul: + return h.consulServiceProvider.UpdateWorkload(old, new) + default: + return fmt.Errorf("unknown service registration provider for update: %q", newProvider) + } + } + + // If we have new services, call the relevant provider. Registering can + // return an error. Do this before RemoveWorkload, so we can halt the + // process if needed, otherwise we may leave the task/group + // registration-less. + if len(new.Services) > 0 { + if err := h.RegisterWorkload(new); err != nil { + return err + } + } + + if len(old.Services) > 0 { + h.RemoveWorkload(old) + } + + return nil +} diff --git a/client/serviceregistration/wrapper/wrapper_test.go b/client/serviceregistration/wrapper/wrapper_test.go new file mode 100644 index 000000000000..81dfd7c1f67e --- /dev/null +++ b/client/serviceregistration/wrapper/wrapper_test.go @@ -0,0 +1,384 @@ +package wrapper + +import ( + "testing" + + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/nomad/client/serviceregistration" + regMock "github.com/hashicorp/nomad/client/serviceregistration/mock" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/stretchr/testify/require" +) + +func Test_NewHandlerWrapper(t *testing.T) { + log := hclog.NewNullLogger() + mockProvider := regMock.NewServiceRegistrationHandler(log) + wrapper := NewHandlerWrapper(log, mockProvider, mockProvider) + require.NotNil(t, wrapper) + require.NotNil(t, wrapper.log) + require.NotNil(t, wrapper.nomadServiceProvider) + require.NotNil(t, wrapper.consulServiceProvider) +} + +func TestHandlerWrapper_RegisterWorkload(t *testing.T) { + testCases := []struct { + testFn func(t *testing.T) + name string + }{ + { + testFn: func(t *testing.T) { + + // Generate the test wrapper and provider mocks. + wrapper, consul, nomad := setupTestWrapper() + + // Call the function with no services and check that nothing is + // registered. + require.NoError(t, wrapper.RegisterWorkload(&serviceregistration.WorkloadServices{})) + require.Len(t, consul.GetOps(), 0) + require.Len(t, nomad.GetOps(), 0) + }, + name: "zero services", + }, + { + testFn: func(t *testing.T) { + + // Generate the test wrapper and provider mocks. + wrapper, consul, nomad := setupTestWrapper() + + // Generate a minimal workload with an unknown provider. + workload := serviceregistration.WorkloadServices{ + Services: []*structs.Service{ + { + Provider: "istio", + }, + }, + } + + // Call register and ensure an error is returned along with + // nothing registered in the providers. + err := wrapper.RegisterWorkload(&workload) + require.Error(t, err) + require.Contains(t, err.Error(), "unknown service registration provider: \"istio\"") + require.Len(t, consul.GetOps(), 0) + require.Len(t, nomad.GetOps(), 0) + + }, + name: "unknown provider", + }, + { + testFn: func(t *testing.T) { + + // Generate the test wrapper and provider mocks. + wrapper, consul, nomad := setupTestWrapper() + + // Generate a minimal workload with the nomad provider. + workload := serviceregistration.WorkloadServices{ + Services: []*structs.Service{ + { + Provider: structs.ServiceProviderNomad, + }, + }, + } + + // Call register and ensure no error is returned along with the + // correct operations. + require.NoError(t, wrapper.RegisterWorkload(&workload)) + require.Len(t, consul.GetOps(), 0) + require.Len(t, nomad.GetOps(), 1) + + }, + name: "nomad provider", + }, + { + testFn: func(t *testing.T) { + + // Generate the test wrapper and provider mocks. + wrapper, consul, nomad := setupTestWrapper() + + // Generate a minimal workload with the consul provider. + workload := serviceregistration.WorkloadServices{ + Services: []*structs.Service{ + { + Provider: structs.ServiceProviderConsul, + }, + }, + } + + // Call register and ensure no error is returned along with the + // correct operations. + require.NoError(t, wrapper.RegisterWorkload(&workload)) + require.Len(t, consul.GetOps(), 1) + require.Len(t, nomad.GetOps(), 0) + }, + name: "consul provider", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tc.testFn(t) + }) + } +} + +func TestHandlerWrapper_RemoveWorkload(t *testing.T) { + testCases := []struct { + testFn func(t *testing.T) + name string + }{ + { + testFn: func(t *testing.T) { + // Generate the test wrapper and provider mocks. + wrapper, consul, nomad := setupTestWrapper() + + // Call the function with no services and check that nothing is + // registered. + wrapper.RemoveWorkload(&serviceregistration.WorkloadServices{}) + require.Len(t, consul.GetOps(), 0) + require.Len(t, nomad.GetOps(), 0) + }, + name: "zero services", + }, + { + testFn: func(t *testing.T) { + + // Generate the test wrapper and provider mocks. + wrapper, consul, nomad := setupTestWrapper() + + // Generate a minimal workload with an unknown provider. + workload := serviceregistration.WorkloadServices{ + Services: []*structs.Service{ + { + Provider: "istio", + }, + }, + } + + // Call remove and ensure nothing registered in the providers. + wrapper.RemoveWorkload(&workload) + require.Len(t, consul.GetOps(), 0) + require.Len(t, nomad.GetOps(), 0) + }, + name: "unknown provider", + }, + { + testFn: func(t *testing.T) { + + // Generate the test wrapper and provider mocks. + wrapper, consul, nomad := setupTestWrapper() + + // Generate a minimal workload with the consul provider. + workload := serviceregistration.WorkloadServices{ + Services: []*structs.Service{ + { + Provider: structs.ServiceProviderConsul, + }, + }, + } + + // Call remove and ensure the correct backend includes + // operations. + wrapper.RemoveWorkload(&workload) + require.Len(t, consul.GetOps(), 1) + require.Len(t, nomad.GetOps(), 0) + }, + name: "consul provider", + }, + { + testFn: func(t *testing.T) { + + // Generate the test wrapper and provider mocks. + wrapper, consul, nomad := setupTestWrapper() + + // Generate a minimal workload with the nomad provider. + workload := serviceregistration.WorkloadServices{ + Services: []*structs.Service{ + { + Provider: structs.ServiceProviderNomad, + }, + }, + } + + // Call remove and ensure the correct backend includes + // operations. + wrapper.RemoveWorkload(&workload) + require.Len(t, consul.GetOps(), 0) + require.Len(t, nomad.GetOps(), 1) + }, + name: "nomad provider", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tc.testFn(t) + }) + } +} + +func TestHandlerWrapper_UpdateWorkload(t *testing.T) { + testCases := []struct { + testFn func(t *testing.T) + name string + }{ + { + testFn: func(t *testing.T) { + + // Generate the test wrapper and provider mocks. + wrapper, consul, nomad := setupTestWrapper() + + // Call the function with no services and check that nothing is + // registered in either mock backend. + err := wrapper.UpdateWorkload(&serviceregistration.WorkloadServices{}, + &serviceregistration.WorkloadServices{}) + require.NoError(t, err) + require.Len(t, consul.GetOps(), 0) + require.Len(t, nomad.GetOps(), 0) + + }, + name: "zero new or old", + }, + { + testFn: func(t *testing.T) { + + // Generate the test wrapper and provider mocks. + wrapper, consul, nomad := setupTestWrapper() + + // Create a single workload that we can use twice, using the + // consul provider. + workload := serviceregistration.WorkloadServices{ + Services: []*structs.Service{ + { + Provider: structs.ServiceProviderConsul, + }, + }, + } + + // Call the function and ensure the consul backend has the + // expected operations. + require.NoError(t, wrapper.UpdateWorkload(&workload, &workload)) + require.Len(t, nomad.GetOps(), 0) + + consulOps := consul.GetOps() + require.Len(t, consulOps, 1) + require.Equal(t, "update", consulOps[0].Op) + }, + name: "consul new and old", + }, + { + testFn: func(t *testing.T) { + + // Generate the test wrapper and provider mocks. + wrapper, consul, nomad := setupTestWrapper() + + // Create a single workload that we can use twice, using the + // nomad provider. + workload := serviceregistration.WorkloadServices{ + Services: []*structs.Service{ + { + Provider: structs.ServiceProviderNomad, + }, + }, + } + + // Call the function and ensure the nomad backend has the + // expected operations. + require.NoError(t, wrapper.UpdateWorkload(&workload, &workload)) + require.Len(t, consul.GetOps(), 0) + + nomadOps := nomad.GetOps() + require.Len(t, nomadOps, 1) + require.Equal(t, "update", nomadOps[0].Op) + }, + name: "nomad new and old", + }, + { + testFn: func(t *testing.T) { + + // Generate the test wrapper and provider mocks. + wrapper, consul, nomad := setupTestWrapper() + + // Create each workload. + newWorkload := serviceregistration.WorkloadServices{ + Services: []*structs.Service{ + { + Provider: structs.ServiceProviderNomad, + }, + }, + } + + oldWorkload := serviceregistration.WorkloadServices{ + Services: []*structs.Service{ + { + Provider: structs.ServiceProviderConsul, + }, + }, + } + + // Call the function and ensure the backends have the expected + // operations. + require.NoError(t, wrapper.UpdateWorkload(&oldWorkload, &newWorkload)) + + nomadOps := nomad.GetOps() + require.Len(t, nomadOps, 1) + require.Equal(t, "add", nomadOps[0].Op) + + consulOps := consul.GetOps() + require.Len(t, consulOps, 1) + require.Equal(t, "remove", consulOps[0].Op) + }, + name: "nomad new and consul old", + }, + { + testFn: func(t *testing.T) { + + // Generate the test wrapper and provider mocks. + wrapper, consul, nomad := setupTestWrapper() + + // Create each workload. + newWorkload := serviceregistration.WorkloadServices{ + Services: []*structs.Service{ + { + Provider: structs.ServiceProviderConsul, + }, + }, + } + + oldWorkload := serviceregistration.WorkloadServices{ + Services: []*structs.Service{ + { + Provider: structs.ServiceProviderNomad, + }, + }, + } + + // Call the function and ensure the backends have the expected + // operations. + require.NoError(t, wrapper.UpdateWorkload(&oldWorkload, &newWorkload)) + + nomadOps := nomad.GetOps() + require.Len(t, nomadOps, 1) + require.Equal(t, "remove", nomadOps[0].Op) + + consulOps := consul.GetOps() + require.Len(t, consulOps, 1) + require.Equal(t, "add", consulOps[0].Op) + }, + name: "consul new and nomad old", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tc.testFn(t) + }) + } +} + +func setupTestWrapper() (*HandlerWrapper, *regMock.ServiceRegistrationHandler, *regMock.ServiceRegistrationHandler) { + log := hclog.NewNullLogger() + consulMock := regMock.NewServiceRegistrationHandler(log) + nomadMock := regMock.NewServiceRegistrationHandler(log) + wrapper := NewHandlerWrapper(log, consulMock, nomadMock) + return wrapper, consulMock, nomadMock +} From 9185316857d61ff2af2cc0cedc7735d644bbdac7 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Thu, 17 Mar 2022 09:17:13 +0100 Subject: [PATCH 24/31] client: add Nomad service registration implementation. --- client/serviceregistration/nsd/doc.go | 4 + client/serviceregistration/nsd/nsd.go | 304 +++++++++++ client/serviceregistration/nsd/nsd_test.go | 553 +++++++++++++++++++++ 3 files changed, 861 insertions(+) create mode 100644 client/serviceregistration/nsd/doc.go create mode 100644 client/serviceregistration/nsd/nsd.go create mode 100644 client/serviceregistration/nsd/nsd_test.go diff --git a/client/serviceregistration/nsd/doc.go b/client/serviceregistration/nsd/doc.go new file mode 100644 index 000000000000..f86c8458d840 --- /dev/null +++ b/client/serviceregistration/nsd/doc.go @@ -0,0 +1,4 @@ +// Package nsd provides Nomad service registration and therefore discovery +// capabilities for Nomad clients. The name nsd was used instead of Nomad to +// avoid conflict with the existing nomad package. +package nsd diff --git a/client/serviceregistration/nsd/nsd.go b/client/serviceregistration/nsd/nsd.go new file mode 100644 index 000000000000..312e1e4e2270 --- /dev/null +++ b/client/serviceregistration/nsd/nsd.go @@ -0,0 +1,304 @@ +package nsd + +import ( + "errors" + "fmt" + "strings" + + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/go-multierror" + "github.com/hashicorp/nomad/client/serviceregistration" + "github.com/hashicorp/nomad/nomad/structs" +) + +type ServiceRegistrationHandler struct { + log hclog.Logger + cfg *ServiceRegistrationHandlerCfg + + // registrationEnabled tracks whether this handler is enabled for + // registrations. This is needed as it's possible a client has its config + // changed whilst allocations using this provider are running on it. In + // this situation we need to be able to deregister services, but disallow + // registering new ones. + registrationEnabled bool + + // shutDownCh coordinates shutting down the handler and any long-running + // processes, such as the RPC retry. + shutDownCh chan struct{} +} + +// ServiceRegistrationHandlerCfg holds critical information used during the +// normal process of the ServiceRegistrationHandler. It is used to keep the +// NewServiceRegistrationHandler function signature small and easy to modify. +type ServiceRegistrationHandlerCfg struct { + + // Enabled tracks whether this client feature is enabled. + Enabled bool + + // Datacenter, NodeID, and Region are all properties of the Nomad client + // and are used to perform RPC requests. + Datacenter string + NodeID string + Region string + + // NodeSecret is the secret ID of the node and is used to authenticate RPC + // requests. + NodeSecret string + + // RPCFn is the client RPC function which is used to perform client to + // server service registration RPC calls. This RPC function has basic retry + // functionality. + RPCFn func(method string, args, resp interface{}) error +} + +// NewServiceRegistrationHandler returns a ready to use +// ServiceRegistrationHandler which implements the serviceregistration.Handler +// interface. +func NewServiceRegistrationHandler( + log hclog.Logger, cfg *ServiceRegistrationHandlerCfg) serviceregistration.Handler { + return &ServiceRegistrationHandler{ + cfg: cfg, + log: log.Named("service_registration.nomad"), + registrationEnabled: cfg.Enabled, + shutDownCh: make(chan struct{}), + } +} + +func (s *ServiceRegistrationHandler) RegisterWorkload(workload *serviceregistration.WorkloadServices) error { + + // Check whether we are enabled or not first. Hitting this likely means + // there is a bug within the implicit constraint, or process using it, as + // that should guard ever placing an allocation on this client. + if !s.registrationEnabled { + return errors.New(`service registration provider "nomad" not enabled`) + } + + // Collect all errors generating service registrations. + var mErr multierror.Error + + registrations := make([]*structs.ServiceRegistration, len(workload.Services)) + + // Iterate over the services and generate a hydrated registration object for + // each. All services are part of a single allocation, therefore we cannot + // have one failure without all becoming a failure. + for i, serviceSpec := range workload.Services { + serviceRegistration, err := s.generateNomadServiceRegistration(serviceSpec, workload) + if err != nil { + mErr.Errors = append(mErr.Errors, err) + } else if mErr.ErrorOrNil() == nil { + registrations[i] = serviceRegistration + } + } + + // If we generated any errors, return this to the caller. + if err := mErr.ErrorOrNil(); err != nil { + return err + } + + args := structs.ServiceRegistrationUpsertRequest{ + Services: registrations, + WriteRequest: structs.WriteRequest{ + Region: s.cfg.Region, + AuthToken: s.cfg.NodeSecret, + }, + } + + var resp structs.ServiceRegistrationUpsertResponse + + return s.cfg.RPCFn(structs.ServiceRegistrationUpsertRPCMethod, &args, &resp) +} + +// RemoveWorkload iterates the services and removes them from the service +// registration state. +// +// This function works regardless of whether the client has this feature +// enabled. This covers situations where the feature is disabled, yet still has +// allocations which, when stopped need their registrations removed. +func (s *ServiceRegistrationHandler) RemoveWorkload(workload *serviceregistration.WorkloadServices) { + for _, serviceSpec := range workload.Services { + go s.removeWorkload(workload, serviceSpec) + } +} + +func (s *ServiceRegistrationHandler) removeWorkload( + workload *serviceregistration.WorkloadServices, serviceSpec *structs.Service) { + + // Generate the consistent ID for this service, so we know what to remove. + id := serviceregistration.MakeAllocServiceID(workload.AllocID, workload.Name(), serviceSpec) + + deleteArgs := structs.ServiceRegistrationDeleteByIDRequest{ + ID: id, + WriteRequest: structs.WriteRequest{ + Region: s.cfg.Region, + Namespace: workload.Namespace, + AuthToken: s.cfg.NodeSecret, + }, + } + + var deleteResp structs.ServiceRegistrationDeleteByIDResponse + + err := s.cfg.RPCFn(structs.ServiceRegistrationDeleteByIDRPCMethod, &deleteArgs, &deleteResp) + if err == nil { + return + } + + // The Nomad API exposes service registration deletion to handle + // orphaned service registrations. In the event a service is removed + // accidentally that is still running, we will hit this error when we + // eventually want to remove it. We therefore want to handle this, + // while ensuring the operator can see. + if strings.Contains(err.Error(), "service registration not found") { + s.log.Info("attempted to delete non-existent service registration", + "service_id", id, "namespace", workload.Namespace) + return + } + + // Log the error as there is nothing left to do, so the operator can see it + // and identify any problems. + s.log.Error("failed to delete service registration", + "error", err, "service_id", id, "namespace", workload.Namespace) +} + +func (s *ServiceRegistrationHandler) UpdateWorkload(old, new *serviceregistration.WorkloadServices) error { + + // Overwrite the workload with the deduplicated versions. + old, new = s.dedupUpdatedWorkload(old, new) + + // Use the register error as an update protection and only ever deregister + // when this has completed successfully. In the event of an error, we can + // return this to the caller stack without modifying state in a weird half + // manner. + if len(new.Services) > 0 { + if err := s.RegisterWorkload(new); err != nil { + return err + } + } + + if len(old.Services) > 0 { + s.RemoveWorkload(old) + } + + return nil +} + +// dedupUpdatedWorkload works through the request old and new workload to +// return a deduplicated set of services. +// +// This is within its own function to make testing easier. +func (s *ServiceRegistrationHandler) dedupUpdatedWorkload( + oldWork, newWork *serviceregistration.WorkloadServices) ( + *serviceregistration.WorkloadServices, *serviceregistration.WorkloadServices) { + + // Create copies of the old and new workload services. These specifically + // ignore the services array so this can be populated as the function + // decides what is needed. + oldCopy := oldWork.Copy() + oldCopy.Services = make([]*structs.Service, 0) + + newCopy := newWork.Copy() + newCopy.Services = make([]*structs.Service, 0) + + // Generate and populate a mapping of the new service registration IDs. + newIDs := make(map[string]*structs.Service, len(newWork.Services)) + + for _, s := range newWork.Services { + newIDs[serviceregistration.MakeAllocServiceID(newWork.AllocID, newWork.Name(), s)] = s + } + + // Iterate through the old services in order to identify whether they can + // be modified solely via upsert, or whether they need to be deleted. + for _, oldService := range oldWork.Services { + + // Generate the service ID of the old service. If this is not found + // within the new mapping then we need to remove it. + oldID := serviceregistration.MakeAllocServiceID(oldWork.AllocID, oldWork.Name(), oldService) + newSvc, ok := newIDs[oldID] + if !ok { + oldCopy.Services = append(oldCopy.Services, oldService) + continue + } + + // Add the new service into the array for upserting and remove its + // entry for the map. Doing it here is efficient as we are already + // inside a loop. + // + // There isn't much point in hashing the old/new services as we would + // still need to ensure the service has previously been registered + // before discarding it from future RPC calls. The Nomad state handles + // performing the diff gracefully, therefore this will still be a + // single RPC. + newCopy.Services = append(newCopy.Services, newSvc) + delete(newIDs, oldID) + } + + // Iterate the remaining new IDs to add them to the registration array. It + // catches any that didn't get added via the previous loop. + for _, newSvc := range newIDs { + newCopy.Services = append(newCopy.Services, newSvc) + } + + return oldCopy, newCopy +} + +// AllocRegistrations is currently a noop implementation as the Nomad provider +// does not support health check which is the sole subsystem caller of this +// function. +func (s *ServiceRegistrationHandler) AllocRegistrations(_ string) (*serviceregistration.AllocRegistration, error) { + return nil, nil +} + +// UpdateTTL is currently a noop implementation as the Nomad provider does not +// support health check which is the sole subsystem caller of this function. +func (s *ServiceRegistrationHandler) UpdateTTL(_, _, _, _ string) error { + return nil +} + +// Shutdown is used to initiate shutdown of the handler. This is specifically +// used to exit any routines running retry functions without leaving them +// orphaned. +func (s *ServiceRegistrationHandler) Shutdown() { close(s.shutDownCh) } + +// generateNomadServiceRegistration is a helper to build the Nomad specific +// registration object on a per-service basis. +func (s *ServiceRegistrationHandler) generateNomadServiceRegistration( + serviceSpec *structs.Service, workload *serviceregistration.WorkloadServices) (*structs.ServiceRegistration, error) { + + // Service address modes default to auto. + addrMode := serviceSpec.AddressMode + if addrMode == "" { + addrMode = structs.AddressModeAuto + } + + // Determine the address to advertise based on the mode. + ip, port, err := serviceregistration.GetAddress( + addrMode, serviceSpec.PortLabel, workload.Networks, + workload.DriverNetwork, workload.Ports, workload.NetworkStatus) + if err != nil { + return nil, fmt.Errorf("unable to get address for service %q: %v", serviceSpec.Name, err) + } + + // Build the tags to use for this registration which is a result of whether + // this is a canary, or not. + var tags []string + + if workload.Canary && len(serviceSpec.CanaryTags) > 0 { + tags = make([]string, len(serviceSpec.CanaryTags)) + copy(tags, serviceSpec.CanaryTags) + } else { + tags = make([]string, len(serviceSpec.Tags)) + copy(tags, serviceSpec.Tags) + } + + return &structs.ServiceRegistration{ + ID: serviceregistration.MakeAllocServiceID(workload.AllocID, workload.Name(), serviceSpec), + ServiceName: serviceSpec.Name, + NodeID: s.cfg.NodeID, + JobID: workload.JobID, + AllocID: workload.AllocID, + Namespace: workload.Namespace, + Datacenter: s.cfg.Datacenter, + Tags: tags, + Address: ip, + Port: port, + }, nil +} diff --git a/client/serviceregistration/nsd/nsd_test.go b/client/serviceregistration/nsd/nsd_test.go new file mode 100644 index 000000000000..935c247b73bf --- /dev/null +++ b/client/serviceregistration/nsd/nsd_test.go @@ -0,0 +1,553 @@ +package nsd + +import ( + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/nomad/client/serviceregistration" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestServiceRegistrationHandler_RegisterWorkload(t *testing.T) { + testCases := []struct { + inputCfg *ServiceRegistrationHandlerCfg + inputWorkload *serviceregistration.WorkloadServices + expectedRPCs map[string]int + expectedError error + name string + }{ + { + inputCfg: &ServiceRegistrationHandlerCfg{ + Enabled: false, + }, + inputWorkload: mockWorkload(), + expectedRPCs: map[string]int{}, + expectedError: errors.New(`service registration provider "nomad" not enabled`), + name: "registration disabled", + }, + { + inputCfg: &ServiceRegistrationHandlerCfg{ + Enabled: true, + }, + inputWorkload: mockWorkload(), + expectedRPCs: map[string]int{structs.ServiceRegistrationUpsertRPCMethod: 1}, + expectedError: nil, + name: "registration enabled", + }, + } + + // Create a logger we can use for all tests. + log := hclog.NewNullLogger() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + + // Add the mock RPC functionality. + mockRPC := mockRPC{callCounts: map[string]int{}} + tc.inputCfg.RPCFn = mockRPC.RPC + + // Create the handler and run the tests. + h := NewServiceRegistrationHandler(log, tc.inputCfg) + + actualErr := h.RegisterWorkload(tc.inputWorkload) + require.Equal(t, tc.expectedError, actualErr) + require.Equal(t, tc.expectedRPCs, mockRPC.calls()) + }) + } +} + +func TestServiceRegistrationHandler_RemoveWorkload(t *testing.T) { + testCases := []struct { + inputCfg *ServiceRegistrationHandlerCfg + inputWorkload *serviceregistration.WorkloadServices + expectedRPCs map[string]int + expectedError error + name string + }{ + { + inputCfg: &ServiceRegistrationHandlerCfg{ + Enabled: false, + }, + inputWorkload: mockWorkload(), + expectedRPCs: map[string]int{structs.ServiceRegistrationDeleteByIDRPCMethod: 2}, + expectedError: nil, + name: "registration disabled multiple services", + }, + { + inputCfg: &ServiceRegistrationHandlerCfg{ + Enabled: true, + }, + inputWorkload: mockWorkload(), + expectedRPCs: map[string]int{structs.ServiceRegistrationDeleteByIDRPCMethod: 2}, + expectedError: nil, + name: "registration enabled multiple services", + }, + } + + // Create a logger we can use for all tests. + log := hclog.NewNullLogger() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + + // Add the mock RPC functionality. + mockRPC := mockRPC{callCounts: map[string]int{}} + tc.inputCfg.RPCFn = mockRPC.RPC + + // Create the handler and run the tests. + h := NewServiceRegistrationHandler(log, tc.inputCfg) + + h.RemoveWorkload(tc.inputWorkload) + + require.Eventually(t, func() bool { + return assert.Equal(t, tc.expectedRPCs, mockRPC.calls()) + }, 100*time.Millisecond, 10*time.Millisecond) + }) + } +} + +func TestServiceRegistrationHandler_UpdateWorkload(t *testing.T) { + testCases := []struct { + inputCfg *ServiceRegistrationHandlerCfg + inputOldWorkload *serviceregistration.WorkloadServices + inputNewWorkload *serviceregistration.WorkloadServices + expectedRPCs map[string]int + expectedError error + name string + }{ + { + inputCfg: &ServiceRegistrationHandlerCfg{ + Enabled: true, + }, + inputOldWorkload: mockWorkload(), + inputNewWorkload: &serviceregistration.WorkloadServices{ + AllocID: "98ea220b-7ebe-4662-6d74-9868e797717c", + Task: "redis", + Group: "cache", + JobID: "example", + Canary: false, + Namespace: "default", + Services: []*structs.Service{ + { + Name: "changed-redis-db", + AddressMode: structs.AddressModeHost, + PortLabel: "db", + }, + { + Name: "changed-redis-http", + AddressMode: structs.AddressModeHost, + PortLabel: "http", + }, + }, + Ports: []structs.AllocatedPortMapping{ + { + Label: "db", + HostIP: "10.10.13.2", + Value: 23098, + }, + { + Label: "http", + HostIP: "10.10.13.2", + Value: 24098, + }, + }, + }, + expectedRPCs: map[string]int{ + structs.ServiceRegistrationUpsertRPCMethod: 1, + structs.ServiceRegistrationDeleteByIDRPCMethod: 2, + }, + expectedError: nil, + name: "delete and upsert", + }, + { + inputCfg: &ServiceRegistrationHandlerCfg{ + Enabled: true, + }, + inputOldWorkload: mockWorkload(), + inputNewWorkload: &serviceregistration.WorkloadServices{ + AllocID: "98ea220b-7ebe-4662-6d74-9868e797717c", + Task: "redis", + Group: "cache", + JobID: "example", + Canary: false, + Namespace: "default", + Services: []*structs.Service{ + { + Name: "redis-db", + AddressMode: structs.AddressModeHost, + PortLabel: "db", + Tags: []string{"foo"}, + }, + { + Name: "redis-http", + AddressMode: structs.AddressModeHost, + PortLabel: "http", + Tags: []string{"bar"}, + }, + }, + Ports: []structs.AllocatedPortMapping{ + { + Label: "db", + HostIP: "10.10.13.2", + Value: 23098, + }, + { + Label: "http", + HostIP: "10.10.13.2", + Value: 24098, + }, + }, + }, + expectedRPCs: map[string]int{ + structs.ServiceRegistrationUpsertRPCMethod: 1, + }, + expectedError: nil, + name: "upsert only", + }, + } + + // Create a logger we can use for all tests. + log := hclog.NewNullLogger() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + + // Add the mock RPC functionality. + mockRPC := mockRPC{callCounts: map[string]int{}} + tc.inputCfg.RPCFn = mockRPC.RPC + + // Create the handler and run the tests. + h := NewServiceRegistrationHandler(log, tc.inputCfg) + + require.Equal(t, tc.expectedError, h.UpdateWorkload(tc.inputOldWorkload, tc.inputNewWorkload)) + + require.Eventually(t, func() bool { + return assert.Equal(t, tc.expectedRPCs, mockRPC.calls()) + }, 100*time.Millisecond, 10*time.Millisecond) + }) + } + +} + +func TestServiceRegistrationHandler_dedupUpdatedWorkload(t *testing.T) { + testCases := []struct { + inputOldWorkload *serviceregistration.WorkloadServices + inputNewWorkload *serviceregistration.WorkloadServices + expectedOldOutput *serviceregistration.WorkloadServices + expectedNewOutput *serviceregistration.WorkloadServices + name string + }{ + { + inputOldWorkload: mockWorkload(), + inputNewWorkload: &serviceregistration.WorkloadServices{ + AllocID: "98ea220b-7ebe-4662-6d74-9868e797717c", + Task: "redis", + Group: "cache", + JobID: "example", + Canary: false, + Namespace: "default", + Services: []*structs.Service{ + { + Name: "changed-redis-db", + AddressMode: structs.AddressModeHost, + PortLabel: "db", + }, + { + Name: "changed-redis-http", + AddressMode: structs.AddressModeHost, + PortLabel: "http", + }, + }, + Ports: []structs.AllocatedPortMapping{ + { + Label: "db", + HostIP: "10.10.13.2", + Value: 23098, + }, + { + Label: "http", + HostIP: "10.10.13.2", + Value: 24098, + }, + }, + }, + expectedOldOutput: mockWorkload(), + expectedNewOutput: &serviceregistration.WorkloadServices{ + AllocID: "98ea220b-7ebe-4662-6d74-9868e797717c", + Task: "redis", + Group: "cache", + JobID: "example", + Canary: false, + Namespace: "default", + Services: []*structs.Service{ + { + Name: "changed-redis-db", + AddressMode: structs.AddressModeHost, + PortLabel: "db", + }, + { + Name: "changed-redis-http", + AddressMode: structs.AddressModeHost, + PortLabel: "http", + }, + }, + Ports: []structs.AllocatedPortMapping{ + { + Label: "db", + HostIP: "10.10.13.2", + Value: 23098, + }, + { + Label: "http", + HostIP: "10.10.13.2", + Value: 24098, + }, + }, + }, + name: "service names changed", + }, + { + inputOldWorkload: mockWorkload(), + inputNewWorkload: &serviceregistration.WorkloadServices{ + AllocID: "98ea220b-7ebe-4662-6d74-9868e797717c", + Task: "redis", + Group: "cache", + JobID: "example", + Canary: false, + Namespace: "default", + Services: []*structs.Service{ + { + Name: "redis-db", + AddressMode: structs.AddressModeHost, + PortLabel: "db", + Tags: []string{"foo"}, + }, + { + Name: "redis-http", + AddressMode: structs.AddressModeHost, + PortLabel: "http", + Tags: []string{"bar"}, + }, + }, + Ports: []structs.AllocatedPortMapping{ + { + Label: "db", + HostIP: "10.10.13.2", + Value: 23098, + }, + { + Label: "http", + HostIP: "10.10.13.2", + Value: 24098, + }, + }, + }, + expectedOldOutput: &serviceregistration.WorkloadServices{ + AllocID: "98ea220b-7ebe-4662-6d74-9868e797717c", + Task: "redis", + Group: "cache", + JobID: "example", + Canary: false, + Namespace: "default", + Services: []*structs.Service{}, + Ports: []structs.AllocatedPortMapping{ + { + Label: "db", + HostIP: "10.10.13.2", + Value: 23098, + }, + { + Label: "http", + HostIP: "10.10.13.2", + Value: 24098, + }, + }, + }, + expectedNewOutput: &serviceregistration.WorkloadServices{ + AllocID: "98ea220b-7ebe-4662-6d74-9868e797717c", + Task: "redis", + Group: "cache", + JobID: "example", + Canary: false, + Namespace: "default", + Services: []*structs.Service{ + { + Name: "redis-db", + AddressMode: structs.AddressModeHost, + PortLabel: "db", + Tags: []string{"foo"}, + }, + { + Name: "redis-http", + AddressMode: structs.AddressModeHost, + PortLabel: "http", + Tags: []string{"bar"}, + }, + }, + Ports: []structs.AllocatedPortMapping{ + { + Label: "db", + HostIP: "10.10.13.2", + Value: 23098, + }, + { + Label: "http", + HostIP: "10.10.13.2", + Value: 24098, + }, + }, + }, + name: "tags updated", + }, + { + inputOldWorkload: mockWorkload(), + inputNewWorkload: &serviceregistration.WorkloadServices{ + AllocID: "98ea220b-7ebe-4662-6d74-9868e797717c", + Task: "redis", + Group: "cache", + JobID: "example", + Canary: false, + Namespace: "default", + Services: []*structs.Service{ + { + Name: "redis-db", + AddressMode: structs.AddressModeHost, + PortLabel: "dbs", + }, + { + Name: "redis-http", + AddressMode: structs.AddressModeHost, + PortLabel: "https", + }, + }, + Ports: []structs.AllocatedPortMapping{ + { + Label: "dbs", + HostIP: "10.10.13.2", + Value: 23098, + }, + { + Label: "https", + HostIP: "10.10.13.2", + Value: 24098, + }, + }, + }, + expectedOldOutput: mockWorkload(), + expectedNewOutput: &serviceregistration.WorkloadServices{ + AllocID: "98ea220b-7ebe-4662-6d74-9868e797717c", + Task: "redis", + Group: "cache", + JobID: "example", + Canary: false, + Namespace: "default", + Services: []*structs.Service{ + { + Name: "redis-db", + AddressMode: structs.AddressModeHost, + PortLabel: "dbs", + }, + { + Name: "redis-http", + AddressMode: structs.AddressModeHost, + PortLabel: "https", + }, + }, + Ports: []structs.AllocatedPortMapping{ + { + Label: "dbs", + HostIP: "10.10.13.2", + Value: 23098, + }, + { + Label: "https", + HostIP: "10.10.13.2", + Value: 24098, + }, + }, + }, + name: "canary tags updated", + }, + } + + s := &ServiceRegistrationHandler{} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualOld, actualNew := s.dedupUpdatedWorkload(tc.inputOldWorkload, tc.inputNewWorkload) + require.ElementsMatch(t, tc.expectedOldOutput.Services, actualOld.Services) + require.ElementsMatch(t, tc.expectedNewOutput.Services, actualNew.Services) + }) + } +} + +func mockWorkload() *serviceregistration.WorkloadServices { + return &serviceregistration.WorkloadServices{ + AllocID: "98ea220b-7ebe-4662-6d74-9868e797717c", + Task: "redis", + Group: "cache", + JobID: "example", + Canary: false, + Namespace: "default", + Services: []*structs.Service{ + { + Name: "redis-db", + AddressMode: structs.AddressModeHost, + PortLabel: "db", + }, + { + Name: "redis-http", + AddressMode: structs.AddressModeHost, + PortLabel: "http", + }, + }, + Ports: []structs.AllocatedPortMapping{ + { + Label: "db", + HostIP: "10.10.13.2", + Value: 23098, + }, + { + Label: "http", + HostIP: "10.10.13.2", + Value: 24098, + }, + }, + } +} + +// mockRPC mocks and tracks RPC calls made for testing. +type mockRPC struct { + + // callCounts tracks how many times each RPC method has been called. The + // lock should be used to access this. + callCounts map[string]int + l sync.RWMutex +} + +// calls returns the mapping counting the number of calls made to each RPC +// method. +func (mr *mockRPC) calls() map[string]int { + mr.l.RLock() + defer mr.l.RUnlock() + return mr.callCounts +} + +// RPC mocks the server RPCs, acting as though any request succeeds. +func (mr *mockRPC) RPC(method string, _, _ interface{}) error { + switch method { + case structs.ServiceRegistrationUpsertRPCMethod, structs.ServiceRegistrationDeleteByIDRPCMethod: + mr.l.Lock() + mr.callCounts[method]++ + mr.l.Unlock() + return nil + default: + return fmt.Errorf("unexpected RPC method: %v", method) + } +} From 131cda28242956a2c4eb18f77256d1daba067935 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Mon, 21 Mar 2022 09:49:39 +0100 Subject: [PATCH 25/31] client: modify service wrapper to accomodate restore behaviour. --- client/serviceregistration/wrapper/wrapper.go | 26 ++++++++++++------- .../wrapper/wrapper_test.go | 6 ++--- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/client/serviceregistration/wrapper/wrapper.go b/client/serviceregistration/wrapper/wrapper.go index 37b5e4f614be..1c52dfd1987f 100644 --- a/client/serviceregistration/wrapper/wrapper.go +++ b/client/serviceregistration/wrapper/wrapper.go @@ -66,20 +66,28 @@ func (h *HandlerWrapper) RegisterWorkload(workload *serviceregistration.Workload // workload unless the provider is unknown. func (h *HandlerWrapper) RemoveWorkload(services *serviceregistration.WorkloadServices) { - // Don't rely on callers to check there are no services to remove. - if len(services.Services) == 0 { - return - } + var provider string - provider := services.RegistrationProvider() + // It is possible the services field is empty depending on the exact + // situation which resulted in the call. + if len(services.Services) > 0 { + provider = services.RegistrationProvider() + } - // Call the correct provider. In the case it is not supported, we can't do - // much apart from log, although we should never reach this point because - // of job validation. + // Call the correct provider, if we have managed to identify it. An empty + // string means you didn't find a provider, therefore default to consul. + // + // In certain situations this function is called with zero services, + // therefore meaning we make an assumption on the provider. When this + // happens, we need to ensure the allocation is removed from the Consul + // implementation. This tracking (allocRegistrations) is used by the + // allochealth tracker and so is critical to be removed. The test + // allocrunner.TestAllocRunner_Restore_RunningTerminal covers the case + // described here. switch provider { case structs.ServiceProviderNomad: h.nomadServiceProvider.RemoveWorkload(services) - case structs.ServiceProviderConsul: + case structs.ServiceProviderConsul, "": h.consulServiceProvider.RemoveWorkload(services) default: h.log.Error("unknown service registration provider", "provider", provider) diff --git a/client/serviceregistration/wrapper/wrapper_test.go b/client/serviceregistration/wrapper/wrapper_test.go index 81dfd7c1f67e..2acb82376262 100644 --- a/client/serviceregistration/wrapper/wrapper_test.go +++ b/client/serviceregistration/wrapper/wrapper_test.go @@ -131,10 +131,10 @@ func TestHandlerWrapper_RemoveWorkload(t *testing.T) { // Generate the test wrapper and provider mocks. wrapper, consul, nomad := setupTestWrapper() - // Call the function with no services and check that nothing is - // registered. + // Call the function with no services and check that consul is + // defaulted to. wrapper.RemoveWorkload(&serviceregistration.WorkloadServices{}) - require.Len(t, consul.GetOps(), 0) + require.Len(t, consul.GetOps(), 1) require.Len(t, nomad.GetOps(), 0) }, name: "zero services", From f0be952cb50ba58f20b3a2c18818da3d6dc479fa Mon Sep 17 00:00:00 2001 From: James Rasell Date: Mon, 21 Mar 2022 10:29:57 +0100 Subject: [PATCH 26/31] client: hookup service wrapper for use within client hooks. --- client/allocrunner/alloc_runner.go | 10 +- client/allocrunner/alloc_runner_hooks.go | 4 +- client/allocrunner/alloc_runner_test.go | 4 +- client/allocrunner/alloc_runner_unix_test.go | 1 + client/allocrunner/config.go | 5 + client/allocrunner/groupservice_hook.go | 36 ++-- client/allocrunner/groupservice_hook_test.go | 165 +++++++++++++----- .../taskrunner/script_check_hook_test.go | 10 +- client/allocrunner/taskrunner/service_hook.go | 70 +++++--- .../taskrunner/service_hook_test.go | 137 +++++++++++++-- client/allocrunner/taskrunner/task_runner.go | 10 ++ .../taskrunner/task_runner_hooks.go | 18 +- .../taskrunner/task_runner_test.go | 14 +- client/allocrunner/testing.go | 11 +- client/client.go | 42 ++++- command/agent/agent.go | 4 +- command/agent/consul/int_test.go | 5 + nomad/structs/alloc.go | 34 ++++ nomad/structs/alloc_test.go | 102 +++++++++++ 19 files changed, 566 insertions(+), 116 deletions(-) diff --git a/client/allocrunner/alloc_runner.go b/client/allocrunner/alloc_runner.go index 17971c72bb9b..fa01c618b2ed 100644 --- a/client/allocrunner/alloc_runner.go +++ b/client/allocrunner/alloc_runner.go @@ -6,8 +6,6 @@ import ( "sync" "time" - "github.com/hashicorp/nomad/client/lib/cgutil" - log "github.com/hashicorp/go-hclog" multierror "github.com/hashicorp/go-multierror" "github.com/hashicorp/nomad/client/allocdir" @@ -20,9 +18,11 @@ import ( "github.com/hashicorp/nomad/client/devicemanager" "github.com/hashicorp/nomad/client/dynamicplugins" cinterfaces "github.com/hashicorp/nomad/client/interfaces" + "github.com/hashicorp/nomad/client/lib/cgutil" "github.com/hashicorp/nomad/client/pluginmanager/csimanager" "github.com/hashicorp/nomad/client/pluginmanager/drivermanager" "github.com/hashicorp/nomad/client/serviceregistration" + "github.com/hashicorp/nomad/client/serviceregistration/wrapper" cstate "github.com/hashicorp/nomad/client/state" cstructs "github.com/hashicorp/nomad/client/structs" "github.com/hashicorp/nomad/client/vaultclient" @@ -178,6 +178,10 @@ type allocRunner struct { // rpcClient is the RPC Client that should be used by the allocrunner and its // hooks to communicate with Nomad Servers. rpcClient RPCer + + // serviceRegWrapper is the handler wrapper that is used by service hooks + // to perform service and check registration and deregistration. + serviceRegWrapper *wrapper.HandlerWrapper } // RPCer is the interface needed by hooks to make RPC calls. @@ -221,6 +225,7 @@ func NewAllocRunner(config *Config) (*allocRunner, error) { driverManager: config.DriverManager, serversContactedCh: config.ServersContactedCh, rpcClient: config.RPCClient, + serviceRegWrapper: config.ServiceRegWrapper, } // Create the logger based on the allocation ID @@ -274,6 +279,7 @@ func (ar *allocRunner) initTaskRunners(tasks []*structs.Task) error { ServersContactedCh: ar.serversContactedCh, StartConditionMetCtx: ar.taskHookCoordinator.startConditionForTask(task), ShutdownDelayCtx: ar.shutdownDelayCtx, + ServiceRegWrapper: ar.serviceRegWrapper, } if ar.cpusetManager != nil { diff --git a/client/allocrunner/alloc_runner_hooks.go b/client/allocrunner/alloc_runner_hooks.go index 52252f08313f..30611b394aae 100644 --- a/client/allocrunner/alloc_runner_hooks.go +++ b/client/allocrunner/alloc_runner_hooks.go @@ -153,8 +153,8 @@ func (ar *allocRunner) initRunnerHooks(config *clientconfig.Config) error { newNetworkHook(hookLogger, ns, alloc, nm, nc, ar, builtTaskEnv), newGroupServiceHook(groupServiceHookConfig{ alloc: alloc, - consul: ar.consulClient, - consulNamespace: alloc.ConsulNamespace(), + namespace: alloc.ServiceProviderNamespace(), + serviceRegWrapper: ar.serviceRegWrapper, restarter: ar, taskEnvBuilder: envBuilder, networkStatusGetter: ar, diff --git a/client/allocrunner/alloc_runner_test.go b/client/allocrunner/alloc_runner_test.go index 23d59dc6915b..93f0e399876c 100644 --- a/client/allocrunner/alloc_runner_test.go +++ b/client/allocrunner/alloc_runner_test.go @@ -489,7 +489,8 @@ func TestAllocRunner_TaskGroup_ShutdownDelay(t *testing.T) { tg := alloc.Job.TaskGroups[0] tg.Services = []*structs.Service{ { - Name: "shutdown_service", + Name: "shutdown_service", + Provider: structs.ServiceProviderConsul, }, } @@ -1314,6 +1315,7 @@ func TestAllocRunner_TaskFailed_KillTG(t *testing.T) { { Name: "fakservice", PortLabel: "http", + Provider: structs.ServiceProviderConsul, Checks: []*structs.ServiceCheck{ { Name: "fakecheck", diff --git a/client/allocrunner/alloc_runner_unix_test.go b/client/allocrunner/alloc_runner_unix_test.go index 63d06fd3fea5..a321f54cc3af 100644 --- a/client/allocrunner/alloc_runner_unix_test.go +++ b/client/allocrunner/alloc_runner_unix_test.go @@ -38,6 +38,7 @@ func TestAllocRunner_Restore_RunningTerminal(t *testing.T) { { Name: "foo", PortLabel: "8888", + Provider: structs.ServiceProviderConsul, }, } task := alloc.Job.TaskGroups[0].Tasks[0] diff --git a/client/allocrunner/config.go b/client/allocrunner/config.go index d1500c906744..0ec3ba51c3ad 100644 --- a/client/allocrunner/config.go +++ b/client/allocrunner/config.go @@ -12,6 +12,7 @@ import ( "github.com/hashicorp/nomad/client/pluginmanager/csimanager" "github.com/hashicorp/nomad/client/pluginmanager/drivermanager" "github.com/hashicorp/nomad/client/serviceregistration" + "github.com/hashicorp/nomad/client/serviceregistration/wrapper" cstate "github.com/hashicorp/nomad/client/state" "github.com/hashicorp/nomad/client/vaultclient" "github.com/hashicorp/nomad/nomad/structs" @@ -81,4 +82,8 @@ type Config struct { // RPCClient is the RPC Client that should be used by the allocrunner and its // hooks to communicate with Nomad Servers. RPCClient RPCer + + // ServiceRegWrapper is the handler wrapper that is used by service hooks + // to perform service and check registration and deregistration. + ServiceRegWrapper *wrapper.HandlerWrapper } diff --git a/client/allocrunner/groupservice_hook.go b/client/allocrunner/groupservice_hook.go index 1d5a6205373f..d94de2db6637 100644 --- a/client/allocrunner/groupservice_hook.go +++ b/client/allocrunner/groupservice_hook.go @@ -8,6 +8,7 @@ import ( log "github.com/hashicorp/go-hclog" "github.com/hashicorp/nomad/client/allocrunner/interfaces" "github.com/hashicorp/nomad/client/serviceregistration" + "github.com/hashicorp/nomad/client/serviceregistration/wrapper" "github.com/hashicorp/nomad/client/taskenv" agentconsul "github.com/hashicorp/nomad/command/agent/consul" "github.com/hashicorp/nomad/nomad/structs" @@ -25,15 +26,22 @@ type networkStatusGetter interface { // deregistration. type groupServiceHook struct { allocID string + jobID string group string restarter agentconsul.WorkloadRestarter - consulClient serviceregistration.Handler - consulNamespace string prerun bool deregistered bool networkStatusGetter networkStatusGetter shutdownDelayCtx context.Context + // namespace is the Nomad or Consul namespace in which service + // registrations will be made. + namespace string + + // serviceRegWrapper is the handler wrapper that is used to perform service + // and check registration and deregistration. + serviceRegWrapper *wrapper.HandlerWrapper + logger log.Logger // The following fields may be updated @@ -51,13 +59,19 @@ type groupServiceHook struct { type groupServiceHookConfig struct { alloc *structs.Allocation - consul serviceregistration.Handler - consulNamespace string restarter agentconsul.WorkloadRestarter taskEnvBuilder *taskenv.Builder networkStatusGetter networkStatusGetter shutdownDelayCtx context.Context logger log.Logger + + // namespace is the Nomad or Consul namespace in which service + // registrations will be made. + namespace string + + // serviceRegWrapper is the handler wrapper that is used to perform service + // and check registration and deregistration. + serviceRegWrapper *wrapper.HandlerWrapper } func newGroupServiceHook(cfg groupServiceHookConfig) *groupServiceHook { @@ -70,15 +84,16 @@ func newGroupServiceHook(cfg groupServiceHookConfig) *groupServiceHook { h := &groupServiceHook{ allocID: cfg.alloc.ID, + jobID: cfg.alloc.JobID, group: cfg.alloc.TaskGroup, restarter: cfg.restarter, - consulClient: cfg.consul, - consulNamespace: cfg.consulNamespace, + namespace: cfg.namespace, taskEnvBuilder: cfg.taskEnvBuilder, delay: shutdownDelay, networkStatusGetter: cfg.networkStatusGetter, logger: cfg.logger.Named(groupServiceHookName), services: cfg.alloc.Job.LookupTaskGroup(cfg.alloc.TaskGroup).Services, + serviceRegWrapper: cfg.serviceRegWrapper, shutdownDelayCtx: cfg.shutdownDelayCtx, } @@ -114,7 +129,7 @@ func (h *groupServiceHook) prerunLocked() error { } services := h.getWorkloadServices() - return h.consulClient.RegisterWorkload(services) + return h.serviceRegWrapper.RegisterWorkload(services) } func (h *groupServiceHook) Update(req *interfaces.RunnerUpdateRequest) error { @@ -157,7 +172,7 @@ func (h *groupServiceHook) Update(req *interfaces.RunnerUpdateRequest) error { return nil } - return h.consulClient.UpdateWorkload(oldWorkloadServices, newWorkloadServices) + return h.serviceRegWrapper.UpdateWorkload(oldWorkloadServices, newWorkloadServices) } func (h *groupServiceHook) PreTaskRestart() error { @@ -213,7 +228,7 @@ func (h *groupServiceHook) Postrun() error { func (h *groupServiceHook) deregister() { if len(h.services) > 0 { workloadServices := h.getWorkloadServices() - h.consulClient.RemoveWorkload(workloadServices) + h.serviceRegWrapper.RemoveWorkload(workloadServices) } } @@ -229,8 +244,9 @@ func (h *groupServiceHook) getWorkloadServices() *serviceregistration.WorkloadSe // Create task services struct with request's driver metadata return &serviceregistration.WorkloadServices{ AllocID: h.allocID, + JobID: h.jobID, Group: h.group, - Namespace: h.consulNamespace, + Namespace: h.namespace, Restarter: h.restarter, Services: interpolatedServices, Networks: h.networks, diff --git a/client/allocrunner/groupservice_hook_test.go b/client/allocrunner/groupservice_hook_test.go index 4fae46a300af..79cd122c10e5 100644 --- a/client/allocrunner/groupservice_hook_test.go +++ b/client/allocrunner/groupservice_hook_test.go @@ -9,6 +9,7 @@ import ( ctestutil "github.com/hashicorp/consul/sdk/testutil" "github.com/hashicorp/nomad/client/allocrunner/interfaces" regMock "github.com/hashicorp/nomad/client/serviceregistration/mock" + "github.com/hashicorp/nomad/client/serviceregistration/wrapper" "github.com/hashicorp/nomad/client/taskenv" agentconsul "github.com/hashicorp/nomad/command/agent/consul" "github.com/hashicorp/nomad/helper" @@ -32,17 +33,24 @@ func TestGroupServiceHook_NoGroupServices(t *testing.T) { alloc := mock.Alloc() alloc.Job.TaskGroups[0].Services = []*structs.Service{{ Name: "foo", + Provider: "consul", PortLabel: "9999", }} logger := testlog.HCLogger(t) - consulClient := regMock.NewServiceRegistrationHandler(logger) + + consulMockClient := regMock.NewServiceRegistrationHandler(logger) + + regWrapper := wrapper.NewHandlerWrapper( + logger, + consulMockClient, + regMock.NewServiceRegistrationHandler(logger)) h := newGroupServiceHook(groupServiceHookConfig{ - alloc: alloc, - consul: consulClient, - restarter: agentconsul.NoopRestarter(), - taskEnvBuilder: taskenv.NewBuilder(mock.Node(), alloc, nil, alloc.Job.Region), - logger: logger, + alloc: alloc, + serviceRegWrapper: regWrapper, + restarter: agentconsul.NoopRestarter(), + taskEnvBuilder: taskenv.NewBuilder(mock.Node(), alloc, nil, alloc.Job.Region), + logger: logger, }) require.NoError(t, h.Prerun()) @@ -53,7 +61,7 @@ func TestGroupServiceHook_NoGroupServices(t *testing.T) { require.NoError(t, h.PreTaskRestart()) - ops := consulClient.GetOps() + ops := consulMockClient.GetOps() require.Len(t, ops, 5) require.Equal(t, "add", ops[0].Op) // Prerun require.Equal(t, "update", ops[1].Op) // Update @@ -71,14 +79,20 @@ func TestGroupServiceHook_ShutdownDelayUpdate(t *testing.T) { alloc.Job.TaskGroups[0].ShutdownDelay = helper.TimeToPtr(10 * time.Second) logger := testlog.HCLogger(t) - consulClient := regMock.NewServiceRegistrationHandler(logger) + consulMockClient := regMock.NewServiceRegistrationHandler(logger) + + regWrapper := wrapper.NewHandlerWrapper( + logger, + consulMockClient, + regMock.NewServiceRegistrationHandler(logger), + ) h := newGroupServiceHook(groupServiceHookConfig{ - alloc: alloc, - consul: consulClient, - restarter: agentconsul.NoopRestarter(), - taskEnvBuilder: taskenv.NewBuilder(mock.Node(), alloc, nil, alloc.Job.Region), - logger: logger, + alloc: alloc, + serviceRegWrapper: regWrapper, + restarter: agentconsul.NoopRestarter(), + taskEnvBuilder: taskenv.NewBuilder(mock.Node(), alloc, nil, alloc.Job.Region), + logger: logger, }) require.NoError(t, h.Prerun()) @@ -105,15 +119,21 @@ func TestGroupServiceHook_GroupServices(t *testing.T) { t.Parallel() alloc := mock.ConnectAlloc() + alloc.Job.Canonicalize() logger := testlog.HCLogger(t) - consulClient := regMock.NewServiceRegistrationHandler(logger) + consulMockClient := regMock.NewServiceRegistrationHandler(logger) + + regWrapper := wrapper.NewHandlerWrapper( + logger, + consulMockClient, + regMock.NewServiceRegistrationHandler(logger)) h := newGroupServiceHook(groupServiceHookConfig{ - alloc: alloc, - consul: consulClient, - restarter: agentconsul.NoopRestarter(), - taskEnvBuilder: taskenv.NewBuilder(mock.Node(), alloc, nil, alloc.Job.Region), - logger: logger, + alloc: alloc, + serviceRegWrapper: regWrapper, + restarter: agentconsul.NoopRestarter(), + taskEnvBuilder: taskenv.NewBuilder(mock.Node(), alloc, nil, alloc.Job.Region), + logger: logger, }) require.NoError(t, h.Prerun()) @@ -124,7 +144,7 @@ func TestGroupServiceHook_GroupServices(t *testing.T) { require.NoError(t, h.PreTaskRestart()) - ops := consulClient.GetOps() + ops := consulMockClient.GetOps() require.Len(t, ops, 5) require.Equal(t, "add", ops[0].Op) // Prerun require.Equal(t, "update", ops[1].Op) // Update @@ -133,6 +153,55 @@ func TestGroupServiceHook_GroupServices(t *testing.T) { require.Equal(t, "add", ops[4].Op) // Restart -> preRun } +// TestGroupServiceHook_GroupServices_Nomad asserts group service hooks with +// group services does not error when using the Nomad provider. +func TestGroupServiceHook_GroupServices_Nomad(t *testing.T) { + t.Parallel() + + // Create a mock alloc, and add a group service using provider Nomad. + alloc := mock.Alloc() + alloc.Job.TaskGroups[0].Services = []*structs.Service{ + { + Name: "nomad-provider-service", + Provider: structs.ServiceProviderNomad, + }, + } + + // Create our base objects and our subsequent wrapper. + logger := testlog.HCLogger(t) + consulMockClient := regMock.NewServiceRegistrationHandler(logger) + nomadMockClient := regMock.NewServiceRegistrationHandler(logger) + + regWrapper := wrapper.NewHandlerWrapper(logger, consulMockClient, nomadMockClient) + + h := newGroupServiceHook(groupServiceHookConfig{ + alloc: alloc, + serviceRegWrapper: regWrapper, + restarter: agentconsul.NoopRestarter(), + taskEnvBuilder: taskenv.NewBuilder(mock.Node(), alloc, nil, alloc.Job.Region), + logger: logger, + }) + require.NoError(t, h.Prerun()) + + // Trigger our hook requests. + req := &interfaces.RunnerUpdateRequest{Alloc: alloc} + require.NoError(t, h.Update(req)) + require.NoError(t, h.Postrun()) + require.NoError(t, h.PreTaskRestart()) + + // Ensure the Nomad mock provider has the expected operations. + ops := nomadMockClient.GetOps() + require.Len(t, ops, 5) + require.Equal(t, "add", ops[0].Op) // Prerun + require.Equal(t, "update", ops[1].Op) // Update + require.Equal(t, "remove", ops[2].Op) // Postrun + require.Equal(t, "remove", ops[3].Op) // Restart -> preKill + require.Equal(t, "add", ops[4].Op) // Restart -> preRun + + // Ensure the Consul mock provider has zero operations. + require.Len(t, consulMockClient.GetOps(), 0) +} + // TestGroupServiceHook_Error asserts group service hooks with group // services but no group network is handled gracefully. func TestGroupServiceHook_NoNetwork(t *testing.T) { @@ -144,6 +213,7 @@ func TestGroupServiceHook_NoNetwork(t *testing.T) { tg.Services = []*structs.Service{ { Name: "testconnect", + Provider: "consul", PortLabel: "9999", Connect: &structs.ConsulConnect{ SidecarService: &structs.ConsulSidecarService{}, @@ -152,14 +222,19 @@ func TestGroupServiceHook_NoNetwork(t *testing.T) { } logger := testlog.HCLogger(t) - consulClient := regMock.NewServiceRegistrationHandler(logger) + consulMockClient := regMock.NewServiceRegistrationHandler(logger) + + regWrapper := wrapper.NewHandlerWrapper( + logger, + consulMockClient, + regMock.NewServiceRegistrationHandler(logger)) h := newGroupServiceHook(groupServiceHookConfig{ - alloc: alloc, - consul: consulClient, - restarter: agentconsul.NoopRestarter(), - taskEnvBuilder: taskenv.NewBuilder(mock.Node(), alloc, nil, alloc.Job.Region), - logger: logger, + alloc: alloc, + serviceRegWrapper: regWrapper, + restarter: agentconsul.NoopRestarter(), + taskEnvBuilder: taskenv.NewBuilder(mock.Node(), alloc, nil, alloc.Job.Region), + logger: logger, }) require.NoError(t, h.Prerun()) @@ -170,7 +245,7 @@ func TestGroupServiceHook_NoNetwork(t *testing.T) { require.NoError(t, h.PreTaskRestart()) - ops := consulClient.GetOps() + ops := consulMockClient.GetOps() require.Len(t, ops, 5) require.Equal(t, "add", ops[0].Op) // Prerun require.Equal(t, "update", ops[1].Op) // Update @@ -196,14 +271,19 @@ func TestGroupServiceHook_getWorkloadServices(t *testing.T) { } logger := testlog.HCLogger(t) - consulClient := regMock.NewServiceRegistrationHandler(logger) + consulMockClient := regMock.NewServiceRegistrationHandler(logger) + + regWrapper := wrapper.NewHandlerWrapper( + logger, + consulMockClient, + regMock.NewServiceRegistrationHandler(logger)) h := newGroupServiceHook(groupServiceHookConfig{ - alloc: alloc, - consul: consulClient, - restarter: agentconsul.NoopRestarter(), - taskEnvBuilder: taskenv.NewBuilder(mock.Node(), alloc, nil, alloc.Job.Region), - logger: logger, + alloc: alloc, + serviceRegWrapper: regWrapper, + restarter: agentconsul.NoopRestarter(), + taskEnvBuilder: taskenv.NewBuilder(mock.Node(), alloc, nil, alloc.Job.Region), + logger: logger, }) services := h.getWorkloadServices() @@ -234,8 +314,15 @@ func TestGroupServiceHook_Update08Alloc(t *testing.T) { require.NoError(t, err) namespacesClient := agentconsul.NewNamespacesClient(consulClient.Namespaces(), consulClient.Agent()) + logger := testlog.HCLogger(t) serviceClient := agentconsul.NewServiceClient(consulClient.Agent(), namespacesClient, testlog.HCLogger(t), true) + regWrapper := wrapper.NewHandlerWrapper( + logger, + serviceClient, + regMock.NewServiceRegistrationHandler(logger), + ) + // Lower periodicInterval to ensure periodic syncing doesn't improperly // remove Connect services. //const interval = 50 * time.Millisecond @@ -266,6 +353,7 @@ func TestGroupServiceHook_Update08Alloc(t *testing.T) { tg.Services = []*structs.Service{ { Name: "testconnect", + Provider: "consul", PortLabel: "9999", Connect: &structs.ConsulConnect{ SidecarService: &structs.ConsulSidecarService{ @@ -284,11 +372,11 @@ func TestGroupServiceHook_Update08Alloc(t *testing.T) { // Create the group service hook h := newGroupServiceHook(groupServiceHookConfig{ - alloc: oldAlloc, - consul: serviceClient, - restarter: agentconsul.NoopRestarter(), - taskEnvBuilder: taskenv.NewBuilder(mock.Node(), oldAlloc, nil, oldAlloc.Job.Region), - logger: testlog.HCLogger(t), + alloc: oldAlloc, + serviceRegWrapper: regWrapper, + restarter: agentconsul.NoopRestarter(), + taskEnvBuilder: taskenv.NewBuilder(mock.Node(), oldAlloc, nil, oldAlloc.Job.Region), + logger: testlog.HCLogger(t), }) require.NoError(t, h.Prerun()) @@ -300,5 +388,4 @@ func TestGroupServiceHook_Update08Alloc(t *testing.T) { require.NoError(t, err) return len(services) == 2 }, 3*time.Second, 100*time.Millisecond) - } diff --git a/client/allocrunner/taskrunner/script_check_hook_test.go b/client/allocrunner/taskrunner/script_check_hook_test.go index 5463a0b2e1ae..2c149ae15454 100644 --- a/client/allocrunner/taskrunner/script_check_hook_test.go +++ b/client/allocrunner/taskrunner/script_check_hook_test.go @@ -12,6 +12,7 @@ import ( "github.com/hashicorp/nomad/client/allocrunner/taskrunner/interfaces" "github.com/hashicorp/nomad/client/serviceregistration" regMock "github.com/hashicorp/nomad/client/serviceregistration/mock" + "github.com/hashicorp/nomad/client/serviceregistration/wrapper" "github.com/hashicorp/nomad/client/taskenv" agentconsul "github.com/hashicorp/nomad/command/agent/consul" "github.com/hashicorp/nomad/helper/testlog" @@ -228,6 +229,7 @@ func TestScript_TaskEnvInterpolation(t *testing.T) { logger := testlog.HCLogger(t) consulClient := regMock.NewServiceRegistrationHandler(logger) + regWrap := wrapper.NewHandlerWrapper(logger, consulClient, nil) exec, cancel := newBlockingScriptExec() defer cancel() @@ -243,10 +245,10 @@ func TestScript_TaskEnvInterpolation(t *testing.T) { map[string]string{"SVC_NAME": "frontend"}).Build() svcHook := newServiceHook(serviceHookConfig{ - alloc: alloc, - task: task, - consulServices: consulClient, - logger: logger, + alloc: alloc, + task: task, + serviceRegWrapper: regWrap, + logger: logger, }) // emulate prestart having been fired svcHook.taskEnv = env diff --git a/client/allocrunner/taskrunner/service_hook.go b/client/allocrunner/taskrunner/service_hook.go index 9684317e5411..872453527061 100644 --- a/client/allocrunner/taskrunner/service_hook.go +++ b/client/allocrunner/taskrunner/service_hook.go @@ -9,6 +9,7 @@ import ( "github.com/hashicorp/nomad/client/allocrunner/interfaces" tinterfaces "github.com/hashicorp/nomad/client/allocrunner/taskrunner/interfaces" "github.com/hashicorp/nomad/client/serviceregistration" + "github.com/hashicorp/nomad/client/serviceregistration/wrapper" "github.com/hashicorp/nomad/client/taskenv" agentconsul "github.com/hashicorp/nomad/command/agent/consul" "github.com/hashicorp/nomad/nomad/structs" @@ -21,11 +22,21 @@ var _ interfaces.TaskExitedHook = &serviceHook{} var _ interfaces.TaskStopHook = &serviceHook{} var _ interfaces.TaskUpdateHook = &serviceHook{} +const ( + taskServiceHookName = "task_services" +) + type serviceHookConfig struct { - alloc *structs.Allocation - task *structs.Task - consulServices serviceregistration.Handler - consulNamespace string + alloc *structs.Allocation + task *structs.Task + + // namespace is the Nomad or Consul namespace in which service + // registrations will be made. + namespace string + + // serviceRegWrapper is the handler wrapper that is used to perform service + // and check registration and deregistration. + serviceRegWrapper *wrapper.HandlerWrapper // Restarter is a subset of the TaskLifecycle interface restarter agentconsul.WorkloadRestarter @@ -34,12 +45,11 @@ type serviceHookConfig struct { } type serviceHook struct { - allocID string - taskName string - consulNamespace string - consulServices serviceregistration.Handler - restarter agentconsul.WorkloadRestarter - logger log.Logger + allocID string + jobID string + taskName string + restarter agentconsul.WorkloadRestarter + logger log.Logger // The following fields may be updated driverExec tinterfaces.ScriptExecutor @@ -50,6 +60,14 @@ type serviceHook struct { ports structs.AllocatedPorts taskEnv *taskenv.TaskEnv + // namespace is the Nomad or Consul namespace in which service + // registrations will be made. + namespace string + + // serviceRegWrapper is the handler wrapper that is used to perform service + // and check registration and deregistration. + serviceRegWrapper *wrapper.HandlerWrapper + // initialRegistrations tracks if Poststart has completed, initializing // fields required in other lifecycle funcs initialRegistration bool @@ -65,13 +83,14 @@ type serviceHook struct { func newServiceHook(c serviceHookConfig) *serviceHook { h := &serviceHook{ - allocID: c.alloc.ID, - taskName: c.task.Name, - consulServices: c.consulServices, - consulNamespace: c.consulNamespace, - services: c.task.Services, - restarter: c.restarter, - ports: c.alloc.AllocatedResources.Shared.Ports, + allocID: c.alloc.ID, + jobID: c.alloc.JobID, + taskName: c.task.Name, + namespace: c.namespace, + serviceRegWrapper: c.serviceRegWrapper, + services: c.task.Services, + restarter: c.restarter, + ports: c.alloc.AllocatedResources.Shared.Ports, } if res := c.alloc.AllocatedResources.Tasks[c.task.Name]; res != nil { @@ -86,9 +105,7 @@ func newServiceHook(c serviceHookConfig) *serviceHook { return h } -func (h *serviceHook) Name() string { - return "consul_services" -} +func (h *serviceHook) Name() string { return taskServiceHookName } func (h *serviceHook) Poststart(ctx context.Context, req *interfaces.TaskPoststartRequest, _ *interfaces.TaskPoststartResponse) error { h.mu.Lock() @@ -106,15 +123,15 @@ func (h *serviceHook) Poststart(ctx context.Context, req *interfaces.TaskPoststa // Create task services struct with request's driver metadata workloadServices := h.getWorkloadServices() - return h.consulServices.RegisterWorkload(workloadServices) + return h.serviceRegWrapper.RegisterWorkload(workloadServices) } func (h *serviceHook) Update(ctx context.Context, req *interfaces.TaskUpdateRequest, _ *interfaces.TaskUpdateResponse) error { h.mu.Lock() defer h.mu.Unlock() if !h.initialRegistration { - // no op Consul since initial registration has not finished - // only update hook fields + // no op since initial registration has not finished only update hook + // fields. return h.updateHookFields(req) } @@ -129,7 +146,7 @@ func (h *serviceHook) Update(ctx context.Context, req *interfaces.TaskUpdateRequ // Create new task services struct with those new values newWorkloadServices := h.getWorkloadServices() - return h.consulServices.UpdateWorkload(oldWorkloadServices, newWorkloadServices) + return h.serviceRegWrapper.UpdateWorkload(oldWorkloadServices, newWorkloadServices) } func (h *serviceHook) updateHookFields(req *interfaces.TaskUpdateRequest) error { @@ -180,7 +197,7 @@ func (h *serviceHook) Exited(context.Context, *interfaces.TaskExitedRequest, *in func (h *serviceHook) deregister() { if len(h.services) > 0 && !h.deregistered { workloadServices := h.getWorkloadServices() - h.consulServices.RemoveWorkload(workloadServices) + h.serviceRegWrapper.RemoveWorkload(workloadServices) } h.initialRegistration = false h.deregistered = true @@ -200,8 +217,9 @@ func (h *serviceHook) getWorkloadServices() *serviceregistration.WorkloadService // Create task services struct with request's driver metadata return &serviceregistration.WorkloadServices{ AllocID: h.allocID, + JobID: h.jobID, Task: h.taskName, - Namespace: h.consulNamespace, + Namespace: h.namespace, Restarter: h.restarter, Services: interpolatedServices, DriverExec: h.driverExec, diff --git a/client/allocrunner/taskrunner/service_hook_test.go b/client/allocrunner/taskrunner/service_hook_test.go index 4489e722008c..494fb61370d4 100644 --- a/client/allocrunner/taskrunner/service_hook_test.go +++ b/client/allocrunner/taskrunner/service_hook_test.go @@ -6,8 +6,12 @@ import ( "github.com/hashicorp/nomad/client/allocrunner/interfaces" regMock "github.com/hashicorp/nomad/client/serviceregistration/mock" + "github.com/hashicorp/nomad/client/serviceregistration/wrapper" + "github.com/hashicorp/nomad/client/taskenv" + agentconsul "github.com/hashicorp/nomad/command/agent/consul" "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/nomad/mock" + "github.com/hashicorp/nomad/nomad/structs" "github.com/stretchr/testify/require" ) @@ -19,20 +23,38 @@ var _ interfaces.TaskUpdateHook = (*serviceHook)(nil) func TestUpdate_beforePoststart(t *testing.T) { alloc := mock.Alloc() + alloc.Job.Canonicalize() logger := testlog.HCLogger(t) + c := regMock.NewServiceRegistrationHandler(logger) + regWrap := wrapper.NewHandlerWrapper(logger, c, nil) + + // Interpolating workload services performs a check on the task env, if it + // is nil, nil is returned meaning no services. This does not work with the + // wrapper len protections, so we need a dummy taskenv. + spoofTaskEnv := taskenv.TaskEnv{NodeAttrs: map[string]string{}} hook := newServiceHook(serviceHookConfig{ - alloc: alloc, - task: alloc.LookupTask("web"), - consulServices: c, - logger: logger, + alloc: alloc, + task: alloc.LookupTask("web"), + serviceRegWrapper: regWrap, + logger: logger, }) - require.NoError(t, hook.Update(context.Background(), &interfaces.TaskUpdateRequest{Alloc: alloc}, &interfaces.TaskUpdateResponse{})) + require.NoError(t, hook.Update(context.Background(), &interfaces.TaskUpdateRequest{ + Alloc: alloc, + TaskEnv: &spoofTaskEnv, + }, &interfaces.TaskUpdateResponse{})) require.Len(t, c.GetOps(), 0) - require.NoError(t, hook.Poststart(context.Background(), &interfaces.TaskPoststartRequest{}, &interfaces.TaskPoststartResponse{})) + + require.NoError(t, hook.Poststart(context.Background(), &interfaces.TaskPoststartRequest{ + TaskEnv: &spoofTaskEnv, + }, &interfaces.TaskPoststartResponse{})) require.Len(t, c.GetOps(), 1) - require.NoError(t, hook.Update(context.Background(), &interfaces.TaskUpdateRequest{Alloc: alloc}, &interfaces.TaskUpdateResponse{})) + + require.NoError(t, hook.Update(context.Background(), &interfaces.TaskUpdateRequest{ + Alloc: alloc, + TaskEnv: &spoofTaskEnv, + }, &interfaces.TaskUpdateResponse{})) require.Len(t, c.GetOps(), 2) // When a task exits it could be restarted with new driver info @@ -40,15 +62,31 @@ func TestUpdate_beforePoststart(t *testing.T) { require.NoError(t, hook.Exited(context.Background(), &interfaces.TaskExitedRequest{}, &interfaces.TaskExitedResponse{})) require.Len(t, c.GetOps(), 3) - require.NoError(t, hook.Update(context.Background(), &interfaces.TaskUpdateRequest{Alloc: alloc}, &interfaces.TaskUpdateResponse{})) + + require.NoError(t, hook.Update(context.Background(), &interfaces.TaskUpdateRequest{ + Alloc: alloc, + TaskEnv: &spoofTaskEnv, + }, &interfaces.TaskUpdateResponse{})) require.Len(t, c.GetOps(), 3) - require.NoError(t, hook.Poststart(context.Background(), &interfaces.TaskPoststartRequest{}, &interfaces.TaskPoststartResponse{})) + + require.NoError(t, hook.Poststart(context.Background(), &interfaces.TaskPoststartRequest{ + TaskEnv: &spoofTaskEnv, + }, &interfaces.TaskPoststartResponse{})) require.Len(t, c.GetOps(), 4) - require.NoError(t, hook.Update(context.Background(), &interfaces.TaskUpdateRequest{Alloc: alloc}, &interfaces.TaskUpdateResponse{})) + + require.NoError(t, hook.Update(context.Background(), &interfaces.TaskUpdateRequest{ + Alloc: alloc, + TaskEnv: &spoofTaskEnv, + }, &interfaces.TaskUpdateResponse{})) require.Len(t, c.GetOps(), 5) + require.NoError(t, hook.PreKilling(context.Background(), &interfaces.TaskPreKillRequest{}, &interfaces.TaskPreKillResponse{})) require.Len(t, c.GetOps(), 6) - require.NoError(t, hook.Update(context.Background(), &interfaces.TaskUpdateRequest{Alloc: alloc}, &interfaces.TaskUpdateResponse{})) + + require.NoError(t, hook.Update(context.Background(), &interfaces.TaskUpdateRequest{ + Alloc: alloc, + TaskEnv: &spoofTaskEnv, + }, &interfaces.TaskUpdateResponse{})) require.Len(t, c.GetOps(), 6) } @@ -56,17 +94,26 @@ func Test_serviceHook_multipleDeRegisterCall(t *testing.T) { alloc := mock.Alloc() logger := testlog.HCLogger(t) + c := regMock.NewServiceRegistrationHandler(logger) + regWrap := wrapper.NewHandlerWrapper(logger, c, nil) hook := newServiceHook(serviceHookConfig{ - alloc: alloc, - task: alloc.LookupTask("web"), - consulServices: c, - logger: logger, + alloc: alloc, + task: alloc.LookupTask("web"), + serviceRegWrapper: regWrap, + logger: logger, }) + // Interpolating workload services performs a check on the task env, if it + // is nil, nil is returned meaning no services. This does not work with the + // wrapper len protections, so we need a dummy taskenv. + spoofTaskEnv := taskenv.TaskEnv{NodeAttrs: map[string]string{}} + // Add a registration, as we would in normal operation. - require.NoError(t, hook.Poststart(context.Background(), &interfaces.TaskPoststartRequest{}, &interfaces.TaskPoststartResponse{})) + require.NoError(t, hook.Poststart(context.Background(), &interfaces.TaskPoststartRequest{ + TaskEnv: &spoofTaskEnv, + }, &interfaces.TaskPoststartResponse{})) require.Len(t, c.GetOps(), 1) // Call all three deregister backed functions in a row. Ensure the number @@ -84,7 +131,9 @@ func Test_serviceHook_multipleDeRegisterCall(t *testing.T) { require.Equal(t, c.GetOps()[1].Op, "remove") // Now we act like a restart. - require.NoError(t, hook.Poststart(context.Background(), &interfaces.TaskPoststartRequest{}, &interfaces.TaskPoststartResponse{})) + require.NoError(t, hook.Poststart(context.Background(), &interfaces.TaskPoststartRequest{ + TaskEnv: &spoofTaskEnv, + }, &interfaces.TaskPoststartResponse{})) require.Len(t, c.GetOps(), 3) require.Equal(t, c.GetOps()[2].Op, "add") @@ -101,3 +150,57 @@ func Test_serviceHook_multipleDeRegisterCall(t *testing.T) { require.Len(t, c.GetOps(), 4) require.Equal(t, c.GetOps()[3].Op, "remove") } + +// Test_serviceHook_Nomad performs a normal operation test of the serviceHook +// when using task services which utilise the Nomad provider. +func Test_serviceHook_Nomad(t *testing.T) { + t.Parallel() + + // Create a mock alloc, and add a task service using provider Nomad. + alloc := mock.Alloc() + alloc.Job.TaskGroups[0].Tasks[0].Services = []*structs.Service{ + { + Name: "nomad-provider-service", + Provider: structs.ServiceProviderNomad, + }, + } + + // Create our base objects and our subsequent wrapper. + logger := testlog.HCLogger(t) + consulMockClient := regMock.NewServiceRegistrationHandler(logger) + nomadMockClient := regMock.NewServiceRegistrationHandler(logger) + + regWrapper := wrapper.NewHandlerWrapper(logger, consulMockClient, nomadMockClient) + + h := newServiceHook(serviceHookConfig{ + alloc: alloc, + task: alloc.LookupTask("web"), + namespace: "default", + serviceRegWrapper: regWrapper, + restarter: agentconsul.NoopRestarter(), + logger: logger, + }) + + // Create a taskEnv builder to use in requests, otherwise interpolation of + // services will always return nil. + taskEnvBuilder := taskenv.NewBuilder(mock.Node(), alloc, nil, alloc.Job.Region) + + // Trigger our initial hook function. + require.NoError(t, h.Poststart(context.Background(), &interfaces.TaskPoststartRequest{ + TaskEnv: taskEnvBuilder.Build()}, nil)) + + // Trigger all the possible stop functions to ensure we only deregister + // once. + require.NoError(t, h.PreKilling(context.Background(), nil, nil)) + require.NoError(t, h.Exited(context.Background(), nil, nil)) + require.NoError(t, h.Stop(context.Background(), nil, nil)) + + // Ensure the Nomad mock provider has the expected operations. + nomadOps := nomadMockClient.GetOps() + require.Len(t, nomadOps, 2) + require.Equal(t, "add", nomadOps[0].Op) // Poststart + require.Equal(t, "remove", nomadOps[1].Op) // PreKilling,Exited,Stop + + // Ensure the Consul mock provider has zero operations. + require.Len(t, consulMockClient.GetOps(), 0) +} diff --git a/client/allocrunner/taskrunner/task_runner.go b/client/allocrunner/taskrunner/task_runner.go index 41938aea33fd..b8c3b270c329 100644 --- a/client/allocrunner/taskrunner/task_runner.go +++ b/client/allocrunner/taskrunner/task_runner.go @@ -26,6 +26,7 @@ import ( "github.com/hashicorp/nomad/client/pluginmanager/csimanager" "github.com/hashicorp/nomad/client/pluginmanager/drivermanager" "github.com/hashicorp/nomad/client/serviceregistration" + "github.com/hashicorp/nomad/client/serviceregistration/wrapper" cstate "github.com/hashicorp/nomad/client/state" cstructs "github.com/hashicorp/nomad/client/structs" "github.com/hashicorp/nomad/client/taskenv" @@ -239,6 +240,10 @@ type TaskRunner struct { networkIsolationSpec *drivers.NetworkIsolationSpec allocHookResources *cstructs.AllocHookResources + + // serviceRegWrapper is the handler wrapper that is used by service hooks + // to perform service and check registration and deregistration. + serviceRegWrapper *wrapper.HandlerWrapper } type Config struct { @@ -300,6 +305,10 @@ type Config struct { // ShutdownDelayCancelFn should only be used in testing. ShutdownDelayCancelFn context.CancelFunc + + // ServiceRegWrapper is the handler wrapper that is used by service hooks + // to perform service and check registration and deregistration. + ServiceRegWrapper *wrapper.HandlerWrapper } func NewTaskRunner(config *Config) (*TaskRunner, error) { @@ -357,6 +366,7 @@ func NewTaskRunner(config *Config) (*TaskRunner, error) { startConditionMetCtx: config.StartConditionMetCtx, shutdownDelayCtx: config.ShutdownDelayCtx, shutdownDelayCancelFn: config.ShutdownDelayCancelFn, + serviceRegWrapper: config.ServiceRegWrapper, } // Create the logger based on the allocation ID diff --git a/client/allocrunner/taskrunner/task_runner_hooks.go b/client/allocrunner/taskrunner/task_runner_hooks.go index 62ff26c4b625..86c8ff37ad6a 100644 --- a/client/allocrunner/taskrunner/task_runner_hooks.go +++ b/client/allocrunner/taskrunner/task_runner_hooks.go @@ -96,9 +96,13 @@ func (tr *TaskRunner) initHooks() { })) } - // Get the consul namespace for the TG of the allocation + // Get the consul namespace for the TG of the allocation. consulNamespace := tr.alloc.ConsulNamespace() + // Identify the service registration provider, which can differ from the + // Consul namespace depending on which provider is used. + serviceProviderNamespace := tr.alloc.ServiceProviderNamespace() + // If there are templates is enabled, add the hook if len(task.Templates) != 0 { tr.runnerHooks = append(tr.runnerHooks, newTemplateHook(&templateHookConfig{ @@ -115,12 +119,12 @@ func (tr *TaskRunner) initHooks() { // Always add the service hook. A task with no services on initial registration // may be updated to include services, which must be handled with this hook. tr.runnerHooks = append(tr.runnerHooks, newServiceHook(serviceHookConfig{ - alloc: tr.Alloc(), - task: tr.Task(), - consulServices: tr.consulServiceClient, - consulNamespace: consulNamespace, - restarter: tr, - logger: hookLogger, + alloc: tr.Alloc(), + task: tr.Task(), + namespace: serviceProviderNamespace, + serviceRegWrapper: tr.serviceRegWrapper, + restarter: tr, + logger: hookLogger, })) // If this is a Connect sidecar proxy (or a Connect Native) service, diff --git a/client/allocrunner/taskrunner/task_runner_test.go b/client/allocrunner/taskrunner/task_runner_test.go index 139808d913e4..415e4436cdde 100644 --- a/client/allocrunner/taskrunner/task_runner_test.go +++ b/client/allocrunner/taskrunner/task_runner_test.go @@ -25,6 +25,7 @@ import ( "github.com/hashicorp/nomad/client/devicemanager" "github.com/hashicorp/nomad/client/pluginmanager/drivermanager" regMock "github.com/hashicorp/nomad/client/serviceregistration/mock" + "github.com/hashicorp/nomad/client/serviceregistration/wrapper" cstate "github.com/hashicorp/nomad/client/state" ctestutil "github.com/hashicorp/nomad/client/testutil" "github.com/hashicorp/nomad/client/vaultclient" @@ -104,13 +105,18 @@ func testTaskRunnerConfig(t *testing.T, alloc *structs.Allocation, taskName stri closedCh := make(chan struct{}) close(closedCh) + // Set up the Nomad and Consul registration providers along with the wrapper. + consulRegMock := regMock.NewServiceRegistrationHandler(logger) + nomadRegMock := regMock.NewServiceRegistrationHandler(logger) + wrapperMock := wrapper.NewHandlerWrapper(logger, consulRegMock, nomadRegMock) + conf := &Config{ Alloc: alloc, ClientConfig: clientConf, Task: thisTask, TaskDir: taskDir, Logger: clientConf.Logger, - Consul: regMock.NewServiceRegistrationHandler(logger), + Consul: consulRegMock, ConsulSI: consulapi.NewMockServiceIdentitiesClient(), Vault: vaultclient.NewMockVaultClient(), StateDB: cstate.NoopDB{}, @@ -121,6 +127,7 @@ func testTaskRunnerConfig(t *testing.T, alloc *structs.Allocation, taskName stri StartConditionMetCtx: closedCh, ShutdownDelayCtx: shutdownDelayCtx, ShutdownDelayCancelFn: shutdownDelayCancelFn, + ServiceRegWrapper: wrapperMock, } return conf, trCleanup } @@ -1229,6 +1236,7 @@ func TestTaskRunner_CheckWatcher_Restart(t *testing.T) { Grace: 100 * time.Millisecond, }, } + task.Services[0].Provider = structs.ServiceProviderConsul conf, cleanup := testTaskRunnerConfig(t, alloc, task.Name) defer cleanup() @@ -1246,6 +1254,7 @@ func TestTaskRunner_CheckWatcher_Restart(t *testing.T) { defer consulClient.Shutdown() conf.Consul = consulClient + conf.ServiceRegWrapper = wrapper.NewHandlerWrapper(conf.Logger, consulClient, nil) tr, err := NewTaskRunner(conf) require.NoError(t, err) @@ -1885,6 +1894,7 @@ func TestTaskRunner_DriverNetwork(t *testing.T) { Name: "host-service", PortLabel: "http", AddressMode: "host", + Provider: structs.ServiceProviderConsul, Checks: []*structs.ServiceCheck{ { Name: "driver-check", @@ -1898,6 +1908,7 @@ func TestTaskRunner_DriverNetwork(t *testing.T) { Name: "driver-service", PortLabel: "5678", AddressMode: "driver", + Provider: structs.ServiceProviderConsul, Checks: []*structs.ServiceCheck{ { Name: "host-check", @@ -1928,6 +1939,7 @@ func TestTaskRunner_DriverNetwork(t *testing.T) { go consulClient.Run() conf.Consul = consulClient + conf.ServiceRegWrapper = wrapper.NewHandlerWrapper(conf.Logger, consulClient, nil) tr, err := NewTaskRunner(conf) require.NoError(t, err) diff --git a/client/allocrunner/testing.go b/client/allocrunner/testing.go index 4fde2cd02348..4910d292490f 100644 --- a/client/allocrunner/testing.go +++ b/client/allocrunner/testing.go @@ -7,14 +7,14 @@ import ( "sync" "testing" - "github.com/hashicorp/nomad/client/lib/cgutil" - "github.com/hashicorp/nomad/client/allocwatcher" clientconfig "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/consul" "github.com/hashicorp/nomad/client/devicemanager" + "github.com/hashicorp/nomad/client/lib/cgutil" "github.com/hashicorp/nomad/client/pluginmanager/drivermanager" "github.com/hashicorp/nomad/client/serviceregistration/mock" + "github.com/hashicorp/nomad/client/serviceregistration/wrapper" "github.com/hashicorp/nomad/client/state" "github.com/hashicorp/nomad/client/vaultclient" "github.com/hashicorp/nomad/nomad/structs" @@ -57,13 +57,17 @@ func (m *MockStateUpdater) Reset() { func testAllocRunnerConfig(t *testing.T, alloc *structs.Allocation) (*Config, func()) { clientConf, cleanup := clientconfig.TestClientConfig(t) + + consulRegMock := mock.NewServiceRegistrationHandler(clientConf.Logger) + nomadRegMock := mock.NewServiceRegistrationHandler(clientConf.Logger) + conf := &Config{ // Copy the alloc in case the caller edits and reuses it Alloc: alloc.Copy(), Logger: clientConf.Logger, ClientConfig: clientConf, StateDB: state.NoopDB{}, - Consul: mock.NewServiceRegistrationHandler(clientConf.Logger), + Consul: consulRegMock, ConsulSI: consul.NewMockServiceIdentitiesClient(), Vault: vaultclient.NewMockVaultClient(), StateUpdater: &MockStateUpdater{}, @@ -73,6 +77,7 @@ func testAllocRunnerConfig(t *testing.T, alloc *structs.Allocation) (*Config, fu DriverManager: drivermanager.TestDriverManager(t), CpusetManager: cgutil.NoopCpusetManager(), ServersContactedCh: make(chan struct{}), + ServiceRegWrapper: wrapper.NewHandlerWrapper(clientConf.Logger, consulRegMock, nomadRegMock), } return conf, cleanup } diff --git a/client/client.go b/client/client.go index c70a1547daac..1b52582699b6 100644 --- a/client/client.go +++ b/client/client.go @@ -40,6 +40,8 @@ import ( "github.com/hashicorp/nomad/client/pluginmanager/drivermanager" "github.com/hashicorp/nomad/client/servers" "github.com/hashicorp/nomad/client/serviceregistration" + "github.com/hashicorp/nomad/client/serviceregistration/nsd" + "github.com/hashicorp/nomad/client/serviceregistration/wrapper" "github.com/hashicorp/nomad/client/state" "github.com/hashicorp/nomad/client/stats" cstructs "github.com/hashicorp/nomad/client/structs" @@ -226,10 +228,19 @@ type Client struct { // allocUpdates stores allocations that need to be synced to the server. allocUpdates chan *structs.Allocation - // consulService is Nomad's custom Consul client for managing services + // consulService is the Consul handler implementation for managing services // and checks. consulService serviceregistration.Handler + // nomadService is the Nomad handler implementation for managing service + // registrations. + nomadService serviceregistration.Handler + + // serviceRegWrapper wraps the consulService and nomadService + // implementations so that the alloc and task runner service hooks can call + // this without needing to identify which backend provider should be used. + serviceRegWrapper *wrapper.HandlerWrapper + // consulProxies is Nomad's custom Consul client for looking up supported // envoy versions consulProxies consulApi.SupportedProxiesAPI @@ -472,6 +483,12 @@ func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulProxie c.devicemanager = devManager c.pluginManagers.RegisterAndRun(devManager) + // Set up the service registration wrapper using the Consul and Nomad + // implementations. The Nomad implementation is only ever used on the + // client, so we do that here rather than within the agent. + c.setupNomadServiceRegistrationHandler() + c.serviceRegWrapper = wrapper.NewHandlerWrapper(c.logger, c.consulService, c.nomadService) + // Batching of initial fingerprints is done to reduce the number of node // updates sent to the server on startup. This is the first RPC to the servers go c.batchFirstFingerprints() @@ -785,6 +802,13 @@ func (c *Client) Shutdown() error { } arGroup.Wait() + // Assert the implementation, so we can trigger the shutdown call. This is + // the only place this occurs, so it's OK to store the interface rather + // than the implementation. + if h, ok := c.nomadService.(*nsd.ServiceRegistrationHandler); ok { + h.Shutdown() + } + // Shutdown the plugin managers c.pluginManagers.Shutdown() @@ -1141,6 +1165,7 @@ func (c *Client) restoreState() error { DeviceManager: c.devicemanager, DriverManager: c.drivermanager, ServersContactedCh: c.serversContactedCh, + ServiceRegWrapper: c.serviceRegWrapper, RPCClient: c, } c.configLock.RUnlock() @@ -2462,6 +2487,7 @@ func (c *Client) addAlloc(alloc *structs.Allocation, migrateToken string) error CpusetManager: c.cpusetManager, DeviceManager: c.devicemanager, DriverManager: c.drivermanager, + ServiceRegWrapper: c.serviceRegWrapper, RPCClient: c, } c.configLock.RUnlock() @@ -2509,6 +2535,20 @@ func (c *Client) setupVaultClient() error { return nil } +// setupNomadServiceRegistrationHandler sets up the registration handler to use +// for native service discovery. +func (c *Client) setupNomadServiceRegistrationHandler() { + cfg := nsd.ServiceRegistrationHandlerCfg{ + Datacenter: c.Datacenter(), + Enabled: c.config.NomadServiceDiscovery, + NodeID: c.NodeID(), + NodeSecret: c.secretNodeID(), + Region: c.Region(), + RPCFn: c.RPC, + } + c.nomadService = nsd.NewServiceRegistrationHandler(c.logger, &cfg) +} + // deriveToken takes in an allocation and a set of tasks and derives vault // tokens for each of the tasks, unwraps all of them using the supplied vault // client and returns a map of unwrapped tokens, indexed by the task name. diff --git a/command/agent/agent.go b/command/agent/agent.go index b65e101ef658..9d2668d88428 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -14,8 +14,6 @@ import ( "sync" "time" - "github.com/hashicorp/nomad/lib/cpuset" - metrics "github.com/armon/go-metrics" consulapi "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/lib" @@ -28,6 +26,7 @@ import ( "github.com/hashicorp/nomad/command/agent/event" "github.com/hashicorp/nomad/helper/pluginutils/loader" "github.com/hashicorp/nomad/helper/uuid" + "github.com/hashicorp/nomad/lib/cpuset" "github.com/hashicorp/nomad/nomad" "github.com/hashicorp/nomad/nomad/deploymentwatcher" "github.com/hashicorp/nomad/nomad/structs" @@ -891,7 +890,6 @@ func (a *Agent) setupClient() error { if !a.config.Client.Enabled { return nil } - // Setup the configuration conf, err := a.clientConfig() if err != nil { diff --git a/command/agent/consul/int_test.go b/command/agent/consul/int_test.go index 12aa80f8f358..5ec318354b17 100644 --- a/command/agent/consul/int_test.go +++ b/command/agent/consul/int_test.go @@ -15,6 +15,8 @@ import ( "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/devicemanager" "github.com/hashicorp/nomad/client/pluginmanager/drivermanager" + regMock "github.com/hashicorp/nomad/client/serviceregistration/mock" + "github.com/hashicorp/nomad/client/serviceregistration/wrapper" "github.com/hashicorp/nomad/client/state" "github.com/hashicorp/nomad/client/vaultclient" "github.com/hashicorp/nomad/command/agent/consul" @@ -93,6 +95,7 @@ func TestConsul_Integration(t *testing.T) { Name: "httpd", PortLabel: "http", Tags: []string{"nomad", "test", "http"}, + Provider: structs.ServiceProviderConsul, Checks: []*structs.ServiceCheck{ { Name: "httpd-http-check", @@ -114,6 +117,7 @@ func TestConsul_Integration(t *testing.T) { { Name: "httpd2", PortLabel: "http", + Provider: structs.ServiceProviderConsul, Tags: []string{ "test", // Use URL-unfriendly tags to test #3620 @@ -162,6 +166,7 @@ func TestConsul_Integration(t *testing.T) { DeviceManager: devicemanager.NoopMockManager(), DriverManager: drivermanager.TestDriverManager(t), StartConditionMetCtx: closedCh, + ServiceRegWrapper: wrapper.NewHandlerWrapper(logger, serviceClient, regMock.NewServiceRegistrationHandler(logger)), } tr, err := taskrunner.NewTaskRunner(config) diff --git a/nomad/structs/alloc.go b/nomad/structs/alloc.go index 9f1dee8d079c..a0277477373c 100644 --- a/nomad/structs/alloc.go +++ b/nomad/structs/alloc.go @@ -22,3 +22,37 @@ type AllocServiceRegistrationsResponse struct { Services []*ServiceRegistration QueryMeta } + +// ServiceProviderNamespace returns the namespace within which the allocations +// services should be registered. This takes into account the different +// providers that can provide service registrations. In the event no services +// are found, the function will return the Consul namespace which allows hooks +// to work as they did before this feature. +// +// It currently assumes that all services within an allocation use the same +// provider and therefore the same namespace. +func (a *Allocation) ServiceProviderNamespace() string { + tg := a.Job.LookupTaskGroup(a.TaskGroup) + + if len(tg.Services) > 0 { + switch tg.Services[0].Provider { + case ServiceProviderNomad: + return a.Job.Namespace + default: + return tg.Consul.GetNamespace() + } + } + + if len(tg.Tasks) > 0 { + if len(tg.Tasks[0].Services) > 0 { + switch tg.Tasks[0].Services[0].Provider { + case ServiceProviderNomad: + return a.Job.Namespace + default: + return tg.Consul.GetNamespace() + } + } + } + + return tg.Consul.GetNamespace() +} diff --git a/nomad/structs/alloc_test.go b/nomad/structs/alloc_test.go index ce2ce52dab8f..92bc61615502 100644 --- a/nomad/structs/alloc_test.go +++ b/nomad/structs/alloc_test.go @@ -10,3 +10,105 @@ func TestAllocServiceRegistrationsRequest_StaleReadSupport(t *testing.T) { req := &AllocServiceRegistrationsRequest{} require.True(t, req.IsRead()) } + +func Test_Allocation_ServiceProviderNamespace(t *testing.T) { + testCases := []struct { + inputAllocation *Allocation + expectedOutput string + name string + }{ + { + inputAllocation: &Allocation{ + Job: &Job{ + TaskGroups: []*TaskGroup{ + { + Name: "test-group", + Services: []*Service{ + { + Provider: ServiceProviderConsul, + }, + }, + }, + }, + }, + TaskGroup: "test-group", + }, + expectedOutput: "", + name: "consul task group service", + }, + { + inputAllocation: &Allocation{ + Job: &Job{ + TaskGroups: []*TaskGroup{ + { + Name: "test-group", + Tasks: []*Task{ + { + Services: []*Service{ + { + Provider: ServiceProviderConsul, + }, + }, + }, + }, + }, + }, + }, + TaskGroup: "test-group", + }, + expectedOutput: "", + name: "consul task service", + }, + { + inputAllocation: &Allocation{ + Job: &Job{ + Namespace: "platform", + TaskGroups: []*TaskGroup{ + { + Name: "test-group", + Services: []*Service{ + { + Provider: ServiceProviderNomad, + }, + }, + }, + }, + }, + TaskGroup: "test-group", + }, + expectedOutput: "platform", + name: "nomad task group service", + }, + { + inputAllocation: &Allocation{ + Job: &Job{ + Namespace: "platform", + TaskGroups: []*TaskGroup{ + { + Name: "test-group", + Tasks: []*Task{ + { + Services: []*Service{ + { + Provider: ServiceProviderNomad, + }, + }, + }, + }, + }, + }, + }, + TaskGroup: "test-group", + }, + expectedOutput: "platform", + name: "nomad task service", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualOutput := tc.inputAllocation.ServiceProviderNamespace() + require.Equal(t, tc.expectedOutput, actualOutput) + }) + } +} From 9944777919885a0c3dce33430151a2b2f9d86c22 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Mon, 21 Mar 2022 11:59:03 +0100 Subject: [PATCH 27/31] cli: add service commands for list, info, and delete. --- command/commands.go | 20 ++++ command/service.go | 40 +++++++ command/service_delete.go | 68 ++++++++++++ command/service_delete_test.go | 74 +++++++++++++ command/service_info.go | 186 +++++++++++++++++++++++++++++++++ command/service_info_test.go | 121 +++++++++++++++++++++ command/service_list.go | 180 +++++++++++++++++++++++++++++++ command/service_list_test.go | 111 ++++++++++++++++++++ 8 files changed, 800 insertions(+) create mode 100644 command/service.go create mode 100644 command/service_delete.go create mode 100644 command/service_delete_test.go create mode 100644 command/service_info.go create mode 100644 command/service_info_test.go create mode 100644 command/service_list.go create mode 100644 command/service_list_test.go diff --git a/command/commands.go b/command/commands.go index 287cdfaf1934..9334847a3411 100644 --- a/command/commands.go +++ b/command/commands.go @@ -768,6 +768,26 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory { Meta: meta, }, nil }, + "service": func() (cli.Command, error) { + return &ServiceCommand{ + Meta: meta, + }, nil + }, + "service list": func() (cli.Command, error) { + return &ServiceListCommand{ + Meta: meta, + }, nil + }, + "service info": func() (cli.Command, error) { + return &ServiceInfoCommand{ + Meta: meta, + }, nil + }, + "service delete": func() (cli.Command, error) { + return &ServiceDeleteCommand{ + Meta: meta, + }, nil + }, "status": func() (cli.Command, error) { return &StatusCommand{ Meta: meta, diff --git a/command/service.go b/command/service.go new file mode 100644 index 000000000000..fbc213111996 --- /dev/null +++ b/command/service.go @@ -0,0 +1,40 @@ +package command + +import ( + "strings" + + "github.com/mitchellh/cli" +) + +type ServiceCommand struct { + Meta +} + +func (c *ServiceCommand) Help() string { + helpText := ` +Usage: nomad service [options] + + This command groups subcommands for interacting with the services API. + + List services: + + $ nomad service list + + Detail an individual service: + + $ nomad service info + + Delete an individual service registration: + + $ nomad service delete + + Please see the individual subcommand help for detailed usage information. +` + return strings.TrimSpace(helpText) +} + +func (c *ServiceCommand) Name() string { return "service" } + +func (c *ServiceCommand) Synopsis() string { return "Interact with registered services" } + +func (c *ServiceCommand) Run(_ []string) int { return cli.RunResultHelp } diff --git a/command/service_delete.go b/command/service_delete.go new file mode 100644 index 000000000000..8971a05b02c9 --- /dev/null +++ b/command/service_delete.go @@ -0,0 +1,68 @@ +package command + +import ( + "fmt" + "strings" + + "github.com/posener/complete" +) + +type ServiceDeleteCommand struct { + Meta +} + +func (s *ServiceDeleteCommand) Help() string { + helpText := ` +Usage: nomad service delete [options] + + Delete is used to deregister the specified service registration. It should be + used with caution and can only remove a single registration, via the service + name and service ID, at a time. + + When ACLs are enabled, this command requires a token with the 'submit-job' + capability for the service registration namespace. + +General Options: + + ` + generalOptionsUsage(usageOptsDefault) + + return strings.TrimSpace(helpText) +} + +func (s *ServiceDeleteCommand) Name() string { return "service delete" } + +func (s *ServiceDeleteCommand) Synopsis() string { return "Deregister a registered service" } + +func (s *ServiceDeleteCommand) AutocompleteFlags() complete.Flags { + return mergeAutocompleteFlags(s.Meta.AutocompleteFlags(FlagSetClient), + complete.Flags{}) +} + +func (s *ServiceDeleteCommand) Run(args []string) int { + + flags := s.Meta.FlagSet(s.Name(), FlagSetClient) + if err := flags.Parse(args); err != nil { + return 1 + } + args = flags.Args() + + if len(args) != 2 { + s.Ui.Error("This command takes two arguments: and ") + s.Ui.Error(commandErrorText(s)) + return 1 + } + + client, err := s.Meta.Client() + if err != nil { + s.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) + return 1 + } + + if _, err := client.ServiceRegistrations().Delete(args[0], args[1], nil); err != nil { + s.Ui.Error(fmt.Sprintf("Error deleting service registration: %s", err)) + return 1 + } + + s.Ui.Output("Successfully deleted service registration") + return 0 +} diff --git a/command/service_delete_test.go b/command/service_delete_test.go new file mode 100644 index 000000000000..fc7768fe08f2 --- /dev/null +++ b/command/service_delete_test.go @@ -0,0 +1,74 @@ +package command + +import ( + "fmt" + "testing" + + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/testutil" + "github.com/mitchellh/cli" + "github.com/stretchr/testify/require" +) + +func TestServiceDeleteCommand_Run(t *testing.T) { + t.Parallel() + + srv, client, url := testServer(t, true, nil) + defer srv.Shutdown() + + // Wait until our test node is ready. + testutil.WaitForResult(func() (bool, error) { + nodes, _, err := client.Nodes().List(nil) + if err != nil { + return false, err + } + if len(nodes) == 0 { + return false, fmt.Errorf("missing node") + } + if _, ok := nodes[0].Drivers["mock_driver"]; !ok { + return false, fmt.Errorf("mock_driver not ready") + } + return true, nil + }, func(err error) { + require.NoError(t, err) + }) + + ui := cli.NewMockUi() + cmd := &ServiceDeleteCommand{ + Meta: Meta{ + Ui: ui, + flagAddress: url, + }, + } + + // Run the command without any arguments to ensure we are performing this + // check. + require.Equal(t, 1, cmd.Run([]string{"-address=" + url})) + require.Contains(t, ui.ErrorWriter.String(), + "This command takes two arguments: and ") + ui.ErrorWriter.Reset() + + // Create a test job with a Nomad service. + testJob := testJob("service-discovery-nomad-delete") + testJob.TaskGroups[0].Tasks[0].Services = []*api.Service{ + {Name: "service-discovery-nomad-delete", Provider: "nomad"}} + + // Register that job. + regResp, _, err := client.Jobs().Register(testJob, nil) + require.NoError(t, err) + registerCode := waitForSuccess(ui, client, fullId, t, regResp.EvalID) + require.Equal(t, 0, registerCode) + + // Detail the service as we need the ID. + serviceList, _, err := client.ServiceRegistrations().Get("service-discovery-nomad-delete", nil) + require.NoError(t, err) + require.Len(t, serviceList, 1) + + // Attempt to manually delete the service registration. + code := cmd.Run([]string{"-address=" + url, "service-discovery-nomad-delete", serviceList[0].ID}) + require.Equal(t, 0, code) + require.Contains(t, ui.OutputWriter.String(), "Successfully deleted service registration") + + ui.OutputWriter.Reset() + ui.ErrorWriter.Reset() +} diff --git a/command/service_info.go b/command/service_info.go new file mode 100644 index 000000000000..2f54e5edab44 --- /dev/null +++ b/command/service_info.go @@ -0,0 +1,186 @@ +package command + +import ( + "fmt" + "sort" + "strings" + + "github.com/hashicorp/nomad/api" + "github.com/mitchellh/cli" + "github.com/posener/complete" +) + +// Ensure ServiceInfoCommand satisfies the cli.Command interface. +var _ cli.Command = &ServiceInfoCommand{} + +// ServiceInfoCommand implements cli.Command. +type ServiceInfoCommand struct { + Meta +} + +// Help satisfies the cli.Command Help function. +func (s *ServiceInfoCommand) Help() string { + helpText := ` +Usage: nomad service info [options] + + Info is used to read the services registered to a single service name. + + When ACLs are enabled, this command requires a token with the 'read-job' + capability for the service namespace. + +General Options: + + ` + generalOptionsUsage(usageOptsDefault) + ` + +Service Info Options: + + -verbose + Display full information. + + -json + Output the service in JSON format. + + -t + Format and display the service using a Go template. +` + return strings.TrimSpace(helpText) +} + +// Synopsis satisfies the cli.Command Synopsis function. +func (s *ServiceInfoCommand) Synopsis() string { + return "Display an individual Nomad service registration" +} + +func (s *ServiceInfoCommand) AutocompleteFlags() complete.Flags { + return mergeAutocompleteFlags(s.Meta.AutocompleteFlags(FlagSetClient), + complete.Flags{ + "-json": complete.PredictNothing, + "-t": complete.PredictAnything, + "-verbose": complete.PredictNothing, + }) +} + +// Name returns the name of this command. +func (s *ServiceInfoCommand) Name() string { return "service info" } + +// Run satisfies the cli.Command Run function. +func (s *ServiceInfoCommand) Run(args []string) int { + var ( + json, verbose bool + tmpl string + ) + + flags := s.Meta.FlagSet(s.Name(), FlagSetClient) + flags.Usage = func() { s.Ui.Output(s.Help()) } + flags.BoolVar(&json, "json", false, "") + flags.BoolVar(&verbose, "verbose", false, "") + flags.StringVar(&tmpl, "t", "", "") + if err := flags.Parse(args); err != nil { + return 1 + } + args = flags.Args() + + if len(args) != 1 { + s.Ui.Error("This command takes one argument: ") + s.Ui.Error(commandErrorText(s)) + return 1 + } + + client, err := s.Meta.Client() + if err != nil { + s.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) + return 1 + } + + serviceInfo, _, err := client.ServiceRegistrations().Get(args[0], nil) + if err != nil { + s.Ui.Error(fmt.Sprintf("Error listing service registrations: %s", err)) + return 1 + } + + if len(serviceInfo) == 0 { + s.Ui.Output("No service registrations found") + return 0 + } + + if json || len(tmpl) > 0 { + out, err := Format(json, tmpl, serviceInfo) + if err != nil { + s.Ui.Error(err.Error()) + return 1 + } + s.Ui.Output(out) + return 0 + } + + // It is possible for multiple jobs to register a service with the same + // name. In order to provide consistency, sort the output by job ID. + sortedJobID := []string{} + jobIDServices := make(map[string][]*api.ServiceRegistration) + + // Populate the objects, ensuring we do not add duplicate job IDs to the + // array which will be sorted. + for _, service := range serviceInfo { + if _, ok := jobIDServices[service.JobID]; ok { + jobIDServices[service.JobID] = append(jobIDServices[service.JobID], service) + } else { + jobIDServices[service.JobID] = []*api.ServiceRegistration{service} + sortedJobID = append(sortedJobID, service.JobID) + } + } + + // Sort the jobIDs. + sort.Strings(sortedJobID) + + if verbose { + s.formatVerboseOutput(sortedJobID, jobIDServices) + } else { + s.formatOutput(sortedJobID, jobIDServices) + } + return 0 +} + +// formatOutput produces the non-verbose output of service registration info +// for a specific service by its name. +func (s *ServiceInfoCommand) formatOutput(jobIDs []string, jobServices map[string][]*api.ServiceRegistration) { + + // Create the output table header. + outputTable := []string{"Job ID|Address|Tags|Node ID|Alloc ID"} + + // Populate the list. + for _, jobID := range jobIDs { + for _, service := range jobServices[jobID] { + outputTable = append(outputTable, fmt.Sprintf( + "%s|%s|[%s]|%s|%s", + service.JobID, + fmt.Sprintf("%s:%v", service.Address, service.Port), + strings.Join(service.Tags, ","), + limit(service.NodeID, shortId), + limit(service.AllocID, shortId), + )) + } + } + s.Ui.Output(formatList(outputTable)) +} + +// formatOutput produces the verbose output of service registration info for a +// specific service by its name. +func (s *ServiceInfoCommand) formatVerboseOutput(jobIDs []string, jobServices map[string][]*api.ServiceRegistration) { + for _, jobID := range jobIDs { + for _, service := range jobServices[jobID] { + out := []string{ + fmt.Sprintf("ID|%s", service.ID), + fmt.Sprintf("Service Name|%s", service.ServiceName), + fmt.Sprintf("Namespace|%s", service.Namespace), + fmt.Sprintf("Job ID|%s", service.JobID), + fmt.Sprintf("Alloc ID|%s", service.AllocID), + fmt.Sprintf("Node ID|%s", service.NodeID), + fmt.Sprintf("Datacenter|%s", service.Datacenter), + fmt.Sprintf("Address|%v", fmt.Sprintf("%s:%v", service.Address, service.Port)), + fmt.Sprintf("Tags|[%s]\n", strings.Join(service.Tags, ",")), + } + s.Ui.Output(formatKV(out)) + s.Ui.Output("") + } + } +} diff --git a/command/service_info_test.go b/command/service_info_test.go new file mode 100644 index 000000000000..70fdb6df015b --- /dev/null +++ b/command/service_info_test.go @@ -0,0 +1,121 @@ +package command + +import ( + "fmt" + "testing" + "time" + + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/testutil" + "github.com/mitchellh/cli" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestServiceInfoCommand_Run(t *testing.T) { + t.Parallel() + + srv, client, url := testServer(t, true, nil) + defer srv.Shutdown() + + // Wait until our test node is ready. + testutil.WaitForResult(func() (bool, error) { + nodes, _, err := client.Nodes().List(nil) + if err != nil { + return false, err + } + if len(nodes) == 0 { + return false, fmt.Errorf("missing node") + } + if _, ok := nodes[0].Drivers["mock_driver"]; !ok { + return false, fmt.Errorf("mock_driver not ready") + } + return true, nil + }, func(err error) { + require.NoError(t, err) + }) + + ui := cli.NewMockUi() + cmd := &ServiceInfoCommand{ + Meta: Meta{ + Ui: ui, + flagAddress: url, + }, + } + + // Run the command without any arguments to ensure we are performing this + // check. + require.Equal(t, 1, cmd.Run([]string{"-address=" + url})) + require.Contains(t, ui.ErrorWriter.String(), + "This command takes one argument: ") + ui.ErrorWriter.Reset() + + // Create a test job with a Nomad service. + testJob := testJob("service-discovery-nomad-info") + testJob.TaskGroups[0].Tasks[0].Services = []*api.Service{ + {Name: "service-discovery-nomad-info", Provider: "nomad", Tags: []string{"foo", "bar"}}} + + // Register that job. + regResp, _, err := client.Jobs().Register(testJob, nil) + require.NoError(t, err) + registerCode := waitForSuccess(ui, client, fullId, t, regResp.EvalID) + require.Equal(t, 0, registerCode) + + // Reset the output writer, otherwise we will have additional information here. + ui.OutputWriter.Reset() + + // Job register doesn't assure the service registration has completed. It + // therefore needs this wrapper to account for eventual service + // registration. One this has completed, we can perform lookups without + // similar wraps. + require.Eventually(t, func() bool { + + defer ui.OutputWriter.Reset() + + // Perform a standard lookup. + if code := cmd.Run([]string{"-address=" + url, "service-discovery-nomad-info"}); code != 0 { + return false + } + + // Test each header and data entry. + s := ui.OutputWriter.String() + if !assert.Contains(t, s, "Job ID") { + return false + } + if !assert.Contains(t, s, "Address") { + return false + } + if !assert.Contains(t, s, "Node ID") { + return false + } + if !assert.Contains(t, s, "Alloc ID") { + return false + } + if !assert.Contains(t, s, "service-discovery-nomad-info") { + return false + } + if !assert.Contains(t, s, ":0") { + return false + } + if !assert.Contains(t, s, "[foo,bar]") { + return false + } + return true + }, 5*time.Second, 100*time.Millisecond) + + // Perform a verbose lookup. + code := cmd.Run([]string{"-address=" + url, "-verbose", "service-discovery-nomad-info"}) + require.Equal(t, 0, code) + + // Test KV entries. + s := ui.OutputWriter.String() + require.Contains(t, s, "Service Name = service-discovery-nomad-info") + require.Contains(t, s, "Namespace = default") + require.Contains(t, s, "Job ID = service-discovery-nomad-info") + require.Contains(t, s, "Datacenter = dc1") + require.Contains(t, s, "Address = :0") + require.Contains(t, s, "Tags = [foo,bar]") + + ui.OutputWriter.Reset() + ui.ErrorWriter.Reset() +} diff --git a/command/service_list.go b/command/service_list.go new file mode 100644 index 000000000000..8e132494f6db --- /dev/null +++ b/command/service_list.go @@ -0,0 +1,180 @@ +package command + +import ( + "fmt" + "sort" + "strings" + + "github.com/hashicorp/nomad/api" + "github.com/mitchellh/cli" + "github.com/posener/complete" +) + +// Ensure ServiceListCommand satisfies the cli.Command interface. +var _ cli.Command = &ServiceListCommand{} + +// ServiceListCommand implements cli.Command. +type ServiceListCommand struct { + Meta +} + +// Help satisfies the cli.Command Help function. +func (s *ServiceListCommand) Help() string { + helpText := ` +Usage: nomad service list [options] + + List is used to list the currently registered services. + + If ACLs are enabled, this command requires a token with the 'read-job' + capabilities for the namespace of all services. Any namespaces that the token + does not have access to will have its services filtered from the results. + +General Options: + + ` + generalOptionsUsage(usageOptsDefault) + ` + +Service List Options: + + -json + Output the services in JSON format. + + -t + Format and display the services using a Go template. +` + return strings.TrimSpace(helpText) +} + +// Synopsis satisfies the cli.Command Synopsis function. +func (s *ServiceListCommand) Synopsis() string { + return "Display all registered Nomad services" +} + +func (s *ServiceListCommand) AutocompleteFlags() complete.Flags { + return mergeAutocompleteFlags(s.Meta.AutocompleteFlags(FlagSetClient), + complete.Flags{ + "-json": complete.PredictNothing, + "-t": complete.PredictAnything, + }) +} + +// Name returns the name of this command. +func (s *ServiceListCommand) Name() string { return "service list" } + +// Run satisfies the cli.Command Run function. +func (s *ServiceListCommand) Run(args []string) int { + + var ( + json bool + tmpl, name string + ) + + flags := s.Meta.FlagSet(s.Name(), FlagSetClient) + flags.Usage = func() { s.Ui.Output(s.Help()) } + flags.BoolVar(&json, "json", false, "") + flags.StringVar(&name, "name", "", "") + flags.StringVar(&tmpl, "t", "", "") + if err := flags.Parse(args); err != nil { + return 1 + } + + if args = flags.Args(); len(args) > 0 { + s.Ui.Error("This command takes no arguments") + s.Ui.Error(commandErrorText(s)) + return 1 + } + + client, err := s.Meta.Client() + if err != nil { + s.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) + return 1 + } + + list, _, err := client.ServiceRegistrations().List(nil) + if err != nil { + s.Ui.Error(fmt.Sprintf("Error listing service registrations: %s", err)) + return 1 + } + + if len(list) == 0 { + s.Ui.Output("No service registrations found") + return 0 + } + + if json || len(tmpl) > 0 { + out, err := Format(json, tmpl, list) + if err != nil { + s.Ui.Error(err.Error()) + return 1 + } + s.Ui.Output(out) + return 0 + } + + s.formatOutput(list) + return 0 +} + +func (s *ServiceListCommand) formatOutput(regs []*api.ServiceRegistrationListStub) { + + // Create objects to hold sorted a sorted namespace array and a mapping, so + // we can perform service lookups on a namespace basis. + sortedNamespaces := make([]string, len(regs)) + namespacedServices := make(map[string][]*api.ServiceRegistrationStub) + + for i, namespaceServices := range regs { + sortedNamespaces[i] = namespaceServices.Namespace + namespacedServices[namespaceServices.Namespace] = namespaceServices.Services + } + + // Sort the namespaces. + sort.Strings(sortedNamespaces) + + // The table always starts with the service name. + outputTable := []string{"Service Name"} + + // If the request was made using the wildcard namespace, include this in + // the output. + if s.Meta.namespace == api.AllNamespacesNamespace { + outputTable[0] += "|Namespace" + } + + // The tags come last and are always present. + outputTable[0] += "|Tags" + + for _, ns := range sortedNamespaces { + + // Grab the services belonging to this namespace. + services := namespacedServices[ns] + + // Create objects to hold sorted a sorted service name array and a + // mapping, so we can perform service tag lookups on a name basis. + sortedNames := make([]string, len(services)) + serviceTags := make(map[string][]string) + + for i, service := range services { + sortedNames[i] = service.ServiceName + serviceTags[service.ServiceName] = service.Tags + } + + // Sort the service names. + sort.Strings(sortedNames) + + for _, serviceName := range sortedNames { + + // Grab the service tags, and sort these for good measure. + tags := serviceTags[serviceName] + sort.Strings(tags) + + // Build the output array entry. + regOutput := serviceName + + if s.Meta.namespace == api.AllNamespacesNamespace { + regOutput += "|" + ns + } + regOutput += "|" + fmt.Sprintf("[%s]", strings.Join(tags, ",")) + outputTable = append(outputTable, regOutput) + } + } + + s.Ui.Output(formatList(outputTable)) +} diff --git a/command/service_list_test.go b/command/service_list_test.go new file mode 100644 index 000000000000..8295963f96bf --- /dev/null +++ b/command/service_list_test.go @@ -0,0 +1,111 @@ +package command + +import ( + "fmt" + "testing" + "time" + + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/testutil" + "github.com/mitchellh/cli" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestServiceListCommand_Run(t *testing.T) { + t.Parallel() + + srv, client, url := testServer(t, true, nil) + defer srv.Shutdown() + + // Wait until our test node is ready. + testutil.WaitForResult(func() (bool, error) { + nodes, _, err := client.Nodes().List(nil) + if err != nil { + return false, err + } + if len(nodes) == 0 { + return false, fmt.Errorf("missing node") + } + if _, ok := nodes[0].Drivers["mock_driver"]; !ok { + return false, fmt.Errorf("mock_driver not ready") + } + return true, nil + }, func(err error) { + require.NoError(t, err) + }) + + ui := cli.NewMockUi() + cmd := &ServiceListCommand{ + Meta: Meta{ + Ui: ui, + flagAddress: url, + }, + } + + // Run the command with some random arguments to ensure we are performing + // this check. + require.Equal(t, 1, cmd.Run([]string{"-address=" + url, "pretty-please"})) + require.Contains(t, ui.ErrorWriter.String(), "This command takes no arguments") + ui.ErrorWriter.Reset() + + // Create a test job with a Nomad service. + testJob := testJob("service-discovery-nomad-list") + testJob.TaskGroups[0].Tasks[0].Services = []*api.Service{ + {Name: "service-discovery-nomad-list", Provider: "nomad", Tags: []string{"foo", "bar"}}} + + // Register that job. + regResp, _, err := client.Jobs().Register(testJob, nil) + require.NoError(t, err) + registerCode := waitForSuccess(ui, client, fullId, t, regResp.EvalID) + require.Equal(t, 0, registerCode) + + // Reset the output writer, otherwise we will have additional information here. + ui.OutputWriter.Reset() + + // Job register doesn't assure the service registration has completed. It + // therefore needs this wrapper to account for eventual service + // registration. One this has completed, we can perform lookups without + // similar wraps. + require.Eventually(t, func() bool { + + defer ui.OutputWriter.Reset() + + // Perform a standard lookup. + if code := cmd.Run([]string{"-address=" + url}); code != 0 { + return false + } + + // Test each header and data entry. + s := ui.OutputWriter.String() + if !assert.Contains(t, s, "Service Name") { + return false + } + if !assert.Contains(t, s, "Tags") { + return false + } + if !assert.Contains(t, s, "service-discovery-nomad-list") { + return false + } + if !assert.Contains(t, s, "[bar,foo]") { + return false + } + return true + }, 5*time.Second, 100*time.Millisecond) + + // Perform a wildcard namespace lookup. + code := cmd.Run([]string{"-address=" + url, "-namespace", "*"}) + require.Equal(t, 0, code) + + // Test each header and data entry. + s := ui.OutputWriter.String() + require.Contains(t, s, "Service Name") + require.Contains(t, s, "Namespace") + require.Contains(t, s, "Tags") + require.Contains(t, s, "service-discovery-nomad-list") + require.Contains(t, s, "default") + require.Contains(t, s, "[bar,foo]") + + ui.OutputWriter.Reset() + ui.ErrorWriter.Reset() +} From 6d002e9698d169c62f79cb8b639522a3c1a89adf Mon Sep 17 00:00:00 2001 From: James Rasell Date: Wed, 23 Mar 2022 09:42:46 +0100 Subject: [PATCH 28/31] core: remove node service registrations when node is down. When a node fails its heart beating a number of actions are taken to ensure state is cleaned. Service registrations a loosely tied to nodes, therefore we should remove these from state when a node is considered terminally down. --- nomad/node_endpoint.go | 26 ++++++++++++++++++++ nomad/node_endpoint_test.go | 49 +++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/nomad/node_endpoint.go b/nomad/node_endpoint.go index 2a6b7dc9457d..02d8c31a1f44 100644 --- a/nomad/node_endpoint.go +++ b/nomad/node_endpoint.go @@ -522,6 +522,32 @@ func (n *Node) UpdateStatus(args *structs.NodeUpdateStatusRequest, reply *struct n.logger.Debug("revoking SI accessors on node due to down state", "num_accessors", l, "node_id", args.NodeID) _ = n.srv.consulACLs.RevokeTokens(context.Background(), accessors, true) } + + // Identify the service registrations current placed on the downed + // node. + serviceRegistrations, err := n.srv.State().GetServiceRegistrationsByNodeID(ws, args.NodeID) + if err != nil { + n.logger.Error("looking up service registrations for node failed", + "node_id", args.NodeID, "error", err) + return err + } + + // If the node has service registrations assigned to it, delete these + // via Raft. + if l := len(serviceRegistrations); l > 0 { + n.logger.Debug("deleting service registrations on node due to down state", + "num_service_registrations", l, "node_id", args.NodeID) + + deleteRegReq := structs.ServiceRegistrationDeleteByNodeIDRequest{NodeID: args.NodeID} + + _, index, err = n.srv.raftApply(structs.ServiceRegistrationDeleteByNodeIDRequestType, &deleteRegReq) + if err != nil { + n.logger.Error("failed to delete service registrations for node", + "node_id", args.NodeID, "error", err) + return err + } + } + default: ttl, err := n.srv.resetHeartbeatTimer(args.NodeID) if err != nil { diff --git a/nomad/node_endpoint_test.go b/nomad/node_endpoint_test.go index 8d1cd12cc408..0ce68721d2cc 100644 --- a/nomad/node_endpoint_test.go +++ b/nomad/node_endpoint_test.go @@ -870,6 +870,55 @@ func TestClientEndpoint_UpdateStatus_HeartbeatOnly_Advertise(t *testing.T) { require.Equal(resp.Servers[0].RPCAdvertiseAddr, advAddr) } +func TestNode_UpdateStatus_ServiceRegistrations(t *testing.T) { + t.Parallel() + + testServer, serverCleanup := TestServer(t, nil) + defer serverCleanup() + testutil.WaitForLeader(t, testServer.RPC) + + // Create a node and upsert this into state. + node := mock.Node() + require.NoError(t, testServer.State().UpsertNode(structs.MsgTypeTestSetup, 10, node)) + + // Generate service registrations, ensuring the nodeID is set to the + // generated node from above. + services := mock.ServiceRegistrations() + + for _, s := range services { + s.NodeID = node.ID + } + + // Upsert the service registrations into state. + require.NoError(t, testServer.State().UpsertServiceRegistrations(structs.MsgTypeTestSetup, 20, services)) + + // Check the service registrations are in state as we expect, so we can + // have confidence in the rest of the test. + ws := memdb.NewWatchSet() + nodeRegs, err := testServer.State().GetServiceRegistrationsByNodeID(ws, node.ID) + require.NoError(t, err) + require.Len(t, nodeRegs, 2) + require.Equal(t, nodeRegs[0].NodeID, node.ID) + require.Equal(t, nodeRegs[1].NodeID, node.ID) + + // Generate and trigger a node down status update. This mimics what happens + // when the node fails its heart-beating. + args := structs.NodeUpdateStatusRequest{ + NodeID: node.ID, + Status: structs.NodeStatusDown, + WriteRequest: structs.WriteRequest{Region: "global"}, + } + + var reply structs.NodeUpdateResponse + require.NoError(t, testServer.staticEndpoints.Node.UpdateStatus(&args, &reply)) + + // Query our state, to ensure the node service registrations have been + // removed. + nodeRegs, err = testServer.State().GetServiceRegistrationsByNodeID(ws, node.ID) + require.NoError(t, err) + require.Len(t, nodeRegs, 0) +} + // TestClientEndpoint_UpdateDrain asserts the ability to initiate drain // against a node and cancel that drain. It also asserts: // * an evaluation is created when the node becomes eligible From e89dd5dcf7d75fa6093ad798aba4f2cfaf90d998 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Thu, 24 Mar 2022 08:45:13 +0100 Subject: [PATCH 29/31] test: move remaining tests to use ci.Parallel. --- client/allocrunner/groupservice_hook_test.go | 2 +- .../taskrunner/service_hook_test.go | 2 +- command/agent/alloc_endpoint_test.go | 2 +- command/agent/eval_endpoint_test.go | 2 +- command/agent/job_endpoint_test.go | 2 +- .../service_registration_endpoint_test.go | 5 +++-- command/service_delete_test.go | 3 ++- command/service_info_test.go | 3 ++- command/service_list_test.go | 3 ++- nomad/alloc_endpoint_test.go | 2 +- nomad/fsm_test.go | 6 +++--- nomad/job_endpoint_hooks_test.go | 3 ++- nomad/job_endpoint_test.go | 2 +- nomad/node_endpoint_test.go | 2 +- nomad/service_registration_endpoint_test.go | 9 ++++---- nomad/state/events_test.go | 2 +- nomad/state/state_store_restore_test.go | 2 +- .../state_store_service_regisration_test.go | 21 ++++++++++--------- nomad/state/state_store_test.go | 2 +- nomad/structs/services_test.go | 4 ++-- 20 files changed, 43 insertions(+), 36 deletions(-) diff --git a/client/allocrunner/groupservice_hook_test.go b/client/allocrunner/groupservice_hook_test.go index abec8a1315da..12a46358da13 100644 --- a/client/allocrunner/groupservice_hook_test.go +++ b/client/allocrunner/groupservice_hook_test.go @@ -157,7 +157,7 @@ func TestGroupServiceHook_GroupServices(t *testing.T) { // TestGroupServiceHook_GroupServices_Nomad asserts group service hooks with // group services does not error when using the Nomad provider. func TestGroupServiceHook_GroupServices_Nomad(t *testing.T) { - t.Parallel() + ci.Parallel(t) // Create a mock alloc, and add a group service using provider Nomad. alloc := mock.Alloc() diff --git a/client/allocrunner/taskrunner/service_hook_test.go b/client/allocrunner/taskrunner/service_hook_test.go index 655a2e1bfd18..ae59fb649f30 100644 --- a/client/allocrunner/taskrunner/service_hook_test.go +++ b/client/allocrunner/taskrunner/service_hook_test.go @@ -156,7 +156,7 @@ func Test_serviceHook_multipleDeRegisterCall(t *testing.T) { // Test_serviceHook_Nomad performs a normal operation test of the serviceHook // when using task services which utilise the Nomad provider. func Test_serviceHook_Nomad(t *testing.T) { - t.Parallel() + ci.Parallel(t) // Create a mock alloc, and add a task service using provider Nomad. alloc := mock.Alloc() diff --git a/command/agent/alloc_endpoint_test.go b/command/agent/alloc_endpoint_test.go index 01aebdd555e7..cfae04dd9917 100644 --- a/command/agent/alloc_endpoint_test.go +++ b/command/agent/alloc_endpoint_test.go @@ -435,7 +435,7 @@ func TestHTTP_AllocStop(t *testing.T) { } func TestHTTP_allocServiceRegistrations(t *testing.T) { - t.Parallel() + ci.Parallel(t) testCases := []struct { testFn func(srv *TestAgent) diff --git a/command/agent/eval_endpoint_test.go b/command/agent/eval_endpoint_test.go index df651d462ae1..2c9665d91050 100644 --- a/command/agent/eval_endpoint_test.go +++ b/command/agent/eval_endpoint_test.go @@ -202,7 +202,7 @@ func TestHTTP_EvalQuery(t *testing.T) { } func TestHTTP_EvalQueryWithRelated(t *testing.T) { - t.Parallel() + ci.Parallel(t) httpTest(t, nil, func(s *TestAgent) { // Directly manipulate the state state := s.Agent.server.State() diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index 6cee319dbddc..3dd9197ecb65 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -2250,7 +2250,7 @@ func TestJobs_NamespaceForJob(t *testing.T) { } func TestHTTPServer_jobServiceRegistrations(t *testing.T) { - t.Parallel() + ci.Parallel(t) testCases := []struct { testFn func(srv *TestAgent) diff --git a/command/agent/service_registration_endpoint_test.go b/command/agent/service_registration_endpoint_test.go index e02edc0607a3..d6e6954377fd 100644 --- a/command/agent/service_registration_endpoint_test.go +++ b/command/agent/service_registration_endpoint_test.go @@ -7,13 +7,14 @@ import ( "testing" "github.com/hashicorp/go-memdb" + "github.com/hashicorp/nomad/ci" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" "github.com/stretchr/testify/require" ) func TestHTTPServer_ServiceRegistrationListRequest(t *testing.T) { - t.Parallel() + ci.Parallel(t) testCases := []struct { testFn func(srv *TestAgent) @@ -149,7 +150,7 @@ func TestHTTPServer_ServiceRegistrationListRequest(t *testing.T) { } func TestHTTPServer_ServiceRegistrationRequest(t *testing.T) { - t.Parallel() + ci.Parallel(t) testCases := []struct { testFn func(srv *TestAgent) diff --git a/command/service_delete_test.go b/command/service_delete_test.go index fc7768fe08f2..6e17d5f14924 100644 --- a/command/service_delete_test.go +++ b/command/service_delete_test.go @@ -5,13 +5,14 @@ import ( "testing" "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/ci" "github.com/hashicorp/nomad/testutil" "github.com/mitchellh/cli" "github.com/stretchr/testify/require" ) func TestServiceDeleteCommand_Run(t *testing.T) { - t.Parallel() + ci.Parallel(t) srv, client, url := testServer(t, true, nil) defer srv.Shutdown() diff --git a/command/service_info_test.go b/command/service_info_test.go index 70fdb6df015b..08990deed1b3 100644 --- a/command/service_info_test.go +++ b/command/service_info_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/ci" "github.com/hashicorp/nomad/testutil" "github.com/mitchellh/cli" "github.com/stretchr/testify/assert" @@ -13,7 +14,7 @@ import ( ) func TestServiceInfoCommand_Run(t *testing.T) { - t.Parallel() + ci.Parallel(t) srv, client, url := testServer(t, true, nil) defer srv.Shutdown() diff --git a/command/service_list_test.go b/command/service_list_test.go index 8295963f96bf..b26479e5ed26 100644 --- a/command/service_list_test.go +++ b/command/service_list_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/ci" "github.com/hashicorp/nomad/testutil" "github.com/mitchellh/cli" "github.com/stretchr/testify/assert" @@ -13,7 +14,7 @@ import ( ) func TestServiceListCommand_Run(t *testing.T) { - t.Parallel() + ci.Parallel(t) srv, client, url := testServer(t, true, nil) defer srv.Shutdown() diff --git a/nomad/alloc_endpoint_test.go b/nomad/alloc_endpoint_test.go index 7a2103b004e0..b15fae6e30da 100644 --- a/nomad/alloc_endpoint_test.go +++ b/nomad/alloc_endpoint_test.go @@ -1362,7 +1362,7 @@ func TestAllocEndpoint_List_AllNamespaces_ACL_OSS(t *testing.T) { } func TestAlloc_GetServiceRegistrations(t *testing.T) { - t.Parallel() + ci.Parallel(t) // This function is a helper function to set up an allocation and service // which can be queried. diff --git a/nomad/fsm_test.go b/nomad/fsm_test.go index ecf1f4696f16..cab4a636615a 100644 --- a/nomad/fsm_test.go +++ b/nomad/fsm_test.go @@ -3260,7 +3260,7 @@ func TestFSM_SnapshotRestore_Namespaces(t *testing.T) { } func TestFSM_UpsertServiceRegistrations(t *testing.T) { - t.Parallel() + ci.Parallel(t) fsm := testFSM(t) // Generate our test service registrations. @@ -3284,7 +3284,7 @@ func TestFSM_UpsertServiceRegistrations(t *testing.T) { } func TestFSM_DeleteServiceRegistrationsByID(t *testing.T) { - t.Parallel() + ci.Parallel(t) fsm := testFSM(t) // Generate our test service registrations. @@ -3312,7 +3312,7 @@ func TestFSM_DeleteServiceRegistrationsByID(t *testing.T) { } func TestFSM_DeleteServiceRegistrationsByNodeID(t *testing.T) { - t.Parallel() + ci.Parallel(t) fsm := testFSM(t) // Generate our test service registrations. Set them both to have the same diff --git a/nomad/job_endpoint_hooks_test.go b/nomad/job_endpoint_hooks_test.go index 8e146972a277..f93637287273 100644 --- a/nomad/job_endpoint_hooks_test.go +++ b/nomad/job_endpoint_hooks_test.go @@ -3,12 +3,13 @@ package nomad import ( "testing" + "github.com/hashicorp/nomad/ci" "github.com/hashicorp/nomad/nomad/structs" "github.com/stretchr/testify/require" ) func Test_jobImpliedConstraints_Mutate(t *testing.T) { - t.Parallel() + ci.Parallel(t) testCases := []struct { inputJob *structs.Job diff --git a/nomad/job_endpoint_test.go b/nomad/job_endpoint_test.go index ca49b7aeb575..6802e2bf65bf 100644 --- a/nomad/job_endpoint_test.go +++ b/nomad/job_endpoint_test.go @@ -7920,7 +7920,7 @@ func TestJobEndpoint_GetScaleStatus_ACL(t *testing.T) { } func TestJob_GetServiceRegistrations(t *testing.T) { - t.Parallel() + ci.Parallel(t) // This function is a helper function to set up job and service which can // be queried. diff --git a/nomad/node_endpoint_test.go b/nomad/node_endpoint_test.go index 0ce68721d2cc..222de1ae2b95 100644 --- a/nomad/node_endpoint_test.go +++ b/nomad/node_endpoint_test.go @@ -871,7 +871,7 @@ func TestClientEndpoint_UpdateStatus_HeartbeatOnly_Advertise(t *testing.T) { } func TestNode_UpdateStatus_ServiceRegistrations(t *testing.T) { - t.Parallel() + ci.Parallel(t) testServer, serverCleanup := TestServer(t, nil) defer serverCleanup() diff --git a/nomad/service_registration_endpoint_test.go b/nomad/service_registration_endpoint_test.go index cff3473119d5..49831eefdf39 100644 --- a/nomad/service_registration_endpoint_test.go +++ b/nomad/service_registration_endpoint_test.go @@ -6,6 +6,7 @@ import ( "github.com/hashicorp/go-memdb" msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" "github.com/hashicorp/nomad/acl" + "github.com/hashicorp/nomad/ci" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/testutil" @@ -13,7 +14,7 @@ import ( ) func TestServiceRegistration_Upsert(t *testing.T) { - t.Parallel() + ci.Parallel(t) testCases := []struct { serverFn func(t *testing.T) (*Server, *structs.ACLToken, func()) @@ -198,7 +199,7 @@ func TestServiceRegistration_Upsert(t *testing.T) { } func TestServiceRegistration_DeleteByID(t *testing.T) { - t.Parallel() + ci.Parallel(t) testCases := []struct { serverFn func(t *testing.T) (*Server, *structs.ACLToken, func()) @@ -401,7 +402,7 @@ func TestServiceRegistration_DeleteByID(t *testing.T) { } func TestServiceRegistration_List(t *testing.T) { - t.Parallel() + ci.Parallel(t) testCases := []struct { serverFn func(t *testing.T) (*Server, *structs.ACLToken, func()) @@ -811,7 +812,7 @@ func TestServiceRegistration_List(t *testing.T) { } func TestServiceRegistration_GetService(t *testing.T) { - t.Parallel() + ci.Parallel(t) testCases := []struct { serverFn func(t *testing.T) (*Server, *structs.ACLToken, func()) diff --git a/nomad/state/events_test.go b/nomad/state/events_test.go index b3d9d468132e..7e25a6194bb5 100644 --- a/nomad/state/events_test.go +++ b/nomad/state/events_test.go @@ -956,7 +956,7 @@ func TestNodeDrainEventFromChanges(t *testing.T) { } func Test_eventsFromChanges_ServiceRegistration(t *testing.T) { - t.Parallel() + ci.Parallel(t) testState := TestStateStoreCfg(t, TestStateStorePublisher(t)) defer testState.StopEventBroker() diff --git a/nomad/state/state_store_restore_test.go b/nomad/state/state_store_restore_test.go index ea92c29ee278..8bc0fe2e61e3 100644 --- a/nomad/state/state_store_restore_test.go +++ b/nomad/state/state_store_restore_test.go @@ -544,7 +544,7 @@ func TestStateStore_RestoreSchedulerConfig(t *testing.T) { } func TestStateStore_ServiceRegistrationRestore(t *testing.T) { - t.Parallel() + ci.Parallel(t) testState := testStateStore(t) // Set up our test registrations and index. diff --git a/nomad/state/state_store_service_regisration_test.go b/nomad/state/state_store_service_regisration_test.go index 84ca6c93d27a..25cc2887550b 100644 --- a/nomad/state/state_store_service_regisration_test.go +++ b/nomad/state/state_store_service_regisration_test.go @@ -5,13 +5,14 @@ import ( "testing" "github.com/hashicorp/go-memdb" + "github.com/hashicorp/nomad/ci" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" "github.com/stretchr/testify/require" ) func TestStateStore_UpsertServiceRegistrations(t *testing.T) { - t.Parallel() + ci.Parallel(t) testState := testStateStore(t) // SubTest Marker: This ensures new service registrations are inserted as @@ -143,7 +144,7 @@ func TestStateStore_UpsertServiceRegistrations(t *testing.T) { } func TestStateStore_DeleteServiceRegistrationByID(t *testing.T) { - t.Parallel() + ci.Parallel(t) testState := testStateStore(t) // Generate some test services that we will use and modify throughout. @@ -216,7 +217,7 @@ func TestStateStore_DeleteServiceRegistrationByID(t *testing.T) { } func TestStateStore_DeleteServiceRegistrationByNodeID(t *testing.T) { - t.Parallel() + ci.Parallel(t) testState := testStateStore(t) // Generate some test services that we will use and modify throughout. @@ -312,7 +313,7 @@ func TestStateStore_DeleteServiceRegistrationByNodeID(t *testing.T) { } func TestStateStore_GetServiceRegistrations(t *testing.T) { - t.Parallel() + ci.Parallel(t) testState := testStateStore(t) // Generate some test services and upsert them. @@ -347,7 +348,7 @@ func TestStateStore_GetServiceRegistrations(t *testing.T) { } func TestStateStore_GetServiceRegistrationsByNamespace(t *testing.T) { - t.Parallel() + ci.Parallel(t) testState := testStateStore(t) // Generate some test services and upsert them. @@ -400,7 +401,7 @@ func TestStateStore_GetServiceRegistrationsByNamespace(t *testing.T) { } func TestStateStore_GetServiceRegistrationByName(t *testing.T) { - t.Parallel() + ci.Parallel(t) testState := testStateStore(t) // Generate some test services and upsert them. @@ -473,7 +474,7 @@ func TestStateStore_GetServiceRegistrationByName(t *testing.T) { } func TestStateStore_GetServiceRegistrationByID(t *testing.T) { - t.Parallel() + ci.Parallel(t) testState := testStateStore(t) // Generate some test services and upsert them. @@ -499,7 +500,7 @@ func TestStateStore_GetServiceRegistrationByID(t *testing.T) { } func TestStateStore_GetServiceRegistrationsByAllocID(t *testing.T) { - t.Parallel() + ci.Parallel(t) testState := testStateStore(t) // Generate some test services and upsert them. @@ -543,7 +544,7 @@ func TestStateStore_GetServiceRegistrationsByAllocID(t *testing.T) { } func TestStateStore_GetServiceRegistrationsByJobID(t *testing.T) { - t.Parallel() + ci.Parallel(t) testState := testStateStore(t) // Generate some test services and upsert them. @@ -593,7 +594,7 @@ func TestStateStore_GetServiceRegistrationsByJobID(t *testing.T) { } func TestStateStore_GetServiceRegistrationsByNodeID(t *testing.T) { - t.Parallel() + ci.Parallel(t) testState := testStateStore(t) // Generate some test services and upsert them. diff --git a/nomad/state/state_store_test.go b/nomad/state/state_store_test.go index 0859b6e54cbf..25df0e62594e 100644 --- a/nomad/state/state_store_test.go +++ b/nomad/state/state_store_test.go @@ -4599,7 +4599,7 @@ func TestStateStore_EvalsByIDPrefix_Namespaces(t *testing.T) { } func TestStateStore_EvalsRelatedToID(t *testing.T) { - t.Parallel() + ci.Parallel(t) state := testStateStore(t) diff --git a/nomad/structs/services_test.go b/nomad/structs/services_test.go index 51088425383d..915a5a9698e0 100644 --- a/nomad/structs/services_test.go +++ b/nomad/structs/services_test.go @@ -2,10 +2,10 @@ package structs import ( "errors" - "github.com/hashicorp/go-multierror" "testing" "time" + "github.com/hashicorp/go-multierror" "github.com/hashicorp/nomad/ci" "github.com/hashicorp/nomad/helper" "github.com/stretchr/testify/require" @@ -1479,7 +1479,7 @@ func TestConsulMeshGateway_Validate(t *testing.T) { } func TestService_validateNomadService(t *testing.T) { - t.Parallel() + ci.Parallel(t) testCases := []struct { inputService *Service From 2ac318b6f76146057aa750424a5a9fefabec727f Mon Sep 17 00:00:00 2001 From: James Rasell Date: Thu, 24 Mar 2022 09:08:45 +0100 Subject: [PATCH 30/31] api: move serviceregistration client to servics to match CLI. The service registration client name was used to provide a distinction between the service block and the service client. This however creates new wording to understand and does not match the CLI, therefore this change fixes that so we have a Services client. Consul specific objects within the service file have been moved to the consul location to create a clearer separation. --- api/consul.go | 557 ++++++++++++++++++++++++ api/consul_test.go | 453 ++++++++++++++++++++ api/service_registrations.go | 129 ------ api/service_registrations_test.go | 17 - api/services.go | 679 ++++++------------------------ api/services_test.go | 467 +------------------- command/service_delete.go | 2 +- command/service_delete_test.go | 2 +- command/service_info.go | 2 +- command/service_list.go | 2 +- 10 files changed, 1153 insertions(+), 1157 deletions(-) delete mode 100644 api/service_registrations.go delete mode 100644 api/service_registrations_test.go diff --git a/api/consul.go b/api/consul.go index 64e085e618dd..9a11187c08cb 100644 --- a/api/consul.go +++ b/api/consul.go @@ -1,5 +1,7 @@ package api +import "time" + // Consul represents configuration related to consul. type Consul struct { // (Enterprise-only) Namespace represents a Consul namespace. @@ -33,3 +35,558 @@ func (c *Consul) MergeNamespace(namespace *string) { c.Namespace = *namespace } } + +// ConsulConnect represents a Consul Connect jobspec stanza. +type ConsulConnect struct { + Native bool `hcl:"native,optional"` + Gateway *ConsulGateway `hcl:"gateway,block"` + SidecarService *ConsulSidecarService `mapstructure:"sidecar_service" hcl:"sidecar_service,block"` + SidecarTask *SidecarTask `mapstructure:"sidecar_task" hcl:"sidecar_task,block"` +} + +func (cc *ConsulConnect) Canonicalize() { + if cc == nil { + return + } + + cc.SidecarService.Canonicalize() + cc.SidecarTask.Canonicalize() + cc.Gateway.Canonicalize() +} + +// ConsulSidecarService represents a Consul Connect SidecarService jobspec +// stanza. +type ConsulSidecarService struct { + Tags []string `hcl:"tags,optional"` + Port string `hcl:"port,optional"` + Proxy *ConsulProxy `hcl:"proxy,block"` + DisableDefaultTCPCheck bool `mapstructure:"disable_default_tcp_check" hcl:"disable_default_tcp_check,optional"` +} + +func (css *ConsulSidecarService) Canonicalize() { + if css == nil { + return + } + + if len(css.Tags) == 0 { + css.Tags = nil + } + + css.Proxy.Canonicalize() +} + + +// SidecarTask represents a subset of Task fields that can be set to override +// the fields of the Task generated for the sidecar +type SidecarTask struct { + Name string `hcl:"name,optional"` + Driver string `hcl:"driver,optional"` + User string `hcl:"user,optional"` + Config map[string]interface{} `hcl:"config,block"` + Env map[string]string `hcl:"env,block"` + Resources *Resources `hcl:"resources,block"` + Meta map[string]string `hcl:"meta,block"` + KillTimeout *time.Duration `mapstructure:"kill_timeout" hcl:"kill_timeout,optional"` + LogConfig *LogConfig `mapstructure:"logs" hcl:"logs,block"` + ShutdownDelay *time.Duration `mapstructure:"shutdown_delay" hcl:"shutdown_delay,optional"` + KillSignal string `mapstructure:"kill_signal" hcl:"kill_signal,optional"` +} + +func (st *SidecarTask) Canonicalize() { + if st == nil { + return + } + + if len(st.Config) == 0 { + st.Config = nil + } + + if len(st.Env) == 0 { + st.Env = nil + } + + if st.Resources == nil { + st.Resources = DefaultResources() + } else { + st.Resources.Canonicalize() + } + + if st.LogConfig == nil { + st.LogConfig = DefaultLogConfig() + } else { + st.LogConfig.Canonicalize() + } + + if len(st.Meta) == 0 { + st.Meta = nil + } + + if st.KillTimeout == nil { + st.KillTimeout = timeToPtr(5 * time.Second) + } + + if st.ShutdownDelay == nil { + st.ShutdownDelay = timeToPtr(0) + } +} + +// ConsulProxy represents a Consul Connect sidecar proxy jobspec stanza. +type ConsulProxy struct { + LocalServiceAddress string `mapstructure:"local_service_address" hcl:"local_service_address,optional"` + LocalServicePort int `mapstructure:"local_service_port" hcl:"local_service_port,optional"` + ExposeConfig *ConsulExposeConfig `mapstructure:"expose" hcl:"expose,block"` + Upstreams []*ConsulUpstream `hcl:"upstreams,block"` + Config map[string]interface{} `hcl:"config,block"` +} + +func (cp *ConsulProxy) Canonicalize() { + if cp == nil { + return + } + + cp.ExposeConfig.Canonicalize() + + if len(cp.Upstreams) == 0 { + cp.Upstreams = nil + } + + for _, upstream := range cp.Upstreams { + upstream.Canonicalize() + } + + if len(cp.Config) == 0 { + cp.Config = nil + } +} + +// ConsulMeshGateway is used to configure mesh gateway usage when connecting to +// a connect upstream in another datacenter. +type ConsulMeshGateway struct { + // Mode configures how an upstream should be accessed with regard to using + // mesh gateways. + // + // local - the connect proxy makes outbound connections through mesh gateway + // originating in the same datacenter. + // + // remote - the connect proxy makes outbound connections to a mesh gateway + // in the destination datacenter. + // + // none (default) - no mesh gateway is used, the proxy makes outbound connections + // directly to destination services. + // + // https://www.consul.io/docs/connect/gateways/mesh-gateway#modes-of-operation + Mode string `mapstructure:"mode" hcl:"mode,optional"` +} + +func (c *ConsulMeshGateway) Canonicalize() { + // Mode may be empty string, indicating behavior will defer to Consul + // service-defaults config entry. + return +} + +func (c *ConsulMeshGateway) Copy() *ConsulMeshGateway { + if c == nil { + return nil + } + + return &ConsulMeshGateway{ + Mode: c.Mode, + } +} + +// ConsulUpstream represents a Consul Connect upstream jobspec stanza. +type ConsulUpstream struct { + DestinationName string `mapstructure:"destination_name" hcl:"destination_name,optional"` + LocalBindPort int `mapstructure:"local_bind_port" hcl:"local_bind_port,optional"` + Datacenter string `mapstructure:"datacenter" hcl:"datacenter,optional"` + LocalBindAddress string `mapstructure:"local_bind_address" hcl:"local_bind_address,optional"` + MeshGateway *ConsulMeshGateway `mapstructure:"mesh_gateway" hcl:"mesh_gateway,block"` +} + +func (cu *ConsulUpstream) Copy() *ConsulUpstream { + if cu == nil { + return nil + } + return &ConsulUpstream{ + DestinationName: cu.DestinationName, + LocalBindPort: cu.LocalBindPort, + Datacenter: cu.Datacenter, + LocalBindAddress: cu.LocalBindAddress, + MeshGateway: cu.MeshGateway.Copy(), + } +} + +func (cu *ConsulUpstream) Canonicalize() { + if cu == nil { + return + } + cu.MeshGateway.Canonicalize() +} + +type ConsulExposeConfig struct { + Path []*ConsulExposePath `mapstructure:"path" hcl:"path,block"` +} + +func (cec *ConsulExposeConfig) Canonicalize() { + if cec == nil { + return + } + + if len(cec.Path) == 0 { + cec.Path = nil + } +} + +type ConsulExposePath struct { + Path string `hcl:"path,optional"` + Protocol string `hcl:"protocol,optional"` + LocalPathPort int `mapstructure:"local_path_port" hcl:"local_path_port,optional"` + ListenerPort string `mapstructure:"listener_port" hcl:"listener_port,optional"` +} + +// ConsulGateway is used to configure one of the Consul Connect Gateway types. +type ConsulGateway struct { + // Proxy is used to configure the Envoy instance acting as the gateway. + Proxy *ConsulGatewayProxy `hcl:"proxy,block"` + + // Ingress represents the Consul Configuration Entry for an Ingress Gateway. + Ingress *ConsulIngressConfigEntry `hcl:"ingress,block"` + + // Terminating represents the Consul Configuration Entry for a Terminating Gateway. + Terminating *ConsulTerminatingConfigEntry `hcl:"terminating,block"` + + // Mesh indicates the Consul service should be a Mesh Gateway. + Mesh *ConsulMeshConfigEntry `hcl:"mesh,block"` +} + +func (g *ConsulGateway) Canonicalize() { + if g == nil { + return + } + g.Proxy.Canonicalize() + g.Ingress.Canonicalize() + g.Terminating.Canonicalize() +} + +func (g *ConsulGateway) Copy() *ConsulGateway { + if g == nil { + return nil + } + + return &ConsulGateway{ + Proxy: g.Proxy.Copy(), + Ingress: g.Ingress.Copy(), + Terminating: g.Terminating.Copy(), + } +} + +type ConsulGatewayBindAddress struct { + Name string `hcl:",label"` + Address string `mapstructure:"address" hcl:"address,optional"` + Port int `mapstructure:"port" hcl:"port,optional"` +} + +var ( + // defaultGatewayConnectTimeout is the default amount of time connections to + // upstreams are allowed before timing out. + defaultGatewayConnectTimeout = 5 * time.Second +) + +// ConsulGatewayProxy is used to tune parameters of the proxy instance acting as +// one of the forms of Connect gateways that Consul supports. +// +// https://www.consul.io/docs/connect/proxies/envoy#gateway-options +type ConsulGatewayProxy struct { + ConnectTimeout *time.Duration `mapstructure:"connect_timeout" hcl:"connect_timeout,optional"` + EnvoyGatewayBindTaggedAddresses bool `mapstructure:"envoy_gateway_bind_tagged_addresses" hcl:"envoy_gateway_bind_tagged_addresses,optional"` + EnvoyGatewayBindAddresses map[string]*ConsulGatewayBindAddress `mapstructure:"envoy_gateway_bind_addresses" hcl:"envoy_gateway_bind_addresses,block"` + EnvoyGatewayNoDefaultBind bool `mapstructure:"envoy_gateway_no_default_bind" hcl:"envoy_gateway_no_default_bind,optional"` + EnvoyDNSDiscoveryType string `mapstructure:"envoy_dns_discovery_type" hcl:"envoy_dns_discovery_type,optional"` + Config map[string]interface{} `hcl:"config,block"` // escape hatch envoy config +} + +func (p *ConsulGatewayProxy) Canonicalize() { + if p == nil { + return + } + + if p.ConnectTimeout == nil { + // same as the default from consul + p.ConnectTimeout = timeToPtr(defaultGatewayConnectTimeout) + } + + if len(p.EnvoyGatewayBindAddresses) == 0 { + p.EnvoyGatewayBindAddresses = nil + } + + if len(p.Config) == 0 { + p.Config = nil + } +} + +func (p *ConsulGatewayProxy) Copy() *ConsulGatewayProxy { + if p == nil { + return nil + } + + var binds map[string]*ConsulGatewayBindAddress = nil + if p.EnvoyGatewayBindAddresses != nil { + binds = make(map[string]*ConsulGatewayBindAddress, len(p.EnvoyGatewayBindAddresses)) + for k, v := range p.EnvoyGatewayBindAddresses { + binds[k] = v + } + } + + var config map[string]interface{} = nil + if p.Config != nil { + config = make(map[string]interface{}, len(p.Config)) + for k, v := range p.Config { + config[k] = v + } + } + + return &ConsulGatewayProxy{ + ConnectTimeout: timeToPtr(*p.ConnectTimeout), + EnvoyGatewayBindTaggedAddresses: p.EnvoyGatewayBindTaggedAddresses, + EnvoyGatewayBindAddresses: binds, + EnvoyGatewayNoDefaultBind: p.EnvoyGatewayNoDefaultBind, + EnvoyDNSDiscoveryType: p.EnvoyDNSDiscoveryType, + Config: config, + } +} + +// ConsulGatewayTLSConfig is used to configure TLS for a gateway. +type ConsulGatewayTLSConfig struct { + Enabled bool `hcl:"enabled,optional"` +} + +func (tc *ConsulGatewayTLSConfig) Canonicalize() { +} + +func (tc *ConsulGatewayTLSConfig) Copy() *ConsulGatewayTLSConfig { + if tc == nil { + return nil + } + + return &ConsulGatewayTLSConfig{ + Enabled: tc.Enabled, + } +} + +// ConsulIngressService is used to configure a service fronted by the ingress gateway. +type ConsulIngressService struct { + // Namespace is not yet supported. + // Namespace string + Name string `hcl:"name,optional"` + + Hosts []string `hcl:"hosts,optional"` +} + +func (s *ConsulIngressService) Canonicalize() { + if s == nil { + return + } + + if len(s.Hosts) == 0 { + s.Hosts = nil + } +} + +func (s *ConsulIngressService) Copy() *ConsulIngressService { + if s == nil { + return nil + } + + var hosts []string = nil + if n := len(s.Hosts); n > 0 { + hosts = make([]string, n) + copy(hosts, s.Hosts) + } + + return &ConsulIngressService{ + Name: s.Name, + Hosts: hosts, + } +} + +const ( + defaultIngressListenerProtocol = "tcp" +) + +// ConsulIngressListener is used to configure a listener on a Consul Ingress +// Gateway. +type ConsulIngressListener struct { + Port int `hcl:"port,optional"` + Protocol string `hcl:"protocol,optional"` + Services []*ConsulIngressService `hcl:"service,block"` +} + +func (l *ConsulIngressListener) Canonicalize() { + if l == nil { + return + } + + if l.Protocol == "" { + // same as default from consul + l.Protocol = defaultIngressListenerProtocol + } + + if len(l.Services) == 0 { + l.Services = nil + } +} + +func (l *ConsulIngressListener) Copy() *ConsulIngressListener { + if l == nil { + return nil + } + + var services []*ConsulIngressService = nil + if n := len(l.Services); n > 0 { + services = make([]*ConsulIngressService, n) + for i := 0; i < n; i++ { + services[i] = l.Services[i].Copy() + } + } + + return &ConsulIngressListener{ + Port: l.Port, + Protocol: l.Protocol, + Services: services, + } +} + +// ConsulIngressConfigEntry represents the Consul Configuration Entry type for +// an Ingress Gateway. +// +// https://www.consul.io/docs/agent/config-entries/ingress-gateway#available-fields +type ConsulIngressConfigEntry struct { + // Namespace is not yet supported. + // Namespace string + + TLS *ConsulGatewayTLSConfig `hcl:"tls,block"` + Listeners []*ConsulIngressListener `hcl:"listener,block"` +} + +func (e *ConsulIngressConfigEntry) Canonicalize() { + if e == nil { + return + } + + e.TLS.Canonicalize() + + if len(e.Listeners) == 0 { + e.Listeners = nil + } + + for _, listener := range e.Listeners { + listener.Canonicalize() + } +} + +func (e *ConsulIngressConfigEntry) Copy() *ConsulIngressConfigEntry { + if e == nil { + return nil + } + + var listeners []*ConsulIngressListener = nil + if n := len(e.Listeners); n > 0 { + listeners = make([]*ConsulIngressListener, n) + for i := 0; i < n; i++ { + listeners[i] = e.Listeners[i].Copy() + } + } + + return &ConsulIngressConfigEntry{ + TLS: e.TLS.Copy(), + Listeners: listeners, + } +} + +type ConsulLinkedService struct { + Name string `hcl:"name,optional"` + CAFile string `hcl:"ca_file,optional" mapstructure:"ca_file"` + CertFile string `hcl:"cert_file,optional" mapstructure:"cert_file"` + KeyFile string `hcl:"key_file,optional" mapstructure:"key_file"` + SNI string `hcl:"sni,optional"` +} + +func (s *ConsulLinkedService) Canonicalize() { + // nothing to do for now +} + +func (s *ConsulLinkedService) Copy() *ConsulLinkedService { + if s == nil { + return nil + } + + return &ConsulLinkedService{ + Name: s.Name, + CAFile: s.CAFile, + CertFile: s.CertFile, + KeyFile: s.KeyFile, + SNI: s.SNI, + } +} + +// ConsulTerminatingConfigEntry represents the Consul Configuration Entry type +// for a Terminating Gateway. +// +// https://www.consul.io/docs/agent/config-entries/terminating-gateway#available-fields +type ConsulTerminatingConfigEntry struct { + // Namespace is not yet supported. + // Namespace string + + Services []*ConsulLinkedService `hcl:"service,block"` +} + +func (e *ConsulTerminatingConfigEntry) Canonicalize() { + if e == nil { + return + } + + if len(e.Services) == 0 { + e.Services = nil + } + + for _, service := range e.Services { + service.Canonicalize() + } +} + +func (e *ConsulTerminatingConfigEntry) Copy() *ConsulTerminatingConfigEntry { + if e == nil { + return nil + } + + var services []*ConsulLinkedService = nil + if n := len(e.Services); n > 0 { + services = make([]*ConsulLinkedService, n) + for i := 0; i < n; i++ { + services[i] = e.Services[i].Copy() + } + } + + return &ConsulTerminatingConfigEntry{ + Services: services, + } +} + +// ConsulMeshConfigEntry is a stub used to represent that the gateway service type +// should be for a Mesh Gateway. Unlike Ingress and Terminating, there is no +// actual Consul Config Entry type for mesh-gateway, at least for now. We still +// create a type for future proofing, instead just using a bool for example. +type ConsulMeshConfigEntry struct { + // nothing in here +} + +func (e *ConsulMeshConfigEntry) Canonicalize() { + return +} + +func (e *ConsulMeshConfigEntry) Copy() *ConsulMeshConfigEntry { + if e == nil { + return nil + } + return new(ConsulMeshConfigEntry) +} diff --git a/api/consul_test.go b/api/consul_test.go index 81ebaf48b8a0..0c32c4c8168d 100644 --- a/api/consul_test.go +++ b/api/consul_test.go @@ -2,6 +2,7 @@ package api import ( "testing" + "time" "github.com/hashicorp/nomad/api/internal/testutil" "github.com/stretchr/testify/require" @@ -60,3 +61,455 @@ func TestConsul_MergeNamespace(t *testing.T) { require.Nil(t, ns) }) } + +func TestConsulConnect_Canonicalize(t *testing.T) { + testutil.Parallel(t) + + t.Run("nil connect", func(t *testing.T) { + cc := (*ConsulConnect)(nil) + cc.Canonicalize() + require.Nil(t, cc) + }) + + t.Run("empty connect", func(t *testing.T) { + cc := new(ConsulConnect) + cc.Canonicalize() + require.Empty(t, cc.Native) + require.Nil(t, cc.SidecarService) + require.Nil(t, cc.SidecarTask) + }) +} + +func TestConsulSidecarService_Canonicalize(t *testing.T) { + testutil.Parallel(t) + + t.Run("nil sidecar_service", func(t *testing.T) { + css := (*ConsulSidecarService)(nil) + css.Canonicalize() + require.Nil(t, css) + }) + + t.Run("empty sidecar_service", func(t *testing.T) { + css := new(ConsulSidecarService) + css.Canonicalize() + require.Empty(t, css.Tags) + require.Nil(t, css.Proxy) + }) + + t.Run("non-empty sidecar_service", func(t *testing.T) { + css := &ConsulSidecarService{ + Tags: make([]string, 0), + Port: "port", + Proxy: &ConsulProxy{ + LocalServiceAddress: "lsa", + LocalServicePort: 80, + }, + } + css.Canonicalize() + require.Equal(t, &ConsulSidecarService{ + Tags: nil, + Port: "port", + Proxy: &ConsulProxy{ + LocalServiceAddress: "lsa", + LocalServicePort: 80}, + }, css) + }) +} + +func TestConsulProxy_Canonicalize(t *testing.T) { + testutil.Parallel(t) + + t.Run("nil proxy", func(t *testing.T) { + cp := (*ConsulProxy)(nil) + cp.Canonicalize() + require.Nil(t, cp) + }) + + t.Run("empty proxy", func(t *testing.T) { + cp := new(ConsulProxy) + cp.Canonicalize() + require.Empty(t, cp.LocalServiceAddress) + require.Zero(t, cp.LocalServicePort) + require.Nil(t, cp.ExposeConfig) + require.Nil(t, cp.Upstreams) + require.Empty(t, cp.Config) + }) + + t.Run("non empty proxy", func(t *testing.T) { + cp := &ConsulProxy{ + LocalServiceAddress: "127.0.0.1", + LocalServicePort: 80, + ExposeConfig: new(ConsulExposeConfig), + Upstreams: make([]*ConsulUpstream, 0), + Config: make(map[string]interface{}), + } + cp.Canonicalize() + require.Equal(t, "127.0.0.1", cp.LocalServiceAddress) + require.Equal(t, 80, cp.LocalServicePort) + require.Equal(t, &ConsulExposeConfig{}, cp.ExposeConfig) + require.Nil(t, cp.Upstreams) + require.Nil(t, cp.Config) + }) +} + +func TestConsulUpstream_Copy(t *testing.T) { + testutil.Parallel(t) + + t.Run("nil upstream", func(t *testing.T) { + cu := (*ConsulUpstream)(nil) + result := cu.Copy() + require.Nil(t, result) + }) + + t.Run("complete upstream", func(t *testing.T) { + cu := &ConsulUpstream{ + DestinationName: "dest1", + Datacenter: "dc2", + LocalBindPort: 2000, + LocalBindAddress: "10.0.0.1", + MeshGateway: &ConsulMeshGateway{Mode: "remote"}, + } + result := cu.Copy() + require.Equal(t, cu, result) + }) +} + +func TestConsulUpstream_Canonicalize(t *testing.T) { + testutil.Parallel(t) + + t.Run("nil upstream", func(t *testing.T) { + cu := (*ConsulUpstream)(nil) + cu.Canonicalize() + require.Nil(t, cu) + }) + + t.Run("complete", func(t *testing.T) { + cu := &ConsulUpstream{ + DestinationName: "dest1", + Datacenter: "dc2", + LocalBindPort: 2000, + LocalBindAddress: "10.0.0.1", + MeshGateway: &ConsulMeshGateway{Mode: ""}, + } + cu.Canonicalize() + require.Equal(t, &ConsulUpstream{ + DestinationName: "dest1", + Datacenter: "dc2", + LocalBindPort: 2000, + LocalBindAddress: "10.0.0.1", + MeshGateway: &ConsulMeshGateway{Mode: ""}, + }, cu) + }) +} + +func TestSidecarTask_Canonicalize(t *testing.T) { + testutil.Parallel(t) + + t.Run("nil sidecar_task", func(t *testing.T) { + st := (*SidecarTask)(nil) + st.Canonicalize() + require.Nil(t, st) + }) + + t.Run("empty sidecar_task", func(t *testing.T) { + st := new(SidecarTask) + st.Canonicalize() + require.Nil(t, st.Config) + require.Nil(t, st.Env) + require.Equal(t, DefaultResources(), st.Resources) + require.Equal(t, DefaultLogConfig(), st.LogConfig) + require.Nil(t, st.Meta) + require.Equal(t, 5*time.Second, *st.KillTimeout) + require.Equal(t, 0*time.Second, *st.ShutdownDelay) + }) + + t.Run("non empty sidecar_task resources", func(t *testing.T) { + exp := DefaultResources() + exp.MemoryMB = intToPtr(333) + st := &SidecarTask{ + Resources: &Resources{MemoryMB: intToPtr(333)}, + } + st.Canonicalize() + require.Equal(t, exp, st.Resources) + }) +} + +func TestConsulGateway_Canonicalize(t *testing.T) { + testutil.Parallel(t) + + t.Run("nil", func(t *testing.T) { + cg := (*ConsulGateway)(nil) + cg.Canonicalize() + require.Nil(t, cg) + }) + + t.Run("set defaults", func(t *testing.T) { + cg := &ConsulGateway{ + Proxy: &ConsulGatewayProxy{ + ConnectTimeout: nil, + EnvoyGatewayBindTaggedAddresses: true, + EnvoyGatewayBindAddresses: make(map[string]*ConsulGatewayBindAddress, 0), + EnvoyGatewayNoDefaultBind: true, + Config: make(map[string]interface{}, 0), + }, + Ingress: &ConsulIngressConfigEntry{ + TLS: &ConsulGatewayTLSConfig{ + Enabled: false, + }, + Listeners: make([]*ConsulIngressListener, 0), + }, + } + cg.Canonicalize() + require.Equal(t, timeToPtr(5*time.Second), cg.Proxy.ConnectTimeout) + require.True(t, cg.Proxy.EnvoyGatewayBindTaggedAddresses) + require.Nil(t, cg.Proxy.EnvoyGatewayBindAddresses) + require.True(t, cg.Proxy.EnvoyGatewayNoDefaultBind) + require.Empty(t, cg.Proxy.EnvoyDNSDiscoveryType) + require.Nil(t, cg.Proxy.Config) + require.Nil(t, cg.Ingress.Listeners) + }) +} + +func TestConsulGateway_Copy(t *testing.T) { + testutil.Parallel(t) + + t.Run("nil", func(t *testing.T) { + result := (*ConsulGateway)(nil).Copy() + require.Nil(t, result) + }) + + gateway := &ConsulGateway{ + Proxy: &ConsulGatewayProxy{ + ConnectTimeout: timeToPtr(3 * time.Second), + EnvoyGatewayBindTaggedAddresses: true, + EnvoyGatewayBindAddresses: map[string]*ConsulGatewayBindAddress{ + "listener1": {Address: "10.0.0.1", Port: 2000}, + "listener2": {Address: "10.0.0.1", Port: 2001}, + }, + EnvoyGatewayNoDefaultBind: true, + EnvoyDNSDiscoveryType: "STRICT_DNS", + Config: map[string]interface{}{ + "foo": "bar", + "baz": 3, + }, + }, + Ingress: &ConsulIngressConfigEntry{ + TLS: &ConsulGatewayTLSConfig{ + Enabled: true, + }, + Listeners: []*ConsulIngressListener{{ + Port: 3333, + Protocol: "tcp", + Services: []*ConsulIngressService{{ + Name: "service1", + Hosts: []string{ + "127.0.0.1", "127.0.0.1:3333", + }}, + }}, + }, + }, + Terminating: &ConsulTerminatingConfigEntry{ + Services: []*ConsulLinkedService{{ + Name: "linked-service1", + }}, + }, + } + + t.Run("complete", func(t *testing.T) { + result := gateway.Copy() + require.Equal(t, gateway, result) + }) +} + +func TestConsulIngressConfigEntry_Canonicalize(t *testing.T) { + testutil.Parallel(t) + + t.Run("nil", func(t *testing.T) { + c := (*ConsulIngressConfigEntry)(nil) + c.Canonicalize() + require.Nil(t, c) + }) + + t.Run("empty fields", func(t *testing.T) { + c := &ConsulIngressConfigEntry{ + TLS: nil, + Listeners: []*ConsulIngressListener{}, + } + c.Canonicalize() + require.Nil(t, c.TLS) + require.Nil(t, c.Listeners) + }) + + t.Run("complete", func(t *testing.T) { + c := &ConsulIngressConfigEntry{ + TLS: &ConsulGatewayTLSConfig{Enabled: true}, + Listeners: []*ConsulIngressListener{{ + Port: 9090, + Protocol: "http", + Services: []*ConsulIngressService{{ + Name: "service1", + Hosts: []string{"1.1.1.1"}, + }}, + }}, + } + c.Canonicalize() + require.Equal(t, &ConsulIngressConfigEntry{ + TLS: &ConsulGatewayTLSConfig{Enabled: true}, + Listeners: []*ConsulIngressListener{{ + Port: 9090, + Protocol: "http", + Services: []*ConsulIngressService{{ + Name: "service1", + Hosts: []string{"1.1.1.1"}, + }}, + }}, + }, c) + }) +} + +func TestConsulIngressConfigEntry_Copy(t *testing.T) { + testutil.Parallel(t) + + t.Run("nil", func(t *testing.T) { + result := (*ConsulIngressConfigEntry)(nil).Copy() + require.Nil(t, result) + }) + + entry := &ConsulIngressConfigEntry{ + TLS: &ConsulGatewayTLSConfig{ + Enabled: true, + }, + Listeners: []*ConsulIngressListener{{ + Port: 1111, + Protocol: "http", + Services: []*ConsulIngressService{{ + Name: "service1", + Hosts: []string{"1.1.1.1", "1.1.1.1:9000"}, + }, { + Name: "service2", + Hosts: []string{"2.2.2.2"}, + }}, + }}, + } + + t.Run("complete", func(t *testing.T) { + result := entry.Copy() + require.Equal(t, entry, result) + }) +} + +func TestConsulTerminatingConfigEntry_Canonicalize(t *testing.T) { + testutil.Parallel(t) + + t.Run("nil", func(t *testing.T) { + c := (*ConsulTerminatingConfigEntry)(nil) + c.Canonicalize() + require.Nil(t, c) + }) + + t.Run("empty services", func(t *testing.T) { + c := &ConsulTerminatingConfigEntry{ + Services: []*ConsulLinkedService{}, + } + c.Canonicalize() + require.Nil(t, c.Services) + }) +} + +func TestConsulTerminatingConfigEntry_Copy(t *testing.T) { + testutil.Parallel(t) + + t.Run("nil", func(t *testing.T) { + result := (*ConsulIngressConfigEntry)(nil).Copy() + require.Nil(t, result) + }) + + entry := &ConsulTerminatingConfigEntry{ + Services: []*ConsulLinkedService{{ + Name: "servic1", + }, { + Name: "service2", + CAFile: "ca_file.pem", + CertFile: "cert_file.pem", + KeyFile: "key_file.pem", + SNI: "sni.terminating.consul", + }}, + } + + t.Run("complete", func(t *testing.T) { + result := entry.Copy() + require.Equal(t, entry, result) + }) +} + +func TestConsulMeshConfigEntry_Canonicalize(t *testing.T) { + testutil.Parallel(t) + + t.Run("nil", func(t *testing.T) { + ce := (*ConsulMeshConfigEntry)(nil) + ce.Canonicalize() + require.Nil(t, ce) + }) + + t.Run("instantiated", func(t *testing.T) { + ce := new(ConsulMeshConfigEntry) + ce.Canonicalize() + require.NotNil(t, ce) + }) +} + +func TestConsulMeshConfigEntry_Copy(t *testing.T) { + testutil.Parallel(t) + + t.Run("nil", func(t *testing.T) { + ce := (*ConsulMeshConfigEntry)(nil) + ce2 := ce.Copy() + require.Nil(t, ce2) + }) + + t.Run("instantiated", func(t *testing.T) { + ce := new(ConsulMeshConfigEntry) + ce2 := ce.Copy() + require.NotNil(t, ce2) + }) +} + +func TestConsulMeshGateway_Canonicalize(t *testing.T) { + testutil.Parallel(t) + + t.Run("nil", func(t *testing.T) { + c := (*ConsulMeshGateway)(nil) + c.Canonicalize() + require.Nil(t, c) + }) + + t.Run("unset mode", func(t *testing.T) { + c := &ConsulMeshGateway{Mode: ""} + c.Canonicalize() + require.Equal(t, "", c.Mode) + }) + + t.Run("set mode", func(t *testing.T) { + c := &ConsulMeshGateway{Mode: "remote"} + c.Canonicalize() + require.Equal(t, "remote", c.Mode) + }) +} + +func TestConsulMeshGateway_Copy(t *testing.T) { + testutil.Parallel(t) + + t.Run("nil", func(t *testing.T) { + c := (*ConsulMeshGateway)(nil) + result := c.Copy() + require.Nil(t, result) + }) + + t.Run("instantiated", func(t *testing.T) { + c := &ConsulMeshGateway{ + Mode: "local", + } + result := c.Copy() + require.Equal(t, c, result) + }) +} diff --git a/api/service_registrations.go b/api/service_registrations.go deleted file mode 100644 index ebf1418b41fe..000000000000 --- a/api/service_registrations.go +++ /dev/null @@ -1,129 +0,0 @@ -package api - -import ( - "fmt" - "net/url" -) - -// ServiceRegistrations is used to query the service endpoints. -type ServiceRegistrations struct { - client *Client -} - -// ServiceRegistration is an instance of a single allocation advertising itself -// as a named service with a specific address. Each registration is constructed -// from the job specification Service block. Whether the service is registered -// within Nomad, and therefore generates a ServiceRegistration is controlled by -// the Service.Provider parameter. -type ServiceRegistration struct { - - // ID is the unique identifier for this registration. It currently follows - // the Consul service registration format to provide consistency between - // the two solutions. - ID string - - // ServiceName is the human friendly identifier for this service - // registration. - ServiceName string - - // Namespace represents the namespace within which this service is - // registered. - Namespace string - - // NodeID is Node.ID on which this service registration is currently - // running. - NodeID string - - // Datacenter is the DC identifier of the node as identified by - // Node.Datacenter. - Datacenter string - - // JobID is Job.ID and represents the job which contained the service block - // which resulted in this service registration. - JobID string - - // AllocID is Allocation.ID and represents the allocation within which this - // service is running. - AllocID string - - // Tags are determined from either Service.Tags or Service.CanaryTags and - // help identify this service. Tags can also be used to perform lookups of - // services depending on their state and role. - Tags []string - - // Address is the IP address of this service registration. This information - // comes from the client and is not guaranteed to be routable; this depends - // on cluster network topology. - Address string - - // Port is the port number on which this service registration is bound. It - // is determined by a combination of factors on the client. - Port int - - CreateIndex uint64 - ModifyIndex uint64 -} - -// ServiceRegistrationListStub represents all service registrations held within a -// single namespace. -type ServiceRegistrationListStub struct { - - // Namespace details the namespace in which these services have been - // registered. - Namespace string - - // Services is a list of services found within the namespace. - Services []*ServiceRegistrationStub -} - -// ServiceRegistrationStub is the stub object describing an individual -// namespaced service. The object is built in a manner which would allow us to -// add additional fields in the future, if we wanted. -type ServiceRegistrationStub struct { - - // ServiceName is the human friendly name for this service as specified - // within Service.Name. - ServiceName string - - // Tags is a list of unique tags found for this service. The list is - // de-duplicated automatically by Nomad. - Tags []string -} - -// ServiceRegistrations returns a new handle on the services endpoints. -func (c *Client) ServiceRegistrations() *ServiceRegistrations { - return &ServiceRegistrations{client: c} -} - -// List can be used to list all service registrations currently stored within -// the target namespace. It returns a stub response object. -func (s *ServiceRegistrations) List(q *QueryOptions) ([]*ServiceRegistrationListStub, *QueryMeta, error) { - var resp []*ServiceRegistrationListStub - qm, err := s.client.query("/v1/services", &resp, q) - if err != nil { - return nil, qm, err - } - return resp, qm, nil -} - -// Get is used to return a list of service registrations whose name matches the -// specified parameter. -func (s *ServiceRegistrations) Get(serviceName string, q *QueryOptions) ([]*ServiceRegistration, *QueryMeta, error) { - var resp []*ServiceRegistration - qm, err := s.client.query("/v1/service/"+url.PathEscape(serviceName), &resp, q) - if err != nil { - return nil, qm, err - } - return resp, qm, nil -} - -// Delete can be used to delete an individual service registration as defined -// by its service name and service ID. -func (s *ServiceRegistrations) Delete(serviceName, serviceID string, q *WriteOptions) (*WriteMeta, error) { - path := fmt.Sprintf("/v1/service/%s/%s", url.PathEscape(serviceName), url.PathEscape(serviceID)) - wm, err := s.client.delete(path, nil, q) - if err != nil { - return nil, err - } - return wm, nil -} diff --git a/api/service_registrations_test.go b/api/service_registrations_test.go deleted file mode 100644 index b957e194b432..000000000000 --- a/api/service_registrations_test.go +++ /dev/null @@ -1,17 +0,0 @@ -package api - -import ( - "testing" -) - -func TestServiceRegistrations_List(t *testing.T) { - // TODO(jrasell) add tests once registration process is in place. -} - -func TestServiceRegistrations_Get(t *testing.T) { - // TODO(jrasell) add tests once registration process is in place. -} - -func TestServiceRegistrations_Delete(t *testing.T) { - // TODO(jrasell) add tests once registration process is in place. -} diff --git a/api/services.go b/api/services.go index d927c514ec2e..f531c2964416 100644 --- a/api/services.go +++ b/api/services.go @@ -2,9 +2,134 @@ package api import ( "fmt" + "net/url" "time" ) +// ServiceRegistration is an instance of a single allocation advertising itself +// as a named service with a specific address. Each registration is constructed +// from the job specification Service block. Whether the service is registered +// within Nomad, and therefore generates a ServiceRegistration is controlled by +// the Service.Provider parameter. +type ServiceRegistration struct { + + // ID is the unique identifier for this registration. It currently follows + // the Consul service registration format to provide consistency between + // the two solutions. + ID string + + // ServiceName is the human friendly identifier for this service + // registration. + ServiceName string + + // Namespace represents the namespace within which this service is + // registered. + Namespace string + + // NodeID is Node.ID on which this service registration is currently + // running. + NodeID string + + // Datacenter is the DC identifier of the node as identified by + // Node.Datacenter. + Datacenter string + + // JobID is Job.ID and represents the job which contained the service block + // which resulted in this service registration. + JobID string + + // AllocID is Allocation.ID and represents the allocation within which this + // service is running. + AllocID string + + // Tags are determined from either Service.Tags or Service.CanaryTags and + // help identify this service. Tags can also be used to perform lookups of + // services depending on their state and role. + Tags []string + + // Address is the IP address of this service registration. This information + // comes from the client and is not guaranteed to be routable; this depends + // on cluster network topology. + Address string + + // Port is the port number on which this service registration is bound. It + // is determined by a combination of factors on the client. + Port int + + CreateIndex uint64 + ModifyIndex uint64 +} + +// ServiceRegistrationListStub represents all service registrations held within a +// single namespace. +type ServiceRegistrationListStub struct { + + // Namespace details the namespace in which these services have been + // registered. + Namespace string + + // Services is a list of services found within the namespace. + Services []*ServiceRegistrationStub +} + +// ServiceRegistrationStub is the stub object describing an individual +// namespaced service. The object is built in a manner which would allow us to +// add additional fields in the future, if we wanted. +type ServiceRegistrationStub struct { + + // ServiceName is the human friendly name for this service as specified + // within Service.Name. + ServiceName string + + // Tags is a list of unique tags found for this service. The list is + // de-duplicated automatically by Nomad. + Tags []string +} + + +// Services is used to query the service endpoints. +type Services struct { + client *Client +} + +// Services returns a new handle on the services endpoints. +func (c *Client) Services() *Services { + return &Services{client: c} +} + +// List can be used to list all service registrations currently stored within +// the target namespace. It returns a stub response object. +func (s *Services) List(q *QueryOptions) ([]*ServiceRegistrationListStub, *QueryMeta, error) { + var resp []*ServiceRegistrationListStub + qm, err := s.client.query("/v1/services", &resp, q) + if err != nil { + return nil, qm, err + } + return resp, qm, nil +} + +// Get is used to return a list of service registrations whose name matches the +// specified parameter. +func (s *Services) Get(serviceName string, q *QueryOptions) ([]*ServiceRegistration, *QueryMeta, error) { + var resp []*ServiceRegistration + qm, err := s.client.query("/v1/service/"+url.PathEscape(serviceName), &resp, q) + if err != nil { + return nil, qm, err + } + return resp, qm, nil +} + +// Delete can be used to delete an individual service registration as defined +// by its service name and service ID. +func (s *Services) Delete(serviceName, serviceID string, q *WriteOptions) (*WriteMeta, error) { + path := fmt.Sprintf("/v1/service/%s/%s", url.PathEscape(serviceName), url.PathEscape(serviceID)) + wm, err := s.client.delete(path, nil, q) + if err != nil { + return nil, err + } + return wm, nil +} + // CheckRestart describes if and when a task should be restarted based on // failing health checks. type CheckRestart struct { @@ -181,557 +306,3 @@ func (s *Service) Canonicalize(t *Task, tg *TaskGroup, job *Job) { } } } - -// ConsulConnect represents a Consul Connect jobspec stanza. -type ConsulConnect struct { - Native bool `hcl:"native,optional"` - Gateway *ConsulGateway `hcl:"gateway,block"` - SidecarService *ConsulSidecarService `mapstructure:"sidecar_service" hcl:"sidecar_service,block"` - SidecarTask *SidecarTask `mapstructure:"sidecar_task" hcl:"sidecar_task,block"` -} - -func (cc *ConsulConnect) Canonicalize() { - if cc == nil { - return - } - - cc.SidecarService.Canonicalize() - cc.SidecarTask.Canonicalize() - cc.Gateway.Canonicalize() -} - -// ConsulSidecarService represents a Consul Connect SidecarService jobspec -// stanza. -type ConsulSidecarService struct { - Tags []string `hcl:"tags,optional"` - Port string `hcl:"port,optional"` - Proxy *ConsulProxy `hcl:"proxy,block"` - DisableDefaultTCPCheck bool `mapstructure:"disable_default_tcp_check" hcl:"disable_default_tcp_check,optional"` -} - -func (css *ConsulSidecarService) Canonicalize() { - if css == nil { - return - } - - if len(css.Tags) == 0 { - css.Tags = nil - } - - css.Proxy.Canonicalize() -} - -// SidecarTask represents a subset of Task fields that can be set to override -// the fields of the Task generated for the sidecar -type SidecarTask struct { - Name string `hcl:"name,optional"` - Driver string `hcl:"driver,optional"` - User string `hcl:"user,optional"` - Config map[string]interface{} `hcl:"config,block"` - Env map[string]string `hcl:"env,block"` - Resources *Resources `hcl:"resources,block"` - Meta map[string]string `hcl:"meta,block"` - KillTimeout *time.Duration `mapstructure:"kill_timeout" hcl:"kill_timeout,optional"` - LogConfig *LogConfig `mapstructure:"logs" hcl:"logs,block"` - ShutdownDelay *time.Duration `mapstructure:"shutdown_delay" hcl:"shutdown_delay,optional"` - KillSignal string `mapstructure:"kill_signal" hcl:"kill_signal,optional"` -} - -func (st *SidecarTask) Canonicalize() { - if st == nil { - return - } - - if len(st.Config) == 0 { - st.Config = nil - } - - if len(st.Env) == 0 { - st.Env = nil - } - - if st.Resources == nil { - st.Resources = DefaultResources() - } else { - st.Resources.Canonicalize() - } - - if st.LogConfig == nil { - st.LogConfig = DefaultLogConfig() - } else { - st.LogConfig.Canonicalize() - } - - if len(st.Meta) == 0 { - st.Meta = nil - } - - if st.KillTimeout == nil { - st.KillTimeout = timeToPtr(5 * time.Second) - } - - if st.ShutdownDelay == nil { - st.ShutdownDelay = timeToPtr(0) - } -} - -// ConsulProxy represents a Consul Connect sidecar proxy jobspec stanza. -type ConsulProxy struct { - LocalServiceAddress string `mapstructure:"local_service_address" hcl:"local_service_address,optional"` - LocalServicePort int `mapstructure:"local_service_port" hcl:"local_service_port,optional"` - ExposeConfig *ConsulExposeConfig `mapstructure:"expose" hcl:"expose,block"` - Upstreams []*ConsulUpstream `hcl:"upstreams,block"` - Config map[string]interface{} `hcl:"config,block"` -} - -func (cp *ConsulProxy) Canonicalize() { - if cp == nil { - return - } - - cp.ExposeConfig.Canonicalize() - - if len(cp.Upstreams) == 0 { - cp.Upstreams = nil - } - - for _, upstream := range cp.Upstreams { - upstream.Canonicalize() - } - - if len(cp.Config) == 0 { - cp.Config = nil - } -} - -// ConsulMeshGateway is used to configure mesh gateway usage when connecting to -// a connect upstream in another datacenter. -type ConsulMeshGateway struct { - // Mode configures how an upstream should be accessed with regard to using - // mesh gateways. - // - // local - the connect proxy makes outbound connections through mesh gateway - // originating in the same datacenter. - // - // remote - the connect proxy makes outbound connections to a mesh gateway - // in the destination datacenter. - // - // none (default) - no mesh gateway is used, the proxy makes outbound connections - // directly to destination services. - // - // https://www.consul.io/docs/connect/gateways/mesh-gateway#modes-of-operation - Mode string `mapstructure:"mode" hcl:"mode,optional"` -} - -func (c *ConsulMeshGateway) Canonicalize() { - // Mode may be empty string, indicating behavior will defer to Consul - // service-defaults config entry. - return -} - -func (c *ConsulMeshGateway) Copy() *ConsulMeshGateway { - if c == nil { - return nil - } - - return &ConsulMeshGateway{ - Mode: c.Mode, - } -} - -// ConsulUpstream represents a Consul Connect upstream jobspec stanza. -type ConsulUpstream struct { - DestinationName string `mapstructure:"destination_name" hcl:"destination_name,optional"` - LocalBindPort int `mapstructure:"local_bind_port" hcl:"local_bind_port,optional"` - Datacenter string `mapstructure:"datacenter" hcl:"datacenter,optional"` - LocalBindAddress string `mapstructure:"local_bind_address" hcl:"local_bind_address,optional"` - MeshGateway *ConsulMeshGateway `mapstructure:"mesh_gateway" hcl:"mesh_gateway,block"` -} - -func (cu *ConsulUpstream) Copy() *ConsulUpstream { - if cu == nil { - return nil - } - return &ConsulUpstream{ - DestinationName: cu.DestinationName, - LocalBindPort: cu.LocalBindPort, - Datacenter: cu.Datacenter, - LocalBindAddress: cu.LocalBindAddress, - MeshGateway: cu.MeshGateway.Copy(), - } -} - -func (cu *ConsulUpstream) Canonicalize() { - if cu == nil { - return - } - cu.MeshGateway.Canonicalize() -} - -type ConsulExposeConfig struct { - Path []*ConsulExposePath `mapstructure:"path" hcl:"path,block"` -} - -func (cec *ConsulExposeConfig) Canonicalize() { - if cec == nil { - return - } - - if len(cec.Path) == 0 { - cec.Path = nil - } -} - -type ConsulExposePath struct { - Path string `hcl:"path,optional"` - Protocol string `hcl:"protocol,optional"` - LocalPathPort int `mapstructure:"local_path_port" hcl:"local_path_port,optional"` - ListenerPort string `mapstructure:"listener_port" hcl:"listener_port,optional"` -} - -// ConsulGateway is used to configure one of the Consul Connect Gateway types. -type ConsulGateway struct { - // Proxy is used to configure the Envoy instance acting as the gateway. - Proxy *ConsulGatewayProxy `hcl:"proxy,block"` - - // Ingress represents the Consul Configuration Entry for an Ingress Gateway. - Ingress *ConsulIngressConfigEntry `hcl:"ingress,block"` - - // Terminating represents the Consul Configuration Entry for a Terminating Gateway. - Terminating *ConsulTerminatingConfigEntry `hcl:"terminating,block"` - - // Mesh indicates the Consul service should be a Mesh Gateway. - Mesh *ConsulMeshConfigEntry `hcl:"mesh,block"` -} - -func (g *ConsulGateway) Canonicalize() { - if g == nil { - return - } - g.Proxy.Canonicalize() - g.Ingress.Canonicalize() - g.Terminating.Canonicalize() -} - -func (g *ConsulGateway) Copy() *ConsulGateway { - if g == nil { - return nil - } - - return &ConsulGateway{ - Proxy: g.Proxy.Copy(), - Ingress: g.Ingress.Copy(), - Terminating: g.Terminating.Copy(), - } -} - -type ConsulGatewayBindAddress struct { - Name string `hcl:",label"` - Address string `mapstructure:"address" hcl:"address,optional"` - Port int `mapstructure:"port" hcl:"port,optional"` -} - -var ( - // defaultGatewayConnectTimeout is the default amount of time connections to - // upstreams are allowed before timing out. - defaultGatewayConnectTimeout = 5 * time.Second -) - -// ConsulGatewayProxy is used to tune parameters of the proxy instance acting as -// one of the forms of Connect gateways that Consul supports. -// -// https://www.consul.io/docs/connect/proxies/envoy#gateway-options -type ConsulGatewayProxy struct { - ConnectTimeout *time.Duration `mapstructure:"connect_timeout" hcl:"connect_timeout,optional"` - EnvoyGatewayBindTaggedAddresses bool `mapstructure:"envoy_gateway_bind_tagged_addresses" hcl:"envoy_gateway_bind_tagged_addresses,optional"` - EnvoyGatewayBindAddresses map[string]*ConsulGatewayBindAddress `mapstructure:"envoy_gateway_bind_addresses" hcl:"envoy_gateway_bind_addresses,block"` - EnvoyGatewayNoDefaultBind bool `mapstructure:"envoy_gateway_no_default_bind" hcl:"envoy_gateway_no_default_bind,optional"` - EnvoyDNSDiscoveryType string `mapstructure:"envoy_dns_discovery_type" hcl:"envoy_dns_discovery_type,optional"` - Config map[string]interface{} `hcl:"config,block"` // escape hatch envoy config -} - -func (p *ConsulGatewayProxy) Canonicalize() { - if p == nil { - return - } - - if p.ConnectTimeout == nil { - // same as the default from consul - p.ConnectTimeout = timeToPtr(defaultGatewayConnectTimeout) - } - - if len(p.EnvoyGatewayBindAddresses) == 0 { - p.EnvoyGatewayBindAddresses = nil - } - - if len(p.Config) == 0 { - p.Config = nil - } -} - -func (p *ConsulGatewayProxy) Copy() *ConsulGatewayProxy { - if p == nil { - return nil - } - - var binds map[string]*ConsulGatewayBindAddress = nil - if p.EnvoyGatewayBindAddresses != nil { - binds = make(map[string]*ConsulGatewayBindAddress, len(p.EnvoyGatewayBindAddresses)) - for k, v := range p.EnvoyGatewayBindAddresses { - binds[k] = v - } - } - - var config map[string]interface{} = nil - if p.Config != nil { - config = make(map[string]interface{}, len(p.Config)) - for k, v := range p.Config { - config[k] = v - } - } - - return &ConsulGatewayProxy{ - ConnectTimeout: timeToPtr(*p.ConnectTimeout), - EnvoyGatewayBindTaggedAddresses: p.EnvoyGatewayBindTaggedAddresses, - EnvoyGatewayBindAddresses: binds, - EnvoyGatewayNoDefaultBind: p.EnvoyGatewayNoDefaultBind, - EnvoyDNSDiscoveryType: p.EnvoyDNSDiscoveryType, - Config: config, - } -} - -// ConsulGatewayTLSConfig is used to configure TLS for a gateway. -type ConsulGatewayTLSConfig struct { - Enabled bool `hcl:"enabled,optional"` -} - -func (tc *ConsulGatewayTLSConfig) Canonicalize() { -} - -func (tc *ConsulGatewayTLSConfig) Copy() *ConsulGatewayTLSConfig { - if tc == nil { - return nil - } - - return &ConsulGatewayTLSConfig{ - Enabled: tc.Enabled, - } -} - -// ConsulIngressService is used to configure a service fronted by the ingress gateway. -type ConsulIngressService struct { - // Namespace is not yet supported. - // Namespace string - Name string `hcl:"name,optional"` - - Hosts []string `hcl:"hosts,optional"` -} - -func (s *ConsulIngressService) Canonicalize() { - if s == nil { - return - } - - if len(s.Hosts) == 0 { - s.Hosts = nil - } -} - -func (s *ConsulIngressService) Copy() *ConsulIngressService { - if s == nil { - return nil - } - - var hosts []string = nil - if n := len(s.Hosts); n > 0 { - hosts = make([]string, n) - copy(hosts, s.Hosts) - } - - return &ConsulIngressService{ - Name: s.Name, - Hosts: hosts, - } -} - -const ( - defaultIngressListenerProtocol = "tcp" -) - -// ConsulIngressListener is used to configure a listener on a Consul Ingress -// Gateway. -type ConsulIngressListener struct { - Port int `hcl:"port,optional"` - Protocol string `hcl:"protocol,optional"` - Services []*ConsulIngressService `hcl:"service,block"` -} - -func (l *ConsulIngressListener) Canonicalize() { - if l == nil { - return - } - - if l.Protocol == "" { - // same as default from consul - l.Protocol = defaultIngressListenerProtocol - } - - if len(l.Services) == 0 { - l.Services = nil - } -} - -func (l *ConsulIngressListener) Copy() *ConsulIngressListener { - if l == nil { - return nil - } - - var services []*ConsulIngressService = nil - if n := len(l.Services); n > 0 { - services = make([]*ConsulIngressService, n) - for i := 0; i < n; i++ { - services[i] = l.Services[i].Copy() - } - } - - return &ConsulIngressListener{ - Port: l.Port, - Protocol: l.Protocol, - Services: services, - } -} - -// ConsulIngressConfigEntry represents the Consul Configuration Entry type for -// an Ingress Gateway. -// -// https://www.consul.io/docs/agent/config-entries/ingress-gateway#available-fields -type ConsulIngressConfigEntry struct { - // Namespace is not yet supported. - // Namespace string - - TLS *ConsulGatewayTLSConfig `hcl:"tls,block"` - Listeners []*ConsulIngressListener `hcl:"listener,block"` -} - -func (e *ConsulIngressConfigEntry) Canonicalize() { - if e == nil { - return - } - - e.TLS.Canonicalize() - - if len(e.Listeners) == 0 { - e.Listeners = nil - } - - for _, listener := range e.Listeners { - listener.Canonicalize() - } -} - -func (e *ConsulIngressConfigEntry) Copy() *ConsulIngressConfigEntry { - if e == nil { - return nil - } - - var listeners []*ConsulIngressListener = nil - if n := len(e.Listeners); n > 0 { - listeners = make([]*ConsulIngressListener, n) - for i := 0; i < n; i++ { - listeners[i] = e.Listeners[i].Copy() - } - } - - return &ConsulIngressConfigEntry{ - TLS: e.TLS.Copy(), - Listeners: listeners, - } -} - -type ConsulLinkedService struct { - Name string `hcl:"name,optional"` - CAFile string `hcl:"ca_file,optional" mapstructure:"ca_file"` - CertFile string `hcl:"cert_file,optional" mapstructure:"cert_file"` - KeyFile string `hcl:"key_file,optional" mapstructure:"key_file"` - SNI string `hcl:"sni,optional"` -} - -func (s *ConsulLinkedService) Canonicalize() { - // nothing to do for now -} - -func (s *ConsulLinkedService) Copy() *ConsulLinkedService { - if s == nil { - return nil - } - - return &ConsulLinkedService{ - Name: s.Name, - CAFile: s.CAFile, - CertFile: s.CertFile, - KeyFile: s.KeyFile, - SNI: s.SNI, - } -} - -// ConsulTerminatingConfigEntry represents the Consul Configuration Entry type -// for a Terminating Gateway. -// -// https://www.consul.io/docs/agent/config-entries/terminating-gateway#available-fields -type ConsulTerminatingConfigEntry struct { - // Namespace is not yet supported. - // Namespace string - - Services []*ConsulLinkedService `hcl:"service,block"` -} - -func (e *ConsulTerminatingConfigEntry) Canonicalize() { - if e == nil { - return - } - - if len(e.Services) == 0 { - e.Services = nil - } - - for _, service := range e.Services { - service.Canonicalize() - } -} - -func (e *ConsulTerminatingConfigEntry) Copy() *ConsulTerminatingConfigEntry { - if e == nil { - return nil - } - - var services []*ConsulLinkedService = nil - if n := len(e.Services); n > 0 { - services = make([]*ConsulLinkedService, n) - for i := 0; i < n; i++ { - services[i] = e.Services[i].Copy() - } - } - - return &ConsulTerminatingConfigEntry{ - Services: services, - } -} - -// ConsulMeshConfigEntry is a stub used to represent that the gateway service type -// should be for a Mesh Gateway. Unlike Ingress and Terminating, there is no -// actual Consul Config Entry type for mesh-gateway, at least for now. We still -// create a type for future proofing, instead just using a bool for example. -type ConsulMeshConfigEntry struct { - // nothing in here -} - -func (e *ConsulMeshConfigEntry) Canonicalize() { - return -} - -func (e *ConsulMeshConfigEntry) Copy() *ConsulMeshConfigEntry { - if e == nil { - return nil - } - return new(ConsulMeshConfigEntry) -} diff --git a/api/services_test.go b/api/services_test.go index 02b606eb02af..c9ef884e6974 100644 --- a/api/services_test.go +++ b/api/services_test.go @@ -9,6 +9,19 @@ import ( "github.com/stretchr/testify/require" ) +func TestServiceRegistrations_List(t *testing.T) { + // TODO(jrasell) add tests once registration process is in place. +} + +func TestServiceRegistrations_Get(t *testing.T) { + // TODO(jrasell) add tests once registration process is in place. +} + +func TestServiceRegistrations_Delete(t *testing.T) { + // TODO(jrasell) add tests once registration process is in place. +} + + func TestService_Canonicalize(t *testing.T) { testutil.Parallel(t) @@ -128,146 +141,6 @@ func TestService_CheckRestart(t *testing.T) { require.True(t, service.Checks[2].CheckRestart.IgnoreWarnings) } -func TestService_Connect_Canonicalize(t *testing.T) { - testutil.Parallel(t) - - t.Run("nil connect", func(t *testing.T) { - cc := (*ConsulConnect)(nil) - cc.Canonicalize() - require.Nil(t, cc) - }) - - t.Run("empty connect", func(t *testing.T) { - cc := new(ConsulConnect) - cc.Canonicalize() - require.Empty(t, cc.Native) - require.Nil(t, cc.SidecarService) - require.Nil(t, cc.SidecarTask) - }) -} - -func TestService_Connect_ConsulSidecarService_Canonicalize(t *testing.T) { - testutil.Parallel(t) - - t.Run("nil sidecar_service", func(t *testing.T) { - css := (*ConsulSidecarService)(nil) - css.Canonicalize() - require.Nil(t, css) - }) - - t.Run("empty sidecar_service", func(t *testing.T) { - css := new(ConsulSidecarService) - css.Canonicalize() - require.Empty(t, css.Tags) - require.Nil(t, css.Proxy) - }) - - t.Run("non-empty sidecar_service", func(t *testing.T) { - css := &ConsulSidecarService{ - Tags: make([]string, 0), - Port: "port", - Proxy: &ConsulProxy{ - LocalServiceAddress: "lsa", - LocalServicePort: 80, - }, - } - css.Canonicalize() - require.Equal(t, &ConsulSidecarService{ - Tags: nil, - Port: "port", - Proxy: &ConsulProxy{ - LocalServiceAddress: "lsa", - LocalServicePort: 80}, - }, css) - }) -} - -func TestService_Connect_ConsulProxy_Canonicalize(t *testing.T) { - testutil.Parallel(t) - - t.Run("nil proxy", func(t *testing.T) { - cp := (*ConsulProxy)(nil) - cp.Canonicalize() - require.Nil(t, cp) - }) - - t.Run("empty proxy", func(t *testing.T) { - cp := new(ConsulProxy) - cp.Canonicalize() - require.Empty(t, cp.LocalServiceAddress) - require.Zero(t, cp.LocalServicePort) - require.Nil(t, cp.ExposeConfig) - require.Nil(t, cp.Upstreams) - require.Empty(t, cp.Config) - }) - - t.Run("non empty proxy", func(t *testing.T) { - cp := &ConsulProxy{ - LocalServiceAddress: "127.0.0.1", - LocalServicePort: 80, - ExposeConfig: new(ConsulExposeConfig), - Upstreams: make([]*ConsulUpstream, 0), - Config: make(map[string]interface{}), - } - cp.Canonicalize() - require.Equal(t, "127.0.0.1", cp.LocalServiceAddress) - require.Equal(t, 80, cp.LocalServicePort) - require.Equal(t, &ConsulExposeConfig{}, cp.ExposeConfig) - require.Nil(t, cp.Upstreams) - require.Nil(t, cp.Config) - }) -} - -func TestService_Connect_ConsulUpstream_Copy(t *testing.T) { - testutil.Parallel(t) - - t.Run("nil upstream", func(t *testing.T) { - cu := (*ConsulUpstream)(nil) - result := cu.Copy() - require.Nil(t, result) - }) - - t.Run("complete upstream", func(t *testing.T) { - cu := &ConsulUpstream{ - DestinationName: "dest1", - Datacenter: "dc2", - LocalBindPort: 2000, - LocalBindAddress: "10.0.0.1", - MeshGateway: &ConsulMeshGateway{Mode: "remote"}, - } - result := cu.Copy() - require.Equal(t, cu, result) - }) -} - -func TestService_Connect_ConsulUpstream_Canonicalize(t *testing.T) { - testutil.Parallel(t) - - t.Run("nil upstream", func(t *testing.T) { - cu := (*ConsulUpstream)(nil) - cu.Canonicalize() - require.Nil(t, cu) - }) - - t.Run("complete", func(t *testing.T) { - cu := &ConsulUpstream{ - DestinationName: "dest1", - Datacenter: "dc2", - LocalBindPort: 2000, - LocalBindAddress: "10.0.0.1", - MeshGateway: &ConsulMeshGateway{Mode: ""}, - } - cu.Canonicalize() - require.Equal(t, &ConsulUpstream{ - DestinationName: "dest1", - Datacenter: "dc2", - LocalBindPort: 2000, - LocalBindAddress: "10.0.0.1", - MeshGateway: &ConsulMeshGateway{Mode: ""}, - }, cu) - }) -} - func TestService_Connect_proxy_settings(t *testing.T) { testutil.Parallel(t) @@ -319,316 +192,4 @@ func TestService_Tags(t *testing.T) { r.True(service.EnableTagOverride) r.Equal([]string{"a", "b"}, service.Tags) r.Equal([]string{"c", "d"}, service.CanaryTags) -} - -func TestService_Connect_SidecarTask_Canonicalize(t *testing.T) { - testutil.Parallel(t) - - t.Run("nil sidecar_task", func(t *testing.T) { - st := (*SidecarTask)(nil) - st.Canonicalize() - require.Nil(t, st) - }) - - t.Run("empty sidecar_task", func(t *testing.T) { - st := new(SidecarTask) - st.Canonicalize() - require.Nil(t, st.Config) - require.Nil(t, st.Env) - require.Equal(t, DefaultResources(), st.Resources) - require.Equal(t, DefaultLogConfig(), st.LogConfig) - require.Nil(t, st.Meta) - require.Equal(t, 5*time.Second, *st.KillTimeout) - require.Equal(t, 0*time.Second, *st.ShutdownDelay) - }) - - t.Run("non empty sidecar_task resources", func(t *testing.T) { - exp := DefaultResources() - exp.MemoryMB = intToPtr(333) - st := &SidecarTask{ - Resources: &Resources{MemoryMB: intToPtr(333)}, - } - st.Canonicalize() - require.Equal(t, exp, st.Resources) - }) -} - -func TestService_ConsulGateway_Canonicalize(t *testing.T) { - testutil.Parallel(t) - - t.Run("nil", func(t *testing.T) { - cg := (*ConsulGateway)(nil) - cg.Canonicalize() - require.Nil(t, cg) - }) - - t.Run("set defaults", func(t *testing.T) { - cg := &ConsulGateway{ - Proxy: &ConsulGatewayProxy{ - ConnectTimeout: nil, - EnvoyGatewayBindTaggedAddresses: true, - EnvoyGatewayBindAddresses: make(map[string]*ConsulGatewayBindAddress, 0), - EnvoyGatewayNoDefaultBind: true, - Config: make(map[string]interface{}, 0), - }, - Ingress: &ConsulIngressConfigEntry{ - TLS: &ConsulGatewayTLSConfig{ - Enabled: false, - }, - Listeners: make([]*ConsulIngressListener, 0), - }, - } - cg.Canonicalize() - require.Equal(t, timeToPtr(5*time.Second), cg.Proxy.ConnectTimeout) - require.True(t, cg.Proxy.EnvoyGatewayBindTaggedAddresses) - require.Nil(t, cg.Proxy.EnvoyGatewayBindAddresses) - require.True(t, cg.Proxy.EnvoyGatewayNoDefaultBind) - require.Empty(t, cg.Proxy.EnvoyDNSDiscoveryType) - require.Nil(t, cg.Proxy.Config) - require.Nil(t, cg.Ingress.Listeners) - }) -} - -func TestService_ConsulGateway_Copy(t *testing.T) { - testutil.Parallel(t) - - t.Run("nil", func(t *testing.T) { - result := (*ConsulGateway)(nil).Copy() - require.Nil(t, result) - }) - - gateway := &ConsulGateway{ - Proxy: &ConsulGatewayProxy{ - ConnectTimeout: timeToPtr(3 * time.Second), - EnvoyGatewayBindTaggedAddresses: true, - EnvoyGatewayBindAddresses: map[string]*ConsulGatewayBindAddress{ - "listener1": {Address: "10.0.0.1", Port: 2000}, - "listener2": {Address: "10.0.0.1", Port: 2001}, - }, - EnvoyGatewayNoDefaultBind: true, - EnvoyDNSDiscoveryType: "STRICT_DNS", - Config: map[string]interface{}{ - "foo": "bar", - "baz": 3, - }, - }, - Ingress: &ConsulIngressConfigEntry{ - TLS: &ConsulGatewayTLSConfig{ - Enabled: true, - }, - Listeners: []*ConsulIngressListener{{ - Port: 3333, - Protocol: "tcp", - Services: []*ConsulIngressService{{ - Name: "service1", - Hosts: []string{ - "127.0.0.1", "127.0.0.1:3333", - }}, - }}, - }, - }, - Terminating: &ConsulTerminatingConfigEntry{ - Services: []*ConsulLinkedService{{ - Name: "linked-service1", - }}, - }, - } - - t.Run("complete", func(t *testing.T) { - result := gateway.Copy() - require.Equal(t, gateway, result) - }) -} - -func TestService_ConsulIngressConfigEntry_Canonicalize(t *testing.T) { - testutil.Parallel(t) - - t.Run("nil", func(t *testing.T) { - c := (*ConsulIngressConfigEntry)(nil) - c.Canonicalize() - require.Nil(t, c) - }) - - t.Run("empty fields", func(t *testing.T) { - c := &ConsulIngressConfigEntry{ - TLS: nil, - Listeners: []*ConsulIngressListener{}, - } - c.Canonicalize() - require.Nil(t, c.TLS) - require.Nil(t, c.Listeners) - }) - - t.Run("complete", func(t *testing.T) { - c := &ConsulIngressConfigEntry{ - TLS: &ConsulGatewayTLSConfig{Enabled: true}, - Listeners: []*ConsulIngressListener{{ - Port: 9090, - Protocol: "http", - Services: []*ConsulIngressService{{ - Name: "service1", - Hosts: []string{"1.1.1.1"}, - }}, - }}, - } - c.Canonicalize() - require.Equal(t, &ConsulIngressConfigEntry{ - TLS: &ConsulGatewayTLSConfig{Enabled: true}, - Listeners: []*ConsulIngressListener{{ - Port: 9090, - Protocol: "http", - Services: []*ConsulIngressService{{ - Name: "service1", - Hosts: []string{"1.1.1.1"}, - }}, - }}, - }, c) - }) -} - -func TestService_ConsulIngressConfigEntry_Copy(t *testing.T) { - testutil.Parallel(t) - - t.Run("nil", func(t *testing.T) { - result := (*ConsulIngressConfigEntry)(nil).Copy() - require.Nil(t, result) - }) - - entry := &ConsulIngressConfigEntry{ - TLS: &ConsulGatewayTLSConfig{ - Enabled: true, - }, - Listeners: []*ConsulIngressListener{{ - Port: 1111, - Protocol: "http", - Services: []*ConsulIngressService{{ - Name: "service1", - Hosts: []string{"1.1.1.1", "1.1.1.1:9000"}, - }, { - Name: "service2", - Hosts: []string{"2.2.2.2"}, - }}, - }}, - } - - t.Run("complete", func(t *testing.T) { - result := entry.Copy() - require.Equal(t, entry, result) - }) -} - -func TestService_ConsulTerminatingConfigEntry_Canonicalize(t *testing.T) { - testutil.Parallel(t) - - t.Run("nil", func(t *testing.T) { - c := (*ConsulTerminatingConfigEntry)(nil) - c.Canonicalize() - require.Nil(t, c) - }) - - t.Run("empty services", func(t *testing.T) { - c := &ConsulTerminatingConfigEntry{ - Services: []*ConsulLinkedService{}, - } - c.Canonicalize() - require.Nil(t, c.Services) - }) -} - -func TestService_ConsulTerminatingConfigEntry_Copy(t *testing.T) { - testutil.Parallel(t) - - t.Run("nil", func(t *testing.T) { - result := (*ConsulIngressConfigEntry)(nil).Copy() - require.Nil(t, result) - }) - - entry := &ConsulTerminatingConfigEntry{ - Services: []*ConsulLinkedService{{ - Name: "servic1", - }, { - Name: "service2", - CAFile: "ca_file.pem", - CertFile: "cert_file.pem", - KeyFile: "key_file.pem", - SNI: "sni.terminating.consul", - }}, - } - - t.Run("complete", func(t *testing.T) { - result := entry.Copy() - require.Equal(t, entry, result) - }) -} - -func TestService_ConsulMeshConfigEntry_Canonicalize(t *testing.T) { - testutil.Parallel(t) - - t.Run("nil", func(t *testing.T) { - ce := (*ConsulMeshConfigEntry)(nil) - ce.Canonicalize() - require.Nil(t, ce) - }) - - t.Run("instantiated", func(t *testing.T) { - ce := new(ConsulMeshConfigEntry) - ce.Canonicalize() - require.NotNil(t, ce) - }) -} - -func TestService_ConsulMeshConfigEntry_Copy(t *testing.T) { - testutil.Parallel(t) - - t.Run("nil", func(t *testing.T) { - ce := (*ConsulMeshConfigEntry)(nil) - ce2 := ce.Copy() - require.Nil(t, ce2) - }) - - t.Run("instantiated", func(t *testing.T) { - ce := new(ConsulMeshConfigEntry) - ce2 := ce.Copy() - require.NotNil(t, ce2) - }) -} - -func TestService_ConsulMeshGateway_Canonicalize(t *testing.T) { - testutil.Parallel(t) - - t.Run("nil", func(t *testing.T) { - c := (*ConsulMeshGateway)(nil) - c.Canonicalize() - require.Nil(t, c) - }) - - t.Run("unset mode", func(t *testing.T) { - c := &ConsulMeshGateway{Mode: ""} - c.Canonicalize() - require.Equal(t, "", c.Mode) - }) - - t.Run("set mode", func(t *testing.T) { - c := &ConsulMeshGateway{Mode: "remote"} - c.Canonicalize() - require.Equal(t, "remote", c.Mode) - }) -} - -func TestService_ConsulMeshGateway_Copy(t *testing.T) { - testutil.Parallel(t) - - t.Run("nil", func(t *testing.T) { - c := (*ConsulMeshGateway)(nil) - result := c.Copy() - require.Nil(t, result) - }) - - t.Run("instantiated", func(t *testing.T) { - c := &ConsulMeshGateway{ - Mode: "local", - } - result := c.Copy() - require.Equal(t, c, result) - }) -} +} \ No newline at end of file diff --git a/command/service_delete.go b/command/service_delete.go index 8971a05b02c9..0c1570a0adc1 100644 --- a/command/service_delete.go +++ b/command/service_delete.go @@ -58,7 +58,7 @@ func (s *ServiceDeleteCommand) Run(args []string) int { return 1 } - if _, err := client.ServiceRegistrations().Delete(args[0], args[1], nil); err != nil { + if _, err := client.Services().Delete(args[0], args[1], nil); err != nil { s.Ui.Error(fmt.Sprintf("Error deleting service registration: %s", err)) return 1 } diff --git a/command/service_delete_test.go b/command/service_delete_test.go index 6e17d5f14924..b1862fb6a6e0 100644 --- a/command/service_delete_test.go +++ b/command/service_delete_test.go @@ -61,7 +61,7 @@ func TestServiceDeleteCommand_Run(t *testing.T) { require.Equal(t, 0, registerCode) // Detail the service as we need the ID. - serviceList, _, err := client.ServiceRegistrations().Get("service-discovery-nomad-delete", nil) + serviceList, _, err := client.Services().Get("service-discovery-nomad-delete", nil) require.NoError(t, err) require.Len(t, serviceList, 1) diff --git a/command/service_info.go b/command/service_info.go index 2f54e5edab44..95f413c362fe 100644 --- a/command/service_info.go +++ b/command/service_info.go @@ -92,7 +92,7 @@ func (s *ServiceInfoCommand) Run(args []string) int { return 1 } - serviceInfo, _, err := client.ServiceRegistrations().Get(args[0], nil) + serviceInfo, _, err := client.Services().Get(args[0], nil) if err != nil { s.Ui.Error(fmt.Sprintf("Error listing service registrations: %s", err)) return 1 diff --git a/command/service_list.go b/command/service_list.go index 8e132494f6db..b64d56e30424 100644 --- a/command/service_list.go +++ b/command/service_list.go @@ -89,7 +89,7 @@ func (s *ServiceListCommand) Run(args []string) int { return 1 } - list, _, err := client.ServiceRegistrations().List(nil) + list, _, err := client.Services().List(nil) if err != nil { s.Ui.Error(fmt.Sprintf("Error listing service registrations: %s", err)) return 1 From dfe185467e46062c26262aa4ac494dae00982644 Mon Sep 17 00:00:00 2001 From: Luiz Aoqui Date: Fri, 25 Mar 2022 12:00:48 -0400 Subject: [PATCH 31/31] ci: fix semgrep rule for RPC authentication --- .semgrep/rpc_endpoint.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.semgrep/rpc_endpoint.yml b/.semgrep/rpc_endpoint.yml index 9f22f67a2dca..af94a45ec969 100644 --- a/.semgrep/rpc_endpoint.yml +++ b/.semgrep/rpc_endpoint.yml @@ -30,6 +30,7 @@ rules: # Pattern used by endpoints called exclusively between agents # (server -> server or client -> server) - pattern-not-inside: | + ... ... := validateTLSCertificateLevel(...) ... if done, err := $A.$B.forward($METHOD, ...); done {