From 8ab37d691fc41ac67599405a940338f0c33b86dc Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Thu, 22 Jul 2021 15:15:25 -0400 Subject: [PATCH 1/7] test parsing more service fields --- jobspec/parse_test.go | 11 +++++++---- jobspec/test-fixtures/basic.hcl | 10 +++++++--- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index 48d8b547598a..34bfd1446364 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -134,10 +134,12 @@ func TestParse(t *testing.T) { ExtraKeysHCL: nil, }, "bar": { - Name: "bar", - Type: "csi", - Source: "bar-vol", - ReadOnly: true, + Name: "bar", + Type: "csi", + Source: "bar-vol", + ReadOnly: true, + AccessMode: "single-mode-writer", + AttachmentMode: "file-system", MountOptions: &api.CSIMountOptions{ FSType: "ext4", }, @@ -152,6 +154,7 @@ func TestParse(t *testing.T) { "ro", }, }, + PerAlloc: true, ExtraKeysHCL: nil, }, }, diff --git a/jobspec/test-fixtures/basic.hcl b/jobspec/test-fixtures/basic.hcl index 107ffaca1ccf..325c5e3624a7 100644 --- a/jobspec/test-fixtures/basic.hcl +++ b/jobspec/test-fixtures/basic.hcl @@ -76,9 +76,11 @@ job "binstore-storagelocker" { } volume "bar" { - type = "csi" - source = "bar-vol" - read_only = true + type = "csi" + source = "bar-vol" + read_only = true + attachment_mode = "file-system" + access_mode = "single-mode-writer" mount_options { fs_type = "ext4" @@ -92,6 +94,8 @@ job "binstore-storagelocker" { mount_options { mount_flags = ["ro"] } + + per_alloc = true } restart { From 25b480f7b93c6590864d5a8bd90fd727d5c78369 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Thu, 22 Jul 2021 15:15:50 -0400 Subject: [PATCH 2/7] hclv1: parse service upstreams --- jobspec/parse_service.go | 47 ++++++++++++++++++++++++++++ jobspec/parse_test.go | 10 ++++-- jobspec/test-fixtures/tg-network.hcl | 10 ++++-- 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/jobspec/parse_service.go b/jobspec/parse_service.go index 974acae6e742..c42cd7e8b27c 100644 --- a/jobspec/parse_service.go +++ b/jobspec/parse_service.go @@ -891,6 +891,9 @@ func parseUpstream(uo *ast.ObjectItem) (*api.ConsulUpstream, error) { valid := []string{ "destination_name", "local_bind_port", + "local_bind_address", + "datacenter", + "mesh_gateway", } if err := checkHCLKeys(uo.Val, valid); err != nil { @@ -903,6 +906,8 @@ func parseUpstream(uo *ast.ObjectItem) (*api.ConsulUpstream, error) { return nil, err } + delete(m, "mesh_gateway") + dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ DecodeHook: mapstructure.StringToTimeDurationHookFunc(), WeaklyTypedInput: true, @@ -916,9 +921,51 @@ func parseUpstream(uo *ast.ObjectItem) (*api.ConsulUpstream, error) { return nil, err } + var listVal *ast.ObjectList + if ot, ok := uo.Val.(*ast.ObjectType); ok { + listVal = ot.List + } else { + return nil, fmt.Errorf("'%s': should be an object", upstream.DestinationName) + } + + if mgO := listVal.Filter("mesh_gateway"); len(mgO.Items) > 0 { + if len(mgO.Items) > 1 { + return nil, fmt.Errorf("upstream '%s': cannot have more than 1 mesh_gateway", upstream.DestinationName) + } + + mgw, err := parseMeshGateway(mgO.Items[0]) + if err != nil { + return nil, multierror.Prefix(err, fmt.Sprintf("'%s',", upstream.DestinationName)) + } + + upstream.MeshGateway = mgw + + } return &upstream, nil } +func parseMeshGateway(gwo *ast.ObjectItem) (*api.ConsulMeshGateway, error) { + valid := []string{ + "mode", + } + + if err := checkHCLKeys(gwo.Val, valid); err != nil { + return nil, multierror.Prefix(err, "mesh_gateway ->") + } + + var m map[string]interface{} + if err := hcl.DecodeObject(&m, gwo.Val); err != nil { + return nil, err + } + + var mgw api.ConsulMeshGateway + if err := mapstructure.WeakDecode(m, &mgw); err != nil { + return nil, err + } + + return &mgw, nil +} + func parseChecks(service *api.Service, checkObjs *ast.ObjectList) error { service.Checks = make([]api.ServiceCheck, len(checkObjs.Items)) for idx, co := range checkObjs.Items { diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index 34bfd1446364..3673cae79ca6 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -1115,8 +1115,14 @@ func TestParse(t *testing.T) { LocalServicePort: 8080, Upstreams: []*api.ConsulUpstream{ { - DestinationName: "other-service", - LocalBindPort: 4567, + DestinationName: "other-service", + LocalBindPort: 4567, + LocalBindAddress: "0.0.0.0", + Datacenter: "dc1", + + MeshGateway: &api.ConsulMeshGateway{ + Mode: "local", + }, }, }, }, diff --git a/jobspec/test-fixtures/tg-network.hcl b/jobspec/test-fixtures/tg-network.hcl index 7d23e7d1b2ed..50e1a93568d0 100644 --- a/jobspec/test-fixtures/tg-network.hcl +++ b/jobspec/test-fixtures/tg-network.hcl @@ -34,8 +34,14 @@ job "foo" { local_service_port = 8080 upstreams { - destination_name = "other-service" - local_bind_port = 4567 + destination_name = "other-service" + local_bind_port = 4567 + local_bind_address = "0.0.0.0" + datacenter = "dc1" + + mesh_gateway { + mode = "local" + } } } } From dd6ee3f27b389d225569489f4db4657120ae31ce Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Thu, 22 Jul 2021 15:25:36 -0400 Subject: [PATCH 3/7] services OnUpdate --- jobspec/parse_service.go | 2 ++ jobspec/parse_test.go | 2 ++ jobspec/test-fixtures/tg-service-check.hcl | 6 ++++-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/jobspec/parse_service.go b/jobspec/parse_service.go index c42cd7e8b27c..22347cd80043 100644 --- a/jobspec/parse_service.go +++ b/jobspec/parse_service.go @@ -50,6 +50,7 @@ func parseService(o *ast.ObjectItem) (*api.Service, error) { "task", "meta", "canary_meta", + "on_update", } if err := checkHCLKeys(o.Val, valid); err != nil { return nil, err @@ -992,6 +993,7 @@ func parseChecks(service *api.Service, checkObjs *ast.ObjectList) error { "task", "success_before_passing", "failures_before_critical", + "on_update", } if err := checkHCLKeys(co.Val, valid); err != nil { return multierror.Prefix(err, "check ->") diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index 3673cae79ca6..ef3d3180509b 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -1187,6 +1187,7 @@ func TestParse(t *testing.T) { { Name: "foo-service", PortLabel: "http", + OnUpdate: "ignore", Checks: []api.ServiceCheck{ { Name: "check-name", @@ -1196,6 +1197,7 @@ func TestParse(t *testing.T) { Timeout: time.Duration(2 * time.Second), InitialStatus: "passing", TaskName: "foo", + OnUpdate: "ignore", }, }, }, diff --git a/jobspec/test-fixtures/tg-service-check.hcl b/jobspec/test-fixtures/tg-service-check.hcl index c0d97b91b6a8..721e7ec4b6eb 100644 --- a/jobspec/test-fixtures/tg-service-check.hcl +++ b/jobspec/test-fixtures/tg-service-check.hcl @@ -12,8 +12,9 @@ job "group_service_check_script" { } service { - name = "foo-service" - port = "http" + name = "foo-service" + port = "http" + on_update = "ignore" check { name = "check-name" @@ -23,6 +24,7 @@ job "group_service_check_script" { timeout = "2s" initial_status = "passing" task = "foo" + on_update = "ignore" } } From f352490687a976efb2a06f4aa96802dc7b74bf91 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Thu, 22 Jul 2021 15:34:23 -0400 Subject: [PATCH 4/7] parse service check body --- jobspec/parse_service.go | 1 + jobspec/parse_test.go | 1 + jobspec/test-fixtures/tg-service-check.hcl | 1 + 3 files changed, 3 insertions(+) diff --git a/jobspec/parse_service.go b/jobspec/parse_service.go index 22347cd80043..b702f9b8c199 100644 --- a/jobspec/parse_service.go +++ b/jobspec/parse_service.go @@ -994,6 +994,7 @@ func parseChecks(service *api.Service, checkObjs *ast.ObjectList) error { "success_before_passing", "failures_before_critical", "on_update", + "body", } if err := checkHCLKeys(co.Val, valid); err != nil { return multierror.Prefix(err, "check ->") diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index ef3d3180509b..4603b0a6191a 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -1198,6 +1198,7 @@ func TestParse(t *testing.T) { InitialStatus: "passing", TaskName: "foo", OnUpdate: "ignore", + Body: "post body", }, }, }, diff --git a/jobspec/test-fixtures/tg-service-check.hcl b/jobspec/test-fixtures/tg-service-check.hcl index 721e7ec4b6eb..77934bb311a2 100644 --- a/jobspec/test-fixtures/tg-service-check.hcl +++ b/jobspec/test-fixtures/tg-service-check.hcl @@ -25,6 +25,7 @@ job "group_service_check_script" { initial_status = "passing" task = "foo" on_update = "ignore" + body = "post body" } } From f5069abf0230b5f9b8a17f0f7875a5bc5fe0cbc6 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Thu, 22 Jul 2021 15:40:03 -0400 Subject: [PATCH 5/7] add a test for envoy_dns_discovery_type --- jobspec/parse_test.go | 1 + jobspec/test-fixtures/tg-service-connect-gateway-terminating.hcl | 1 + 2 files changed, 2 insertions(+) diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index 4603b0a6191a..9e8171421c7f 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -1563,6 +1563,7 @@ func TestParse(t *testing.T) { "listener2": {Name: "listener2", Address: "10.0.0.2", Port: 8889}, }, EnvoyGatewayNoDefaultBind: true, + EnvoyDNSDiscoveryType: "LOGICAL_DNS", Config: map[string]interface{}{"foo": "bar"}, }, Terminating: &api.ConsulTerminatingConfigEntry{ diff --git a/jobspec/test-fixtures/tg-service-connect-gateway-terminating.hcl b/jobspec/test-fixtures/tg-service-connect-gateway-terminating.hcl index 82a8bfec0510..24e0f8aecd09 100644 --- a/jobspec/test-fixtures/tg-service-connect-gateway-terminating.hcl +++ b/jobspec/test-fixtures/tg-service-connect-gateway-terminating.hcl @@ -20,6 +20,7 @@ job "connect_gateway_terminating" { } envoy_gateway_no_default_bind = true + envoy_dns_discovery_type = "LOGICAL_DNS" config { foo = "bar" From 09903f4454f0423c7817f162de7be9a85afd0d43 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Thu, 22 Jul 2021 15:57:02 -0400 Subject: [PATCH 6/7] support gateway mesh --- jobspec/parse_service.go | 31 +++++++++++++------ jobspec/parse_test.go | 22 +++++++++++++ .../tg-service-connect-gateway-mesh.hcl | 19 ++++++++++++ 3 files changed, 63 insertions(+), 9 deletions(-) create mode 100644 jobspec/test-fixtures/tg-service-connect-gateway-mesh.hcl diff --git a/jobspec/parse_service.go b/jobspec/parse_service.go index b702f9b8c199..c4c1d33d8780 100644 --- a/jobspec/parse_service.go +++ b/jobspec/parse_service.go @@ -227,6 +227,7 @@ func parseGateway(o *ast.ObjectItem) (*api.ConsulGateway, error) { "proxy", "ingress", "terminating", + "mesh", } if err := checkHCLKeys(o.Val, valid); err != nil { @@ -242,6 +243,7 @@ func parseGateway(o *ast.ObjectItem) (*api.ConsulGateway, error) { delete(m, "proxy") delete(m, "ingress") delete(m, "terminating") + delete(m, "mesh") dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ DecodeHook: mapstructure.StringToTimeDurationHookFunc(), @@ -275,8 +277,11 @@ func parseGateway(o *ast.ObjectItem) (*api.ConsulGateway, error) { gateway.Proxy = proxy // extract and parse the ingress block - io := listVal.Filter("ingress") - if len(io.Items) == 1 { + if io := listVal.Filter("ingress"); len(io.Items) > 0 { + if len(io.Items) > 1 { + return nil, fmt.Errorf("ingress, %s", "multiple ingress stanzas not allowed") + } + ingress, err := parseIngressConfigEntry(io.Items[0]) if err != nil { return nil, fmt.Errorf("ingress, %v", err) @@ -284,12 +289,11 @@ func parseGateway(o *ast.ObjectItem) (*api.ConsulGateway, error) { gateway.Ingress = ingress } - if len(io.Items) > 1 { - return nil, fmt.Errorf("ingress, %s", "multiple ingress stanzas not allowed") - } + if to := listVal.Filter("terminating"); len(to.Items) > 0 { + if len(to.Items) > 1 { + return nil, fmt.Errorf("terminating, %s", "multiple terminating stanzas not allowed") + } - to := listVal.Filter("terminating") - if len(to.Items) == 1 { terminating, err := parseTerminatingConfigEntry(to.Items[0]) if err != nil { return nil, fmt.Errorf("terminating, %v", err) @@ -297,8 +301,17 @@ func parseGateway(o *ast.ObjectItem) (*api.ConsulGateway, error) { gateway.Terminating = terminating } - if len(to.Items) > 1 { - return nil, fmt.Errorf("terminating, %s", "multiple terminating stanzas not allowed") + if mo := listVal.Filter("mesh"); len(mo.Items) > 0 { + if len(mo.Items) > 1 { + return nil, fmt.Errorf("mesh, %s", "multiple mesh stanzas not allowed") + } + + // mesh should have no keys + if err := checkHCLKeys(mo.Items[0].Val, []string{}); err != nil { + return nil, fmt.Errorf("mesh, %s", err) + } + + gateway.Mesh = &api.ConsulMeshConfigEntry{} } return &gateway, nil diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index 9e8171421c7f..852a4f669aa9 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -1584,6 +1584,28 @@ func TestParse(t *testing.T) { }, false, }, + { + "tg-service-connect-gateway-mesh.hcl", + &api.Job{ + ID: stringToPtr("connect_gateway_mesh"), + Name: stringToPtr("connect_gateway_mesh"), + TaskGroups: []*api.TaskGroup{{ + Name: stringToPtr("group"), + Services: []*api.Service{{ + Name: "mesh-gateway-service", + Connect: &api.ConsulConnect{ + Gateway: &api.ConsulGateway{ + Proxy: &api.ConsulGatewayProxy{ + Config: map[string]interface{}{"foo": "bar"}, + }, + Mesh: &api.ConsulMeshConfigEntry{}, + }, + }, + }}, + }}, + }, + false, + }, { "tg-scaling-policy-minimal.hcl", &api.Job{ diff --git a/jobspec/test-fixtures/tg-service-connect-gateway-mesh.hcl b/jobspec/test-fixtures/tg-service-connect-gateway-mesh.hcl new file mode 100644 index 000000000000..ef0ebc18a929 --- /dev/null +++ b/jobspec/test-fixtures/tg-service-connect-gateway-mesh.hcl @@ -0,0 +1,19 @@ +job "connect_gateway_mesh" { + group "group" { + service { + name = "mesh-gateway-service" + + connect { + gateway { + proxy { + config { + foo = "bar" + } + } + + mesh {} + } + } + } + } +} From a4e9c11ef9cdf304c40035888d8bc88766a4f5ba Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Mon, 26 Jul 2021 11:25:28 -0400 Subject: [PATCH 7/7] hcl: add new cores resources field --- jobspec/parse_task.go | 1 + jobspec/parse_test.go | 24 +++++++++++++++++++++++ jobspec/test-fixtures/resources-cores.hcl | 14 +++++++++++++ 3 files changed, 39 insertions(+) create mode 100644 jobspec/test-fixtures/resources-cores.hcl diff --git a/jobspec/parse_task.go b/jobspec/parse_task.go index 71e6af26b448..ff81b6ba66a8 100644 --- a/jobspec/parse_task.go +++ b/jobspec/parse_task.go @@ -551,6 +551,7 @@ func parseResources(result *api.Resources, list *ast.ObjectList) error { "memory_max", "network", "device", + "cores", } if err := checkHCLKeys(listVal, valid); err != nil { return multierror.Prefix(err, "resources ->") diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index 852a4f669aa9..0f563045cfda 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -1699,6 +1699,30 @@ func TestParse(t *testing.T) { }, false, }, + { + "resources-cores.hcl", + &api.Job{ + ID: stringToPtr("cores-test"), + Name: stringToPtr("cores-test"), + TaskGroups: []*api.TaskGroup{ + { + Count: intToPtr(5), + Name: stringToPtr("group"), + Tasks: []*api.Task{ + { + Name: "task", + Driver: "docker", + Resources: &api.Resources{ + Cores: intToPtr(4), + MemoryMB: intToPtr(128), + }, + }, + }, + }, + }, + }, + false, + }, } for _, tc := range cases { diff --git a/jobspec/test-fixtures/resources-cores.hcl b/jobspec/test-fixtures/resources-cores.hcl new file mode 100644 index 000000000000..f3e11e93e1f7 --- /dev/null +++ b/jobspec/test-fixtures/resources-cores.hcl @@ -0,0 +1,14 @@ +job "cores-test" { + group "group" { + count = 5 + + task "task" { + driver = "docker" + + resources { + cores = 4 + memory = 128 + } + } + } +}