From 45f0a3a532eeb7da4d8931d43b3b472c7c144520 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Thu, 8 Apr 2021 10:42:19 -0400 Subject: [PATCH] CSI: capability block is required for volume registration --- CHANGELOG.md | 8 +- command/agent/csi_endpoint_test.go | 20 ++-- nomad/core_sched_test.go | 14 +-- nomad/csi_endpoint_test.go | 97 ++++++++++++------- nomad/structs/csi.go | 3 + .../content/docs/commands/volume/register.mdx | 21 +++- .../content/docs/job-specification/volume.mdx | 20 ++-- .../content/docs/upgrade/upgrade-specific.mdx | 16 +++ 8 files changed, 136 insertions(+), 63 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88a129dff029..31988abcec67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,13 @@ ## 1.1.0 (Unreleased) FEATURES: - * **Consul Namespaces (Enterprise)**: Adds support for Consul Namespaces [[GH-10235](https://github.com/hashicorp/nomad/pull/10235)] + * **Consul Namespaces (Enterprise)**: Added support for Consul Namespaces [[GH-10235](https://github.com/hashicorp/nomad/pull/10235)] + * **CSI Volume Create and Snapshot**: Added support for creating, deleting, listing, and snapshotting volumes managed by Container Storage Interface (CSI) plugins. [[GH-8212](https://github.com/hashicorp/nomad/issues/8212)] * **Licensing (Enterprise)**: Support loading Enterprise license from disk or environment. [[GH-10216](https://github.com/hashicorp/nomad/issues/10216)] - * **Readiness Checks**: Adds `service` and `check` `on_update` configuration to support liveness and readiness checks. [[GH-9955](https://github.com/hashicorp/nomad/issues/9955)] + * **Readiness Checks**: Added `service` and `check` `on_update` configuration to support liveness and readiness checks. [[GH-9955](https://github.com/hashicorp/nomad/issues/9955)] + +__BACKWARDS INCOMPATIBILITIES:__ + * csi: The `attachment_mode` and `access_mode` field are required for `volume` blocks in job specifications. Registering a volume requires at least one `capability` block with the `attachment_mode` and `access_mode` fields set. [[GH-10330](https://github.com/hashicorp/nomad/issues/10330)] IMPROVEMENTS: * api: Removed unimplemented `CSIVolumes.PluginList` API. [[GH-10158](https://github.com/hashicorp/nomad/issues/10158)] diff --git a/command/agent/csi_endpoint_test.go b/command/agent/csi_endpoint_test.go index 87aea933fe71..d8de9cb88650 100644 --- a/command/agent/csi_endpoint_test.go +++ b/command/agent/csi_endpoint_test.go @@ -67,10 +67,12 @@ func TestHTTP_CSIEndpointRegisterVolume(t *testing.T) { args := structs.CSIVolumeRegisterRequest{ Volumes: []*structs.CSIVolume{{ - ID: "bar", - PluginID: "foo", - AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter, - AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, + ID: "bar", + PluginID: "foo", + RequestedCapabilities: []*structs.CSIVolumeCapability{{ + AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter, + AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, + }}, }}, } body := encodeReq(args) @@ -107,10 +109,12 @@ func TestHTTP_CSIEndpointCreateVolume(t *testing.T) { args := structs.CSIVolumeCreateRequest{ Volumes: []*structs.CSIVolume{{ - ID: "baz", - PluginID: "foo", - AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter, - AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, + ID: "baz", + PluginID: "foo", + RequestedCapabilities: []*structs.CSIVolumeCapability{{ + AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter, + AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, + }}, }}, } body := encodeReq(args) diff --git a/nomad/core_sched_test.go b/nomad/core_sched_test.go index 744800bce94f..a19d4395b07d 100644 --- a/nomad/core_sched_test.go +++ b/nomad/core_sched_test.go @@ -2293,12 +2293,14 @@ func TestCoreScheduler_CSIVolumeClaimGC(t *testing.T) { // Register a volume vols := []*structs.CSIVolume{{ - ID: volID, - Namespace: ns, - PluginID: pluginID, - AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter, - AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, - Topologies: []*structs.CSITopology{}, + ID: volID, + Namespace: ns, + PluginID: pluginID, + Topologies: []*structs.CSITopology{}, + RequestedCapabilities: []*structs.CSIVolumeCapability{{ + AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter, + AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, + }}, }} volReq := &structs.CSIVolumeRegisterRequest{Volumes: vols} volReq.Namespace = ns diff --git a/nomad/csi_endpoint_test.go b/nomad/csi_endpoint_test.go index effa1c2215e2..cbcd74695ce2 100644 --- a/nomad/csi_endpoint_test.go +++ b/nomad/csi_endpoint_test.go @@ -36,12 +36,14 @@ func TestCSIVolumeEndpoint_Get(t *testing.T) { // Create the volume vols := []*structs.CSIVolume{{ - ID: id0, - Namespace: ns, - AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter, - AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, - PluginID: "minnie", - Secrets: structs.CSISecrets{"mysecret": "secretvalue"}, + ID: id0, + Namespace: ns, + PluginID: "minnie", + Secrets: structs.CSISecrets{"mysecret": "secretvalue"}, + RequestedCapabilities: []*structs.CSIVolumeCapability{{ + AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter, + AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, + }}, }} err := state.CSIVolumeRegister(999, vols) require.NoError(t, err) @@ -84,12 +86,14 @@ func TestCSIVolumeEndpoint_Get_ACL(t *testing.T) { // Create the volume vols := []*structs.CSIVolume{{ - ID: id0, - Namespace: ns, - AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter, - AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, - PluginID: "minnie", - Secrets: structs.CSISecrets{"mysecret": "secretvalue"}, + ID: id0, + Namespace: ns, + PluginID: "minnie", + Secrets: structs.CSISecrets{"mysecret": "secretvalue"}, + RequestedCapabilities: []*structs.CSIVolumeCapability{{ + AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter, + AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, + }}, }} err := state.CSIVolumeRegister(999, vols) require.NoError(t, err) @@ -143,13 +147,17 @@ func TestCSIVolumeEndpoint_Register(t *testing.T) { ID: id0, Namespace: "notTheNamespace", PluginID: "minnie", - AccessMode: structs.CSIVolumeAccessModeMultiNodeReader, - AttachmentMode: structs.CSIVolumeAttachmentModeBlockDevice, + AccessMode: structs.CSIVolumeAccessModeSingleNodeReader, // legacy field ignored + AttachmentMode: structs.CSIVolumeAttachmentModeBlockDevice, // legacy field ignored MountOptions: &structs.CSIMountOptions{ FSType: "ext4", MountFlags: []string{"sensitive"}}, Secrets: structs.CSISecrets{"mysecret": "secretvalue"}, Parameters: map[string]string{"myparam": "paramvalue"}, Context: map[string]string{"mycontext": "contextvalue"}, + RequestedCapabilities: []*structs.CSIVolumeCapability{{ + AccessMode: structs.CSIVolumeAccessModeMultiNodeReader, + AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, + }}, }} // Create the register request @@ -262,15 +270,17 @@ func TestCSIVolumeEndpoint_Claim(t *testing.T) { require.NoError(t, err) vols := []*structs.CSIVolume{{ - ID: id0, - Namespace: structs.DefaultNamespace, - PluginID: "minnie", - AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter, - AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, + ID: id0, + Namespace: structs.DefaultNamespace, + PluginID: "minnie", Topologies: []*structs.CSITopology{{ Segments: map[string]string{"foo": "bar"}, }}, Secrets: structs.CSISecrets{"mysecret": "secretvalue"}, + RequestedCapabilities: []*structs.CSIVolumeCapability{{ + AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter, + AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, + }}, }} index++ err = state.CSIVolumeRegister(index, vols) @@ -406,9 +416,11 @@ func TestCSIVolumeEndpoint_ClaimWithController(t *testing.T) { Namespace: ns, PluginID: "minnie", ControllerRequired: true, - AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter, - AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, Secrets: structs.CSISecrets{"mysecret": "secretvalue"}, + RequestedCapabilities: []*structs.CSIVolumeCapability{{ + AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter, + AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, + }}, }} err = state.CSIVolumeRegister(1003, vols) require.NoError(t, err) @@ -510,11 +522,13 @@ func TestCSIVolumeEndpoint_Unpublish(t *testing.T) { vol := &structs.CSIVolume{ ID: volID, Namespace: ns, - AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter, - AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, PluginID: "minnie", Secrets: structs.CSISecrets{"mysecret": "secretvalue"}, ControllerRequired: true, + RequestedCapabilities: []*structs.CSIVolumeCapability{{ + AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter, + AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, + }}, } index++ @@ -608,18 +622,22 @@ func TestCSIVolumeEndpoint_List(t *testing.T) { id0 := uuid.Generate() id1 := uuid.Generate() vols := []*structs.CSIVolume{{ - ID: id0, - Namespace: structs.DefaultNamespace, - AccessMode: structs.CSIVolumeAccessModeMultiNodeReader, - AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, - PluginID: "minnie", - Secrets: structs.CSISecrets{"mysecret": "secretvalue"}, + ID: id0, + Namespace: structs.DefaultNamespace, + PluginID: "minnie", + Secrets: structs.CSISecrets{"mysecret": "secretvalue"}, + RequestedCapabilities: []*structs.CSIVolumeCapability{{ + AccessMode: structs.CSIVolumeAccessModeMultiNodeReader, + AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, + }}, }, { - ID: id1, - Namespace: structs.DefaultNamespace, - AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter, - AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, - PluginID: "adam", + ID: id1, + Namespace: structs.DefaultNamespace, + PluginID: "adam", + RequestedCapabilities: []*structs.CSIVolumeCapability{{ + AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter, + AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, + }}, }} err = state.CSIVolumeRegister(1002, vols) require.NoError(t, err) @@ -730,13 +748,19 @@ func TestCSIVolumeEndpoint_Create(t *testing.T) { Name: "vol", Namespace: "notTheNamespace", // overriden by WriteRequest PluginID: "minnie", - AccessMode: structs.CSIVolumeAccessModeMultiNodeReader, // ignored in create - AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, // ignored in create + AccessMode: structs.CSIVolumeAccessModeSingleNodeReader, // legacy field ignored + AttachmentMode: structs.CSIVolumeAttachmentModeBlockDevice, // legacy field ignored MountOptions: &structs.CSIMountOptions{ FSType: "ext4", MountFlags: []string{"sensitive"}}, // ignored in create Secrets: structs.CSISecrets{"mysecret": "secretvalue"}, Parameters: map[string]string{"myparam": "paramvalue"}, Context: map[string]string{"mycontext": "contextvalue"}, // dropped by create + RequestedCapabilities: []*structs.CSIVolumeCapability{ + { + AccessMode: structs.CSIVolumeAccessModeMultiNodeReader, + AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, + }, + }, }} // Create the create request @@ -773,6 +797,7 @@ func TestCSIVolumeEndpoint_Create(t *testing.T) { require.Equal(t, "csi.CSIOptions(FSType: ext4, MountFlags: [REDACTED])", vol.MountOptions.String()) require.Equal(t, ns, vol.Namespace) + require.Len(t, vol.RequestedCapabilities, 1) // these fields are set from the plugin and should have been written to raft require.Equal(t, "vol-12345", vol.ExternalID) diff --git a/nomad/structs/csi.go b/nomad/structs/csi.go index 9e3561ec0523..1c7845c0a61e 100644 --- a/nomad/structs/csi.go +++ b/nomad/structs/csi.go @@ -664,6 +664,9 @@ func (v *CSIVolume) Validate() error { if v.SnapshotID != "" && v.CloneID != "" { errs = append(errs, "only one of snapshot_id and clone_id is allowed") } + if len(v.RequestedCapabilities) == 0 { + errs = append(errs, "must include at least one capability block") + } // TODO: Volume Topologies are optional - We should check to see if the plugin // the volume is being registered with requires them. diff --git a/website/content/docs/commands/volume/register.mdx b/website/content/docs/commands/volume/register.mdx index d941d74f8d3a..80bdf1b6e669 100644 --- a/website/content/docs/commands/volume/register.mdx +++ b/website/content/docs/commands/volume/register.mdx @@ -77,6 +77,24 @@ context { - `plugin_id` `(string: )` - The ID of the [CSI plugin][csi_plugin] that manages this volume. +- `capability` `(Capability: )` - Option for validating the + capbility of a volume. You must provide at least one `capability` block, and + you must provide a block for each capability you intend to use in a job's + [`volume`] block. Each `capability` block must have the following fields: + + - `access_mode` `(string: )` - Defines whether a volume should be + available concurrently. Can be one of `"single-node-reader-only"`, + `"single-node-writer"`, `"multi-node-reader-only"`, + `"multi-node-single-writer"`, or `"multi-node-multi-writer"`. Most CSI + plugins support only single-node modes. Consult the documentation of the + storage provider and CSI plugin. + + - `attachment_mode` `(string: )` - The storage API that will be used + by the volume. Most storage providers will support `"file-system"`, to mount + volumes using the CSI filesystem API. Some storage providers will support + `"block-device"`, which will mount the volume with the CSI block device API + within the container. + - `secrets` (map:nil) - An optional key-value map of strings used as credentials for publishing and unpublishing volumes. @@ -98,8 +116,7 @@ context { Note that several fields used in the [`volume create`] command are set automatically by the plugin when `volume create` is successful and cannot be set on a pre-existing volume. You should not set the `snapshot_id`, -`clone_id`, `capacity_min`, `capacity_max`, or `capability` fields described -on that page. +`clone_id`, `capacity_min`, or `capacity_max` fields described on that page. [csi]: https://github.com/container-storage-interface/spec [csi_plugins_internals]: /docs/internals/plugins/csi#csi-plugins diff --git a/website/content/docs/job-specification/volume.mdx b/website/content/docs/job-specification/volume.mdx index b215c67c3699..57ba877f27cb 100644 --- a/website/content/docs/job-specification/volume.mdx +++ b/website/content/docs/job-specification/volume.mdx @@ -78,17 +78,19 @@ the [volume_mount][volume_mount] stanza in the `task` configuration. The following fields are only valid for volumes with `type = "csi"`: - `access_mode` `(string: )` - Defines whether a volume should be - available concurrently. Can be one of `"single-node-reader-only"`, - `"single-node-writer"`, `"multi-node-reader-only"`, - `"multi-node-single-writer"`, or `"multi-node-multi-writer"`. Most CSI - plugins support only single-node modes. Consult the documentation of the - storage provider and CSI plugin. + available concurrently. The `access_mode` and `attachment_mode` together + must exactly match one of the volume's `capability` blocks. Can be one of + `"single-node-reader-only"`, `"single-node-writer"`, + `"multi-node-reader-only"`, `"multi-node-single-writer"`, or + `"multi-node-multi-writer"`. Most CSI plugins support only single-node + modes. Consult the documentation of the storage provider and CSI plugin. - `attachment_mode` `(string: )` - The storage API that will be used - by the volume. Most storage providers will support `"file-system"`, to mount - volumes using the CSI filesystem API. Some storage providers will support - `"block-device"`, which will mount the volume with the CSI block device API - within the container. + by the volume. The `access_mode` and `attachment_mode` together must exactly + match one of the volume's `capability` blocks. Most storage providers will + support `"file-system"`, to mount volumes using the CSI filesystem API. Some + storage providers will support `"block-device"`, which will mount the volume + with the CSI block device API within the container. - `per_alloc` `(bool: false)` - Specifies that the `source` of the volume should have the suffix `[n]`, where `n` is the allocation index. This allows diff --git a/website/content/docs/upgrade/upgrade-specific.mdx b/website/content/docs/upgrade/upgrade-specific.mdx index 4dc3b619909c..5780ef7a9c9d 100644 --- a/website/content/docs/upgrade/upgrade-specific.mdx +++ b/website/content/docs/upgrade/upgrade-specific.mdx @@ -22,6 +22,19 @@ The Nomad agent metrics API now respects the configuration value. If this value is set to `false`, which is the default value, calling `/v1/metrics?format=prometheus` will now result in a response error. +#### CSI volumes + +The volume specification for CSI volumes has been updated to support volume +creation. The `access_mode` and `attachment_mode` fields have been moved to a +`capability` block that can be repeated. Existing registered volumes will be +automatically modified the next time that a volume claim is updated. Volume +specification files for new volumes should be updated to the format described +in the [`volume create`] and [`volume register`] commands. + +The [`volume`] block has an `access_mode` and `attachment_mode` field that are +required for CSI volumes. Jobs that use CSI volumes should be updated with +these fields. + #### Connect native tasks Connect native tasks running in host networking mode will now have `CONSUL_HTTP_ADDR` @@ -1024,3 +1037,6 @@ deleted and then Nomad 0.3.0 can be launched. [node drain]: https://www.nomadproject.io/docs/upgrade#5-upgrade-clients [`template.disable_file_sandbox`]: /docs/configuration/client#template-parameters [pki]: https://www.vaultproject.io/docs/secrets/pki +[`volume create`]: /docs/commands/volume/create +[`volume register`]: /docs/commands/volume/register +[`volume`]: /docs/job-specification/volume