diff --git a/api/csi.go b/api/csi.go index 3835c0bdc735..72edd051f6d6 100644 --- a/api/csi.go +++ b/api/csi.go @@ -87,6 +87,8 @@ type CSIMountOptions struct { ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` // report unexpected keys } +type CSISecrets map[string]string + // CSIVolume is used for serialization, see also nomad/structs/csi.go type CSIVolume struct { ID string @@ -97,6 +99,7 @@ type CSIVolume struct { AccessMode CSIVolumeAccessMode `hcl:"access_mode"` AttachmentMode CSIVolumeAttachmentMode `hcl:"attachment_mode"` MountOptions *CSIMountOptions `hcl:"mount_options"` + Secrets CSISecrets `hcl:"secrets"` // Allocations, tracking claim status ReadAllocs map[string]*Allocation diff --git a/client/csi_endpoint.go b/client/csi_endpoint.go index a4251e473b3c..71e42b1f9435 100644 --- a/client/csi_endpoint.go +++ b/client/csi_endpoint.go @@ -61,6 +61,7 @@ func (c *CSI) ControllerValidateVolume(req *structs.ClientCSIControllerValidateV // CSI ValidateVolumeCapabilities errors for timeout, codes.Unavailable and // codes.ResourceExhausted are retried; all other errors are fatal. return plugin.ControllerValidateCapabilities(ctx, req.VolumeID, caps, + req.Secrets, grpc_retry.WithPerRetryTimeout(CSIPluginRequestTimeout), grpc_retry.WithMax(3), grpc_retry.WithBackoff(grpc_retry.BackoffExponential(100*time.Millisecond))) diff --git a/client/pluginmanager/csimanager/volume.go b/client/pluginmanager/csimanager/volume.go index 3012b5362293..4d9c2a4d1227 100644 --- a/client/pluginmanager/csimanager/volume.go +++ b/client/pluginmanager/csimanager/volume.go @@ -170,6 +170,7 @@ func (v *volumeManager) stageVolume(ctx context.Context, vol *structs.CSIVolume, publishContext, pluginStagingPath, capability, + vol.Secrets, grpc_retry.WithPerRetryTimeout(DefaultMountActionTimeout), grpc_retry.WithMax(3), grpc_retry.WithBackoff(grpc_retry.BackoffExponential(100*time.Millisecond)), @@ -208,6 +209,7 @@ func (v *volumeManager) publishVolume(ctx context.Context, vol *structs.CSIVolum TargetPath: pluginTargetPath, VolumeCapability: capabilities, Readonly: usage.ReadOnly, + Secrets: vol.Secrets, }, grpc_retry.WithPerRetryTimeout(DefaultMountActionTimeout), grpc_retry.WithMax(3), diff --git a/client/structs/csi.go b/client/structs/csi.go index 99f0b0773f03..8e59fc12acd7 100644 --- a/client/structs/csi.go +++ b/client/structs/csi.go @@ -35,6 +35,8 @@ type ClientCSIControllerValidateVolumeRequest struct { AttachmentMode structs.CSIVolumeAttachmentMode AccessMode structs.CSIVolumeAccessMode + Secrets structs.CSISecrets + // Parameters map[string]string // TODO: https://github.com/hashicorp/nomad/issues/7670 CSIControllerQuery } @@ -66,6 +68,15 @@ type ClientCSIControllerAttachVolumeRequest struct { // only works when the Controller has the PublishReadonly capability. ReadOnly bool + // Secrets required by plugin to complete the controller publish + // volume request. This field is OPTIONAL. + Secrets structs.CSISecrets + + // TODO https://github.com/hashicorp/nomad/issues/7771 + // Volume context as returned by storage provider in CreateVolumeResponse. + // This field is optional. + // VolumeContext map[string]string + CSIControllerQuery } @@ -82,8 +93,10 @@ func (c *ClientCSIControllerAttachVolumeRequest) ToCSIRequest() (*csi.Controller return &csi.ControllerPublishVolumeRequest{ VolumeID: c.VolumeID, NodeID: c.ClientCSINodeID, - ReadOnly: c.ReadOnly, VolumeCapability: caps, + ReadOnly: c.ReadOnly, + Secrets: c.Secrets, + // VolumeContext: c.VolumeContext, TODO: https://github.com/hashicorp/nomad/issues/7771 }, nil } @@ -117,6 +130,10 @@ type ClientCSIControllerDetachVolumeRequest struct { // by the target node for this plugin name. ClientCSINodeID string + // Secrets required by plugin to complete the controller unpublish + // volume request. This field is OPTIONAL. + Secrets structs.CSISecrets + CSIControllerQuery } diff --git a/command/agent/csi_endpoint.go b/command/agent/csi_endpoint.go index 6a2ce69d9f0f..2d1151ea0b42 100644 --- a/command/agent/csi_endpoint.go +++ b/command/agent/csi_endpoint.go @@ -85,6 +85,10 @@ func (s *HTTPServer) csiVolumeGet(id string, resp http.ResponseWriter, req *http return nil, CodedError(404, "volume not found") } + // remove sensitive fields, as our redaction mechanism doesn't + // help serializing here + out.Volume.Secrets = nil + out.Volume.MountOptions = nil return out.Volume, nil } diff --git a/command/volume_register_test.go b/command/volume_register_test.go index d707f6171eb7..14152be1cebb 100644 --- a/command/volume_register_test.go +++ b/command/volume_register_test.go @@ -57,6 +57,9 @@ namespace = "n" access_mode = "single-node-writer" attachment_mode = "file-system" plugin_id = "p" +secrets { + mysecret = "secretvalue" +} `, q: &api.CSIVolume{ ID: "foo", @@ -64,6 +67,7 @@ plugin_id = "p" AccessMode: "single-node-writer", AttachmentMode: "file-system", PluginID: "p", + Secrets: api.CSISecrets{"mysecret": "secretvalue"}, }, err: "", }, { diff --git a/nomad/csi_endpoint.go b/nomad/csi_endpoint.go index 56e6254beef5..855e3e42512a 100644 --- a/nomad/csi_endpoint.go +++ b/nomad/csi_endpoint.go @@ -237,6 +237,8 @@ func (v *CSIVolume) controllerValidateVolume(req *structs.CSIVolumeRegisterReque VolumeID: vol.RemoteID(), AttachmentMode: vol.AttachmentMode, AccessMode: vol.AccessMode, + Secrets: vol.Secrets, + // Parameters: TODO: https://github.com/hashicorp/nomad/issues/7670 } cReq.PluginID = plugin.ID cResp := &cstructs.ClientCSIControllerValidateVolumeResponse{} @@ -440,6 +442,8 @@ func (v *CSIVolume) controllerPublishVolume(req *structs.CSIVolumeClaimRequest, AttachmentMode: vol.AttachmentMode, AccessMode: vol.AccessMode, ReadOnly: req.Claim == structs.CSIVolumeClaimRead, + Secrets: vol.Secrets, + // VolumeContext: TODO https://github.com/hashicorp/nomad/issues/7771 } cReq.PluginID = plug.ID cResp := &cstructs.ClientCSIControllerAttachVolumeResponse{} diff --git a/nomad/csi_endpoint_test.go b/nomad/csi_endpoint_test.go index fb903cdb38ab..b7e36b4ca0df 100644 --- a/nomad/csi_endpoint_test.go +++ b/nomad/csi_endpoint_test.go @@ -37,6 +37,7 @@ func TestCSIVolumeEndpoint_Get(t *testing.T) { AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter, AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, PluginID: "minnie", + Secrets: structs.CSISecrets{"mysecret": "secretvalue"}, }} err := state.CSIVolumeRegister(999, vols) require.NoError(t, err) @@ -84,6 +85,7 @@ func TestCSIVolumeEndpoint_Get_ACL(t *testing.T) { AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter, AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, PluginID: "minnie", + Secrets: structs.CSISecrets{"mysecret": "secretvalue"}, }} err := state.CSIVolumeRegister(999, vols) require.NoError(t, err) @@ -139,6 +141,7 @@ func TestCSIVolumeEndpoint_Register(t *testing.T) { PluginID: "minnie", AccessMode: structs.CSIVolumeAccessModeMultiNodeReader, AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, + Secrets: structs.CSISecrets{"mysecret": "secretvalue"}, }} // Create the register request @@ -255,6 +258,7 @@ func TestCSIVolumeEndpoint_Claim(t *testing.T) { Topologies: []*structs.CSITopology{{ Segments: map[string]string{"foo": "bar"}, }}, + Secrets: structs.CSISecrets{"mysecret": "secretvalue"}, }} index++ err = state.CSIVolumeRegister(index, vols) @@ -373,6 +377,7 @@ func TestCSIVolumeEndpoint_ClaimWithController(t *testing.T) { ControllerRequired: true, AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter, AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, + Secrets: structs.CSISecrets{"mysecret": "secretvalue"}, }} err = state.CSIVolumeRegister(1003, vols) @@ -439,6 +444,7 @@ func TestCSIVolumeEndpoint_List(t *testing.T) { AccessMode: structs.CSIVolumeAccessModeMultiNodeReader, AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, PluginID: "minnie", + Secrets: structs.CSISecrets{"mysecret": "secretvalue"}, }, { ID: id1, Namespace: structs.DefaultNamespace, diff --git a/nomad/structs/csi.go b/nomad/structs/csi.go index 7cecb7cd9d9d..28721537cf03 100644 --- a/nomad/structs/csi.go +++ b/nomad/structs/csi.go @@ -185,6 +185,27 @@ func (v *CSIMountOptions) GoString() string { return v.String() } +// CSISecrets contain optional additional configuration that can be used +// when specifying that a Volume should be used with VolumeAccessTypeMount. +type CSISecrets map[string]string + +// CSISecrets implements the Stringer and GoStringer interfaces to prevent +// accidental leakage of secrets via logs. +var _ fmt.Stringer = &CSISecrets{} +var _ fmt.GoStringer = &CSISecrets{} + +func (s *CSISecrets) String() string { + redacted := map[string]string{} + for k := range *s { + redacted[k] = "[REDACTED]" + } + return fmt.Sprintf("csi.CSISecrets(%v)", redacted) +} + +func (s *CSISecrets) GoString() string { + return s.String() +} + type CSIVolumeClaim struct { AllocationID string NodeID string @@ -214,6 +235,7 @@ type CSIVolume struct { AccessMode CSIVolumeAccessMode AttachmentMode CSIVolumeAttachmentMode MountOptions *CSIMountOptions + Secrets CSISecrets // Allocations, tracking claim status ReadAllocs map[string]*Allocation // AllocID -> Allocation @@ -279,6 +301,9 @@ func (v *CSIVolume) newStructs() { if v.Topologies == nil { v.Topologies = []*CSITopology{} } + if v.Secrets == nil { + v.Secrets = CSISecrets{} + } v.ReadAllocs = map[string]*Allocation{} v.WriteAllocs = map[string]*Allocation{} @@ -365,6 +390,9 @@ func (v *CSIVolume) Copy() *CSIVolume { copy := *v out := © out.newStructs() + for k, v := range v.Secrets { + out.Secrets[k] = v + } for k, v := range v.ReadAllocs { out.ReadAllocs[k] = v diff --git a/nomad/volumewatcher/volume_watcher.go b/nomad/volumewatcher/volume_watcher.go index 6579564d638b..dc490ad5e1f7 100644 --- a/nomad/volumewatcher/volume_watcher.go +++ b/nomad/volumewatcher/volume_watcher.go @@ -350,6 +350,7 @@ func (vw *volumeWatcher) controllerDetach(vol *structs.CSIVolume, claim *structs cReq := &cstructs.ClientCSIControllerDetachVolumeRequest{ VolumeID: vol.RemoteID(), ClientCSINodeID: targetCSIInfo.NodeInfo.ID, + Secrets: vol.Secrets, } cReq.PluginID = plug.ID err = vw.rpc.ControllerDetachVolume(cReq, diff --git a/plugins/csi/client.go b/plugins/csi/client.go index e17cbef32080..99c0cad3848d 100644 --- a/plugins/csi/client.go +++ b/plugins/csi/client.go @@ -12,6 +12,7 @@ import ( multierror "github.com/hashicorp/go-multierror" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/grpc-middleware/logging" + "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/shared/hclspec" "google.golang.org/grpc" @@ -293,7 +294,7 @@ func (c *client) ControllerUnpublishVolume(ctx context.Context, req *ControllerU return &ControllerUnpublishVolumeResponse{}, nil } -func (c *client) ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *VolumeCapability, opts ...grpc.CallOption) error { +func (c *client) ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error { if c == nil { return fmt.Errorf("Client not initialized") } @@ -314,6 +315,9 @@ func (c *client) ControllerValidateCapabilities(ctx context.Context, volumeID st VolumeCapabilities: []*csipbv1.VolumeCapability{ capabilities.ToCSIRepresentation(), }, + // VolumeContext: map[string]string // TODO: https://github.com/hashicorp/nomad/issues/7771 + // Parameters: map[string]string // TODO: https://github.com/hashicorp/nomad/issues/7670 + Secrets: secrets, } resp, err := c.controllerClient.ValidateVolumeCapabilities(ctx, req, opts...) @@ -461,7 +465,7 @@ func (c *client) NodeGetInfo(ctx context.Context) (*NodeGetInfoResponse, error) return result, nil } -func (c *client) NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *VolumeCapability, opts ...grpc.CallOption) error { +func (c *client) NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error { if c == nil { return fmt.Errorf("Client not initialized") } @@ -483,6 +487,7 @@ func (c *client) NodeStageVolume(ctx context.Context, volumeID string, publishCo PublishContext: publishContext, StagingTargetPath: stagingTargetPath, VolumeCapability: capabilities.ToCSIRepresentation(), + Secrets: secrets, } // NodeStageVolume's response contains no extra data. If err == nil, we were diff --git a/plugins/csi/fake/client.go b/plugins/csi/fake/client.go index 963cad65f23f..77fa5c514817 100644 --- a/plugins/csi/fake/client.go +++ b/plugins/csi/fake/client.go @@ -8,6 +8,7 @@ import ( "fmt" "sync" + "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/csi" "github.com/hashicorp/nomad/plugins/shared/hclspec" @@ -159,7 +160,7 @@ func (c *Client) ControllerUnpublishVolume(ctx context.Context, req *csi.Control return c.NextControllerUnpublishVolumeResponse, c.NextControllerUnpublishVolumeErr } -func (c *Client) ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *csi.VolumeCapability, opts ...grpc.CallOption) error { +func (c *Client) ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *csi.VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error { c.Mu.Lock() defer c.Mu.Unlock() @@ -191,7 +192,7 @@ func (c *Client) NodeGetInfo(ctx context.Context) (*csi.NodeGetInfoResponse, err // NodeStageVolume is used when a plugin has the STAGE_UNSTAGE volume capability // to prepare a volume for usage on a host. If err == nil, the response should // be assumed to be successful. -func (c *Client) NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *csi.VolumeCapability, opts ...grpc.CallOption) error { +func (c *Client) NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *csi.VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error { c.Mu.Lock() defer c.Mu.Unlock() diff --git a/plugins/csi/plugin.go b/plugins/csi/plugin.go index 50bd0bc0032d..90b5ead0a41d 100644 --- a/plugins/csi/plugin.go +++ b/plugins/csi/plugin.go @@ -43,7 +43,7 @@ type CSIPlugin interface { // ControllerValidateCapabilities is used to validate that a volume exists and // supports the requested capability. - ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *VolumeCapability, opts ...grpc.CallOption) error + ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error // NodeGetCapabilities is used to return the available capabilities from the // Node Service. @@ -56,7 +56,7 @@ type CSIPlugin interface { // NodeStageVolume is used when a plugin has the STAGE_UNSTAGE volume capability // to prepare a volume for usage on a host. If err == nil, the response should // be assumed to be successful. - NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *VolumeCapability, opts ...grpc.CallOption) error + NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error // NodeUnstageVolume is used when a plugin has the STAGE_UNSTAGE volume capability // to undo the work performed by NodeStageVolume. If a volume has been staged, @@ -111,8 +111,9 @@ type NodePublishVolumeRequest struct { Readonly bool - // Reserved for future use. - Secrets map[string]string + // Secrets required by plugins to complete the node publish volume + // request. This field is OPTIONAL. + Secrets structs.CSISecrets } func (r *NodePublishVolumeRequest) ToCSIRepresentation() *csipbv1.NodePublishVolumeRequest { @@ -233,6 +234,8 @@ type ControllerPublishVolumeRequest struct { NodeID string ReadOnly bool VolumeCapability *VolumeCapability + Secrets structs.CSISecrets + // VolumeContext map[string]string // TODO: https://github.com/hashicorp/nomad/issues/7771 } func (r *ControllerPublishVolumeRequest) ToCSIRepresentation() *csipbv1.ControllerPublishVolumeRequest { @@ -245,6 +248,8 @@ func (r *ControllerPublishVolumeRequest) ToCSIRepresentation() *csipbv1.Controll NodeId: r.NodeID, Readonly: r.ReadOnly, VolumeCapability: r.VolumeCapability.ToCSIRepresentation(), + Secrets: r.Secrets, + // VolumeContext: r.VolumeContext, https://github.com/hashicorp/nomad/issues/7771 } } @@ -265,6 +270,7 @@ type ControllerPublishVolumeResponse struct { type ControllerUnpublishVolumeRequest struct { VolumeID string NodeID string + Secrets structs.CSISecrets } func (r *ControllerUnpublishVolumeRequest) ToCSIRepresentation() *csipbv1.ControllerUnpublishVolumeRequest { @@ -275,6 +281,7 @@ func (r *ControllerUnpublishVolumeRequest) ToCSIRepresentation() *csipbv1.Contro return &csipbv1.ControllerUnpublishVolumeRequest{ VolumeId: r.VolumeID, NodeId: r.NodeID, + Secrets: r.Secrets, } } diff --git a/website/pages/docs/commands/volume/register.mdx b/website/pages/docs/commands/volume/register.mdx index 90867c26d631..355785521039 100644 --- a/website/pages/docs/commands/volume/register.mdx +++ b/website/pages/docs/commands/volume/register.mdx @@ -46,6 +46,9 @@ mount_options { fs_type = "ext4" mount_flags = ["ro"] } +secrets { + example_secret = "xyzzy" +} ``` ## Volume Specification Parameters @@ -84,6 +87,10 @@ mount_options { - `fs_type`: file system type (ex. `"ext4"`) - `mount_flags`: the flags passed to `mount` (ex. `"ro,noatime"`) +- `secrets` (map:nil) - An optional key-value map of + strings used as credentials for publishing and unpublishing volumes. + + [volume_specification]: #volume-specification [csi]: https://github.com/container-storage-interface/spec [csi_plugin]: /docs/job-specification/csi_plugin