csi: support Secrets parameter in CSI RPCs

CSI plugins can require credentials for some publishing and
unpublishing workflow RPCs. Secrets are configured at the time of
volume registration, stored in the volume struct, and then passed
around as an opaque map by Nomad to the plugins.

api/csi.go

@@ -87,6 +87,8 @@ type CSIMountOptions struct {
 	ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` // report unexpected keys
 }
 
+type CSISecrets map[string]string
+
 // CSIVolume is used for serialization, see also nomad/structs/csi.go
 type CSIVolume struct {
 	ID             string
@@ -97,6 +99,7 @@ type CSIVolume struct {
 	AccessMode     CSIVolumeAccessMode     `hcl:"access_mode"`
 	AttachmentMode CSIVolumeAttachmentMode `hcl:"attachment_mode"`
 	MountOptions   *CSIMountOptions        `hcl:"mount_options"`
+	Secrets        CSISecrets              `hcl:"secrets"`
 
 	// Allocations, tracking claim status
 	ReadAllocs  map[string]*Allocation

client/csi_endpoint.go

@@ -61,6 +61,7 @@ func (c *CSI) ControllerValidateVolume(req *structs.ClientCSIControllerValidateV
 	// CSI ValidateVolumeCapabilities errors for timeout, codes.Unavailable and
 	// codes.ResourceExhausted are retried; all other errors are fatal.
 	return plugin.ControllerValidateCapabilities(ctx, req.VolumeID, caps,
+		req.Secrets,
 		grpc_retry.WithPerRetryTimeout(CSIPluginRequestTimeout),
 		grpc_retry.WithMax(3),
 		grpc_retry.WithBackoff(grpc_retry.BackoffExponential(100*time.Millisecond)))

client/pluginmanager/csimanager/volume.go

@@ -170,6 +170,7 @@ func (v *volumeManager) stageVolume(ctx context.Context, vol *structs.CSIVolume,
 		publishContext,
 		pluginStagingPath,
 		capability,
+		vol.Secrets,
 		grpc_retry.WithPerRetryTimeout(DefaultMountActionTimeout),
 		grpc_retry.WithMax(3),
 		grpc_retry.WithBackoff(grpc_retry.BackoffExponential(100*time.Millisecond)),
@@ -208,6 +209,7 @@ func (v *volumeManager) publishVolume(ctx context.Context, vol *structs.CSIVolum
 		TargetPath:        pluginTargetPath,
 		VolumeCapability:  capabilities,
 		Readonly:          usage.ReadOnly,
+		Secrets:           vol.Secrets,
 	},
 		grpc_retry.WithPerRetryTimeout(DefaultMountActionTimeout),
 		grpc_retry.WithMax(3),

client/structs/csi.go

@@ -35,6 +35,8 @@ type ClientCSIControllerValidateVolumeRequest struct {
 
 	AttachmentMode structs.CSIVolumeAttachmentMode
 	AccessMode     structs.CSIVolumeAccessMode
+	Secrets        structs.CSISecrets
+	// Parameters map[string]string // TODO: https://github.com/hashicorp/nomad/issues/7670
 
 	CSIControllerQuery
 }
@@ -66,6 +68,15 @@ type ClientCSIControllerAttachVolumeRequest struct {
 	// only works when the Controller has the PublishReadonly capability.
 	ReadOnly bool
 
+	// Secrets required by plugin to complete the controller publish
+	// volume request. This field is OPTIONAL.
+	Secrets structs.CSISecrets
+
+	// TODO https://github.com/hashicorp/nomad/issues/7771
+	// Volume context as returned by storage provider in CreateVolumeResponse.
+	// This field is optional.
+	// VolumeContext map[string]string
+
 	CSIControllerQuery
 }
 
@@ -82,8 +93,10 @@ func (c *ClientCSIControllerAttachVolumeRequest) ToCSIRequest() (*csi.Controller
 	return &csi.ControllerPublishVolumeRequest{
 		VolumeID:         c.VolumeID,
 		NodeID:           c.ClientCSINodeID,
-		ReadOnly:         c.ReadOnly,
 		VolumeCapability: caps,
+		ReadOnly:         c.ReadOnly,
+		Secrets:          c.Secrets,
+		// VolumeContext:    c.VolumeContext, TODO: https://github.com/hashicorp/nomad/issues/7771
 	}, nil
 }
 
@@ -117,6 +130,10 @@ type ClientCSIControllerDetachVolumeRequest struct {
 	// by the target node for this plugin name.
 	ClientCSINodeID string
 
+	// Secrets required by plugin to complete the controller unpublish
+	// volume request. This field is OPTIONAL.
+	Secrets structs.CSISecrets
+
 	CSIControllerQuery
 }
 

command/agent/csi_endpoint.go

@@ -85,6 +85,10 @@ func (s *HTTPServer) csiVolumeGet(id string, resp http.ResponseWriter, req *http
 		return nil, CodedError(404, "volume not found")
 	}
 
+	// remove sensitive fields, as our redaction mechanism doesn't
+	// help serializing here
+	out.Volume.Secrets = nil
+	out.Volume.MountOptions = nil
 	return out.Volume, nil
 }
 

command/volume_register_test.go

@@ -57,13 +57,17 @@ namespace = "n"
 access_mode = "single-node-writer"
 attachment_mode = "file-system"
 plugin_id = "p"
+secrets {
+  mysecret = "secretvalue"
+}
 `,
 		q: &api.CSIVolume{
 			ID:             "foo",
 			Namespace:      "n",
 			AccessMode:     "single-node-writer",
 			AttachmentMode: "file-system",
 			PluginID:       "p",
+			Secrets:        api.CSISecrets{"mysecret": "secretvalue"},
 		},
 		err: "",
 	}, {

nomad/csi_endpoint.go

@@ -237,6 +237,8 @@ func (v *CSIVolume) controllerValidateVolume(req *structs.CSIVolumeRegisterReque
 		VolumeID:       vol.RemoteID(),
 		AttachmentMode: vol.AttachmentMode,
 		AccessMode:     vol.AccessMode,
+		Secrets:        vol.Secrets,
+		// Parameters: TODO: https://github.com/hashicorp/nomad/issues/7670
 	}
 	cReq.PluginID = plugin.ID
 	cResp := &cstructs.ClientCSIControllerValidateVolumeResponse{}
@@ -440,6 +442,8 @@ func (v *CSIVolume) controllerPublishVolume(req *structs.CSIVolumeClaimRequest,
 		AttachmentMode:  vol.AttachmentMode,
 		AccessMode:      vol.AccessMode,
 		ReadOnly:        req.Claim == structs.CSIVolumeClaimRead,
+		Secrets:         vol.Secrets,
+		// VolumeContext: TODO https://github.com/hashicorp/nomad/issues/7771
 	}
 	cReq.PluginID = plug.ID
 	cResp := &cstructs.ClientCSIControllerAttachVolumeResponse{}

nomad/csi_endpoint_test.go

@@ -37,6 +37,7 @@ func TestCSIVolumeEndpoint_Get(t *testing.T) {
 		AccessMode:     structs.CSIVolumeAccessModeMultiNodeSingleWriter,
 		AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
 		PluginID:       "minnie",
+		Secrets:        structs.CSISecrets{"mysecret": "secretvalue"},
 	}}
 	err := state.CSIVolumeRegister(999, vols)
 	require.NoError(t, err)
@@ -84,6 +85,7 @@ func TestCSIVolumeEndpoint_Get_ACL(t *testing.T) {
 		AccessMode:     structs.CSIVolumeAccessModeMultiNodeSingleWriter,
 		AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
 		PluginID:       "minnie",
+		Secrets:        structs.CSISecrets{"mysecret": "secretvalue"},
 	}}
 	err := state.CSIVolumeRegister(999, vols)
 	require.NoError(t, err)
@@ -139,6 +141,7 @@ func TestCSIVolumeEndpoint_Register(t *testing.T) {
 		PluginID:       "minnie",
 		AccessMode:     structs.CSIVolumeAccessModeMultiNodeReader,
 		AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
+		Secrets:        structs.CSISecrets{"mysecret": "secretvalue"},
 	}}
 
 	// Create the register request
@@ -255,6 +258,7 @@ func TestCSIVolumeEndpoint_Claim(t *testing.T) {
 		Topologies: []*structs.CSITopology{{
 			Segments: map[string]string{"foo": "bar"},
 		}},
+		Secrets: structs.CSISecrets{"mysecret": "secretvalue"},
 	}}
 	index++
 	err = state.CSIVolumeRegister(index, vols)
@@ -373,6 +377,7 @@ func TestCSIVolumeEndpoint_ClaimWithController(t *testing.T) {
 		ControllerRequired: true,
 		AccessMode:         structs.CSIVolumeAccessModeMultiNodeSingleWriter,
 		AttachmentMode:     structs.CSIVolumeAttachmentModeFilesystem,
+		Secrets:            structs.CSISecrets{"mysecret": "secretvalue"},
 	}}
 	err = state.CSIVolumeRegister(1003, vols)
 
@@ -439,6 +444,7 @@ func TestCSIVolumeEndpoint_List(t *testing.T) {
 		AccessMode:     structs.CSIVolumeAccessModeMultiNodeReader,
 		AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
 		PluginID:       "minnie",
+		Secrets:        structs.CSISecrets{"mysecret": "secretvalue"},
 	}, {
 		ID:             id1,
 		Namespace:      structs.DefaultNamespace,

nomad/structs/csi.go

@@ -185,6 +185,27 @@ func (v *CSIMountOptions) GoString() string {
 	return v.String()
 }
 
+// CSISecrets contain optional additional configuration that can be used
+// when specifying that a Volume should be used with VolumeAccessTypeMount.
+type CSISecrets map[string]string
+
+// CSISecrets implements the Stringer and GoStringer interfaces to prevent
+// accidental leakage of secrets via logs.
+var _ fmt.Stringer = &CSISecrets{}
+var _ fmt.GoStringer = &CSISecrets{}
+
+func (s *CSISecrets) String() string {
+	redacted := map[string]string{}
+	for k := range *s {
+		redacted[k] = "[REDACTED]"
+	}
+	return fmt.Sprintf("csi.CSISecrets(%v)", redacted)
+}
+
+func (s *CSISecrets) GoString() string {
+	return s.String()
+}
+
 type CSIVolumeClaim struct {
 	AllocationID string
 	NodeID       string
@@ -214,6 +235,7 @@ type CSIVolume struct {
 	AccessMode     CSIVolumeAccessMode
 	AttachmentMode CSIVolumeAttachmentMode
 	MountOptions   *CSIMountOptions
+	Secrets        CSISecrets
 
 	// Allocations, tracking claim status
 	ReadAllocs  map[string]*Allocation // AllocID -> Allocation
@@ -279,6 +301,9 @@ func (v *CSIVolume) newStructs() {
 	if v.Topologies == nil {
 		v.Topologies = []*CSITopology{}
 	}
+	if v.Secrets == nil {
+		v.Secrets = CSISecrets{}
+	}
 
 	v.ReadAllocs = map[string]*Allocation{}
 	v.WriteAllocs = map[string]*Allocation{}
@@ -365,6 +390,9 @@ func (v *CSIVolume) Copy() *CSIVolume {
 	copy := *v
 	out := &copy
 	out.newStructs()
+	for k, v := range v.Secrets {
+		out.Secrets[k] = v
+	}
 
 	for k, v := range v.ReadAllocs {
 		out.ReadAllocs[k] = v

nomad/volumewatcher/volume_watcher.go

@@ -350,6 +350,7 @@ func (vw *volumeWatcher) controllerDetach(vol *structs.CSIVolume, claim *structs
 	cReq := &cstructs.ClientCSIControllerDetachVolumeRequest{
 		VolumeID:        vol.RemoteID(),
 		ClientCSINodeID: targetCSIInfo.NodeInfo.ID,
+		Secrets:         vol.Secrets,
 	}
 	cReq.PluginID = plug.ID
 	err = vw.rpc.ControllerDetachVolume(cReq,

plugins/csi/client.go

@@ -12,6 +12,7 @@ import (
 	multierror "github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/nomad/helper"
 	"github.com/hashicorp/nomad/helper/grpc-middleware/logging"
+	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/hashicorp/nomad/plugins/base"
 	"github.com/hashicorp/nomad/plugins/shared/hclspec"
 	"google.golang.org/grpc"
@@ -293,7 +294,7 @@ func (c *client) ControllerUnpublishVolume(ctx context.Context, req *ControllerU
 	return &ControllerUnpublishVolumeResponse{}, nil
 }
 
-func (c *client) ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *VolumeCapability, opts ...grpc.CallOption) error {
+func (c *client) ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error {
 	if c == nil {
 		return fmt.Errorf("Client not initialized")
 	}
@@ -314,6 +315,9 @@ func (c *client) ControllerValidateCapabilities(ctx context.Context, volumeID st
 		VolumeCapabilities: []*csipbv1.VolumeCapability{
 			capabilities.ToCSIRepresentation(),
 		},
+		// VolumeContext: map[string]string // TODO: https://github.com/hashicorp/nomad/issues/7771
+		// Parameters: map[string]string // TODO: https://github.com/hashicorp/nomad/issues/7670
+		Secrets: secrets,
 	}
 
 	resp, err := c.controllerClient.ValidateVolumeCapabilities(ctx, req, opts...)
@@ -461,7 +465,7 @@ func (c *client) NodeGetInfo(ctx context.Context) (*NodeGetInfoResponse, error)
 	return result, nil
 }
 
-func (c *client) NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *VolumeCapability, opts ...grpc.CallOption) error {
+func (c *client) NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error {
 	if c == nil {
 		return fmt.Errorf("Client not initialized")
 	}
@@ -483,6 +487,7 @@ func (c *client) NodeStageVolume(ctx context.Context, volumeID string, publishCo
 		PublishContext:    publishContext,
 		StagingTargetPath: stagingTargetPath,
 		VolumeCapability:  capabilities.ToCSIRepresentation(),
+		Secrets:           secrets,
 	}
 
 	// NodeStageVolume's response contains no extra data. If err == nil, we were

plugins/csi/fake/client.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"sync"
 
+	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/hashicorp/nomad/plugins/base"
 	"github.com/hashicorp/nomad/plugins/csi"
 	"github.com/hashicorp/nomad/plugins/shared/hclspec"
@@ -159,7 +160,7 @@ func (c *Client) ControllerUnpublishVolume(ctx context.Context, req *csi.Control
 	return c.NextControllerUnpublishVolumeResponse, c.NextControllerUnpublishVolumeErr
 }
 
-func (c *Client) ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *csi.VolumeCapability, opts ...grpc.CallOption) error {
+func (c *Client) ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *csi.VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error {
 	c.Mu.Lock()
 	defer c.Mu.Unlock()
 
@@ -191,7 +192,7 @@ func (c *Client) NodeGetInfo(ctx context.Context) (*csi.NodeGetInfoResponse, err
 // NodeStageVolume is used when a plugin has the STAGE_UNSTAGE volume capability
 // to prepare a volume for usage on a host. If err == nil, the response should
 // be assumed to be successful.
-func (c *Client) NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *csi.VolumeCapability, opts ...grpc.CallOption) error {
+func (c *Client) NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *csi.VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error {
 	c.Mu.Lock()
 	defer c.Mu.Unlock()
 

plugins/csi/plugin.go

@@ -43,7 +43,7 @@ type CSIPlugin interface {
 
 	// ControllerValidateCapabilities is used to validate that a volume exists and
 	// supports the requested capability.
-	ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *VolumeCapability, opts ...grpc.CallOption) error
+	ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error
 
 	// NodeGetCapabilities is used to return the available capabilities from the
 	// Node Service.
@@ -56,7 +56,7 @@ type CSIPlugin interface {
 	// NodeStageVolume is used when a plugin has the STAGE_UNSTAGE volume capability
 	// to prepare a volume for usage on a host. If err == nil, the response should
 	// be assumed to be successful.
-	NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *VolumeCapability, opts ...grpc.CallOption) error
+	NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error
 
 	// NodeUnstageVolume is used when a plugin has the STAGE_UNSTAGE volume capability
 	// to undo the work performed by NodeStageVolume. If a volume has been staged,
@@ -111,8 +111,9 @@ type NodePublishVolumeRequest struct {
 
 	Readonly bool
 
-	// Reserved for future use.
-	Secrets map[string]string
+	// Secrets required by plugins to complete the node publish volume
+	// request. This field is OPTIONAL.
+	Secrets structs.CSISecrets
 }
 
 func (r *NodePublishVolumeRequest) ToCSIRepresentation() *csipbv1.NodePublishVolumeRequest {
@@ -233,6 +234,8 @@ type ControllerPublishVolumeRequest struct {
 	NodeID           string
 	ReadOnly         bool
 	VolumeCapability *VolumeCapability
+	Secrets          structs.CSISecrets
+	// VolumeContext    map[string]string  // TODO: https://github.com/hashicorp/nomad/issues/7771
 }
 
 func (r *ControllerPublishVolumeRequest) ToCSIRepresentation() *csipbv1.ControllerPublishVolumeRequest {
@@ -245,6 +248,8 @@ func (r *ControllerPublishVolumeRequest) ToCSIRepresentation() *csipbv1.Controll
 		NodeId:           r.NodeID,
 		Readonly:         r.ReadOnly,
 		VolumeCapability: r.VolumeCapability.ToCSIRepresentation(),
+		Secrets:          r.Secrets,
+		// VolumeContext:    r.VolumeContext, https://github.com/hashicorp/nomad/issues/7771
 	}
 }
 
@@ -265,6 +270,7 @@ type ControllerPublishVolumeResponse struct {
 type ControllerUnpublishVolumeRequest struct {
 	VolumeID string
 	NodeID   string
+	Secrets  structs.CSISecrets
 }
 
 func (r *ControllerUnpublishVolumeRequest) ToCSIRepresentation() *csipbv1.ControllerUnpublishVolumeRequest {
@@ -275,6 +281,7 @@ func (r *ControllerUnpublishVolumeRequest) ToCSIRepresentation() *csipbv1.Contro
 	return &csipbv1.ControllerUnpublishVolumeRequest{
 		VolumeId: r.VolumeID,
 		NodeId:   r.NodeID,
+		Secrets:  r.Secrets,
 	}
 }