Skip to content

Commit

Permalink
csi: support Secrets parameter in CSI RPCs
Browse files Browse the repository at this point in the history
CSI plugins can require credentials for some publishing and
unpublishing workflow RPCs. Secrets are configured at the time of
volume registration, stored in the volume struct, and then passed
around as an opaque map by Nomad to the plugins.
  • Loading branch information
tgross committed May 11, 2020
1 parent 3d6c088 commit c28c4c1
Show file tree
Hide file tree
Showing 16 changed files with 103 additions and 11 deletions.
3 changes: 3 additions & 0 deletions api/csi.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ type CSIMountOptions struct {
ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` // report unexpected keys
}

type CSISecrets map[string]string

// CSIVolume is used for serialization, see also nomad/structs/csi.go
type CSIVolume struct {
ID string
Expand All @@ -97,6 +99,7 @@ type CSIVolume struct {
AccessMode CSIVolumeAccessMode `hcl:"access_mode"`
AttachmentMode CSIVolumeAttachmentMode `hcl:"attachment_mode"`
MountOptions *CSIMountOptions `hcl:"mount_options"`
Secrets CSISecrets `hcl:"secrets"`

// Allocations, tracking claim status
ReadAllocs map[string]*Allocation
Expand Down
1 change: 1 addition & 0 deletions client/csi_endpoint.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ func (c *CSI) ControllerValidateVolume(req *structs.ClientCSIControllerValidateV
// CSI ValidateVolumeCapabilities errors for timeout, codes.Unavailable and
// codes.ResourceExhausted are retried; all other errors are fatal.
return plugin.ControllerValidateCapabilities(ctx, req.VolumeID, caps,
req.Secrets,
grpc_retry.WithPerRetryTimeout(CSIPluginRequestTimeout),
grpc_retry.WithMax(3),
grpc_retry.WithBackoff(grpc_retry.BackoffExponential(100*time.Millisecond)))
Expand Down
2 changes: 2 additions & 0 deletions client/pluginmanager/csimanager/volume.go
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,7 @@ func (v *volumeManager) stageVolume(ctx context.Context, vol *structs.CSIVolume,
publishContext,
pluginStagingPath,
capability,
vol.Secrets,
grpc_retry.WithPerRetryTimeout(DefaultMountActionTimeout),
grpc_retry.WithMax(3),
grpc_retry.WithBackoff(grpc_retry.BackoffExponential(100*time.Millisecond)),
Expand Down Expand Up @@ -208,6 +209,7 @@ func (v *volumeManager) publishVolume(ctx context.Context, vol *structs.CSIVolum
TargetPath: pluginTargetPath,
VolumeCapability: capabilities,
Readonly: usage.ReadOnly,
Secrets: vol.Secrets,
},
grpc_retry.WithPerRetryTimeout(DefaultMountActionTimeout),
grpc_retry.WithMax(3),
Expand Down
19 changes: 18 additions & 1 deletion client/structs/csi.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ type ClientCSIControllerValidateVolumeRequest struct {

AttachmentMode structs.CSIVolumeAttachmentMode
AccessMode structs.CSIVolumeAccessMode
Secrets structs.CSISecrets
// Parameters map[string]string // TODO: https://github.com/hashicorp/nomad/issues/7670

CSIControllerQuery
}
Expand Down Expand Up @@ -66,6 +68,15 @@ type ClientCSIControllerAttachVolumeRequest struct {
// only works when the Controller has the PublishReadonly capability.
ReadOnly bool

// Secrets required by plugin to complete the controller publish
// volume request. This field is OPTIONAL.
Secrets structs.CSISecrets

// TODO https://github.com/hashicorp/nomad/issues/7771
// Volume context as returned by storage provider in CreateVolumeResponse.
// This field is optional.
// VolumeContext map[string]string

CSIControllerQuery
}

Expand All @@ -82,8 +93,10 @@ func (c *ClientCSIControllerAttachVolumeRequest) ToCSIRequest() (*csi.Controller
return &csi.ControllerPublishVolumeRequest{
VolumeID: c.VolumeID,
NodeID: c.ClientCSINodeID,
ReadOnly: c.ReadOnly,
VolumeCapability: caps,
ReadOnly: c.ReadOnly,
Secrets: c.Secrets,
// VolumeContext: c.VolumeContext, TODO: https://github.com/hashicorp/nomad/issues/7771
}, nil
}

Expand Down Expand Up @@ -117,6 +130,10 @@ type ClientCSIControllerDetachVolumeRequest struct {
// by the target node for this plugin name.
ClientCSINodeID string

// Secrets required by plugin to complete the controller unpublish
// volume request. This field is OPTIONAL.
Secrets structs.CSISecrets

CSIControllerQuery
}

Expand Down
4 changes: 4 additions & 0 deletions command/agent/csi_endpoint.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,10 @@ func (s *HTTPServer) csiVolumeGet(id string, resp http.ResponseWriter, req *http
return nil, CodedError(404, "volume not found")
}

// remove sensitive fields, as our redaction mechanism doesn't
// help serializing here
out.Volume.Secrets = nil
out.Volume.MountOptions = nil
return out.Volume, nil
}

Expand Down
4 changes: 4 additions & 0 deletions command/volume_register_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,13 +57,17 @@ namespace = "n"
access_mode = "single-node-writer"
attachment_mode = "file-system"
plugin_id = "p"
secrets {
mysecret = "secretvalue"
}
`,
q: &api.CSIVolume{
ID: "foo",
Namespace: "n",
AccessMode: "single-node-writer",
AttachmentMode: "file-system",
PluginID: "p",
Secrets: api.CSISecrets{"mysecret": "secretvalue"},
},
err: "",
}, {
Expand Down
4 changes: 4 additions & 0 deletions nomad/csi_endpoint.go
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,8 @@ func (v *CSIVolume) controllerValidateVolume(req *structs.CSIVolumeRegisterReque
VolumeID: vol.RemoteID(),
AttachmentMode: vol.AttachmentMode,
AccessMode: vol.AccessMode,
Secrets: vol.Secrets,
// Parameters: TODO: https://github.com/hashicorp/nomad/issues/7670
}
cReq.PluginID = plugin.ID
cResp := &cstructs.ClientCSIControllerValidateVolumeResponse{}
Expand Down Expand Up @@ -440,6 +442,8 @@ func (v *CSIVolume) controllerPublishVolume(req *structs.CSIVolumeClaimRequest,
AttachmentMode: vol.AttachmentMode,
AccessMode: vol.AccessMode,
ReadOnly: req.Claim == structs.CSIVolumeClaimRead,
Secrets: vol.Secrets,
// VolumeContext: TODO https://github.com/hashicorp/nomad/issues/7771
}
cReq.PluginID = plug.ID
cResp := &cstructs.ClientCSIControllerAttachVolumeResponse{}
Expand Down
6 changes: 6 additions & 0 deletions nomad/csi_endpoint_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ func TestCSIVolumeEndpoint_Get(t *testing.T) {
AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter,
AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
PluginID: "minnie",
Secrets: structs.CSISecrets{"mysecret": "secretvalue"},
}}
err := state.CSIVolumeRegister(999, vols)
require.NoError(t, err)
Expand Down Expand Up @@ -84,6 +85,7 @@ func TestCSIVolumeEndpoint_Get_ACL(t *testing.T) {
AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter,
AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
PluginID: "minnie",
Secrets: structs.CSISecrets{"mysecret": "secretvalue"},
}}
err := state.CSIVolumeRegister(999, vols)
require.NoError(t, err)
Expand Down Expand Up @@ -139,6 +141,7 @@ func TestCSIVolumeEndpoint_Register(t *testing.T) {
PluginID: "minnie",
AccessMode: structs.CSIVolumeAccessModeMultiNodeReader,
AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
Secrets: structs.CSISecrets{"mysecret": "secretvalue"},
}}

// Create the register request
Expand Down Expand Up @@ -255,6 +258,7 @@ func TestCSIVolumeEndpoint_Claim(t *testing.T) {
Topologies: []*structs.CSITopology{{
Segments: map[string]string{"foo": "bar"},
}},
Secrets: structs.CSISecrets{"mysecret": "secretvalue"},
}}
index++
err = state.CSIVolumeRegister(index, vols)
Expand Down Expand Up @@ -373,6 +377,7 @@ func TestCSIVolumeEndpoint_ClaimWithController(t *testing.T) {
ControllerRequired: true,
AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter,
AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
Secrets: structs.CSISecrets{"mysecret": "secretvalue"},
}}
err = state.CSIVolumeRegister(1003, vols)

Expand Down Expand Up @@ -439,6 +444,7 @@ func TestCSIVolumeEndpoint_List(t *testing.T) {
AccessMode: structs.CSIVolumeAccessModeMultiNodeReader,
AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
PluginID: "minnie",
Secrets: structs.CSISecrets{"mysecret": "secretvalue"},
}, {
ID: id1,
Namespace: structs.DefaultNamespace,
Expand Down
1 change: 1 addition & 0 deletions nomad/mock/mock.go
Original file line number Diff line number Diff line change
Expand Up @@ -1311,6 +1311,7 @@ func CSIVolume(plugin *structs.CSIPlugin) *structs.CSIVolume {
AccessMode: structs.CSIVolumeAccessModeSingleNodeWriter,
AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
MountOptions: &structs.CSIMountOptions{},
Secrets: structs.CSISecrets{},
ReadAllocs: map[string]*structs.Allocation{},
WriteAllocs: map[string]*structs.Allocation{},
ReadClaims: map[string]*structs.CSIVolumeClaim{},
Expand Down
28 changes: 28 additions & 0 deletions nomad/structs/csi.go
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,27 @@ func (v *CSIMountOptions) GoString() string {
return v.String()
}

// CSISecrets contain optional additional configuration that can be used
// when specifying that a Volume should be used with VolumeAccessTypeMount.
type CSISecrets map[string]string

// CSISecrets implements the Stringer and GoStringer interfaces to prevent
// accidental leakage of secrets via logs.
var _ fmt.Stringer = &CSISecrets{}
var _ fmt.GoStringer = &CSISecrets{}

func (s *CSISecrets) String() string {
redacted := map[string]string{}
for k := range *s {
redacted[k] = "[REDACTED]"
}
return fmt.Sprintf("csi.CSISecrets(%v)", redacted)
}

func (s *CSISecrets) GoString() string {
return s.String()
}

type CSIVolumeClaim struct {
AllocationID string
NodeID string
Expand Down Expand Up @@ -214,6 +235,7 @@ type CSIVolume struct {
AccessMode CSIVolumeAccessMode
AttachmentMode CSIVolumeAttachmentMode
MountOptions *CSIMountOptions
Secrets CSISecrets

// Allocations, tracking claim status
ReadAllocs map[string]*Allocation // AllocID -> Allocation
Expand Down Expand Up @@ -279,6 +301,9 @@ func (v *CSIVolume) newStructs() {
if v.Topologies == nil {
v.Topologies = []*CSITopology{}
}
if v.Secrets == nil {
v.Secrets = CSISecrets{}
}

v.ReadAllocs = map[string]*Allocation{}
v.WriteAllocs = map[string]*Allocation{}
Expand Down Expand Up @@ -365,6 +390,9 @@ func (v *CSIVolume) Copy() *CSIVolume {
copy := *v
out := &copy
out.newStructs()
for k, v := range v.Secrets {
out.Secrets[k] = v
}

for k, v := range v.ReadAllocs {
out.ReadAllocs[k] = v
Expand Down
1 change: 1 addition & 0 deletions nomad/volumewatcher/volume_watcher.go
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,7 @@ func (vw *volumeWatcher) controllerDetach(vol *structs.CSIVolume, claim *structs
cReq := &cstructs.ClientCSIControllerDetachVolumeRequest{
VolumeID: vol.RemoteID(),
ClientCSINodeID: targetCSIInfo.NodeInfo.ID,
Secrets: vol.Secrets,
}
cReq.PluginID = plug.ID
err = vw.rpc.ControllerDetachVolume(cReq,
Expand Down
9 changes: 7 additions & 2 deletions plugins/csi/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
multierror "github.com/hashicorp/go-multierror"
"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/helper/grpc-middleware/logging"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/plugins/base"
"github.com/hashicorp/nomad/plugins/shared/hclspec"
"google.golang.org/grpc"
Expand Down Expand Up @@ -293,7 +294,7 @@ func (c *client) ControllerUnpublishVolume(ctx context.Context, req *ControllerU
return &ControllerUnpublishVolumeResponse{}, nil
}

func (c *client) ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *VolumeCapability, opts ...grpc.CallOption) error {
func (c *client) ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error {
if c == nil {
return fmt.Errorf("Client not initialized")
}
Expand All @@ -314,6 +315,9 @@ func (c *client) ControllerValidateCapabilities(ctx context.Context, volumeID st
VolumeCapabilities: []*csipbv1.VolumeCapability{
capabilities.ToCSIRepresentation(),
},
// VolumeContext: map[string]string // TODO: https://github.com/hashicorp/nomad/issues/7771
// Parameters: map[string]string // TODO: https://github.com/hashicorp/nomad/issues/7670
Secrets: secrets,
}

resp, err := c.controllerClient.ValidateVolumeCapabilities(ctx, req, opts...)
Expand Down Expand Up @@ -461,7 +465,7 @@ func (c *client) NodeGetInfo(ctx context.Context) (*NodeGetInfoResponse, error)
return result, nil
}

func (c *client) NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *VolumeCapability, opts ...grpc.CallOption) error {
func (c *client) NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error {
if c == nil {
return fmt.Errorf("Client not initialized")
}
Expand All @@ -483,6 +487,7 @@ func (c *client) NodeStageVolume(ctx context.Context, volumeID string, publishCo
PublishContext: publishContext,
StagingTargetPath: stagingTargetPath,
VolumeCapability: capabilities.ToCSIRepresentation(),
Secrets: secrets,
}

// NodeStageVolume's response contains no extra data. If err == nil, we were
Expand Down
5 changes: 3 additions & 2 deletions plugins/csi/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -578,7 +578,7 @@ func TestClient_RPC_ControllerValidateVolume(t *testing.T) {
cc.NextErr = c.ResponseErr

err := client.ControllerValidateCapabilities(
context.TODO(), "volumeID", requestedCaps)
context.TODO(), "volumeID", requestedCaps, structs.CSISecrets{})
if c.ExpectedErr != nil {
require.Error(t, c.ExpectedErr, err, c.Name)
} else {
Expand Down Expand Up @@ -616,7 +616,8 @@ func TestClient_RPC_NodeStageVolume(t *testing.T) {
nc.NextErr = c.ResponseErr
nc.NextStageVolumeResponse = c.Response

err := client.NodeStageVolume(context.TODO(), "foo", nil, "/foo", &VolumeCapability{})
err := client.NodeStageVolume(context.TODO(), "foo", nil, "/foo",
&VolumeCapability{}, structs.CSISecrets{})
if c.ExpectedErr != nil {
require.Error(t, c.ExpectedErr, err)
} else {
Expand Down
5 changes: 3 additions & 2 deletions plugins/csi/fake/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"fmt"
"sync"

"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/plugins/base"
"github.com/hashicorp/nomad/plugins/csi"
"github.com/hashicorp/nomad/plugins/shared/hclspec"
Expand Down Expand Up @@ -159,7 +160,7 @@ func (c *Client) ControllerUnpublishVolume(ctx context.Context, req *csi.Control
return c.NextControllerUnpublishVolumeResponse, c.NextControllerUnpublishVolumeErr
}

func (c *Client) ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *csi.VolumeCapability, opts ...grpc.CallOption) error {
func (c *Client) ControllerValidateCapabilities(ctx context.Context, volumeID string, capabilities *csi.VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error {
c.Mu.Lock()
defer c.Mu.Unlock()

Expand Down Expand Up @@ -191,7 +192,7 @@ func (c *Client) NodeGetInfo(ctx context.Context) (*csi.NodeGetInfoResponse, err
// NodeStageVolume is used when a plugin has the STAGE_UNSTAGE volume capability
// to prepare a volume for usage on a host. If err == nil, the response should
// be assumed to be successful.
func (c *Client) NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *csi.VolumeCapability, opts ...grpc.CallOption) error {
func (c *Client) NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *csi.VolumeCapability, secrets structs.CSISecrets, opts ...grpc.CallOption) error {
c.Mu.Lock()
defer c.Mu.Unlock()

Expand Down
Loading

0 comments on commit c28c4c1

Please sign in to comment.