From 4ac580855afa0c28454bfd99eb7cdf0627f0d532 Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Fri, 2 Jul 2021 17:46:41 -0700 Subject: [PATCH] CSI ListSnapshots secrets implementation Signed-off-by: Grant Griffiths --- api/csi.go | 6 +- client/structs/csi.go | 2 + command/agent/csi_endpoint.go | 8 + command/volume_snapshot_list.go | 5 +- nomad/csi_endpoint.go | 1 + nomad/structs/csi.go | 1 + plugins/csi/plugin.go | 2 + vendor/github.com/hashicorp/nomad/api/csi.go | 497 +++++++++++++++++++ 8 files changed, 520 insertions(+), 2 deletions(-) create mode 100644 vendor/github.com/hashicorp/nomad/api/csi.go diff --git a/api/csi.go b/api/csi.go index 836d37d04e42..120c239fde8f 100644 --- a/api/csi.go +++ b/api/csi.go @@ -137,7 +137,7 @@ func (v *CSIVolumes) DeleteSnapshot(snap *CSISnapshot, w *WriteOptions) error { } // ListSnapshots lists external storage volume snapshots. -func (v *CSIVolumes) ListSnapshots(pluginID string, q *QueryOptions) (*CSISnapshotListResponse, *QueryMeta, error) { +func (v *CSIVolumes) ListSnapshots(pluginID string, secrets string, q *QueryOptions) (*CSISnapshotListResponse, *QueryMeta, error) { var resp *CSISnapshotListResponse qp := url.Values{} @@ -150,6 +150,9 @@ func (v *CSIVolumes) ListSnapshots(pluginID string, q *QueryOptions) (*CSISnapsh if q.PerPage != 0 { qp.Set("per_page", fmt.Sprint(q.PerPage)) } + if secrets != "" { + qp.Set("secrets", secrets) + } qm, err := v.client.query("/v1/volumes/snapshot?"+qp.Encode(), &resp, q) if err != nil { @@ -406,6 +409,7 @@ type CSISnapshotCreateResponse struct { // fields type CSISnapshotListRequest struct { PluginID string + Secrets CSISecrets QueryOptions } diff --git a/client/structs/csi.go b/client/structs/csi.go index d4846f7e95cc..ca7f88d41deb 100644 --- a/client/structs/csi.go +++ b/client/structs/csi.go @@ -362,6 +362,7 @@ type ClientCSIControllerListSnapshotsRequest struct { // not Nomad's own fields, for clarity when mapping between the two RPCs MaxEntries int32 StartingToken string + Secrets structs.CSISecrets CSIControllerQuery } @@ -370,6 +371,7 @@ func (req *ClientCSIControllerListSnapshotsRequest) ToCSIRequest() *csi.Controll return &csi.ControllerListSnapshotsRequest{ MaxEntries: req.MaxEntries, StartingToken: req.StartingToken, + Secrets: req.Secrets, } } diff --git a/command/agent/csi_endpoint.go b/command/agent/csi_endpoint.go index 9184bfe0794e..fb8c1a6c0cbb 100644 --- a/command/agent/csi_endpoint.go +++ b/command/agent/csi_endpoint.go @@ -333,6 +333,14 @@ func (s *HTTPServer) csiSnapshotList(resp http.ResponseWriter, req *http.Request query := req.URL.Query() args.PluginID = query.Get("plugin_id") + secrets := query["secrets"] + args.Secrets = make(structs.CSISecrets) + for _, raw := range secrets { + secret := strings.Split(raw, "=") + if len(secret) == 2 { + args.Secrets[secret[0]] = secret[1] + } + } var out structs.CSISnapshotListResponse if err := s.agent.RPC("CSIVolume.ListSnapshots", &args, &out); err != nil { diff --git a/command/volume_snapshot_list.go b/command/volume_snapshot_list.go index 65ea484ca279..b88bdd838adb 100644 --- a/command/volume_snapshot_list.go +++ b/command/volume_snapshot_list.go @@ -35,6 +35,7 @@ List Options: -plugin: Display only snapshots managed by a particular plugin. By default this command will query all plugins for their snapshots. + -secrets: A set of key/value secrets to be used when listing snapshots. ` return strings.TrimSpace(helpText) } @@ -68,11 +69,13 @@ func (c *VolumeSnapshotListCommand) Name() string { return "volume snapshot list func (c *VolumeSnapshotListCommand) Run(args []string) int { var pluginID string var verbose bool + var secrets string flags := c.Meta.FlagSet(c.Name(), FlagSetClient) flags.Usage = func() { c.Ui.Output(c.Help()) } flags.StringVar(&pluginID, "plugin", "", "") flags.BoolVar(&verbose, "verbose", false, "") + flags.StringVar(&secrets, "secrets", "", "") if err := flags.Parse(args); err != nil { c.Ui.Error(fmt.Sprintf("Error parsing arguments %s", err)) @@ -121,7 +124,7 @@ func (c *VolumeSnapshotListCommand) Run(args []string) int { q := &api.QueryOptions{PerPage: 30} // TODO: tune page size for { - resp, _, err := client.CSIVolumes().ListSnapshots(pluginID, q) + resp, _, err := client.CSIVolumes().ListSnapshots(pluginID, secrets, q) if err != nil && !errors.Is(err, io.EOF) { c.Ui.Error(fmt.Sprintf( "Error querying CSI external snapshots for plugin %q: %s", pluginID, err)) diff --git a/nomad/csi_endpoint.go b/nomad/csi_endpoint.go index 30a223cc33a9..eb6954eed2f1 100644 --- a/nomad/csi_endpoint.go +++ b/nomad/csi_endpoint.go @@ -1247,6 +1247,7 @@ func (v *CSIVolume) ListSnapshots(args *structs.CSISnapshotListRequest, reply *s cReq := &cstructs.ClientCSIControllerListSnapshotsRequest{ MaxEntries: args.PerPage, StartingToken: args.NextToken, + Secrets: args.Secrets, } cReq.PluginID = plugin.ID cResp := &cstructs.ClientCSIControllerListSnapshotsResponse{} diff --git a/nomad/structs/csi.go b/nomad/structs/csi.go index 1c7845c0a61e..e0f35aeaca36 100644 --- a/nomad/structs/csi.go +++ b/nomad/structs/csi.go @@ -897,6 +897,7 @@ type CSISnapshotDeleteResponse struct { // fields type CSISnapshotListRequest struct { PluginID string + Secrets CSISecrets QueryOptions } diff --git a/plugins/csi/plugin.go b/plugins/csi/plugin.go index 7019127c54e7..6c85db7a8e09 100644 --- a/plugins/csi/plugin.go +++ b/plugins/csi/plugin.go @@ -756,12 +756,14 @@ func (r *ControllerDeleteSnapshotRequest) Validate() error { type ControllerListSnapshotsRequest struct { MaxEntries int32 StartingToken string + Secrets structs.CSISecrets } func (r *ControllerListSnapshotsRequest) ToCSIRepresentation() *csipbv1.ListSnapshotsRequest { return &csipbv1.ListSnapshotsRequest{ MaxEntries: r.MaxEntries, StartingToken: r.StartingToken, + Secrets: r.Secrets, } } diff --git a/vendor/github.com/hashicorp/nomad/api/csi.go b/vendor/github.com/hashicorp/nomad/api/csi.go new file mode 100644 index 000000000000..120c239fde8f --- /dev/null +++ b/vendor/github.com/hashicorp/nomad/api/csi.go @@ -0,0 +1,497 @@ +package api + +import ( + "fmt" + "net/url" + "sort" + "time" +) + +// CSIVolumes is used to access Container Storage Interface (CSI) endpoints. +type CSIVolumes struct { + client *Client +} + +// CSIVolumes returns a handle on the CSIVolumes endpoint. +func (c *Client) CSIVolumes() *CSIVolumes { + return &CSIVolumes{client: c} +} + +// List returns all CSI volumes. +func (v *CSIVolumes) List(q *QueryOptions) ([]*CSIVolumeListStub, *QueryMeta, error) { + var resp []*CSIVolumeListStub + qm, err := v.client.query("/v1/volumes?type=csi", &resp, q) + if err != nil { + return nil, nil, err + } + sort.Sort(CSIVolumeIndexSort(resp)) + return resp, qm, nil +} + +// ListExternal returns all CSI volumes, as understood by the external storage +// provider. These volumes may or may not be currently registered with Nomad. +// The response is paginated by the plugin and accepts the +// QueryOptions.PerPage and QueryOptions.NextToken fields. +func (v *CSIVolumes) ListExternal(pluginID string, q *QueryOptions) (*CSIVolumeListExternalResponse, *QueryMeta, error) { + var resp *CSIVolumeListExternalResponse + + qp := url.Values{} + qp.Set("plugin_id", pluginID) + if q.NextToken != "" { + qp.Set("next_token", q.NextToken) + } + if q.PerPage != 0 { + qp.Set("per_page", fmt.Sprint(q.PerPage)) + } + + qm, err := v.client.query("/v1/volumes/external?"+qp.Encode(), &resp, q) + if err != nil { + return nil, nil, err + } + + sort.Sort(CSIVolumeExternalStubSort(resp.Volumes)) + return resp, qm, nil +} + +// PluginList returns all CSI volumes for the specified plugin id +func (v *CSIVolumes) PluginList(pluginID string) ([]*CSIVolumeListStub, *QueryMeta, error) { + return v.List(&QueryOptions{Prefix: pluginID}) +} + +// Info is used to retrieve a single CSIVolume +func (v *CSIVolumes) Info(id string, q *QueryOptions) (*CSIVolume, *QueryMeta, error) { + var resp CSIVolume + qm, err := v.client.query("/v1/volume/csi/"+id, &resp, q) + if err != nil { + return nil, nil, err + } + + return &resp, qm, nil +} + +// Register registers a single CSIVolume with Nomad. The volume must already +// exist in the external storage provider. +func (v *CSIVolumes) Register(vol *CSIVolume, w *WriteOptions) (*WriteMeta, error) { + req := CSIVolumeRegisterRequest{ + Volumes: []*CSIVolume{vol}, + } + meta, err := v.client.write("/v1/volume/csi/"+vol.ID, req, nil, w) + return meta, err +} + +// Deregister deregisters a single CSIVolume from Nomad. The volume will not be deleted from the external storage provider. +func (v *CSIVolumes) Deregister(id string, force bool, w *WriteOptions) error { + _, err := v.client.delete(fmt.Sprintf("/v1/volume/csi/%v?force=%t", url.PathEscape(id), force), nil, w) + return err +} + +// Create creates a single CSIVolume in an external storage provider and +// registers it with Nomad. You do not need to call Register if this call is +// successful. +func (v *CSIVolumes) Create(vol *CSIVolume, w *WriteOptions) ([]*CSIVolume, *WriteMeta, error) { + req := CSIVolumeCreateRequest{ + Volumes: []*CSIVolume{vol}, + } + + resp := &CSIVolumeCreateResponse{} + meta, err := v.client.write(fmt.Sprintf("/v1/volume/csi/%v/create", vol.ID), req, resp, w) + return resp.Volumes, meta, err +} + +// Delete deletes a CSI volume from an external storage provider. The ID +// passed as an argument here is for the storage provider's ID, so a volume +// that's already been deregistered can be deleted. +func (v *CSIVolumes) Delete(externalVolID string, w *WriteOptions) error { + _, err := v.client.delete(fmt.Sprintf("/v1/volume/csi/%v/delete", url.PathEscape(externalVolID)), nil, w) + return err +} + +// Detach causes Nomad to attempt to detach a CSI volume from a client +// node. This is used in the case that the node is temporarily lost and the +// allocations are unable to drop their claims automatically. +func (v *CSIVolumes) Detach(volID, nodeID string, w *WriteOptions) error { + _, err := v.client.delete(fmt.Sprintf("/v1/volume/csi/%v/detach?node=%v", url.PathEscape(volID), nodeID), nil, w) + return err +} + +// CreateSnapshot snapshots an external storage volume. +func (v *CSIVolumes) CreateSnapshot(snap *CSISnapshot, w *WriteOptions) (*CSISnapshotCreateResponse, *WriteMeta, error) { + req := &CSISnapshotCreateRequest{ + Snapshots: []*CSISnapshot{snap}, + } + resp := &CSISnapshotCreateResponse{} + meta, err := v.client.write("/v1/volumes/snapshot", req, resp, w) + return resp, meta, err +} + +// DeleteSnapshot deletes an external storage volume snapshot. +func (v *CSIVolumes) DeleteSnapshot(snap *CSISnapshot, w *WriteOptions) error { + qp := url.Values{} + qp.Set("snapshot_id", snap.ID) + qp.Set("plugin_id", snap.PluginID) + for k, v := range snap.Secrets { + qp.Set("secret", fmt.Sprintf("%v=%v", k, v)) + } + _, err := v.client.delete("/v1/volumes/snapshot?"+qp.Encode(), nil, w) + return err +} + +// ListSnapshots lists external storage volume snapshots. +func (v *CSIVolumes) ListSnapshots(pluginID string, secrets string, q *QueryOptions) (*CSISnapshotListResponse, *QueryMeta, error) { + var resp *CSISnapshotListResponse + + qp := url.Values{} + if pluginID != "" { + qp.Set("plugin_id", pluginID) + } + if q.NextToken != "" { + qp.Set("next_token", q.NextToken) + } + if q.PerPage != 0 { + qp.Set("per_page", fmt.Sprint(q.PerPage)) + } + if secrets != "" { + qp.Set("secrets", secrets) + } + + qm, err := v.client.query("/v1/volumes/snapshot?"+qp.Encode(), &resp, q) + if err != nil { + return nil, nil, err + } + + sort.Sort(CSISnapshotSort(resp.Snapshots)) + return resp, qm, nil +} + +// CSIVolumeAttachmentMode chooses the type of storage api that will be used to +// interact with the device. (Duplicated in nomad/structs/csi.go) +type CSIVolumeAttachmentMode string + +const ( + CSIVolumeAttachmentModeUnknown CSIVolumeAttachmentMode = "" + CSIVolumeAttachmentModeBlockDevice CSIVolumeAttachmentMode = "block-device" + CSIVolumeAttachmentModeFilesystem CSIVolumeAttachmentMode = "file-system" +) + +// CSIVolumeAccessMode indicates how a volume should be used in a storage topology +// e.g whether the provider should make the volume available concurrently. (Duplicated in nomad/structs/csi.go) +type CSIVolumeAccessMode string + +const ( + CSIVolumeAccessModeUnknown CSIVolumeAccessMode = "" + CSIVolumeAccessModeSingleNodeReader CSIVolumeAccessMode = "single-node-reader-only" + CSIVolumeAccessModeSingleNodeWriter CSIVolumeAccessMode = "single-node-writer" + CSIVolumeAccessModeMultiNodeReader CSIVolumeAccessMode = "multi-node-reader-only" + CSIVolumeAccessModeMultiNodeSingleWriter CSIVolumeAccessMode = "multi-node-single-writer" + CSIVolumeAccessModeMultiNodeMultiWriter CSIVolumeAccessMode = "multi-node-multi-writer" +) + +// CSIMountOptions contain optional additional configuration that can be used +// when specifying that a Volume should be used with VolumeAccessTypeMount. +type CSIMountOptions struct { + // FSType is an optional field that allows an operator to specify the type + // of the filesystem. + FSType string `hcl:"fs_type,optional"` + + // MountFlags contains additional options that may be used when mounting the + // volume by the plugin. This may contain sensitive data and should not be + // leaked. + MountFlags []string `hcl:"mount_flags,optional"` + + ExtraKeysHCL []string `hcl1:",unusedKeys" json:"-"` // report unexpected keys +} + +// CSISecrets contain optional additional credentials that may be needed by +// the storage provider. These values will be redacted when reported in the +// API or in Nomad's logs. +type CSISecrets map[string]string + +// CSIVolume is used for serialization, see also nomad/structs/csi.go +type CSIVolume struct { + ID string + Name string + ExternalID string `mapstructure:"external_id" hcl:"external_id"` + Namespace string + Topologies []*CSITopology + AccessMode CSIVolumeAccessMode `hcl:"access_mode"` + AttachmentMode CSIVolumeAttachmentMode `hcl:"attachment_mode"` + MountOptions *CSIMountOptions `hcl:"mount_options"` + Secrets CSISecrets `mapstructure:"secrets" hcl:"secrets"` + Parameters map[string]string `mapstructure:"parameters" hcl:"parameters"` + Context map[string]string `mapstructure:"context" hcl:"context"` + Capacity int64 `hcl:"-"` + + // These fields are used as part of the volume creation request + RequestedCapacityMin int64 `hcl:"capacity_min"` + RequestedCapacityMax int64 `hcl:"capacity_max"` + RequestedCapabilities []*CSIVolumeCapability `hcl:"capability"` + CloneID string `mapstructure:"clone_id" hcl:"clone_id"` + SnapshotID string `mapstructure:"snapshot_id" hcl:"snapshot_id"` + + // ReadAllocs is a map of allocation IDs for tracking reader claim status. + // The Allocation value will always be nil; clients can populate this data + // by iterating over the Allocations field. + ReadAllocs map[string]*Allocation + + // WriteAllocs is a map of allocation IDs for tracking writer claim + // status. The Allocation value will always be nil; clients can populate + // this data by iterating over the Allocations field. + WriteAllocs map[string]*Allocation + + // Allocations is a combined list of readers and writers + Allocations []*AllocationListStub + + // Schedulable is true if all the denormalized plugin health fields are true + Schedulable bool + PluginID string `mapstructure:"plugin_id" hcl:"plugin_id"` + Provider string + ProviderVersion string + ControllerRequired bool + ControllersHealthy int + ControllersExpected int + NodesHealthy int + NodesExpected int + ResourceExhausted time.Time + + CreateIndex uint64 + ModifyIndex uint64 + + // ExtraKeysHCL is used by the hcl parser to report unexpected keys + ExtraKeysHCL []string `hcl1:",unusedKeys" json:"-"` +} + +// CSIVolumeCapability is a requested attachment and access mode for a +// volume +type CSIVolumeCapability struct { + AccessMode CSIVolumeAccessMode `mapstructure:"access_mode" hcl:"access_mode"` + AttachmentMode CSIVolumeAttachmentMode `mapstructure:"attachment_mode" hcl:"attachment_mode"` +} + +// CSIVolumeIndexSort is a helper used for sorting volume stubs by creation +// time. +type CSIVolumeIndexSort []*CSIVolumeListStub + +func (v CSIVolumeIndexSort) Len() int { + return len(v) +} + +func (v CSIVolumeIndexSort) Less(i, j int) bool { + return v[i].CreateIndex > v[j].CreateIndex +} + +func (v CSIVolumeIndexSort) Swap(i, j int) { + v[i], v[j] = v[j], v[i] +} + +// CSIVolumeListStub omits allocations. See also nomad/structs/csi.go +type CSIVolumeListStub struct { + ID string + Namespace string + Name string + ExternalID string + Topologies []*CSITopology + AccessMode CSIVolumeAccessMode + AttachmentMode CSIVolumeAttachmentMode + Schedulable bool + PluginID string + Provider string + ControllerRequired bool + ControllersHealthy int + ControllersExpected int + NodesHealthy int + NodesExpected int + ResourceExhausted time.Time + + CreateIndex uint64 + ModifyIndex uint64 +} + +type CSIVolumeListExternalResponse struct { + Volumes []*CSIVolumeExternalStub + NextToken string +} + +// CSIVolumeExternalStub is the storage provider's view of a volume, as +// returned from the controller plugin; all IDs are for external resources +type CSIVolumeExternalStub struct { + ExternalID string + CapacityBytes int64 + VolumeContext map[string]string + CloneID string + SnapshotID string + PublishedExternalNodeIDs []string + IsAbnormal bool + Status string +} + +// CSIVolumeExternalStubSort is a sorting helper for external volumes. We +// can't sort these by creation time because we don't get that data back from +// the storage provider. Sort by External ID within this page. +type CSIVolumeExternalStubSort []*CSIVolumeExternalStub + +func (v CSIVolumeExternalStubSort) Len() int { + return len(v) +} + +func (v CSIVolumeExternalStubSort) Less(i, j int) bool { + return v[i].ExternalID > v[j].ExternalID +} + +func (v CSIVolumeExternalStubSort) Swap(i, j int) { + v[i], v[j] = v[j], v[i] +} + +type CSIVolumeCreateRequest struct { + Volumes []*CSIVolume + WriteRequest +} + +type CSIVolumeCreateResponse struct { + Volumes []*CSIVolume + QueryMeta +} + +type CSIVolumeRegisterRequest struct { + Volumes []*CSIVolume + WriteRequest +} + +type CSIVolumeDeregisterRequest struct { + VolumeIDs []string + WriteRequest +} + +// CSISnapshot is the storage provider's view of a volume snapshot +type CSISnapshot struct { + ID string // storage provider's ID + ExternalSourceVolumeID string // storage provider's ID for volume + SizeBytes int64 // value from storage provider + CreateTime int64 // value from storage provider + IsReady bool // value from storage provider + SourceVolumeID string // Nomad volume ID + PluginID string // CSI plugin ID + + // These field are only used during snapshot creation and will not be + // populated when the snapshot is returned + Name string // suggested name of the snapshot, used for creation + Secrets CSISecrets // secrets needed to create snapshot + Parameters map[string]string // secrets needed to create snapshot +} + +// CSISnapshotSort is a helper used for sorting snapshots by creation time. +type CSISnapshotSort []*CSISnapshot + +func (v CSISnapshotSort) Len() int { + return len(v) +} + +func (v CSISnapshotSort) Less(i, j int) bool { + return v[i].CreateTime > v[j].CreateTime +} + +func (v CSISnapshotSort) Swap(i, j int) { + v[i], v[j] = v[j], v[i] +} + +type CSISnapshotCreateRequest struct { + Snapshots []*CSISnapshot + WriteRequest +} + +type CSISnapshotCreateResponse struct { + Snapshots []*CSISnapshot + QueryMeta +} + +// CSISnapshotListRequest is a request to a controller plugin to list all the +// snapshot known to the the storage provider. This request is paginated by +// the plugin and accepts the QueryOptions.PerPage and QueryOptions.NextToken +// fields +type CSISnapshotListRequest struct { + PluginID string + Secrets CSISecrets + QueryOptions +} + +type CSISnapshotListResponse struct { + Snapshots []*CSISnapshot + NextToken string + QueryMeta +} + +// CSI Plugins are jobs with plugin specific data +type CSIPlugins struct { + client *Client +} + +// CSIPlugin is used for serialization, see also nomad/structs/csi.go +type CSIPlugin struct { + ID string + Provider string + Version string + ControllerRequired bool + // Map Node.ID to CSIInfo fingerprint results + Controllers map[string]*CSIInfo + Nodes map[string]*CSIInfo + Allocations []*AllocationListStub + ControllersHealthy int + ControllersExpected int + NodesHealthy int + NodesExpected int + CreateIndex uint64 + ModifyIndex uint64 +} + +type CSIPluginListStub struct { + ID string + Provider string + ControllerRequired bool + ControllersHealthy int + ControllersExpected int + NodesHealthy int + NodesExpected int + CreateIndex uint64 + ModifyIndex uint64 +} + +// CSIPluginIndexSort is a helper used for sorting plugin stubs by creation +// time. +type CSIPluginIndexSort []*CSIPluginListStub + +func (v CSIPluginIndexSort) Len() int { + return len(v) +} + +func (v CSIPluginIndexSort) Less(i, j int) bool { + return v[i].CreateIndex > v[j].CreateIndex +} + +func (v CSIPluginIndexSort) Swap(i, j int) { + v[i], v[j] = v[j], v[i] +} + +// CSIPlugins returns a handle on the CSIPlugins endpoint +func (c *Client) CSIPlugins() *CSIPlugins { + return &CSIPlugins{client: c} +} + +// List returns all CSI plugins +func (v *CSIPlugins) List(q *QueryOptions) ([]*CSIPluginListStub, *QueryMeta, error) { + var resp []*CSIPluginListStub + qm, err := v.client.query("/v1/plugins?type=csi", &resp, q) + if err != nil { + return nil, nil, err + } + sort.Sort(CSIPluginIndexSort(resp)) + return resp, qm, nil +} + +// Info is used to retrieve a single CSI Plugin Job +func (v *CSIPlugins) Info(id string, q *QueryOptions) (*CSIPlugin, *QueryMeta, error) { + var resp *CSIPlugin + qm, err := v.client.query("/v1/plugin/csi/"+id, &resp, q) + if err != nil { + return nil, nil, err + } + return resp, qm, nil +}