csi: allow for volume detach to work with gc'd nodes (#9057)

When we try to prefix match the `nomad volume detach` node ID argument, the
node may have been already GC'd. The volume unpublish workflow gracefully
handles this case so that we can free the claim. So make a best effort to find
a node ID among the volume's claimed allocations, or otherwise just use the
node ID we've been given by the user as-is.

command/volume_detach.go

@@ -93,18 +93,44 @@ func (c *VolumeDetachCommand) Run(args []string) int {
 		c.Ui.Error(fmt.Sprintf("Error detaching volume: %s", err))
 		return 1
 	}
-	// Return error if no nodes are found
-	if len(nodes) == 0 {
-		c.Ui.Error(fmt.Sprintf("No node(s) with prefix or id %q found", nodeID))
-		return 1
-	}
+
 	if len(nodes) > 1 {
 		c.Ui.Error(fmt.Sprintf("Prefix matched multiple nodes\n\n%s",
 			formatNodeStubList(nodes, true)))
 		return 1
 	}
 
-	err = client.CSIVolumes().Detach(volID, nodes[0].ID, nil)
+	if len(nodes) == 1 {
+		nodeID = nodes[0].ID
+	}
+
+	// If the Nodes.PrefixList doesn't return a node, the node may have been
+	// GC'd. The unpublish workflow gracefully handles this case so that we
+	// can free the claim. Make a best effort to find a node ID among the
+	// volume's claimed allocations, otherwise just use the node ID we've been
+	// given.
+	if len(nodes) == 0 {
+		vol, _, err := client.CSIVolumes().Info(volID, nil)
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("Error querying volume: %s", err))
+			return 1
+		}
+		nodeIDs := []string{}
+		for _, alloc := range vol.Allocations {
+			if strings.HasPrefix(alloc.NodeID, nodeID) {
+				nodeIDs = append(nodeIDs, alloc.NodeID)
+			}
+		}
+		if len(nodeIDs) > 1 {
+			c.Ui.Error(fmt.Sprintf("Prefix matched multiple node IDs\n\n%s",
+				formatList(nodeIDs)))
+		}
+		if len(nodeIDs) == 1 {
+			nodeID = nodeIDs[0]
+		}
+	}
+
+	err = client.CSIVolumes().Detach(volID, nodeID, nil)
 	if err != nil {
 		c.Ui.Error(fmt.Sprintf("Error detaching volume: %s", err))
 		return 1

website/pages/docs/commands/volume/detach.mdx

@@ -21,6 +21,10 @@ The `volume detach` command requires two arguments, specifying the ID of the
 volume to be detached and the node to detach it from. Detaching will fail if
 the volume is still in use by an allocation.
 
+Note that you can use a node ID prefix just as you can with other Nomad
+commands, but if the node has been garbage collected, you may need to pass the
+full node ID.
+
 ## General Options
 
 @include 'general_options.mdx'