consul: handle "not found" errors from Consul when deleting tokens (#…

…17847)

In Consul 1.15.0, the Delete Token API was changed so as to return an error when
deleting a non-existent ACL token. This means that if Nomad successfully deletes
the token but fails to persist that fact, it will get stuck trying to delete a
non-existent token forever.

Update the token deletion function to ignore "not found" errors and treat them
as successful deletions.

Fixes: #17833

.changelog/17847.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+consul: Fixed a bug where Nomad would repeatedly try to revoke successfully revoked SI tokens
+```

nomad/consul.go

@@ -419,8 +419,10 @@ func (c *consulACLsAPI) singleRevoke(ctx context.Context, accessor *structs.SITo
 		return err
 	}
 
-	// Consul will no-op the deletion of a non-existent token (no error)
 	_, err := c.aclClient.TokenDelete(accessor.AccessorID, &api.WriteOptions{Namespace: accessor.ConsulNamespace})
+	if err != nil && strings.Contains(err.Error(), "Cannot find token to delete") {
+		return nil // Consul will error when deleting a non-existent token
+	}
 	return err
 }