feat(kubernetes): support label selectors in deploy manifest stages (s…

…pinnaker#6220)

* feat(kubernetes): support label selectors in deploy manifest stages

via a new labelSelectors pipeline configuration property.  The syntax is the same as what's currently implemented in delete manifest stages.  For example:

{
  "labelSelectors": {
    "selectors": [
      {
        "kind": "EQUALS",
        "key": "my-label-key",
        "values": [
          "my-value"
        ],
      }
    ]
  }
}

See https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ and
KubernetesSelector for more.  Multiple selectors combine with AND (i.e. must all be
satisfied).

Note that kubectl replace doesn't support label selectors, so
KubernetesDeployManifestOperation throws an exception if a deploy manifest stage that
specifies (non-empty) label selectors has a manifest with a strategy.spinnaker.io/replace:
"true" annotation.  Although it's possible to implement the label selector logic in
clouddriver, this PR explicitly avoids that, and leaves the label selector logic to
kubectl.

It's possible that none of the manifests may satisfy the label selectors.  In that case, a
new pipeline configuration property named allowNothingSelected determines the behavior.
If false (the default), KubernetesDeployManifestOperation throws an exception.  If true,
the operation succeeds even though nothing was deployed.

closes spinnaker/spinnaker#3695.

* test(kubernetes): integration test of deploy manifest with label selectors

---------

Co-authored-by: Chuck Lane <clane@salesforce.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

clouddriver-kubernetes/src/integration/java/com/netflix/spinnaker/clouddriver/kubernetes/it/DeployManifestIT.java

@@ -17,6 +17,7 @@
 package com.netflix.spinnaker.clouddriver.kubernetes.it;
 
 import static io.restassured.RestAssured.given;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasSize;
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -95,6 +96,48 @@ public void shouldDeployManifestFromText() throws IOException, InterruptedExcept
         "Expected one ready pod for " + DEPLOYMENT_1_NAME + " deployment. Pods:\n" + pods);
   }
 
+  @DisplayName(
+      ".\n===\n"
+          + "Given mutiple manifests\n"
+          + "  where only one satisfies the given label selector\n"
+          + "When sending deploy manifest request\n"
+          + "  And waiting on manifest stable\n"
+          + "Then only one manifest has been deployed\n===")
+  @Test
+  public void labelSelectors() throws IOException, InterruptedException {
+    // ------------------------- given --------------------------
+    String appName = "deploy-from-text";
+    List<Map<String, Object>> manifest =
+        KubeTestUtils.loadYaml("classpath:manifests/configmaps_with_selectors.yml")
+            .withValue("metadata.namespace", account1Ns)
+            .asList();
+    Map<String, Object> labelSelectors =
+        Map.of(
+            "selectors",
+            List.of(
+                Map.of(
+                    "kind", "EQUALS",
+                    "key", "sample-configmap-selector",
+                    "values", List.of("one"))));
+    System.out.println("> Using namespace: " + account1Ns + ", appName: " + appName);
+
+    // ------------------------- when --------------------------
+    List<Map<String, Object>> body =
+        KubeTestUtils.loadJson("classpath:requests/deploy_manifest.json")
+            .withValue("deployManifest.labelSelectors", labelSelectors)
+            .withValue("deployManifest.account", ACCOUNT1_NAME)
+            .withValue("deployManifest.moniker.app", appName)
+            .withValue("deployManifest.manifests", manifest)
+            .asList();
+    KubeTestUtils.deployAndWaitStable(
+        baseUrl(), body, account1Ns, "configMap sample-config-map-with-selector-one-v000");
+
+    // ------------------------- then --------------------------
+    String configMaps =
+        kubeCluster.execKubectl("-n " + account1Ns + " get configmap -lselector-test=test -o name");
+    assertThat(configMaps).hasLineCount(1);
+  }
+
   @DisplayName(
       ".\n===\n"
           + "Given a deployment manifest with default namespace set\n"

clouddriver-kubernetes/src/integration/resources/manifests/configmaps_with_selectors.yml

@@ -0,0 +1,25 @@
+apiVersion: v1
+data:
+  samplefile.yaml: |-
+    settings:
+      enabled: true
+kind: ConfigMap
+metadata:
+  labels:
+    sample-configmap-selector: one
+    selector-test: test
+  name: sample-config-map-with-selector-one
+  namespace: default
+---
+apiVersion: v1
+data:
+  samplefile2.yaml: |-
+    more-settings:
+      enabled: false
+kind: ConfigMap
+metadata:
+  labels:
+    sample-configmap-selector: two
+    selector-test: test
+  name: sample-config-map-with-selector-two
+  namespace: default

clouddriver-kubernetes/src/main/java/com/netflix/spinnaker/clouddriver/kubernetes/description/manifest/KubernetesDeployManifestDescription.java

@@ -18,6 +18,7 @@
 package com.netflix.spinnaker.clouddriver.kubernetes.description.manifest;
 
 import com.netflix.spinnaker.clouddriver.kubernetes.description.KubernetesAtomicOperationDescription;
+import com.netflix.spinnaker.clouddriver.kubernetes.security.KubernetesSelectorList;
 import com.netflix.spinnaker.kork.artifacts.model.Artifact;
 import com.netflix.spinnaker.moniker.Moniker;
 import java.util.List;
@@ -41,6 +42,14 @@ public class KubernetesDeployManifestDescription extends KubernetesAtomicOperati
   private boolean enableTraffic = true;
   private List<String> services;
   private Strategy strategy;
+  private KubernetesSelectorList labelSelectors = new KubernetesSelectorList();
+
+  /**
+   * If false, and using (non-empty) label selectors, fail if a deploy manifest operation doesn't
+   * deploy anything. If a particular deploy manifest stage intentionally specifies label selectors
+   * that none of the resources satisfy, set this to true to allow the stage to succeed.
+   */
+  private boolean allowNothingSelected = false;
 
   public boolean isBlueGreen() {
     return Strategy.RED_BLACK.equals(this.strategy) || Strategy.BLUE_GREEN.equals(this.strategy);

clouddriver-kubernetes/src/main/java/com/netflix/spinnaker/clouddriver/kubernetes/op/handler/CanDeploy.java

@@ -35,11 +35,12 @@ default OperationResult deploy(
       KubernetesManifestStrategy.DeployStrategy deployStrategy,
       KubernetesManifestStrategy.ServerSideApplyStrategy serverSideApplyStrategy,
       Task task,
-      String opName) {
+      String opName,
+      KubernetesSelectorList labelSelectors) {
     // If the manifest has a generateName, we must apply with kubectl create as all other operations
     // require looking up a manifest by name, which will fail.
     if (manifest.hasGenerateName()) {
-      KubernetesManifest result = credentials.create(manifest, task, opName);
+      KubernetesManifest result = credentials.create(manifest, task, opName, labelSelectors);
       return new OperationResult().addManifest(result);
     }
 
@@ -51,13 +52,13 @@ default OperationResult deploy(
               manifest.getKind(),
               manifest.getNamespace(),
               manifest.getName(),
-              new KubernetesSelectorList(),
+              labelSelectors,
               new V1DeleteOptions(),
               task,
               opName);
         } catch (KubectlJobExecutor.KubectlException ignored) {
         }
-        deployedManifest = credentials.deploy(manifest, task, opName);
+        deployedManifest = credentials.deploy(manifest, task, opName, labelSelectors);
         break;
       case REPLACE:
         deployedManifest = credentials.createOrReplace(manifest, task, opName);
@@ -70,14 +71,23 @@ default OperationResult deploy(
           cmdArgs.add("--force-conflicts=true");
         }
         deployedManifest =
-            credentials.deploy(manifest, task, opName, cmdArgs.toArray(new String[cmdArgs.size()]));
+            credentials.deploy(
+                manifest,
+                task,
+                opName,
+                labelSelectors,
+                cmdArgs.toArray(new String[cmdArgs.size()]));
         break;
       case APPLY:
-        deployedManifest = credentials.deploy(manifest, task, opName);
+        deployedManifest = credentials.deploy(manifest, task, opName, labelSelectors);
         break;
       default:
         throw new AssertionError(String.format("Unknown deploy strategy: %s", deployStrategy));
     }
-    return new OperationResult().addManifest(deployedManifest);
+    OperationResult operationResult = new OperationResult();
+    if (deployedManifest != null) {
+      operationResult.addManifest(deployedManifest);
+    }
+    return operationResult;
   }
 }

clouddriver-kubernetes/src/main/java/com/netflix/spinnaker/clouddriver/kubernetes/op/job/KubectlJobExecutor.java

@@ -66,6 +66,11 @@
 public class KubectlJobExecutor {
   private static final Logger log = LoggerFactory.getLogger(KubectlJobExecutor.class);
   private static final String NOT_FOUND_STRING = "(NotFound)";
+  private static final String NO_OBJECTS_PASSED_TO_STRING = "error: no objects passed to";
+  private static final String NO_OBJECTS_PASSED_TO_APPLY_STRING =
+      NO_OBJECTS_PASSED_TO_STRING + " apply";
+  private static final String NO_OBJECTS_PASSED_TO_CREATE_STRING =
+      NO_OBJECTS_PASSED_TO_STRING + " create";
   private static final String KUBECTL_COMMAND_OPTION_TOKEN = "--token=";
   private static final String KUBECTL_COMMAND_OPTION_KUBECONFIG = "--kubeconfig=";
   private static final String KUBECTL_COMMAND_OPTION_CONTEXT = "--context=";
@@ -522,11 +527,23 @@ public ImmutableList<KubernetesManifest> list(
     return status.getOutput();
   }
 
+  /**
+   * Invoke kubectl apply with the given manifest and (if present) label selectors.
+   *
+   * @param credentials k8s account credentials
+   * @param manifest the manifest to apply
+   * @param task the task performing this kubectl invocation
+   * @param opName the name of the operation performing this kubectl invocation
+   * @param labelSelectors label selectors
+   * @return the manifest parsed from stdout of the kubectl invocation, or null if a label selector
+   *     is present and kubectl returned "no objects passed to apply"
+   */
   public KubernetesManifest deploy(
       KubernetesCredentials credentials,
       KubernetesManifest manifest,
       Task task,
       String opName,
+      KubernetesSelectorList labelSelectors,
       String... cmdArgs) {
     log.info("Deploying manifest {}", manifest.getFullResourceName());
     List<String> command = kubectlAuthPrefix(credentials);
@@ -538,12 +555,22 @@ public KubernetesManifest deploy(
     command.add("json");
     command.add("-f");
     command.add("-");
+    addLabelSelectors(command, labelSelectors);
 
     JobResult<String> status = executeKubectlCommand(credentials, command, Optional.of(manifest));
 
     persistKubectlJobOutput(credentials, status, manifest.getFullResourceName(), task, opName);
 
     if (status.getResult() != JobResult.Result.SUCCESS) {
+      // If the caller provided a label selector, kubectl returns "no objects
+      // passed to apply" if none of the given objects satisfy the selector.
+      // Instead of throwing an exception, leave it to higher level logic to
+      // decide how to behave.
+      if (labelSelectors.isNotEmpty()
+          && status.getError().contains(NO_OBJECTS_PASSED_TO_APPLY_STRING)) {
+        return null;
+      }
+
       throw new KubectlException(
           "Deploy failed for manifest: "
               + manifest.getFullResourceName()
@@ -554,6 +581,16 @@ public KubernetesManifest deploy(
     return getKubernetesManifestFromJobResult(status, manifest);
   }
 
+  /**
+   * Invoke kubectl replace with the given manifest. Note that kubectl replace doesn't support label
+   * selectors.
+   *
+   * @param credentials k8s account credentials
+   * @param manifest the manifest to replace
+   * @param task the task performing this kubectl invocation
+   * @param opName the name of the operation performing this kubectl invocation
+   * @return the manifest parsed from stdout of the kubectl invocation
+   */
   public KubernetesManifest replace(
       KubernetesCredentials credentials, KubernetesManifest manifest, Task task, String opName) {
     log.info("Replacing manifest {}", manifest.getFullResourceName());
@@ -588,8 +625,23 @@ public KubernetesManifest replace(
     return getKubernetesManifestFromJobResult(status, manifest);
   }
 
+  /**
+   * Invoke kubectl create with the given manifest and (if present) label selectors.
+   *
+   * @param credentials k8s account credentials
+   * @param manifest the manifest to create
+   * @param task the task performing this kubectl invocation
+   * @param opName the name of the operation performing this kubectl invocation
+   * @param labelSelectors label selectors
+   * @return the manifest parsed from stdout of the kubectl invocation, or null if a label selector
+   *     is present and kubectl returned "no objects passed to create"
+   */
   public KubernetesManifest create(
-      KubernetesCredentials credentials, KubernetesManifest manifest, Task task, String opName) {
+      KubernetesCredentials credentials,
+      KubernetesManifest manifest,
+      Task task,
+      String opName,
+      KubernetesSelectorList labelSelectors) {
     log.info("Creating manifest {}", manifest.getFullResourceName());
     List<String> command = kubectlAuthPrefix(credentials);
 
@@ -599,12 +651,22 @@ public KubernetesManifest create(
     command.add("json");
     command.add("-f");
     command.add("-");
+    addLabelSelectors(command, labelSelectors);
 
     JobResult<String> status = executeKubectlCommand(credentials, command, Optional.of(manifest));
 
     persistKubectlJobOutput(credentials, status, manifest.getFullResourceName(), task, opName);
 
     if (status.getResult() != JobResult.Result.SUCCESS) {
+      // If the caller provided a label selector, kubectl returns "no objects
+      // passed to create" if none of the given objects satisfy the selector.
+      // Instead of throwing an exception, leave it to higher level logic to
+      // decide how to behave.
+      if (labelSelectors.isNotEmpty()
+          && status.getError().contains(NO_OBJECTS_PASSED_TO_CREATE_STRING)) {
+        return null;
+      }
+
       throw new KubectlException(
           "Create failed for manifest: "
               + manifest.getFullResourceName()
@@ -676,10 +738,7 @@ private List<String> kubectlLookupInfo(
     } else {
       command.add(kind.toString());
     }
-
-    if (labelSelectors != null && !labelSelectors.isEmpty()) {
-      command.add("-l=" + labelSelectors);
-    }
+    addLabelSelectors(command, labelSelectors);
 
     return command;
   }
@@ -1068,6 +1127,12 @@ private void persistKubectlJobOutput(
     }
   }
 
+  private void addLabelSelectors(List<String> command, KubernetesSelectorList labelSelectors) {
+    if (labelSelectors != null && !labelSelectors.isEmpty()) {
+      command.add("-l=" + labelSelectors);
+    }
+  }
+
   public static class KubectlException extends RuntimeException {
     public KubectlException(String message) {
       super(message);

clouddriver-kubernetes/src/main/java/com/netflix/spinnaker/clouddriver/kubernetes/op/manifest/KubernetesDeployManifestOperation.java

@@ -32,6 +32,7 @@
 import com.netflix.spinnaker.clouddriver.kubernetes.op.OperationResult;
 import com.netflix.spinnaker.clouddriver.kubernetes.op.handler.*;
 import com.netflix.spinnaker.clouddriver.kubernetes.security.KubernetesCredentials;
+import com.netflix.spinnaker.clouddriver.kubernetes.security.KubernetesSelectorList;
 import com.netflix.spinnaker.clouddriver.orchestration.AtomicOperation;
 import com.netflix.spinnaker.kork.artifacts.model.Artifact;
 import com.netflix.spinnaker.moniker.Moniker;
@@ -145,6 +146,19 @@ public OperationResult operate(List<OperationResult> _unused) {
 
     checkIfArtifactsBound(result);
 
+    KubernetesSelectorList labelSelectors = this.description.getLabelSelectors();
+
+    // kubectl replace doesn't support selectors, so fail if any manifest uses
+    // the replace strategy
+    if (labelSelectors.isNotEmpty()
+        && toDeploy.stream()
+            .map((holder) -> holder.getStrategy().getDeployStrategy())
+            .anyMatch(
+                (strategy) -> strategy == KubernetesManifestStrategy.DeployStrategy.REPLACE)) {
+      throw new IllegalArgumentException(
+          "label selectors not supported with replace strategy, not deploying");
+    }
+
     toDeploy.forEach(
         holder -> {
           KubernetesResourceProperties properties = findResourceProperties(holder.manifest);
@@ -163,7 +177,8 @@ public OperationResult operate(List<OperationResult> _unused) {
                   strategy.getDeployStrategy(),
                   strategy.getServerSideApplyStrategy(),
                   getTask(),
-                  OP_NAME));
+                  OP_NAME,
+                  labelSelectors));
 
           result.getCreatedArtifacts().add(holder.artifact);
           getTask()
@@ -175,6 +190,17 @@ public OperationResult operate(List<OperationResult> _unused) {
                       + accountName);
         });
 
+    // If a label selector was specified and nothing has been deployed, throw an
+    // exception to fail the task if configured to do so.
+    if (!description.isAllowNothingSelected()
+        && labelSelectors.isNotEmpty()
+        && result.getManifests().isEmpty()) {
+      throw new IllegalStateException(
+          "nothing deployed to account "
+              + accountName
+              + " with label selector(s) "
+              + labelSelectors.toString());
+    }
     result.removeSensitiveKeys(credentials.getResourcePropertyRegistry());
 
     getTask()