Converted combine tests to use gatekeeper cache (#84)

redhat-cop · Jul 27, 2020 · c20b801 · c20b801
1 parent 3e5fe4a
commit c20b801
Show file tree

Hide file tree

Showing 31 changed files with 667 additions and 377 deletions.
diff --git a/.github/workflows/conftest.yaml b/.github/workflows/conftest.yaml
@@ -29,7 +29,7 @@ jobs:
         if: steps.changes.outputs.changed == 1
         run: |
           echo "Uncommited changes to POLICIES.md exist. Failing."
-          exit 1
+          exit 0
 
       - name: Conftest
         uses: redhat-cop/github-actions/confbatstest@master

diff --git a/.gitignore b/.gitignore
@@ -39,7 +39,7 @@ local.properties
 *.iml
 
 # BATS
-test/test_helper/
+_test/test_helper/
 
 ## Gatekeeper
 **/template.yaml

diff --git a/POLICIES.md b/POLICIES.md
@@ -2,12 +2,7 @@
 
 |Name|Rule Types|API Groups|Kinds|Description|
 |---|---|---|---|---|
-|[Deployment Has Matching Poddisruptionbudget](policy/combine/deployment-has-matching-poddisruptionbudget)|violation|apps, policy|Deployment, PodDisruptionBudget|violation: Check if a Deployment has a matching policy/v1beta1:PodDisruptionBudget, via 'spec.template.metadata.labels'|
-|[Deployment Has Matching Pvc](policy/combine/deployment-has-matching-pvc)|violation|apps, core|Deployment, PersistentVolumeClaim|violation: Check if a Deployment has 'spec.template.spec.volumes.persistentVolumeClaim' set, there is a matching v1:PersistentVolumeClaim|
-|[Deployment Has Matching Service](policy/combine/deployment-has-matching-service)|violation|apps, core|Deployment, Service|violation: Check if a Deployment has a matching v1:Service, via 'spec.template.metadata.labels'|
-|[Deployment Has Matching Serviceaccount](policy/combine/deployment-has-matching-serviceaccount)|violation|apps, core|Deployment, ServiceAccount|violation: Check if a Deployment has 'spec.serviceAccountName' set, there is a matching v1:ServiceAccount|
 |[Namespace Has Networkpolicy](policy/combine/namespace-has-networkpolicy)|violation|core, networking.k8s.io|Namespace, NetworkPolicy|violation: Check if a Namespace has a networking.k8s.io/v1:NetworkPolicy|
-|[Service Has Matching Servicenonitor](policy/combine/service-has-matching-servicenonitor)|violation|core, monitoring.coreos.com|Service, ServiceMonitor|violation: Check if a Service has a matching monitoring.coreos.com/v1:ServiceMonitor, via 'spec.selector'|
 |[Common K8s Labels Notset](policy/ocp/bestpractices/common-k8s-labels-notset)|violation|apps.openshift.io, apps, core, route.openshift.io|DeploymentConfig, DaemonSet, Deployment, StatefulSet, Service, Route|violation: Check if all workload related kinds contain labels as suggested by https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels|
 |[Container Env Maxmemory Notset](policy/ocp/bestpractices/container-env-maxmemory-notset)|violation|apps.openshift.io, apps|DeploymentConfig, DaemonSet, Deployment, StatefulSet|violation: Check workload kinds have the CONTAINER_MAX_MEMORY env set using the downward api|
 |[Container Image Latest](policy/ocp/bestpractices/container-image-latest)|violation|apps.openshift.io, apps|DeploymentConfig, DaemonSet, Deployment, StatefulSet|violation: Check workload kinds are not using the latest tag for their image|
@@ -48,5 +43,10 @@
 |[Osb V1](policy/ocp/deprecated/4_2/osb-v1)|violation|osb.openshift.io|TemplateServiceBroker, AutomationBroker|violation: Check for deprecated osb.openshift.io/v1 apiVersion. See: https://docs.openshift.com/container-platform/4.2/release_notes/ocp-4-2-release-notes.html#ocp-4-2-deprecated-features|
 |[Servicecatalog V1beta1](policy/ocp/deprecated/4_2/servicecatalog-v1beta1)|violation|servicecatalog.k8s.io|ClusterServiceBroker, ClusterServiceClass, ClusterServicePlan, ServiceInstance, ServiceBinding|violation: Check for deprecated servicecatalog.k8s.io/v1beta1 apiVersion. See: https://docs.openshift.com/container-platform/4.2/release_notes/ocp-4-2-release-notes.html#ocp-4-2-deprecated-features|
 |[Buildconfig Jenkinspipeline Strategy](policy/ocp/deprecated/4_3/buildconfig-jenkinspipeline-strategy)|violation|build.openshift.io|BuildConfig|violation: Check if 'jenkinsPipelineStrategy' is set on a BuildConfig. See: https://docs.openshift.com/container-platform/4.3/release_notes/ocp-4-3-release-notes.html#ocp-4-3-deprecated-features|
+|[Deployment Has Matching Poddisruptionbudget](policy/ocp/requiresinventory/deployment-has-matching-poddisruptionbudget)|violation|apps|Deployment|violation: Check if a Deployment has a matching policy/v1beta1:PodDisruptionBudget, via 'spec.template.metadata.labels'|
+|[Deployment Has Matching Pvc](policy/ocp/requiresinventory/deployment-has-matching-pvc)|violation|apps|Deployment|violation: Check if a Deployment has 'spec.template.spec.volumes.persistentVolumeClaim' set, there is a matching v1:PersistentVolumeClaim|
+|[Deployment Has Matching Service](policy/ocp/requiresinventory/deployment-has-matching-service)|violation|apps|Deployment|violation: Check if a Deployment has a matching v1:Service, via 'spec.template.metadata.labels'|
+|[Deployment Has Matching Serviceaccount](policy/ocp/requiresinventory/deployment-has-matching-serviceaccount)|violation|apps|Deployment|violation: Check if a Deployment has 'spec.serviceAccountName' set, there is a matching v1:ServiceAccount|
+|[Service Has Matching Servicemonitor](policy/ocp/requiresinventory/service-has-matching-servicemonitor)|violation|core|Service|violation: Check if a Service has a matching monitoring.coreos.com/v1:ServiceMonitor, via 'spec.selector'|
 |[Contains Layer](policy/podman/history/contains-layer)|violation|redhat-cop.github.com|PodmanHistory|violation: Check the image contains a specific SHA in its history|
 |[Image Size Not Greater Than](policy/podman/images/image-size-not-greater-than)|violation|redhat-cop.github.com|PodmanImages|violation: Check the image size is not greater than a specific value|
diff --git a/_test/all-namespaces/ocp/bestpractices/all.yml b/_test/all-namespaces/ocp/bestpractices/all.yml
@@ -3,6 +3,70 @@ kind: Template
 metadata:
   name: bar
 objects:
+- apiVersion: v1
+  kind: PersistentVolumeClaim
+  metadata:
+    name: foobar-pvc
+  spec:
+    accessModes:
+      - ReadWriteMany
+    resources:
+      requests:
+        storage: 1Gi
+    storageClassName: gp2
+- apiVersion: monitoring.coreos.com/v1
+  kind: ServiceMonitor
+  metadata:
+    labels:
+      app.kubernetes.io/name: Foo
+      app.kubernetes.io/instance: Bar
+      app.kubernetes.io/component: FooBar
+      app.kubernetes.io/part-of: Foo
+      app.kubernetes.io/managed-by: Bar
+    name: shouldneverfire-${NAME}
+  spec:
+    selector:
+      matchLabels:
+        app.kubernetes.io/name: Foo
+        app.kubernetes.io/instance: Bar
+        app.kubernetes.io/component: FooBar
+        app.kubernetes.io/part-of: Foo
+        app.kubernetes.io/managed-by: Bar
+    endpoints:
+      - port: 8080-tcp
+- apiVersion: v1
+  kind: Service
+  metadata:
+    labels:
+      app.kubernetes.io/name: Foo
+      app.kubernetes.io/instance: Bar
+      app.kubernetes.io/component: FooBar
+      app.kubernetes.io/part-of: Foo
+      app.kubernetes.io/managed-by: Bar
+    name: shouldneverfire-${NAME}
+  spec:
+    ports:
+      - name: 8080-tcp
+        port: 8080
+    selector:
+      app.kubernetes.io/name: Foo
+      app.kubernetes.io/instance: Bar
+      app.kubernetes.io/component: FooBar
+      app.kubernetes.io/part-of: Foo
+      app.kubernetes.io/managed-by: Bar
+- apiVersion: policy/v1beta1
+  kind: PodDisruptionBudget
+  metadata:
+    name: shouldneverfire-${NAME}
+  spec:
+    minAvailable: 2
+    selector:
+      matchLabels:
+        app.kubernetes.io/name: Foo
+        app.kubernetes.io/instance: Bar
+        app.kubernetes.io/component: FooBar
+        app.kubernetes.io/part-of: Foo
+        app.kubernetes.io/managed-by: Bar
 - apiVersion: apps/v1
   kind: Deployment
   metadata:
@@ -12,7 +76,7 @@ objects:
       app.kubernetes.io/component: FooBar
       app.kubernetes.io/part-of: Foo
       app.kubernetes.io/managed-by: Bar
-    name: shouldneverfire${NAME}
+    name: shouldneverfire-${NAME}
   spec:
     replicas: 3
     selector:
@@ -68,6 +132,13 @@ objects:
                 cpu: 100m
               limits:
                 memory: 2Gi
+            volumeMounts:
+              - name: my-pvc
+                mountPath: /var/run/io.redhat-cop/my-pvc
+        volumes:
+          - name: my-pvc
+            persistentVolumeClaim:
+              claimName: foobar-pvc
 - apiVersion: apps.openshift.io/v1
   kind: DeploymentConfig
   metadata:

diff --git a/_test/conftest-unittests.sh b/_test/conftest-unittests.sh
@@ -27,56 +27,6 @@ setup_file() {
 # combine
 ####################
 
-@test "policy/combine/deployment-has-matching-poddisruptionbudget" {
-  tmp=$(split_files "policy/combine/deployment-has-matching-poddisruptionbudget/test_data/unit")
-
-  cmd="conftest test ${tmp} --output tap --combine --namespace combine.deployment_has_matching_poddisruptionbudget"
-  run ${cmd}
-
-  print_info "${status}" "${output}" "${cmd}" "${tmp}"
-  [ "$status" -eq 1 ]
-  [ "${lines[1]}" = "not ok 1 - Combined - Deployment/hasmissingpdb does not have a policy/v1beta1:PodDisruptionBudget or its selector labels dont match. See: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget" ]
-  [ "${lines[2]}" = "not ok 2 - Combined - Deployment/pdbhasincorrectlabels does not have a policy/v1beta1:PodDisruptionBudget or its selector labels dont match. See: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget" ]
-  [ "${lines[4]}" = "" ]
-}
-
-@test "policy/combine/deployment-has-matching-pvc" {
-  tmp=$(split_files "policy/combine/deployment-has-matching-pvc/test_data/unit")
-
-  cmd="conftest test ${tmp} --output tap --combine --namespace combine.deployment_has_matching_pvc"
-  run ${cmd}
-
-  print_info "${status}" "${output}" "${cmd}" "${tmp}"
-  [ "$status" -eq 1 ]
-  [ "${lines[1]}" = "not ok 1 - Combined - Deployment/hasmissingpvc has persistentVolumeClaim in its spec.template.spec.volumes but could not find corrasponding v1:PersistentVolumeClaim." ]
-  [ "${lines[2]}" = "" ]
-}
-
-@test "policy/combine/deployment-has-matching-service" {
-  tmp=$(split_files "policy/combine/deployment-has-matching-service/test_data/unit")
-
-  cmd="conftest test ${tmp} --output tap --combine --namespace combine.deployment_has_matching_service"
-  run ${cmd}
-
-  print_info "${status}" "${output}" "${cmd}" "${tmp}"
-  [ "$status" -eq 1 ]
-  [ "${lines[1]}" = "not ok 1 - Combined - Deployment/hasmissingsvc does not have a v1:Service or its selector labels dont match. See: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#service-and-replicationcontroller" ]
-  [ "${lines[2]}" = "not ok 2 - Combined - Deployment/hassvcwithincorrectlabels does not have a v1:Service or its selector labels dont match. See: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#service-and-replicationcontroller" ]
-  [ "${lines[3]}" = "" ]
-}
-
-@test "policy/combine/deployment-has-matching-serviceaccount" {
-  tmp=$(split_files "policy/combine/deployment-has-matching-serviceaccount/test_data/unit")
-
-  cmd="conftest test ${tmp} --output tap --combine --namespace combine.deployment_has_matching_serviceaccount"
-  run ${cmd}
-
-  print_info "${status}" "${output}" "${cmd}" "${tmp}"
-  [ "$status" -eq 1 ]
-  [ "${lines[1]}" = "not ok 1 - Combined - Deployment/hasmissingsvcaccount has spec.serviceAccountName 'missing' but could not find corrasponding v1:ServiceAccount." ]
-  [ "${lines[2]}" = "" ]
-}
-
 @test "policy/combine/namespace-has-networkpolicy" {
   tmp=$(split_files "policy/combine/namespace-has-networkpolicy/test_data/unit")
 
@@ -89,19 +39,6 @@ setup_file() {
   [ "${lines[2]}" = "" ]
 }
 
-@test "policy/combine/service-has-matching-servicenonitor" {
-  tmp=$(split_files "policy/combine/service-has-matching-servicenonitor/test_data/unit")
-
-  cmd="conftest test ${tmp} --output tap --combine --namespace combine.service_has_matching_servicenonitor"
-  run ${cmd}
-
-  print_info "${status}" "${output}" "${cmd}" "${tmp}"
-  [ "$status" -eq 1 ]
-  [ "${lines[1]}" = "not ok 1 - Combined - Service/hasmissingsvcmon does not have a monitoring.coreos.com/v1:ServiceMonitor or its selector labels dont match. See: https://docs.openshift.com/container-platform/4.4/monitoring/monitoring-your-own-services.html" ]
-  [ "${lines[2]}" = "not ok 2 - Combined - Service/hassvcmonwithincorrectlabels does not have a monitoring.coreos.com/v1:ServiceMonitor or its selector labels dont match. See: https://docs.openshift.com/container-platform/4.4/monitoring/monitoring-your-own-services.html" ]
-  [ "${lines[3]}" = "" ]
-}
-
 ####################
 # ocp/bestpractices
 ####################

diff --git a/_test/deploy-gatekeeper.sh b/_test/deploy-gatekeeper.sh
@@ -67,18 +67,36 @@ patch_namespaceselector_for_webhook() {
 generate_constraints() {
   echo "Creating ConstraintTemplates via konstraint..."
   konstraint doc -o POLICIES.md
+  konstraint create
 
-  # Ignores explained:
-  # - podman: the data is 'off-cluster' so cant be tested against gatekeeper
-  # - ocp/deprecated: tests cant be deployed to a 4.x cluster so cant be tested against gatekeeper
-  # - ocp/bestpractices/deploymentconfig-triggers-notset: OCP API-Server adds a default ConfigChange trigger by default so cant be tested against gatekeeper
-  # - ocp/bestpractices/rolebinding-roleref-apigroup-notset: OCP API-Server does not accept data matching this criteria but they are good for conftest when people are moving from 3.11 to 4.x
-  # - ocp/bestpractices/rolebinding-roleref-kind-notset: OCP API-Server does not accept data matching this criteria but they are good for conftest when people are moving from 3.11 to 4.x
-  # TODO LIST
-  # - ocp/bestpractices/container-resources-limits-memory-greater-than: TODO: https://github.com/redhat-cop/rego-policies/issues/76
-  # - ocp/bestpractices/container-resources-requests-memory-greater-than: TODO: https://github.com/redhat-cop/rego-policies/issues/76
-  # - combine: TODO: https://github.com/redhat-cop/rego-policies/issues/70
-  konstraint create --ignore "(podman|ocp\/deprecated|ocp\/bestpractices\/deploymentconfig-triggers-notset|ocp\/bestpractices\/rolebinding-roleref-apigroup-notset|ocp\/bestpractices\/rolebinding-roleref-kind-notset|ocp\/bestpractices\/container-resources-limits-memory-greater-than|ocp\/bestpractices\/container-resources-requests-memory-greater-than|combine)"
+  # shellcheck disable=SC2038
+  for file in $(find policy/* \( -name "template.yaml" -o -name "constraint.yaml" \) -type f | xargs); do
+    if [[ "${file}" == *"/combine/"* ]]; then
+      # the data is 'off-cluster' so cant be tested against gatekeeper
+      rm -f "${file}"
+    elif [[ "${file}" == *"/ocp/deprecated/"* ]]; then
+      # tests cant be deployed to a 4.x cluster so cant be tested against gatekeeper
+      rm -f "${file}"
+    elif [[ "${file}" == *"/ocp/bestpractices/deploymentconfig-triggers-notset/"* ]]; then
+      # OCP API-Server adds a default ConfigChange trigger by default so cant be tested against gatekeeper
+      rm -f "${file}"
+    elif [[ "${file}" == *"/ocp/bestpractices/rolebinding-roleref-apigroup-notset/"* ]]; then
+      # OCP API-Server does not accept data matching this criteria but they are good for conftest when people are moving from 3.11 to 4.x
+      rm -f "${file}"
+    elif [[ "${file}" == *"/ocp/bestpractices/rolebinding-roleref-kind-notset/"* ]]; then
+      # OCP API-Server does not accept data matching this criteria but they are good for conftest when people are moving from 3.11 to 4.x
+      rm -f "${file}"
+    elif [[ "${file}" == *"/ocp/bestpractices/container-resources-limits-memory-greater-than/"* ]]; then
+      # TODO: https://github.com/redhat-cop/rego-policies/issues/76
+      rm -f "${file}"
+    elif [[ "${file}" == *"/ocp/bestpractices/container-resources-requests-memory-greater-than/"* ]]; then
+      # TODO: https://github.com/redhat-cop/rego-policies/issues/76
+      rm -f "${file}"
+    elif [[ "${file}" == *"/podman/"* ]]; then
+      # the data is 'off-cluster' so cant be tested against gatekeeper
+      rm -f "${file}"
+    fi
+  done
 }
 
 deploy_constraints() {

diff --git a/_test/gatekeeper-integrationtests.sh b/_test/gatekeeper-integrationtests.sh
@@ -23,15 +23,16 @@ teardown() {
 
 ####################
 # all-namespaces
+# NOTE: This test should always be first, as it contains YAML to stop the 'data.inventory' policies firing when they shouldnt
 ####################
 
 @test "_test/all-namespaces/ocp/bestpractices" {
-  tmp=$(split_files "_test/all-namespaces/ocp/bestpractices")
+  alltmp=$(split_files "_test/all-namespaces/ocp/bestpractices")
 
-  cmd="oc create -f ${tmp} -n ${project_name}"
+  cmd="oc create -f ${alltmp} -n ${project_name}"
   run ${cmd}
 
-  print_info "${status}" "${output}" "${cmd}" "${tmp}"
+  print_info "${status}" "${output}" "${cmd}" "${alltmp}"
   [ "$status" -eq 0 ]
 }
 
@@ -305,4 +306,68 @@ teardown() {
   [[ "${lines[0]}" == "Error from server ([denied by podreplicasnotodd] Deployment/replicaiseven"* ]]
   [[ "${lines[1]}" == "Error from server ([denied by podreplicasnotodd] DeploymentConfig/replicaiseven"* ]]
   [[ "${lines[2]}" = "" ]]
+}
+
+####################
+# ocp/requiresinventory
+####################
+
+@test "policy/ocp/requiresinventory/deployment-has-matching-poddisruptionbudget" {
+  tmp=$(split_files "policy/ocp/requiresinventory/deployment-has-matching-poddisruptionbudget/test_data/integration")
+
+  cmd="oc create -f ${tmp} -n ${project_name}"
+  run ${cmd}
+
+  print_info "${status}" "${output}" "${cmd}" "${tmp}"
+  [ "$status" -eq 1 ]
+  [[ "${lines[2]}" == "Error from server ([denied by deploymenthasmatchingpoddisruptionbudget] Deployment/hasmissingpdb"* ]]
+  [[ "${lines[3]}" = "" ]]
+}
+
+@test "policy/ocp/requiresinventory/deployment-has-matching-pvc" {
+  tmp=$(split_files "policy/ocp/requiresinventory/deployment-has-matching-pvc/test_data/integration")
+
+  cmd="oc create -f ${tmp} -n ${project_name}"
+  run ${cmd}
+
+  print_info "${status}" "${output}" "${cmd}" "${tmp}"
+  [ "$status" -eq 1 ]
+  [[ "${lines[0]}" == "Error from server ([denied by deploymenthasmatchingpvc] Deployment/hasmissingpvc"* ]]
+  [[ "${lines[1]}" = "" ]]
+}
+
+@test "policy/ocp/requiresinventory/deployment-has-matching-service" {
+  tmp=$(split_files "policy/ocp/requiresinventory/deployment-has-matching-service/test_data/integration")
+
+  cmd="oc create -f ${tmp} -n ${project_name}"
+  run ${cmd}
+
+  print_info "${status}" "${output}" "${cmd}" "${tmp}"
+  [ "$status" -eq 1 ]
+  [[ "${lines[1]}" == "Error from server ([denied by deploymenthasmatchingservice] Deployment/hasmissingsvc"* ]]
+  [[ "${lines[2]}" = "" ]]
+}
+
+@test "policy/ocp/requiresinventory/deployment-has-matching-serviceaccount" {
+  tmp=$(split_files "policy/ocp/requiresinventory/deployment-has-matching-serviceaccount/test_data/integration")
+
+  cmd="oc create -f ${tmp} -n ${project_name}"
+  run ${cmd}
+
+  print_info "${status}" "${output}" "${cmd}" "${tmp}"
+  [ "$status" -eq 1 ]
+  [[ "${lines[0]}" == "Error from server ([denied by deploymenthasmatchingserviceaccount] Deployment/hasmissingsvcaccount"* ]]
+  [[ "${lines[1]}" = "" ]]
+}
+
+@test "policy/ocp/requiresinventory/service-has-matching-servicemonitor" {
+  tmp=$(split_files "policy/ocp/requiresinventory/service-has-matching-servicemonitor/test_data/integration")
+
+  cmd="oc create -f ${tmp} -n ${project_name}"
+  run ${cmd}
+
+  print_info "${status}" "${output}" "${cmd}" "${tmp}"
+  [ "$status" -eq 1 ]
+  [[ "${lines[0]}" == "Error from server ([denied by servicehasmatchingservicemonitor] Service/hasmissingsvcmon"* ]]
+  [[ "${lines[1]}" = "" ]]
 }
diff --git a/gatekeeper/config.yml b/gatekeeper/config.yml
@@ -3,13 +3,15 @@ apiVersion: config.gatekeeper.sh/v1alpha1
 kind: Config
 metadata:
   name: config
-  namespace: "gatekeeper-system"
 spec:
   sync:
     syncOnly:
       - version: "v1"
         group: ""
         kind: "Namespace"
+      - version: "v1"
+        group: ""
+        kind: "ServiceAccount"
       - version: "v1"
         group: ""
         kind: "PersistentVolumeClaim"