Updated tests to test against split out items (#17)

* Updated tests to test against split out items * Added tools used by testing
redhat-cop · Jun 24, 2020 · 6ed0e96 · 6ed0e96
1 parent d32cd06
commit 6ed0e96
Show file tree

Hide file tree

Showing 6 changed files with 140 additions and 69 deletions.
diff --git a/README.md b/README.md
@@ -2,7 +2,9 @@
 [Rego](https://www.openpolicyagent.org/docs/latest/policy-language/) policies collection
 
 ## Policies
-Current policies in this repo:
+Current policies in this repo are below.
+
+### Deny Policies
 - [ocp-42-deprecated-apiversions.rego](policy/ocp-42-deprecated-apiversions.rego)
     - [deny rules for OCP 4.2 apiVersion deprecations](https://docs.openshift.com/container-platform/4.2/release_notes/ocp-4-2-release-notes.html#ocp-4-2-deprecated-features)
 

diff --git a/TESTING.md b/TESTING.md
@@ -4,7 +4,20 @@ to validate the rego policies.
 
 ## How do i write a test?
 Each test is expected to have a directory under [_test](_test) which contains the test input data; typically a yaml file 
-containing a OCP Template or k8s List.
+containing a OCP Template or k8s List. Each block of YAML should match exactly 1 policy, due to the order of the failure output
+which needs to be predictable for bats.
 
 The tests are executed by [_test/tests.bats](_test/tests.bats). The test should validate each expected bats output and always
 end with the expected success line.
+
+## Execute Locally
+```bash
+rm -rf /tmp/rego-policies; _test/conftest.sh
+```
+
+## Tools used for testing
+The following tools are required:
+
+- [conftest](https://www.conftest.dev/install)
+- [jq](https://stedolan.github.io/jq/download)
+- [yq](https://pypi.org/project/yq)
diff --git a/_test/_helpers.bash b/_test/_helpers.bash
@@ -0,0 +1,43 @@
+split_via_yq() {
+  SEARCH_PATH=$1
+  SEARCH_DIR=$(dirname "$1")
+  KEY=$2
+
+  mkdir -p /tmp/rego-policies/${SEARCH_DIR}
+
+  for file in $(ls ${SEARCH_PATH} | xargs) ; do
+    yq --yaml-output "${KEY}" ${file} > /tmp/rego-policies/${SEARCH_DIR}/$(basename "${file}")
+  done
+}
+
+split_via_jq() {
+  SEARCH_PATH=$1
+  SEARCH_DIR=$(dirname "$1")
+  KEY=$2
+
+  mkdir -p /tmp/rego-policies/${SEARCH_DIR}
+
+  for file in $(ls ${SEARCH_PATH} | xargs) ; do
+    jq "${KEY}" ${file} > /tmp/rego-policies/${SEARCH_DIR}/$(basename "${file}")
+  done
+}
+
+copy_file_via_yq() {
+  FILE_PATH=$1
+  FILE_DIR=$(dirname "$1")
+
+  mkdir -p /tmp/rego-policies/${FILE_DIR}
+
+  yq --yaml-output "." ${FILE_PATH} > /tmp/rego-policies/${FILE_DIR}/$(basename ${FILE_PATH})
+}
+
+copy_dir_via_jq() {
+  SEARCH_PATH=$1
+  SEARCH_DIR=$(dirname "$1")
+
+  mkdir -p /tmp/rego-policies/${SEARCH_DIR}
+
+  for file in $(ls ${SEARCH_PATH} | xargs) ; do
+    jq '.' ${file} > /tmp/rego-policies/${SEARCH_DIR}/$(basename ${file})
+  done
+}
diff --git a/_test/ocp-43-deprecated-apiversions/list.yml b/_test/ocp-43-deprecated-apiversions/list.yml
@@ -0,0 +1,36 @@
+---
+apiVersion: v1
+kind: List
+metadata:
+  name: Foo
+items:
+# Cluster Scoped
+- apiVersion: v1
+  kind: SecurityContextConstraints
+  metadata:
+    name: Bar
+# Project Scoped
+- apiVersion: v1
+  kind: ProjectRequest
+  metadata:
+    name: Bar
+- apiVersion: v1
+  kind: ImageStream
+  metadata:
+    name: Bar
+- apiVersion: v1
+  kind: BuildConfig
+  metadata:
+    name: Bar
+- apiVersion: v1
+  kind: DeploymentConfig
+  metadata:
+    name: Bar
+- apiVersion: v1
+  kind: RoleBinding
+  metadata:
+    name: Bar
+- apiVersion: v1
+  kind: Route
+  metadata:
+    name: Bar
diff --git a/_test/ocp-43-deprecated-apiversions/template.yml b/_test/ocp-43-deprecated-apiversions/template.yml
@@ -2,40 +2,4 @@
 apiVersion: v1
 kind: Template
 metadata:
-  name: Foo
----
-apiVersion: template.openshift.io/v1
-kind: Template
-metadata:
-  name: Foo
-objects:
-# Project Scoped
-- apiVersion: v1
-  kind: ProjectRequest
-  metadata:
-    name: Bar
-- apiVersion: v1
-  kind: ImageStream
-  metadata:
-    name: Bar
-- apiVersion: v1
-  kind: BuildConfig
-  metadata:
-    name: Bar
-- apiVersion: v1
-  kind: DeploymentConfig
-  metadata:
-    name: Bar
-- apiVersion: v1
-  kind: RoleBinding
-  metadata:
-    name: Bar
-- apiVersion: v1
-  kind: Route
-  metadata:
-    name: Bar
-# Cluster Scoped
-- apiVersion: v1
-  kind: SecurityContextConstraints
-  metadata:
-    name: Bar
+  name: Foo
diff --git a/_test/tests.bats b/_test/tests.bats
@@ -1,38 +1,51 @@
 #!/usr/bin/env bats
 
-@test "k8s-validation-rolebinding" {
-    run conftest test _test/k8s-validation-rolebinding --output tap
+load _helpers
 
-    [ "$status" -eq 1 ]
-    [ "${lines[1]}" = "not ok 1 - _test/k8s-validation-rolebinding/list.yml - RoleBinding/NoApiGroup: RoleBinding roleRef.apiGroup key is null, use rbac.authorization.k8s.io instead." ]
-    [ "${lines[2]}" = "not ok 2 - _test/k8s-validation-rolebinding/list.yml - RoleBinding/NoKind: RoleBinding roleRef.kind key is null, use ClusterRole or Role instead." ]
-    [ "${lines[3]}" = "# Successes" ]
+@test "_test/k8s-validation-rolebinding" {
+  split_via_yq "_test/k8s-validation-rolebinding/*.yml" ".items[]"
+  run conftest test /tmp/rego-policies/_test/k8s-validation-rolebinding --output tap
+
+  [ "$status" -eq 1 ]
+  [ "${lines[1]}" = "not ok 1 - /tmp/rego-policies/_test/k8s-validation-rolebinding/list.yml - RoleBinding/NoApiGroup: RoleBinding roleRef.apiGroup key is null, use rbac.authorization.k8s.io instead." ]
+  [ "${lines[2]}" = "not ok 2 - /tmp/rego-policies/_test/k8s-validation-rolebinding/list.yml - RoleBinding/NoKind: RoleBinding roleRef.kind key is null, use ClusterRole or Role instead." ]
+  [ "${lines[3]}" = "# Successes" ]
+}
+
+@test "_test/ocp-42-deprecated-apiversions" {
+  split_via_yq "_test/ocp-42-deprecated-apiversions/*.yml" ".items[]"
+  run conftest test /tmp/rego-policies/_test/ocp-42-deprecated-apiversions --output tap
+
+  [ "$status" -eq 1 ]
+  [ "${lines[1]}" = "not ok 1 - /tmp/rego-policies/_test/ocp-42-deprecated-apiversions/list.yml - Foo/Bar: servicecatalog.k8s.io/v1beta1 is deprecated." ]
+  [ "${lines[2]}" = "not ok 2 - /tmp/rego-policies/_test/ocp-42-deprecated-apiversions/list.yml - Foo/Bar: automationbroker.io/v1alpha1 is deprecated." ]
+  [ "${lines[3]}" = "not ok 3 - /tmp/rego-policies/_test/ocp-42-deprecated-apiversions/list.yml - Foo/Bar: osb.openshift.io/v1 is deprecated." ]
+  [ "${lines[4]}" = "not ok 4 - /tmp/rego-policies/_test/ocp-42-deprecated-apiversions/list.yml - Foo/Bar: operatorsources.operators.coreos.com/v1 is deprecated." ]
+  [ "${lines[5]}" = "not ok 5 - /tmp/rego-policies/_test/ocp-42-deprecated-apiversions/list.yml - Foo/Bar: catalogsourceconfigs.operators.coreos.com/v1 is deprecated." ]
+  [ "${lines[6]}" = "not ok 6 - /tmp/rego-policies/_test/ocp-42-deprecated-apiversions/list.yml - Foo/Bar: catalogsourceconfigs.operators.coreos.com/v2 is deprecated." ]
+  [ "${lines[7]}" = "# Successes" ]
 }
 
-@test "ocp-42-deprecated-apiversions" {
-    run conftest test _test/ocp-42-deprecated-apiversions --output tap
-
-    [ "$status" -eq 1 ]
-    [ "${lines[1]}" = "not ok 1 - _test/ocp-42-deprecated-apiversions/list.yml - Foo/Bar: servicecatalog.k8s.io/v1beta1 is deprecated." ]
-    [ "${lines[2]}" = "not ok 2 - _test/ocp-42-deprecated-apiversions/list.yml - Foo/Bar: automationbroker.io/v1alpha1 is deprecated." ]
-    [ "${lines[3]}" = "not ok 3 - _test/ocp-42-deprecated-apiversions/list.yml - Foo/Bar: osb.openshift.io/v1 is deprecated." ]
-    [ "${lines[4]}" = "not ok 4 - _test/ocp-42-deprecated-apiversions/list.yml - Foo/Bar: operatorsources.operators.coreos.com/v1 is deprecated." ]
-    [ "${lines[5]}" = "not ok 5 - _test/ocp-42-deprecated-apiversions/list.yml - Foo/Bar: catalogsourceconfigs.operators.coreos.com/v1 is deprecated." ]
-    [ "${lines[6]}" = "not ok 6 - _test/ocp-42-deprecated-apiversions/list.yml - Foo/Bar: catalogsourceconfigs.operators.coreos.com/v2 is deprecated." ]
-    [ "${lines[7]}" = "# Successes" ]
+@test "_test/ocp-43-deprecated-apiversions/template.yml" {
+  copy_file_via_yq "_test/ocp-43-deprecated-apiversions/template.yml"
+  run conftest test _test/ocp-43-deprecated-apiversions/template.yml --output tap
+
+  [ "$status" -eq 1 ]
+  [ "${lines[1]}" = "not ok 1 - _test/ocp-43-deprecated-apiversions/template.yml - Template/Foo: API v1 for Template is no longer served by default, use template.openshift.io/v1 instead." ]
+  [ "${lines[2]}" = "# Successes" ]
 }
 
-@test "ocp-43-deprecated-apiversions" {
-    run conftest test _test/ocp-43-deprecated-apiversions --output tap
-
-    [ "$status" -eq 1 ]
-    [ "${lines[1]}" = "not ok 1 - _test/ocp-43-deprecated-apiversions/template.yml - Template/Foo: API v1 for Template is no longer served by default, use template.openshift.io/v1 instead." ]
-    [ "${lines[2]}" = "not ok 2 - _test/ocp-43-deprecated-apiversions/template.yml - ProjectRequest/Bar: API v1 for ProjectRequest is no longer served by default, use project.openshift.io/v1 instead." ]
-    [ "${lines[3]}" = "not ok 3 - _test/ocp-43-deprecated-apiversions/template.yml - ImageStream/Bar: API v1 for ImageStream is no longer served by default, use image.openshift.io/v1 instead." ]
-    [ "${lines[4]}" = "not ok 4 - _test/ocp-43-deprecated-apiversions/template.yml - BuildConfig/Bar: API v1 for BuildConfig is no longer served by default, use build.openshift.io/v1 instead." ]
-    [ "${lines[5]}" = "not ok 5 - _test/ocp-43-deprecated-apiversions/template.yml - DeploymentConfig/Bar: API v1 for DeploymentConfig is no longer served by default, use apps.openshift.io/v1 instead." ]
-    [ "${lines[6]}" = "not ok 6 - _test/ocp-43-deprecated-apiversions/template.yml - RoleBinding/Bar: API v1 for RoleBinding is no longer served by default, use rbac.authorization.k8s.io/v1 instead." ]
-    [ "${lines[7]}" = "not ok 7 - _test/ocp-43-deprecated-apiversions/template.yml - Route/Bar: API v1 for Route is no longer served by default, use route.openshift.io/v1 instead." ]
-    [ "${lines[8]}" = "not ok 8 - _test/ocp-43-deprecated-apiversions/template.yml - SecurityContextConstraints/Bar: API v1 for SecurityContextConstraints is no longer served by default, use security.openshift.io/v1 instead." ]
-    [ "${lines[9]}" = "# Successes" ]
+@test "_test/ocp-43-deprecated-apiversions/list.yml" {
+  split_via_yq "_test/ocp-43-deprecated-apiversions/list.yml" ".items[]"
+  run conftest test /tmp/rego-policies/_test/ocp-43-deprecated-apiversions/list.yml --output tap
+
+  [ "$status" -eq 1 ]
+  [ "${lines[1]}" = "not ok 1 - /tmp/rego-policies/_test/ocp-43-deprecated-apiversions/list.yml - SecurityContextConstraints/Bar: API v1 for SecurityContextConstraints is no longer served by default, use security.openshift.io/v1 instead." ]
+  [ "${lines[2]}" = "not ok 2 - /tmp/rego-policies/_test/ocp-43-deprecated-apiversions/list.yml - ProjectRequest/Bar: API v1 for ProjectRequest is no longer served by default, use project.openshift.io/v1 instead." ]
+  [ "${lines[3]}" = "not ok 3 - /tmp/rego-policies/_test/ocp-43-deprecated-apiversions/list.yml - ImageStream/Bar: API v1 for ImageStream is no longer served by default, use image.openshift.io/v1 instead." ]
+  [ "${lines[4]}" = "not ok 4 - /tmp/rego-policies/_test/ocp-43-deprecated-apiversions/list.yml - BuildConfig/Bar: API v1 for BuildConfig is no longer served by default, use build.openshift.io/v1 instead." ]
+  [ "${lines[5]}" = "not ok 5 - /tmp/rego-policies/_test/ocp-43-deprecated-apiversions/list.yml - DeploymentConfig/Bar: API v1 for DeploymentConfig is no longer served by default, use apps.openshift.io/v1 instead." ]
+  [ "${lines[6]}" = "not ok 6 - /tmp/rego-policies/_test/ocp-43-deprecated-apiversions/list.yml - RoleBinding/Bar: API v1 for RoleBinding is no longer served by default, use rbac.authorization.k8s.io/v1 instead." ]
+  [ "${lines[7]}" = "not ok 7 - /tmp/rego-policies/_test/ocp-43-deprecated-apiversions/list.yml - Route/Bar: API v1 for Route is no longer served by default, use route.openshift.io/v1 instead." ]
+  [ "${lines[8]}" = "# Successes" ]
 }