From 01a2bddf9638d1f39de2f8cd6a60077f72576276 Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Fri, 1 Dec 2023 19:03:11 -0500 Subject: [PATCH 1/2] ci: adds E2E tests to CI automation Signed-off-by: Jennifer Power --- .github/workflows/e2e.yml | 36 +++++++++++++++++++++++++ .github/workflows/publish.yml | 32 ++++++++++++++++++----- tests/conftest.py | 30 ++++++++++++++------- tests/e2e/README.md | 5 +++- tests/e2e/test_e2e_compdef.py | 16 +++++++----- tests/e2e/test_e2e_ssp.py | 7 ++--- tests/testutils.py | 49 +++++++++++------------------------ 7 files changed, 116 insertions(+), 59 deletions(-) create mode 100644 .github/workflows/e2e.yml diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 00000000..1c1a4883 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,36 @@ +name: E2E + +on: + workflow_call: + inputs: + image: + description: "Name of the trestlebot image you want to test." + type: string + required: true + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Check out + uses: actions/checkout@v3 + + - name: Set up poetry and install + uses: ./.github/actions/setup-poetry + with: + python-version: "3.9" + + # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable + - name: Pull the image + run: | + podman pull "${IMAGE}" + echo "TRESTLEBOT_IMAGE=${IMAGE}" >> "$GITHUB_ENV" + env: + IMAGE: ${{ inputs.image }} + + - name: Run tests + run: make test-e2e diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index bbb48a01..bc459098 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,16 +9,26 @@ on: description: "Name of the tag for the published image" type: string required: true + skip_tests: + description: "Skip end to end tests when publishing an image." + type: boolean + required: false + default: false env: IMAGE_NAME: trestle-bot IMAGE_REGISTRY: quay.io jobs: + publish-image: runs-on: 'ubuntu-latest' permissions: contents: read - packages: write + outputs: + skip_tests: ${{ steps.check_event.outputs.event_type == 'release' + || (steps.check_event.outputs.event_type == 'workflow_dispatch' + && github.event.inputs.skip_tests == 'true') }} + image: ${{ env.IMAGE_REGISTRY }}/${{ vars.QUAY_ORG }}/${{ env.IMAGE_NAME }}@${{ steps.build-image.outputs.digest }} steps: - name: Checkout repository uses: actions/checkout@v4 @@ -42,21 +52,31 @@ jobs: # Using intermediary variable to process event based input - name: Set TAG environment variable for Release - if: steps.check_event.outputs.event_type == 'release' + if: ${{ steps.check_event.outputs.event_type == 'release' }} run: echo "TAG=$RELEASE_VERSION" >> "$GITHUB_ENV" env: RELEASE_VERSION: ${{ github.event.release.tag_name }} - name: Set TAG environment variable for Workflow Dispatch - if: steps.check_event.outputs.event_type == 'workflow_dispatch' + if: ${{ steps.check_event.outputs.event_type == 'workflow_dispatch' }} run: echo "TAG=$INPUT_VERSION" >> "$GITHUB_ENV" env: INPUT_VERSION: ${{ github.event.inputs.tag }} - + - name: Build and Push uses: docker/build-push-action@v5 + id: build-image with: push: true - tags: ${{ env.IMAGE_REGISTRY }}/${{ vars.QUAY_ORG }}/${{ env.IMAGE_NAME }}:${{ env.TAG }} cache-from: type=gha - cache-to: type=gha,mode=max \ No newline at end of file + cache-to: type=gha,mode=max + tags: ${{ env.IMAGE_REGISTRY }}/${{ vars.QUAY_ORG }}/${{ env.IMAGE_NAME }}:${{ env.TAG }} + + test: + permissions: + contents: read + needs: publish-image + if: ${{ needs.publish-image.outputs.skip_tests != 'true' }} + uses: ./.github/workflows/e2e.yml + with: + image: ${{ needs.publish-image.outputs.image }} \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 5b75ef26..8f0c6509 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -30,11 +30,11 @@ from trestle.core.commands.init import InitCmd from tests.testutils import ( + CONTAINER_FILE_NAME, E2E_BUILD_CONTEXT, MOCK_SERVER_IMAGE_NAME, TRESTLEBOT_TEST_IMAGE_NAME, - build_mock_server_image, - build_trestlebot_image, + build_test_image, clean, ) from trestlebot import const @@ -199,17 +199,29 @@ def test_rule() -> TrestleRule: @pytest.fixture(scope="package") -def podman_setup() -> YieldFixture[int]: - """Build the trestlebot container image and run the mock server in a pod.""" - - cleanup_trestlebot_image = build_trestlebot_image() - cleanup_mock_server_image = build_mock_server_image() +def podman_setup() -> YieldFixture[Tuple[int, str]]: + """ + Build the trestlebot container image and run the mock server in a pod. + + Yields: + Tuple[int, str]: The return code from the podman play command and the trestlebot image name. + """ + + # Get the image information from the environment, if present + trestlebot_image = os.environ.get("TRESTLEBOT_IMAGE", TRESTLEBOT_TEST_IMAGE_NAME) + + cleanup_trestlebot_image = build_test_image(trestlebot_image) + cleanup_mock_server_image = build_test_image( + MOCK_SERVER_IMAGE_NAME, + f"{E2E_BUILD_CONTEXT}/{CONTAINER_FILE_NAME}", + E2E_BUILD_CONTEXT, + ) # Create a pod response = subprocess.run( ["podman", "play", "kube", f"{E2E_BUILD_CONTEXT}/play-kube.yml"], check=True ) - yield response.returncode + yield response.returncode, trestlebot_image # Clean up the container image, pod and mock server try: @@ -218,7 +230,7 @@ def podman_setup() -> YieldFixture[int]: check=True, ) if cleanup_trestlebot_image: - subprocess.run(["podman", "rmi", TRESTLEBOT_TEST_IMAGE_NAME], check=True) + subprocess.run(["podman", "rmi", trestlebot_image], check=True) if cleanup_mock_server_image: subprocess.run(["podman", "rmi", MOCK_SERVER_IMAGE_NAME], check=True) except subprocess.CalledProcessError as e: diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 7dbcd418..017abfd4 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -49,11 +49,14 @@ To run the end-to-end tests, follow these steps: ```bash podman build -t localhost/mock-server:latest -f tests/e2e/Dockerfile tests/e2e podman build -t localhost/trestlebot:latest -f Dockerfile . + + # Use a prebuilt image from quay.io + podman pull quay.io/continuouscompliance/trestle-bot:latest + export TRESTLEBOT_IMAGE=quay.io/continuouscompliance/trestle-bot:latest ``` - When created tests that push to a branch, ensure the name is "test". This is because the mock API server is configured to only allow pushes to a branch named "test". ## Future Improvements -- Provide an option to use pre-built trestle-bot container images from a registry instead of building them locally. - Create endpoints that mock GitHub and GitLab API calls for pull request creation. - Add more end-to-end tests to cover more use cases. \ No newline at end of file diff --git a/tests/e2e/test_e2e_compdef.py b/tests/e2e/test_e2e_compdef.py index eb03964b..b8f5c893 100644 --- a/tests/e2e/test_e2e_compdef.py +++ b/tests/e2e/test_e2e_compdef.py @@ -89,7 +89,7 @@ ) def test_rules_transform_e2e( tmp_repo: Tuple[str, Repo], - podman_setup: int, + podman_setup: Tuple[int, str], test_name: str, command_args: Dict[str, str], response: int, @@ -97,7 +97,8 @@ def test_rules_transform_e2e( """Test the trestlebot rules transform command.""" # Check that the container image was built successfully # and the mock server is running - assert podman_setup == 0 + exit_code, image_name = podman_setup + assert exit_code == 0 logger.info(f"Running test: {test_name}") @@ -122,7 +123,9 @@ def test_rules_transform_e2e( remote_url = "http://localhost:8080/test.git" repo.create_remote("origin", url=remote_url) - command = build_test_command(tmp_repo_str, "rules-transform", command_args) + command = build_test_command( + tmp_repo_str, "rules-transform", command_args, image_name + ) run_response = subprocess.run(command, capture_output=True) assert run_response.returncode == response @@ -224,7 +227,7 @@ def test_rules_transform_e2e( ) def test_create_cd_e2e( tmp_repo: Tuple[str, Repo], - podman_setup: int, + podman_setup: Tuple[int, str], test_name: str, command_args: Dict[str, str], response: int, @@ -232,7 +235,8 @@ def test_create_cd_e2e( """Test the trestlebot rules transform command.""" # Check that the container image was built successfully # and the mock server is running - assert podman_setup == 0 + exit_code, image_name = podman_setup + assert exit_code == 0 logger.info(f"Running test: {test_name}") @@ -258,7 +262,7 @@ def test_create_cd_e2e( remote_url = "http://localhost:8080/test.git" repo.create_remote("origin", url=remote_url) - command = build_test_command(tmp_repo_str, "create-cd", command_args) + command = build_test_command(tmp_repo_str, "create-cd", command_args, image_name) run_response = subprocess.run(command, cwd=tmp_repo_path, capture_output=True) assert run_response.returncode == response diff --git a/tests/e2e/test_e2e_ssp.py b/tests/e2e/test_e2e_ssp.py index 2bbf0148..480a4321 100644 --- a/tests/e2e/test_e2e_ssp.py +++ b/tests/e2e/test_e2e_ssp.py @@ -94,7 +94,7 @@ ) def test_ssp_editing_e2e( tmp_repo: Tuple[str, Repo], - podman_setup: int, + podman_setup: Tuple[int, str], test_name: str, command_args: Dict[str, str], response: int, @@ -103,7 +103,8 @@ def test_ssp_editing_e2e( """Test the trestlebot autosync command with SSPs.""" # Check that the container image was built successfully # and the mock server is running - assert podman_setup == 0 + exit_code, image_name = podman_setup + assert exit_code == 0 logger.info(f"Running test: {test_name}") @@ -151,7 +152,7 @@ def test_ssp_editing_e2e( remote_url = "http://localhost:8080/test.git" repo.create_remote("origin", url=remote_url) - command = build_test_command(tmp_repo_str, "autosync", command_args) + command = build_test_command(tmp_repo_str, "autosync", command_args, image_name) run_response = subprocess.run(command, capture_output=True) assert run_response.returncode == response diff --git a/tests/testutils.py b/tests/testutils.py index 3c5d7f23..7c21a116 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -279,9 +279,6 @@ def replace_string_in_file(file_path: str, old_string: str, new_string: str) -> file.write(updated_content) -# E2E test utils - - def _image_exists(image_name: str) -> bool: """Check if the image already exists.""" try: @@ -291,46 +288,27 @@ def _image_exists(image_name: str) -> bool: return False -def build_trestlebot_image() -> bool: - """ - Build the trestlebot image. - - Returns: - Returns true if the image was built, false if it already exists. - """ - if not _image_exists(TRESTLEBOT_TEST_IMAGE_NAME): - subprocess.run( - [ - "podman", - "build", - "-f", - CONTAINER_FILE_NAME, - "-t", - TRESTLEBOT_TEST_IMAGE_NAME, - ], - check=True, - ) - return True - return False - - -def build_mock_server_image() -> bool: +def build_test_image( + image_name: str, + container_file: str = CONTAINER_FILE_NAME, + build_context: str = ".", +) -> bool: """ - Build the mock server image. + Build an image for testing image. Returns: Returns true if the image was built, false if it already exists. """ - if not _image_exists(MOCK_SERVER_IMAGE_NAME): + if not _image_exists(image_name): subprocess.run( [ "podman", "build", "-f", - f"{E2E_BUILD_CONTEXT}/{CONTAINER_FILE_NAME}", + container_file, "-t", - MOCK_SERVER_IMAGE_NAME, - E2E_BUILD_CONTEXT, + image_name, + build_context, ], check=True, ) @@ -339,7 +317,10 @@ def build_mock_server_image() -> bool: def build_test_command( - data_path: str, command_name: str, command_args: Dict[str, str] + data_path: str, + command_name: str, + command_args: Dict[str, str], + image_name: str = TRESTLEBOT_TEST_IMAGE_NAME, ) -> List[str]: """Build a command to be run in the shell for trestlebot""" return [ @@ -354,6 +335,6 @@ def build_test_command( f"{data_path}:/trestle", "-w", "/trestle", - TRESTLEBOT_TEST_IMAGE_NAME, + image_name, *args_dict_to_list(command_args), ] From e6e083e413810f7b31926200639978ca146b1fd1 Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Mon, 4 Dec 2023 17:57:27 -0500 Subject: [PATCH 2/2] feat: adds image scanning between build and push Signed-off-by: Jennifer Power --- .github/workflows/publish.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index bc459098..b08ae529 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -62,6 +62,20 @@ jobs: run: echo "TAG=$INPUT_VERSION" >> "$GITHUB_ENV" env: INPUT_VERSION: ${{ github.event.inputs.tag }} + + - name: Build and export to Docker + uses: docker/build-push-action@v5 + with: + load: true + tags: ${{ env.IMAGE_REGISTRY }}/${{ vars.QUAY_ORG }}/${{ env.IMAGE_NAME }}:${{ env.TAG }} + + - name: Pre-push Image Scan + uses: aquasecurity/trivy-action@0.14.0 + with: + image-ref: ${{ env.IMAGE_REGISTRY }}/${{ vars.QUAY_ORG }}/${{ env.IMAGE_NAME }}:${{ env.TAG }} + exit-code: 1 + scanners: secret + severity: HIGH,CRITICAL,MEDIUM - name: Build and Push uses: docker/build-push-action@v5