test: add integration test

Signed-off-by: Xiaofeng Wang <henrywangxf@me.com>
containers · Mar 5, 2024 · eced96d · eced96d
1 parent 0072e05
commit eced96d
Show file tree

Hide file tree

Showing 16 changed files with 1,228 additions and 0 deletions.
diff --git a/.fmf/version b/.fmf/version
@@ -0,0 +1 @@
+1
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -0,0 +1,111 @@
+---
+name: Integration Test
+
+permissions:
+  pull_requests: read
+  contents: read
+
+on:
+  pull_request:
+    branches: [main]
+
+jobs:
+  pr-info:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Query author repository permissions
+        uses: octokit/request-action@v2.x
+        id: user_permission
+        with:
+          route: GET /repos/${{ github.repository }}/collaborators/${{ github.event.sender.login }}/permission
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      # restrict running of tests to users with admin or write permission for the repository
+      # see https://docs.github.com/en/rest/collaborators/collaborators?apiVersion=2022-11-28#get-repository-permissions-for-a-user
+      - name: Check if user does have correct permissions
+        if: contains('admin write', fromJson(steps.user_permission.outputs.data).permission)
+        id: check_user_perm
+        run: |
+          echo "User '${{ github.event.sender.login }}' has permission '${{ fromJson(steps.user_permission.outputs.data).permission }}' allowed values: 'admin', 'write'"
+          echo "allowed_user=true" >> $GITHUB_OUTPUT
+
+      - name: Get information for pull request
+        uses: octokit/request-action@v2.x
+        id: pr-api
+        with:
+          route: GET /repos/${{ github.repository }}/pulls/${{ github.event.number }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+    outputs:
+      allowed_user: ${{ steps.check_user_perm.outputs.allowed_user }}
+      sha: ${{ fromJson(steps.pr-api.outputs.data).head.sha }}
+      ref: ${{ fromJson(steps.pr-api.outputs.data).head.ref }}
+      repo_url: ${{ fromJson(steps.pr-api.outputs.data).head.repo.html_url }}
+
+  rhel94-integration:
+    needs: pr-info
+    if: ${{ needs.pr-info.outputs.allowed_user == 'true' }}
+    continue-on-error: true
+    strategy:
+      matrix:
+        arch: [x86_64, aarch64]
+        platform: [aws]
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.pr-info.outputs.sha }}
+          fetch-depth: 0
+
+      - name: Run the tests
+        uses: sclorg/testing-farm-as-github-action@v1
+        with:
+          compose: CentOS-Stream-9
+          api_key: ${{ secrets.TF_API_KEY }}
+          git_url: ${{ needs.pr-info.outputs.repo_url }}
+          git_ref: ${{ needs.pr-info.outputs.ref }}
+          arch: ${{ matrix.arch }}
+          tmt_context: "arch=${{ matrix.arch }}"
+          update_pull_request_status: true
+          pull_request_status_name: "Integration-rhel94-${{ matrix.arch }}-${{ matrix.platform }}"
+          tmt_plan_regex: "${{ matrix.platform }}"
+          tf_scope: private
+          secrets: "QUAY_USERNAME=${{ secrets.QUAY_USERNAME }};QUAY_PASSWORD=${{ secrets.QUAY_PASSWORD }};QUAY_SECRET=${{ secrets.QUAY_SECRET }};RHEL_REGISTRY_URL=${{ secrets.RHEL_REGISTRY_URL }};DOWNLOAD_NODE=${{ secrets.DOWNLOAD_NODE }};AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }};AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }};CERT_URL=${{ secrets.CERT_URL }}"
+          variables: "TEST_OS=rhel-9-4;PLATFORM=${{ matrix.platform }};ARCH=${{ matrix.arch }};AWS_REGION=${{ secrets.AWS_REGION }}"
+
+  cs9-dev-integration:
+    needs: pr-info
+    if: ${{ needs.pr-info.outputs.allowed_user == 'true' }}
+    continue-on-error: true
+    strategy:
+      matrix:
+        arch: [x86_64, aarch64]
+        platform: [aws]
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.pr-info.outputs.sha }}
+          fetch-depth: 0
+
+      - name: Run the tests
+        uses: sclorg/testing-farm-as-github-action@v1
+        with:
+          compose: CentOS-Stream-9
+          api_key: ${{ secrets.TF_API_KEY }}
+          git_url: ${{ needs.pr-info.outputs.repo_url }}
+          git_ref: ${{ needs.pr-info.outputs.ref }}
+          arch: ${{ matrix.arch }}
+          tmt_context: "arch=${{ matrix.arch }}"
+          update_pull_request_status: true
+          pull_request_status_name: "Integration-cs9-dev-${{ matrix.arch }}-${{ matrix.platform }}"
+          tmt_plan_regex: "${{ matrix.platform }}"
+          tf_scope: private
+          secrets: "QUAY_USERNAME=${{ secrets.QUAY_USERNAME }};QUAY_PASSWORD=${{ secrets.QUAY_PASSWORD }};QUAY_SECRET=${{ secrets.QUAY_SECRET }};AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }};AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }};CERT_URL=${{ secrets.CERT_URL }}"
+          variables: "TEST_OS=centos-stream-9;PLATFORM=${{ matrix.platform }};ARCH=${{ matrix.arch }};AWS_REGION=${{ secrets.AWS_REGION }}"
diff --git a/plans/install-upgrade.fmf b/plans/install-upgrade.fmf
@@ -0,0 +1,37 @@
+discover:
+  how: fmf
+  test: install-upgrade
+prepare:
+  - how: install
+    package:
+      - ansible-core
+      - gcc
+      - podman
+      - skopeo
+      - jq
+      - python3-devel
+      - unzip
+  - how: shell
+    script: curl -kLO ${CERT_URL}/certs/Current-IT-Root-CAs.pem --output-dir /etc/pki/ca-trust/source/anchors && update-ca-trust
+execute:
+  how: tmt
+
+/aws:
+  summary: Run bootc install and upgrade test on aws
+  tag: aws
+  environment+:
+    PLATFORM: aws
+  discover+:
+    test:
+      - /rpm-build
+      - /bootc-install-upgrade
+  adjust+:
+    - when: arch != x86_64 and arch != aarch64
+      enabled: false
+  prepare+:
+    - how: shell
+      script: |
+        pip install boto3 botocore
+        ansible-galaxy collection install amazon.aws community.general ansible.posix
+    - how: shell
+      script: curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip" -o "awscliv2.zip" && unzip awscliv2.zip && sudo ./aws/install
diff --git a/tests/integration/README.md b/tests/integration/README.md
@@ -0,0 +1,70 @@
+## Integration Test
+
+### Scenarios
+
+Integration test includes two scenarios, `RPM build` and `bootc install/upgrade`.
+
+1. RPM build scenario will build RPM for RHEL 9, CentOS Stream 9, and Fedora with mock.
+
+2. bootc install/upgrade scenario will install and upgrade bootc image and have some system checking, such as check mount point/permission, run podman with root and rootless, check persistent log.
+
+#### Run RPM Build Test
+
+```shell
+    podman run --rm --privileged -v ./:/workdir:z -e TEST_OS=$TEST_OS -e ARCH=$ARCH -e RHEL_REGISTRY_URL=$RHEL_REGISTRY_URL -e DOWNLOAD_NODE=$DOWNLOAD_NODE --workdir /workdir quay.io/fedora/fedora:40 ./tests/integration/mockbuild.sh
+```
+
+#### Run Integartion Test
+
+Run on a shared test infrastructure using the [`testing farm`](https://docs.testing-farm.io/Testing%20Farm/0.1/cli.html) tool. For example, running on AWS.
+
+Run `testing-farm` CLI from `quay.io/testing-farm/cli` container. Don't forget export the `TESTING_FARM_API_TOKEN` in your environment. To run RHEL test, `Red Hat Ranch` has to be used.
+
+```shell
+    export TESTING_FARM_API_TOKEN=<your-token>
+    testing-farm request \
+        --plan "aws" \
+        --environment PLATFORM=$PLATFORM \
+        --environment ARCH=$ARCH \
+        --environment TEST_OS=$TEST_OS \
+        --environment AWS_REGION=us-east-1 \
+        --secret DOWNLOAD_NODE=$DOWNLOAD_NODE \
+        --secret RHEL_REGISTRY_URL=$RHEL_REGISTRY_URL \
+        --secret CERT_URL=$CERT_URL \
+        --secret QUAY_USERNAME=$QUAY_USERNAME \
+        --secret QUAY_PASSWORD=$QUAY_PASSWORD \
+        --secret QUAY_SECRET=$QUAY_SECRET \
+        --secret AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
+        --secret AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
+        --git-url <PR URL> \
+        --git-ref <PR branch> \
+        --compose "CentOS-Stream-9" \
+        --arch $ARCH \
+        --context "arch=$ARCH" \
+        --timeout "120"
+```
+
+* AWS test needs environment variables `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_REGION=us-east-1` have to be configured.
+
+### Required environment variables
+
+    TEST_OS        The OS to run the tests in. Currently supported values:
+                       "rhel-9-4"
+                       "centos-stream-9"
+    ARCH           Test architecture
+                       "x86_64"
+                       "aarch64"
+
+    PLATFORM       Run test on:
+                       "aws"
+    QUAY_USERNAME      quay.io username
+    QUAY_PASSWORD      quay.io password
+    QUAY_SECRET        Save into /etc/ostree/auth.json for authenticated registry
+    DOWNLOAD_NODE      RHEL nightly compose download URL
+    RHEL_REGISTRY_URL  RHEL bootc image URL
+    CERT_URL           CA certificate download URL
+    AWS_ACCESS_KEY_ID           AWS access key id
+    AWS_SECRET_ACCESS_KEY       AWS secrety key
+    AWS_REGION                  AWS region
+                                    "us-east-1" RHEL AWS EC2 image is only available in this region
+    TESTING_FARM_API_TOKEN      Required by Testing Farm API
diff --git a/tests/integration/files/auth.template b/tests/integration/files/auth.template
@@ -0,0 +1,7 @@
+{
+  "auths": {
+    "quay.io": {
+      "auth": "REPLACE_ME"
+    }
+  }
+}
diff --git a/tests/integration/files/rhel-9.template b/tests/integration/files/rhel-9.template
@@ -0,0 +1,10 @@
+[rhel-9x-baseos]
+baseurl=http://REPLACE_ME/rhel-9/nightly/RHEL-9/REPLACE_COMPOSE_ID/compose/BaseOS/$basearch/os/
+enabled=1
+gpgcheck=0
+
+[rhel-9x-appstream]
+baseurl=http://REPLACE_ME/rhel-9/nightly/RHEL-9/REPLACE_COMPOSE_ID/compose/AppStream/$basearch/os/
+enabled=1
+gpgcheck=0
+
diff --git a/tests/integration/install-upgrade.fmf b/tests/integration/install-upgrade.fmf
@@ -0,0 +1,9 @@
+/rpm-build:
+  summary: bootc rpm build test
+  test: podman run --rm --privileged -v ../../:/workdir:z -e TEST_OS=$TEST_OS -e ARCH=$ARCH -e RHEL_REGISTRY_URL=$RHEL_REGISTRY_URL -e DOWNLOAD_NODE=$DOWNLOAD_NODE --workdir /workdir quay.io/fedora/fedora:40 ./tests/integration/mockbuild.sh
+  duration: 40m
+
+/bootc-install-upgrade:
+  summary: bootc install and upgrade test
+  test: ./install-upgrade.sh
+  duration: 40m