Test improvement & kindPath & apiVersionPath #3598
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E CI | |
| on: | |
| release: | |
| types: [created] | |
| pull_request: | |
| branches: | |
| - main | |
| - v* | |
| paths: | |
| - "**.go" | |
| - "!**_test.go" # exclude test files to ignore unit test changes | |
| - "test/**" # include test files in e2e again | |
| - "!**.md" | |
| - "Dockerfile.release" | |
| - ".github/workflows/e2e.yaml" | |
| - "chart/**" | |
| - "manifests/**" | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| REPOSITORY_NAME: ghcr.io/${{ github.repository }}-ci | |
| TAG_NAME: PR${{ github.event.number }} | |
| VCLUSTER_SUFFIX: vcluster | |
| VCLUSTER_NAME: vcluster | |
| VCLUSTER_NAMESPACE: vcluster | |
| jobs: | |
| build-and-push-syncer-image: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - run: git fetch --force --tags | |
| - name: Set up Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: "1.22" | |
| - name: Setup Just | |
| uses: extractions/setup-just@v2 | |
| - name: Setup Syft | |
| uses: anchore/sbom-action/download-syft@v0.17.0 | |
| - name: Setup GoReleaser | |
| uses: goreleaser/goreleaser-action@v6 | |
| with: | |
| install-only: true | |
| version: latest | |
| - name: Build and save syncer image | |
| run: | | |
| set -x | |
| TELEMETRY_PRIVATE_KEY="" goreleaser build --single-target --snapshot --id vcluster --clean --output ./vcluster | |
| docker build -t "${{ env.REPOSITORY_NAME }}:${{ env.TAG_NAME }}" -f Dockerfile.release --build-arg TARGETARCH=amd64 --build-arg TARGETOS=linux . | |
| docker save -o vcluster_syncer "${{ env.REPOSITORY_NAME }}:${{ env.TAG_NAME }}" | |
| - name: Upload syncer image to artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: vcluster_syncer | |
| path: ./vcluster_syncer | |
| retention-days: 7 | |
| build-vcluster-cli: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - run: git fetch --force --tags | |
| - name: Set up Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: "1.22" | |
| - name: Setup Just | |
| uses: extractions/setup-just@v2 | |
| - name: Setup Syft | |
| uses: anchore/sbom-action/download-syft@v0.17.0 | |
| - name: Setup GoReleaser | |
| uses: goreleaser/goreleaser-action@v6 | |
| with: | |
| install-only: true | |
| - name: Build vcluster cli | |
| run: | | |
| set -x | |
| TELEMETRY_PRIVATE_KEY="" goreleaser build --single-target --snapshot --id vcluster-cli --clean --output ./vcluster | |
| - name: Upload vcluster cli to artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: vcluster | |
| path: ./vcluster | |
| retention-days: 7 | |
| get-testsuites-dir: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - id: set-paths-matrix | |
| run: | | |
| set -x | |
| sudo apt-get install -y jq | |
| paths=$(ls -d ./test/e2e*) | |
| echo "matrix=$(printf '%s\n' "${paths}" | jq -R . | jq -cs .)" >> "$GITHUB_OUTPUT" | |
| outputs: | |
| matrix: ${{ steps.set-paths-matrix.outputs.matrix }} | |
| build-e2e: | |
| name: Build e2e binaries | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Configure git | |
| run: git config --global url.https://$GH_ACCESS_TOKEN@github.com/.insteadOf https://github.com/ | |
| env: | |
| GH_ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} | |
| - name: Set up Go | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version-file: go.mod | |
| - name: Build e2e binary | |
| run: | | |
| go run -mod=vendor github.com/onsi/ginkgo/v2/ginkgo build --require-suite -r --mod vendor $(ls -d ./test/e2e* | jq -R . | jq -rcs '. | join(" \\\n")') | |
| env: | |
| GOWORK: off | |
| - name: Upload e2e binaries to artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: e2e-binaries | |
| path: ./test/*/*.test | |
| retention-days: 7 | |
| vcluster-install-delete: | |
| name: Install and delete virtual cluster | |
| needs: | |
| - build-and-push-syncer-image | |
| - build-vcluster-cli | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - uses: azure/setup-helm@v4 | |
| name: Setup Helm | |
| with: | |
| version: "v3.11.0" | |
| - name: Set up kind k8s cluster | |
| uses: engineerd/setup-kind@v0.5.0 | |
| with: | |
| version: "v0.20.0" | |
| image: kindest/node:v1.30.0@sha256:047357ac0cfea04663786a612ba1eaba9702bef25227a794b52890dd8bcd692e | |
| - name: Testing kind cluster set-up | |
| run: | | |
| set -x | |
| kubectl cluster-info | |
| kubectl get pods -n kube-system | |
| echo "kubectl config current-context:" $(kubectl config current-context) | |
| echo "KUBECONFIG env var:" ${KUBECONFIG} | |
| - name: Download vcluster cli | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: vcluster | |
| - name: Download syncer image | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: vcluster_syncer | |
| - name: Setup environment | |
| run: | | |
| kind load image-archive vcluster_syncer | |
| chmod +x vcluster && sudo mv vcluster /usr/bin | |
| sudo apt-get install -y sed | |
| - name: Run tests - install and delete virtual cluster using kubectl | |
| run: | | |
| set -x | |
| ./hack/vcluster-install-scripts/test-kubectl-install.sh | |
| - name: Run tests - install and delete virtual cluster using helm | |
| run: | | |
| set -x | |
| ./hack/vcluster-install-scripts/test-helm-install.sh | |
| download-latest-cli: | |
| name: Execute test suites | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: download current cli | |
| run: | | |
| curl -L -o vcluster-current "https://github.com/loft-sh/vcluster/releases/latest/download/vcluster-linux-amd64" | |
| - name: Upload vcluster cli to artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: vcluster-current | |
| path: ./vcluster-current | |
| retention-days: 7 | |
| upgrade-test: | |
| name: test if we can upgrade from older version | |
| needs: | |
| - build-and-push-syncer-image | |
| - build-vcluster-cli | |
| - download-latest-cli | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| distribution: ["k3s", "k8s", "k0s"] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - uses: azure/setup-helm@v4 | |
| name: Setup Helm | |
| with: | |
| version: "v3.11.0" | |
| - name: Set up kind k8s cluster | |
| uses: engineerd/setup-kind@v0.5.0 | |
| with: | |
| version: "v0.20.0" | |
| image: kindest/node:v1.30.0@sha256:047357ac0cfea04663786a612ba1eaba9702bef25227a794b52890dd8bcd692e | |
| - name: Testing kind cluster set-up | |
| run: | | |
| set -x | |
| kubectl cluster-info | |
| kubectl get pods -n kube-system | |
| echo "kubectl config current-context:" $(kubectl config current-context) | |
| echo "KUBECONFIG env var:" ${KUBECONFIG} | |
| - name: Download vcluster cli | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: vcluster | |
| path: vcluster-dev | |
| - name: Download current cli | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: vcluster-current | |
| - name: Download syncer image | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: vcluster_syncer | |
| - name: install sed | |
| run: | | |
| sudo apt-get install -y sed | |
| - name: create vcluster with current cli | |
| run: | | |
| chmod +x ./vcluster-current | |
| kind load image-archive vcluster_syncer | |
| ./vcluster-current create ${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} \ | |
| --create-namespace \ | |
| --debug \ | |
| --connect=false \ | |
| --distro=${{ matrix.distribution }} | |
| ./hack/wait-for-pod.sh -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} | |
| - name: upgrade with the dev cli | |
| run: | | |
| chmod +x ./vcluster-dev/vcluster | |
| set -x | |
| sed -i "s|REPLACE_REPOSITORY_NAME|${{ env.REPOSITORY_NAME }}|g" test/commonValues.yaml | |
| sed -i "s|REPLACE_TAG_NAME|${{ env.TAG_NAME }}|g" test/commonValues.yaml | |
| ./vcluster-dev/vcluster create vcluster --distro=${{ matrix.distribution }} \ | |
| --connect=false \ | |
| --upgrade \ | |
| --local-chart-dir ./chart \ | |
| -f ./test/commonValues.yaml | |
| ./hack/wait-for-pod.sh -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} | |
| e2e-tests: | |
| name: Execute test suites | |
| needs: | |
| - build-and-push-syncer-image | |
| - build-vcluster-cli | |
| - get-testsuites-dir | |
| - build-e2e | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| distribution: ["k3s", "k8s", "k0s"] | |
| test-suite-path: ${{fromJson(needs.get-testsuites-dir.outputs.matrix)}} | |
| multinamespace-mode: ["false", "true"] | |
| ha: ["false", "true"] | |
| include: | |
| - distribution: "k8s" | |
| ha: "true" | |
| test-suite-path: "./test/e2e" | |
| multinamespace-mode: "false" | |
| - distribution: "k3s" | |
| ha: "true" | |
| test-suite-path: "./test/e2e" | |
| multinamespace-mode: "false" | |
| - distribution: "k0s" | |
| ha: "true" | |
| test-suite-path: "./test/e2e" | |
| multinamespace-mode: "false" | |
| exclude: | |
| - ha: "true" | |
| - distribution: "k8s" | |
| multinamespace-mode: "true" | |
| - distribution: "k0s" | |
| multinamespace-mode: "true" | |
| - distribution: "k3s" | |
| multinamespace-mode: "true" | |
| test-suite-path: "./test/e2e_target_namespace" | |
| - distribution: "k3s" | |
| multinamespace-mode: "true" | |
| test-suite-path: "./test/e2e_plugin" | |
| - distribution: "k3s" | |
| multinamespace-mode: "true" | |
| test-suite-path: "./test/e2e_isolation_mode" | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - uses: azure/setup-helm@v4 | |
| name: Setup Helm | |
| with: | |
| version: "v3.11.0" | |
| - name: Set up kind k8s cluster | |
| uses: engineerd/setup-kind@v0.5.0 | |
| with: | |
| version: "v0.20.0" | |
| image: kindest/node:v1.30.0@sha256:047357ac0cfea04663786a612ba1eaba9702bef25227a794b52890dd8bcd692e | |
| - name: Testing kind cluster set-up | |
| run: | | |
| set -x | |
| kubectl cluster-info | |
| kubectl get pods -n kube-system | |
| echo "kubectl config current-context:" $(kubectl config current-context) | |
| echo "KUBECONFIG env var:" ${KUBECONFIG} | |
| - name: Download vcluster cli | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: vcluster | |
| - name: Download syncer image | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: vcluster_syncer | |
| - name: Download e2e binaries | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: e2e-binaries | |
| path: ./test | |
| # - name: Setup upterm session for debugging | |
| # uses: lhotari/action-upterm@v1 | |
| - name: Create vcluster | |
| id: create-vcluster | |
| run: | | |
| set -x | |
| extraArgs=() | |
| if [ ${{ matrix.multinamespace-mode }} == "true" ]; then | |
| extraArgs+=( -f ./test/multins_values.yaml ) | |
| fi | |
| if [ ${{ matrix.test-suite-path }} == "./test/e2e_metrics_proxy" ]; then | |
| helm repo add metrics-server https://kubernetes-sigs.github.io/metrics-server/ | |
| helm upgrade --install metrics-server metrics-server/metrics-server --set args={--kubelet-insecure-tls} --set containerPort=4443 -n kube-system | |
| fi | |
| if [ ${{ matrix.ha }} == "true" ]; then | |
| haValues="-f ./test/values_ha.yaml" | |
| fi | |
| if [ ${{ matrix.test-suite-path }} == "./test/e2e_target_namespace" ]; then | |
| kubectl apply -f ${{ matrix.test-suite-path }}/role.yaml | |
| fi | |
| sudo apt-get install -y sed | |
| sed -i "s|REPLACE_REPOSITORY_NAME|${{ env.REPOSITORY_NAME }}|g" ${{ matrix.test-suite-path }}/../commonValues.yaml | |
| sed -i "s|REPLACE_TAG_NAME|${{ env.TAG_NAME }}|g" ${{ matrix.test-suite-path }}/../commonValues.yaml | |
| kind load image-archive vcluster_syncer | |
| chmod +x vcluster && sudo mv vcluster /usr/bin | |
| vcluster create ${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} \ | |
| --create-namespace \ | |
| --debug \ | |
| --connect=false \ | |
| --distro=${{ matrix.distribution }} \ | |
| --local-chart-dir ./chart \ | |
| -f ./test/commonValues.yaml \ | |
| $haValues \ | |
| -f ${{ matrix.test-suite-path }}/values.yaml \ | |
| "${extraArgs[@]}" | |
| continue-on-error: true | |
| - name: Wait until vcluster is ready | |
| id: wait-until-vcluster-is-ready | |
| if: steps.create-vcluster.outcome == 'success' | |
| run: | | |
| set -x | |
| ./hack/wait-for-pod.sh -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} | |
| continue-on-error: true | |
| - name: Collect deployment information in case vcluster fails to start | |
| if: steps.wait-until-vcluster-is-ready.outcome != 'success' | |
| run: | | |
| set -x | |
| kubectl get pods -o yaml -n ${{ env.VCLUSTER_NAMESPACE }} | |
| echo "======================================================================================================================" | |
| kubectl get events -n ${{ env.VCLUSTER_NAMESPACE }} --sort-by='.lastTimestamp' | |
| echo "======================================================================================================================" | |
| kubectl logs -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} -c syncer --tail=-1 -p || kubectl logs -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} -c syncer --tail=-1 | |
| echo "======================================================================================================================" | |
| if [[ "${{ matrix.test-suite-path }}" = "./test/e2e_plugin" ]]; then | |
| kubectl logs -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} -c bootstrap-with-deployment --tail=-1 -p || kubectl logs -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} -c bootstrap-with-deployment --tail=-1 | |
| echo "======================================================================================================================" | |
| fi | |
| kubectl describe pods -n ${{ env.VCLUSTER_NAMESPACE }} | |
| exit 1 | |
| # Skips NetworkPolicy tests because they require network plugin with support (e.g. Calico) | |
| - name: Execute e2e tests | |
| id: execute-e2e-tests | |
| working-directory: ${{ matrix.test-suite-path }} | |
| run: | | |
| set -x | |
| sudo chmod +x $(echo "${{ matrix.test-suite-path }}" | sed "s#./test/##g").test | |
| VCLUSTER_SUFFIX=${{ env.VCLUSTER_SUFFIX }} VCLUSTER_NAME=${{ env.VCLUSTER_NAME }} VCLUSTER_NAMESPACE=${{ env.VCLUSTER_NAMESPACE }} MULTINAMESPACE_MODE=${{ matrix.multinamespace-mode }} ./$(echo "${{ matrix.test-suite-path }}" | sed "s#./test/##g").test -test.v --ginkgo.v --ginkgo.skip='.*NetworkPolicy.*' --ginkgo.fail-fast | |
| if kubectl logs -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} -c syncer --tail=-1 -p >/dev/null 2>/dev/null; then | |
| echo "vCluster has restarted during testing, failing..." | |
| exit 1 | |
| fi | |
| continue-on-error: true | |
| - name: Print logs if e2e tests fail | |
| if: steps.execute-e2e-tests.outcome == 'failure' | |
| run: | | |
| set -x | |
| kubectl get pods -o yaml -n ${{ env.VCLUSTER_NAMESPACE }} | |
| echo "======================================================================================================================" | |
| kubectl get events -n ${{ env.VCLUSTER_NAMESPACE }} --sort-by='.lastTimestamp' | |
| echo "======================================================================================================================" | |
| kubectl logs -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} -c syncer --tail=-1 -p || kubectl logs -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} -c syncer --tail=-1 | |
| echo "======================================================================================================================" | |
| kubectl describe pods -n ${{ env.VCLUSTER_NAMESPACE }} | |
| exit 1 | |