enable GMC e2e on Gaudi (#123)

Signed-off-by: Yingchun Guo <yingchun.guo@intel.com>
opea-project · Jun 25, 2024 · d204a77 · d204a77
1 parent 225a507
commit d204a77
Show file tree

Hide file tree

Showing 3 changed files with 107 additions and 4 deletions.
diff --git a/.github/workflows/go-e2e-gaudi.yaml b/.github/workflows/go-e2e-gaudi.yaml
@@ -0,0 +1,102 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: GMC E2e Tests on Gaudi
+
+on:
+  pull_request:
+    branches: [main]
+    types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
+    paths:
+      - microservices-connector/**
+      - manifests/ChatQnA/**
+      - manifests/CodeGen/**
+      - .github/workflows/scripts/e2e/gmc_test.sh
+  workflow_dispatch:
+
+# If there is a new commit, the previous jobs will be canceled
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-gaudi
+  cancel-in-progress: true
+
+env:
+  GOSRC_DIR: "microservices-connector"
+
+jobs:
+  image-build:
+    runs-on: docker-build-gaudi
+    outputs:
+      docker_registry: ${{ steps.set_variables.outputs.DOCKER_REGISTRY }}
+      version: ${{ steps.set_variables.outputs.VERSION }}
+    steps:
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set variables
+        id: set_variables
+        run: |
+          echo "DOCKER_REGISTRY=${OPEA_IMAGE_REPO}opea" >> $GITHUB_ENV
+          echo "VERSION=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          echo "DOCKER_REGISTRY=${OPEA_IMAGE_REPO}opea" >> $GITHUB_OUTPUT
+          echo "VERSION=${{ github.event.pull_request.head.sha }}" >> $GITHUB_OUTPUT
+
+      - name: Build image and push
+        run: |
+          cd $GOSRC_DIR
+          make docker.build
+          make docker.push
+
+      - name: Clean up images
+        if: always()
+        run: |
+          # clean up the images
+          docker rmi ${{ env.DOCKER_REGISTRY }}/gmcrouter:${{ env.VERSION }}
+          docker rmi ${{ env.DOCKER_REGISTRY }}/gmcmanager:${{ env.VERSION }}
+          echo y |  docker image prune
+
+  go-e2e:
+    runs-on: gaudi
+    needs: image-build
+    steps:
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set variables
+        run: |
+          echo "SYSTEM_NAMESPACE=opea-system-$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV
+          echo "APP_NAMESPACE=chatqna-$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV
+          echo "DOCKER_REGISTRY=${{ needs.image-build.outputs.docker_registry }}" >> $GITHUB_ENV
+          echo "VERSION=${{ needs.image-build.outputs.version }}" >> $GITHUB_ENV
+          echo "should_cleanup=false" >> $GITHUB_ENV
+          echo "skip_validate=false" >> $GITHUB_ENV
+
+      - name: Install GMC
+        run: |
+          echo "should_cleanup=true" >> $GITHUB_ENV
+          .github/workflows/scripts/e2e/gmc_test.sh install_gmc
+          exit_status=$$?$$
+          if [ $$exit_status -ne 0 ]; then
+            echo "Failed to install modules"
+            echo "skip_validate=true" >> $GITHUB_ENV
+          fi
+
+      - name: Run e2e tests
+        run: |
+          # export DOCKER_REGISTRY={{ env.DOCKER_REGISTRY }}
+          # export VERSION={{ env.VERSION }}
+          if $skip_validate; then
+            echo "Skip validate"
+          else
+            .github/workflows/scripts/e2e/gmc_test.sh validate_gmc
+          fi
+
+      - name: Cleanup modules
+        if: always()
+        run: |
+          if $should_cleanup; then
+            .github/workflows/scripts/e2e/gmc_test.sh cleanup_gmc
+          fi
diff --git a/.github/workflows/go-e2e.yaml b/.github/workflows/go-e2e.yaml
@@ -1,7 +1,7 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-name: GMC E2e Tests
+name: GMC E2e Tests on Xeon
 
 on:
   pull_request:
@@ -16,7 +16,7 @@ on:
 
 # If there is a new commit, the previous jobs will be canceled
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-xeon
   cancel-in-progress: true
 
 env:
@@ -57,7 +57,7 @@ jobs:
           echo y |  docker image prune
 
   go-e2e:
-    runs-on: k8s
+    runs-on: xeon
     needs: image-build
     steps:
       - name: Checkout out Repo

diff --git a/.github/workflows/scripts/e2e/gmc_test.sh b/.github/workflows/scripts/e2e/gmc_test.sh
@@ -5,7 +5,8 @@
 set -xe
 USER_ID=$(whoami)
 LOG_PATH=/home/$(whoami)/logs
-MOUNT_DIR=/home/$USER_ID/charts-mnt
+MOUNT_DIR=/home/$USER_ID/.cache/huggingface/hub
+
 IMAGE_REPO=${OPEA_IMAGE_REPO:-""}
 CODEGEN_NAMESPACE="${APP_NAMESPACE}-codegen"