Add image build job in docker compose e2e xeon test in CI (#304)

Signed-off-by: Yingchun Guo <yingchun.guo@intel.com>

.github/workflows/E2E_test_with_compose.yml → .github/workflows/docker-compose-e2e.yml

@@ -13,7 +13,7 @@ on:
       - "**/ui/**"
       - "!**.md"
       - "!**.txt"
-      - .github/workflows/E2E_test_with_compose.yml
+      - .github/workflows/docker-compose-e2e.yml
   workflow_dispatch:
 
 # If there is a new commit, the previous jobs will be canceled
@@ -25,10 +25,20 @@ jobs:
   job1:
     uses: ./.github/workflows/reuse-get-test-matrix.yml
     with:
-      diff-excluded-files: '.github|README.md|*.txt|deprecate|kubernetes|manifest|assets'
+      diff_excluded_files: '.github|README.md|*.txt|deprecate|kubernetes|manifest|assets'
 
-  Example-test:
+  mega-image-build:
     needs: job1
+    strategy:
+      matrix: ${{ fromJSON(needs.job1.outputs.run_matrix) }}
+    uses: ./.github/workflows/reuse-image-build.yml
+    with:
+      image_tag: ${{ github.event.pull_request.head.sha }}
+      mega_service: "${{ matrix.example }}"
+      runner_label: "docker-build-${{ matrix.hardware }}"
+
+  Example-test:
+    needs: [job1, mega-image-build]
     strategy:
       matrix: ${{ fromJSON(needs.job1.outputs.run_matrix) }}
     runs-on: ${{ matrix.hardware }}
@@ -51,6 +61,8 @@ jobs:
           HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
           example: ${{ matrix.example }}
           hardware: ${{ matrix.hardware }}
+          IMAGE_REPO: ${{needs.mega-image-build.outputs.image_repo}}
+          IMAGE_TAG: ${{needs.mega-image-build.outputs.image_tag}}
         run: |
           cd ${{ github.workspace }}/$example/tests
           example_l=$(echo $example | tr '[:upper:]' '[:lower:]')

.github/workflows/manifest-e2e.yml

@@ -12,7 +12,6 @@ on:
       - "**/tests/test_manifest**"
       - "!**.md"
       - "!**.txt"
-      - .github/workflows/manifest-e2e.yml
   workflow_dispatch:
 
 concurrency:
@@ -23,18 +22,18 @@ jobs:
   job1:
     uses: ./.github/workflows/reuse-get-test-matrix.yml
     with:
-      diff-excluded-files: '.github|deprecated|docker|assets|*.md|*.txt'
-      xeon-server-lable: 'k8s'
-      gaudi-server-lable: ""
+      diff_excluded_files: '.github|deprecated|docker|assets|*.md|*.txt'
+      xeon_server_label: 'k8s'
+      gaudi_server_label: ""
 
   mega-image-build:
     needs: job1
     strategy:
       matrix: ${{ fromJSON(needs.job1.outputs.run_matrix) }}
     uses: ./.github/workflows/reuse-image-build.yml
     with:
-      image-tag: ${{ github.event.pull_request.head.sha }}
-      mega-service: "${{ matrix.example }}"
+      image_tag: ${{ github.event.pull_request.head.sha }}
+      mega_service: "${{ matrix.example }}"
 
   manifest-test:
     needs: [job1, mega-image-build]
@@ -57,8 +56,8 @@ jobs:
 
       - name: Set variables
         run: |
-          echo "IMAGE_REPO=${OPEA_IMAGE_REPO}/" >> $GITHUB_ENV
-          echo "IMAGE_TAG=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          echo "IMAGE_REPO=${{needs.mega-image-build.outputs.image_repo}}" >> $GITHUB_ENV
+          echo "IMAGE_TAG=${{needs.mega-image-build.outputs.image_tag}}" >> $GITHUB_ENV
           lower_example=$(echo "${{ matrix.example }}" | tr '[:upper:]' '[:lower:]')
           echo "NAMESPACE=$lower_example-$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV
           echo "ROLLOUT_TIMEOUT_SECONDS=1800s" >> $GITHUB_ENV

.github/workflows/reuse-get-test-matrix.yml

@@ -7,15 +7,15 @@ permissions: read-all
 on:
   workflow_call:
     inputs:
-      diff-excluded-files:
+      diff_excluded_files:
         required: false
         type: string
         default: '.github|README.md|*.txt'
-      xeon-server-lable:
+      xeon_server_label:
         required: false
         type: string
         default: 'xeon'
-      gaudi-server-lable:
+      gaudi_server_label:
         required: false
         type: string
         default: 'gaudi'
@@ -57,7 +57,7 @@ jobs:
           fi
           merged_commit=$(git log -1 --format='%H')
           changed_files="$(git diff --name-only ${base_commit} ${merged_commit} | \
-          grep -vE '${{ inputs.diff-excluded-files }}')" || true
+          grep -vE '${{ inputs.diff_excluded_files }}')" || true
           examples=$(printf '%s\n' "${changed_files[@]}" | grep '/' | cut -d'/' -f1 | sort -u)
           run_matrix="{\"include\":["
           for example in ${examples}; do
@@ -66,10 +66,10 @@ jobs:
               if [ $(printf '%s\n' "${changed_files[@]}" | grep ${example} | grep -c xeon) != 0 ]; then run_hardware="xeon ${run_hardware}"; fi
               if [ "$run_hardware" == "" ]; then run_hardware="xeon"; fi
               for hw in ${run_hardware}; do
-                  if [ "$hw" == "gaudi" ] && [ "${{ inputs.gaudi-server-lable }}" != "" ]; then
-                    run_matrix="${run_matrix}{\"example\":\"${example}\",\"hardware\":\"${{ inputs.gaudi-server-lable }}\"},"
-                  elif [ "${{ inputs.xeon-server-lable }}" != "" ]; then
-                    run_matrix="${run_matrix}{\"example\":\"${example}\",\"hardware\":\"${{ inputs.xeon-server-lable }}\"},"
+                  if [ "$hw" == "gaudi" ] && [ "${{ inputs.gaudi_server_label }}" != "" ]; then
+                    run_matrix="${run_matrix}{\"example\":\"${example}\",\"hardware\":\"${{ inputs.gaudi_server_label }}\"},"
+                  elif [ "${{ inputs.xeon_server_label }}" != "" ]; then
+                    run_matrix="${run_matrix}{\"example\":\"${example}\",\"hardware\":\"${{ inputs.xeon_server_label }}\"},"
                   fi
               done
           done

.github/workflows/reuse-image-build.yml

@@ -6,26 +6,33 @@ permissions: read-all
 on:
   workflow_call:
     inputs:
-      image-repo:
+      image_repo:
         required: false
         type: string
-      image-tag:
+      image_tag:
         required: true
         type: string
-      mega-service:
+      mega_service:
         required: true
         type: string
-      runner_lable:
+      runner_label:
         required: false
         type: string
-        default: 'docker-build'
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-image-build
+        default: 'docker-build-xeon'
+    outputs:
+      image_repo:
+        description: "The image reposity used for the image build"
+        value: ${{ jobs.mega-image-build.outputs.image_repo }}
+      image_tag:
+        description: "The image tag used for the image build"
+        value: ${{ jobs.mega-image-build.outputs.image_tag }}
 
 jobs:
   mega-image-build:
-    runs-on: ${{ inputs.runner_lable }}
+    runs-on: ${{ inputs.runner_label }}
+    outputs:
+      image_repo: ${{ steps.build-megaservice-image.outputs.image_repo }}
+      image_tag: ${{ steps.build-megaservice-image.outputs.image_tag }}
     steps:
       - name: Checkout out Repo
         uses: actions/checkout@v4
@@ -35,8 +42,15 @@ jobs:
       - name: Building MegaService Docker Image
         id: build-megaservice-image
         env:
-          IMAGE_REPO: ${{ inputs.image-repo }}
-          IMAGE_TAG: ${{ inputs.image-tag }}
-          mega-service: ${{ inputs.mega-service }}
+          IMAGE_REPO: ${{ inputs.image_repo }}
+          IMAGE_TAG: ${{ inputs.image_tag }}
+          MEGA_SERVICE: ${{ inputs.mega_service }}
         run: |
-          .github/workflows/scripts/build_push.sh ${{ env.mega-service}}
+          .github/workflows/scripts/build_push.sh ${{ env.MEGA_SERVICE}}
+          if [ -z "${{ env.IMAGE_REPO }}" ]; then
+            IMAGE_REPO=$OPEA_IMAGE_REPO
+          fi
+          echo "IMAGE_REPO=${IMAGE_REPO}"
+          echo "IMAGE_TAG=${IMAGE_TAG}"
+          echo "image_repo=$IMAGE_REPO" >> $GITHUB_OUTPUT
+          echo "image_tag=$IMAGE_TAG" >> $GITHUB_OUTPUT

.github/workflows/scripts/build_push.sh

@@ -22,7 +22,7 @@ function checkExist() {
 
 function docker_build() {
     # check if if IMAGE_TAG is not "latest" and the image exists in the registry
-    if [ $IMAGE_TAG != "latest" && $(checkExist $1) == "true" ]; then
+    if [ "$IMAGE_TAG" != "latest" ] && [ "$(checkExist $1)" == "true" ]; then
         echo "Image ${IMAGE_REPO}opea/$1:$IMAGE_TAG already exists in the registry"
         return
     fi
@@ -35,7 +35,8 @@ function docker_build() {
     echo "Building ${IMAGE_REPO}opea/$1:$IMAGE_TAG using Dockerfile $DOCKERFILE_PATH"
     # if https_proxy and http_proxy are set, pass them to docker build
     if [ -z "$https_proxy" ]; then
-        docker build --no-cache -t ${IMAGE_REPO}opea/$1:$IMAGE_TAG -f $DOCKERFILE_PATH .
+        #docker build --no-cache -t ${IMAGE_REPO}opea/$1:$IMAGE_TAG -f $DOCKERFILE_PATH .
+        docker build -t ${IMAGE_REPO}opea/$1:$IMAGE_TAG -f $DOCKERFILE_PATH .
     else
         docker build --no-cache -t ${IMAGE_REPO}opea/$1:$IMAGE_TAG --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f $DOCKERFILE_PATH .
     fi

ChatQnA/tests/test_chatqna_on_xeon.sh

@@ -8,26 +8,6 @@ WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
 ip_address=$(hostname -I | awk '{print $1}')
 
-function build_docker_images() {
-    cd $WORKPATH
-    git clone https://github.com/opea-project/GenAIComps.git
-    cd GenAIComps
-
-    docker build -t opea/embedding-tei:latest -f comps/embeddings/langchain/docker/Dockerfile .
-    docker build -t opea/retriever-redis:latest -f comps/retrievers/langchain/redis/docker/Dockerfile .
-    docker build -t opea/reranking-tei:latest -f comps/reranks/langchain/docker/Dockerfile .
-    docker build -t opea/llm-tgi:latest -f comps/llms/text-generation/tgi/Dockerfile .
-    docker build -t opea/dataprep-redis:latest -f comps/dataprep/redis/langchain/docker/Dockerfile .
-
-    cd $WORKPATH/docker
-    docker build --no-cache -t opea/chatqna:latest -f Dockerfile .
-
-    cd $WORKPATH/docker/ui
-    docker build --no-cache -t opea/chatqna-ui:latest -f docker/Dockerfile .
-
-    docker images
-}
-
 function start_services() {
     cd $WORKPATH/docker/xeon
 
@@ -50,9 +30,12 @@ function start_services() {
 
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env
 
+    # Replace the container name with a test-specific name
+    echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
+    sed -i "s#image: opea/chatqna:latest#image: opea/chatqna:${IMAGE_TAG}#g" docker_compose.yaml
+    sed -i "s#image: opea/chatqna-ui:latest#image: opea/chatqna-ui:${IMAGE_TAG}#g" docker_compose.yaml
+    sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
     # Start Docker Containers
-    # TODO: Replace the container name with a test-specific name
-
     docker compose -f docker_compose.yaml up -d
     n=0
     until [[ "$n" -ge 200 ]]; do
@@ -207,13 +190,10 @@ function main() {
     stop_docker
 
     begin_time=$(date +%s)
-    build_docker_images
-    start_time=$(date +%s)
     start_services
     end_time=$(date +%s)
-    minimal_duration=$((end_time-start_time))
     maximal_duration=$((end_time-begin_time))
-    echo "Mega service start minimal duration is "$minimal_duration"s, maximal duration(including docker image build) is "$maximal_duration"s" && sleep 1s
+    echo "Mega service start duration is "$maximal_duration"s" && sleep 1s
 
     validate_microservices
     validate_megaservice

CodeGen/tests/test_codegen_on_xeon.sh

@@ -8,22 +8,6 @@ WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
 ip_address=$(hostname -I | awk '{print $1}')
 
-function build_docker_images() {
-    cd $WORKPATH
-    git clone https://github.com/opea-project/GenAIComps.git
-    cd GenAIComps
-
-    docker build -t opea/llm-tgi:latest -f comps/llms/text-generation/tgi/Dockerfile .
-
-    cd $WORKPATH/docker
-    docker build --no-cache -t opea/codegen:latest -f Dockerfile .
-
-    cd $WORKPATH/docker/ui
-    docker build --no-cache -t opea/codegen-ui:latest -f docker/Dockerfile .
-
-    docker images
-}
-
 function start_services() {
     cd $WORKPATH/docker/xeon
 
@@ -36,8 +20,12 @@ function start_services() {
 
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env
 
+    # Replace the container name with a test-specific name
+    echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
+    sed -i "s#image: opea/codegen:latest#image: opea/codegen:${IMAGE_TAG}#g" docker_compose.yaml
+    sed -i "s#image: opea/codegen-ui:latest#image: opea/codegen-ui:${IMAGE_TAG}#g" docker_compose.yaml
+    sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
     # Start Docker Containers
-    # TODO: Replace the container name with a test-specific name
     docker compose -f docker_compose.yaml up -d
 
     sleep 2m # Waits 2 minutes
@@ -140,7 +128,6 @@ function main() {
 
     stop_docker
 
-    build_docker_images
     start_services
 
     validate_microservices

CodeTrans/tests/test_codetrans_on_xeon.sh

@@ -8,22 +8,6 @@ WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
 ip_address=$(hostname -I | awk '{print $1}')
 
-function build_docker_images() {
-    cd $WORKPATH
-    git clone https://github.com/opea-project/GenAIComps.git
-    cd GenAIComps
-
-    docker build -t opea/llm-tgi:latest -f comps/llms/text-generation/tgi/Dockerfile .
-
-    cd $WORKPATH/docker
-    docker build --no-cache -t opea/codetrans:latest -f Dockerfile .
-
-    cd $WORKPATH/docker/ui
-    docker build --no-cache -t opea/codetrans-ui:latest -f docker/Dockerfile .
-
-    docker images
-}
-
 function start_services() {
     cd $WORKPATH/docker/xeon
     export http_proxy=${http_proxy}
@@ -37,8 +21,12 @@ function start_services() {
 
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env
 
+    # Replace the container name with a test-specific name
+    echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
+    sed -i "s#image: opea/codetrans:latest#image: opea/codetrans:${IMAGE_TAG}#g" docker_compose.yaml
+    sed -i "s#image: opea/codetrans-ui:latest#image: opea/codetrans-ui:${IMAGE_TAG}#g" docker_compose.yaml
+    sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
     # Start Docker Containers
-    # TODO: Replace the container name with a test-specific name
     docker compose -f docker_compose.yaml up -d
 
     sleep 2m
@@ -140,7 +128,6 @@ function main() {
 
     stop_docker
 
-    build_docker_images
     start_services
 
     validate_microservices