Add build docker image option for test scripts (#312)

Signed-off-by: chensuyue <suyue.chen@intel.com>
opea-project · Jun 24, 2024 · e32a514 · e32a514
1 parent e948a7f
commit e32a514
Show file tree

Hide file tree

Showing 12 changed files with 174 additions and 59 deletions.
diff --git a/.github/workflows/docker-compose-e2e.yml b/.github/workflows/docker-compose-e2e.yml
@@ -61,10 +61,13 @@ jobs:
           HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
           example: ${{ matrix.example }}
           hardware: ${{ matrix.hardware }}
-          IMAGE_REPO: ${{needs.mega-image-build.outputs.image_repo}}
-          IMAGE_TAG: ${{needs.mega-image-build.outputs.image_tag}}
+          IMAGE_TAG: ${{ needs.mega-image-build.outputs.image_tag }}
+          IMAGE_REPO_GAUDI: ${{ vars.IMAGE_REPO_GAUDI }}
+          IMAGE_REPO_XEON: ${{ vars.IMAGE_REPO_XEON }}
         run: |
           cd ${{ github.workspace }}/$example/tests
+          if [ "$hardware" == "gaudi" ]; then IMAGE_REPO=$IMAGE_REPO_GAUDI; else IMAGE_REPO=$IMAGE_REPO_XEON; fi
+          export IMAGE_REPO=${IMAGE_REPO}
           example_l=$(echo $example | tr '[:upper:]' '[:lower:]')
           if [ -f test_${example_l}_on_${hardware}.sh ]; then timeout 30m bash test_${example_l}_on_${hardware}.sh; else echo "Test script not found, skip test!"; fi
 

diff --git a/.github/workflows/manifest-e2e.yml b/.github/workflows/manifest-e2e.yml
@@ -57,7 +57,8 @@ jobs:
 
       - name: Set variables
         run: |
-          echo "IMAGE_REPO=${{needs.mega-image-build.outputs.image_repo}}" >> $GITHUB_ENV
+          if [ ${{ matrix.hardware }} == "gaudi" ]; then IMAGE_REPO=${{ vars.IMAGE_REPO_GAUDI }}; else IMAGE_REPO=${{ vars.IMAGE_REPO_XEON }}; fi
+          echo "IMAGE_REPO=${IMAGE_REPO}" >> $GITHUB_ENV
           echo "IMAGE_TAG=${{needs.mega-image-build.outputs.image_tag}}" >> $GITHUB_ENV
           lower_example=$(echo "${{ matrix.example }}" | tr '[:upper:]' '[:lower:]')
           echo "NAMESPACE=$lower_example-$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV

diff --git a/.github/workflows/reuse-image-build.yml b/.github/workflows/reuse-image-build.yml
@@ -60,7 +60,5 @@ jobs:
           if [ -z "${{ env.IMAGE_REPO }}" ]; then
             IMAGE_REPO=$OPEA_IMAGE_REPO
           fi
-          echo "IMAGE_REPO=${IMAGE_REPO}"
           echo "IMAGE_TAG=${IMAGE_TAG}"
-          echo "image_repo=$IMAGE_REPO" >> $GITHUB_OUTPUT
           echo "image_tag=$IMAGE_TAG" >> $GITHUB_OUTPUT
diff --git a/.github/workflows/scripts/build_push.sh b/.github/workflows/scripts/build_push.sh
@@ -46,7 +46,7 @@ function docker_build() {
 # $1 is like "apple orange pear"
 for MEGA_SVC in $1; do
     case $MEGA_SVC in
-        "ChatQnA"|"CodeGen"|"CodeTrans"|"DocSum"|"SearchQnA")
+        "ChatQnA"|"CodeGen"|"CodeTrans"|"DocSum")
             cd $MEGA_SVC/docker
             IMAGE_NAME="$(getImagenameFromMega $MEGA_SVC)"
             docker_build ${IMAGE_NAME}
@@ -55,7 +55,6 @@ for MEGA_SVC in $1; do
             ;;
         "AudioQnA"|"SearchQnA"|"Translation"|"VisualQnA")
             echo "Not supported yet"
-            exit 1
             ;;
         *)
             echo "Unknown function: $MEGA_SVC"

diff --git a/ChatQnA/tests/test_chatqna_on_gaudi.sh b/ChatQnA/tests/test_chatqna_on_gaudi.sh
@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 set -e
+echo "IMAGE_REPO=${IMAGE_REPO}"
 
 WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
@@ -58,11 +59,14 @@ function start_services() {
 
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env
 
-    # Replace the container name with a test-specific name
-    echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
-    sed -i "s#image: opea/chatqna:latest#image: opea/chatqna:${IMAGE_TAG}#g" docker_compose.yaml
-    sed -i "s#image: opea/chatqna-ui:latest#image: opea/chatqna-ui:${IMAGE_TAG}#g" docker_compose.yaml
-    sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
+    if [[ "$IMAGE_REPO" != "" ]]; then
+        # Replace the container name with a test-specific name
+        echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
+        sed -i "s#image: opea/chatqna:latest#image: opea/chatqna:${IMAGE_TAG}#g" docker_compose.yaml
+        sed -i "s#image: opea/chatqna-ui:latest#image: opea/chatqna-ui:${IMAGE_TAG}#g" docker_compose.yaml
+        sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
+    fi
+
     # Start Docker Containers
     docker compose -f docker_compose.yaml up -d
     n=0
@@ -215,14 +219,12 @@ function stop_docker() {
 function main() {
 
     stop_docker
-    begin_time=$(date +%s)
-    # build_docker_images
-    # start_time=$(date +%s)
+    if [[ "$IMAGE_REPO" == "" ]]; then build_docker_images; fi
+    start_time=$(date +%s)
     start_services
     end_time=$(date +%s)
-    # minimal_duration=$((end_time-start_time))
-    maximal_duration=$((end_time-begin_time))
-    echo "Mega service start duration is "$maximal_duration"s"
+    duration=$((end_time-start_time))
+    echo "Mega service start duration is $duration s"
 
     validate_microservices
     validate_megaservice

diff --git a/ChatQnA/tests/test_chatqna_on_xeon.sh b/ChatQnA/tests/test_chatqna_on_xeon.sh
@@ -3,11 +3,32 @@
 # SPDX-License-Identifier: Apache-2.0
 
 set -e
+echo "IMAGE_REPO=${IMAGE_REPO}"
 
 WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
 ip_address=$(hostname -I | awk '{print $1}')
 
+function build_docker_images() {
+    cd $WORKPATH
+    git clone https://github.com/opea-project/GenAIComps.git
+    cd GenAIComps
+
+    docker build -t opea/embedding-tei:latest -f comps/embeddings/langchain/docker/Dockerfile .
+    docker build -t opea/retriever-redis:latest -f comps/retrievers/langchain/redis/docker/Dockerfile .
+    docker build -t opea/reranking-tei:latest -f comps/reranks/langchain/docker/Dockerfile .
+    docker build -t opea/llm-tgi:latest -f comps/llms/text-generation/tgi/Dockerfile .
+    docker build -t opea/dataprep-redis:latest -f comps/dataprep/redis/langchain/docker/Dockerfile .
+
+    cd $WORKPATH/docker
+    docker build --no-cache -t opea/chatqna:latest -f Dockerfile .
+
+    cd $WORKPATH/docker/ui
+    docker build --no-cache -t opea/chatqna-ui:latest -f docker/Dockerfile .
+
+    docker images
+}
+
 function start_services() {
     cd $WORKPATH/docker/xeon
 
@@ -30,11 +51,14 @@ function start_services() {
 
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env
 
-    # Replace the container name with a test-specific name
-    echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
-    sed -i "s#image: opea/chatqna:latest#image: opea/chatqna:${IMAGE_TAG}#g" docker_compose.yaml
-    sed -i "s#image: opea/chatqna-ui:latest#image: opea/chatqna-ui:${IMAGE_TAG}#g" docker_compose.yaml
-    sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
+    if [[ "$IMAGE_REPO" != "" ]]; then
+        # Replace the container name with a test-specific name
+        echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
+        sed -i "s#image: opea/chatqna:latest#image: opea/chatqna:${IMAGE_TAG}#g" docker_compose.yaml
+        sed -i "s#image: opea/chatqna-ui:latest#image: opea/chatqna-ui:${IMAGE_TAG}#g" docker_compose.yaml
+        sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
+    fi
+
     # Start Docker Containers
     docker compose -f docker_compose.yaml up -d
     n=0
@@ -188,12 +212,12 @@ function stop_docker() {
 function main() {
 
     stop_docker
-
-    begin_time=$(date +%s)
+    if [[ "$IMAGE_REPO" == "" ]]; then build_docker_images; fi
+    start_time=$(date +%s)
     start_services
     end_time=$(date +%s)
-    maximal_duration=$((end_time-begin_time))
-    echo "Mega service start duration is "$maximal_duration"s" && sleep 1s
+    duration=$((end_time-start_time))
+    echo "Mega service start duration is $duration s" && sleep 1s
 
     validate_microservices
     validate_megaservice

diff --git a/CodeGen/tests/test_codegen_on_gaudi.sh b/CodeGen/tests/test_codegen_on_gaudi.sh
@@ -8,6 +8,24 @@ WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
 ip_address=$(hostname -I | awk '{print $1}')
 
+function build_docker_images() {
+    cd $WORKPATH
+    git clone https://github.com/opea-project/GenAIComps.git
+    cd GenAIComps
+
+    docker build -t opea/llm-tgi:latest -f comps/llms/text-generation/tgi/Dockerfile .
+
+    docker pull ghcr.io/huggingface/tgi-gaudi:1.2.1
+
+    cd $WORKPATH/docker
+    docker build --no-cache -t opea/codegen:latest -f Dockerfile .
+
+    cd $WORKPATH/docker/ui
+    docker build --no-cache -t opea/codegen-ui:latest -f docker/Dockerfile .
+
+    docker images
+}
+
 function start_services() {
     cd $WORKPATH/docker/gaudi
 
@@ -20,11 +38,14 @@ function start_services() {
 
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env
 
-    # Replace the container name with a test-specific name
-    echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
-    sed -i "s#image: opea/codegen:latest#image: opea/codegen:${IMAGE_TAG}#g" docker_compose.yaml
-    sed -i "s#image: opea/codegen-ui:latest#image: opea/codegen-ui:${IMAGE_TAG}#g" docker_compose.yaml
-    sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
+    if [[ "$IMAGE_REPO" != "" ]]; then
+        # Replace the container name with a test-specific name
+        echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
+        sed -i "s#image: opea/codegen:latest#image: opea/codegen:${IMAGE_TAG}#g" docker_compose.yaml
+        sed -i "s#image: opea/codegen-ui:latest#image: opea/codegen-ui:${IMAGE_TAG}#g" docker_compose.yaml
+        sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
+    fi
+
     # Start Docker Containers
     docker compose -f docker_compose.yaml up -d
 
@@ -127,6 +148,7 @@ function main() {
 
     stop_docker
 
+    if [[ "$IMAGE_REPO" == "" ]]; then build_docker_images; fi
     start_services
 
     validate_microservices

diff --git a/CodeGen/tests/test_codegen_on_xeon.sh b/CodeGen/tests/test_codegen_on_xeon.sh
@@ -8,6 +8,22 @@ WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
 ip_address=$(hostname -I | awk '{print $1}')
 
+function build_docker_images() {
+    cd $WORKPATH
+    git clone https://github.com/opea-project/GenAIComps.git
+    cd GenAIComps
+
+    docker build -t opea/llm-tgi:latest -f comps/llms/text-generation/tgi/Dockerfile .
+
+    cd $WORKPATH/docker
+    docker build --no-cache -t opea/codegen:latest -f Dockerfile .
+
+    cd $WORKPATH/docker/ui
+    docker build --no-cache -t opea/codegen-ui:latest -f docker/Dockerfile .
+
+    docker images
+}
+
 function start_services() {
     cd $WORKPATH/docker/xeon
 
@@ -20,11 +36,14 @@ function start_services() {
 
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env
 
-    # Replace the container name with a test-specific name
-    echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
-    sed -i "s#image: opea/codegen:latest#image: opea/codegen:${IMAGE_TAG}#g" docker_compose.yaml
-    sed -i "s#image: opea/codegen-ui:latest#image: opea/codegen-ui:${IMAGE_TAG}#g" docker_compose.yaml
-    sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
+    if [[ "$IMAGE_REPO" != "" ]]; then
+        # Replace the container name with a test-specific name
+        echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
+        sed -i "s#image: opea/codegen:latest#image: opea/codegen:${IMAGE_TAG}#g" docker_compose.yaml
+        sed -i "s#image: opea/codegen-ui:latest#image: opea/codegen-ui:${IMAGE_TAG}#g" docker_compose.yaml
+        sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
+    fi
+
     # Start Docker Containers
     docker compose -f docker_compose.yaml up -d
 
@@ -128,6 +147,7 @@ function main() {
 
     stop_docker
 
+    if [[ "$IMAGE_REPO" == "" ]]; then build_docker_images; fi
     start_services
 
     validate_microservices

diff --git a/CodeTrans/tests/test_codetrans_on_gaudi.sh b/CodeTrans/tests/test_codetrans_on_gaudi.sh
@@ -40,11 +40,14 @@ function start_services() {
 
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env
 
-    # Replace the container name with a test-specific name
-    echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
-    sed -i "s#image: opea/codetrans:latest#image: opea/codetrans:${IMAGE_TAG}#g" docker_compose.yaml
-    sed -i "s#image: opea/codetrans-ui:latest#image: opea/codetrans-ui:${IMAGE_TAG}#g" docker_compose.yaml
-    sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
+    if [[ "$IMAGE_REPO" != "" ]]; then
+        # Replace the container name with a test-specific name
+        echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
+        sed -i "s#image: opea/codetrans:latest#image: opea/codetrans:${IMAGE_TAG}#g" docker_compose.yaml
+        sed -i "s#image: opea/codetrans-ui:latest#image: opea/codetrans-ui:${IMAGE_TAG}#g" docker_compose.yaml
+        sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
+    fi
+
     # Start Docker Containers
     docker compose -f docker_compose.yaml up -d
 
@@ -145,7 +148,7 @@ function main() {
 
     stop_docker
 
-    # build_docker_images
+    if [[ "$IMAGE_REPO" == "" ]]; then build_docker_images; fi
     start_services
 
     validate_microservices

diff --git a/CodeTrans/tests/test_codetrans_on_xeon.sh b/CodeTrans/tests/test_codetrans_on_xeon.sh
@@ -8,6 +8,22 @@ WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
 ip_address=$(hostname -I | awk '{print $1}')
 
+function build_docker_images() {
+    cd $WORKPATH
+    git clone https://github.com/opea-project/GenAIComps.git
+    cd GenAIComps
+
+    docker build -t opea/llm-tgi:latest -f comps/llms/text-generation/tgi/Dockerfile .
+
+    cd $WORKPATH/docker
+    docker build --no-cache -t opea/codetrans:latest -f Dockerfile .
+
+    cd $WORKPATH/docker/ui
+    docker build --no-cache -t opea/codetrans-ui:latest -f docker/Dockerfile .
+
+    docker images
+}
+
 function start_services() {
     cd $WORKPATH/docker/xeon
     export http_proxy=${http_proxy}
@@ -21,11 +37,14 @@ function start_services() {
 
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env
 
-    # Replace the container name with a test-specific name
-    echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
-    sed -i "s#image: opea/codetrans:latest#image: opea/codetrans:${IMAGE_TAG}#g" docker_compose.yaml
-    sed -i "s#image: opea/codetrans-ui:latest#image: opea/codetrans-ui:${IMAGE_TAG}#g" docker_compose.yaml
-    sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
+    if [[ "$IMAGE_REPO" != "" ]]; then
+        # Replace the container name with a test-specific name
+        echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
+        sed -i "s#image: opea/codetrans:latest#image: opea/codetrans:${IMAGE_TAG}#g" docker_compose.yaml
+        sed -i "s#image: opea/codetrans-ui:latest#image: opea/codetrans-ui:${IMAGE_TAG}#g" docker_compose.yaml
+        sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
+    fi
+
     # Start Docker Containers
     docker compose -f docker_compose.yaml up -d
 
@@ -128,6 +147,7 @@ function main() {
 
     stop_docker
 
+    if [[ "$IMAGE_REPO" == "" ]]; then build_docker_images; fi
     start_services
 
     validate_microservices

diff --git a/DocSum/tests/test_docsum_on_gaudi.sh b/DocSum/tests/test_docsum_on_gaudi.sh
@@ -38,11 +38,14 @@ function start_services() {
 
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env
 
-    # Replace the container name with a test-specific name
-    echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
-    sed -i "s#image: opea/docsum:latest#image: opea/docsum:${IMAGE_TAG}#g" docker_compose.yaml
-    sed -i "s#image: opea/docsum-ui:latest#image: opea/docsum-ui:${IMAGE_TAG}#g" docker_compose.yaml
-    sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
+    if [[ "$IMAGE_REPO" != "" ]]; then
+        # Replace the container name with a test-specific name
+        echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
+        sed -i "s#image: opea/docsum:latest#image: opea/docsum:${IMAGE_TAG}#g" docker_compose.yaml
+        sed -i "s#image: opea/docsum-ui:latest#image: opea/docsum-ui:${IMAGE_TAG}#g" docker_compose.yaml
+        sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml
+    fi
+
     # Start Docker Containers
     docker compose -f docker_compose.yaml up -d
 
@@ -145,7 +148,7 @@ function main() {
 
     stop_docker
 
-    #build_docker_images
+    if [[ "$IMAGE_REPO" == "" ]]; then build_docker_images; fi
     start_services
 
     validate_microservices