ci(e2e): restore broken e2e test (#1131)

* update e2e * update e2e * mock console build * fix pv error * fix helm error * fix helm error * fix helm error * fix namespace error error * add debug tool * fix error * try * open debug * bug fix * remove debug * test * fix * manually debug * fix image name error * fix pypi host error * fix pypi host error * fix network lost error * restart port forward * restart port forward with sleep * adapt to kill empty error * restore schedule points and strategy matrix
star-whale · Sep 7, 2022 · db2b659 · db2b659
1 parent e0305ae
commit db2b659
Show file tree

Hide file tree

Showing 4 changed files with 43 additions and 10 deletions.
diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml
@@ -1,6 +1,13 @@
 name: End to End test for starwhale
 
 on:
+  workflow_dispatch:
+    inputs:
+      debug_enabled:
+        type: boolean
+        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
+        required: false
+        default: false
   schedule:
     - cron: "34 10,13 * * *"
 
@@ -61,6 +68,9 @@ jobs:
           cache: "maven"
           server-id: starwhale # Value of the distributionManagement/repository/id field of the pom.xml
 
+      - name: Setup tmate session
+        uses: mxschmitt/action-tmate@v3
+        if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
       - name: run e2e
         working-directory: ./scripts/e2e_test
         env:

diff --git a/docker/Makefile b/docker/Makefile
@@ -128,7 +128,7 @@ release-nodejs:
 build-console:
 	docker volume create --name ${YARN_VOLUME} && \
 	docker run --rm -v ${YARN_VOLUME}:/app ${DH_NODEJS_IMAGE} /bin/sh -c "cp -r /root/.npmrc /app/ && chown $(shell id -u):$(shell id -g) -R /app" && \
-	docker run --rm -it \
+	docker run --rm \
 		-u $(shell id -u):$(shell id -g) \
 		-v ${YARN_VOLUME}:/var/yarn-cache \
 		-v ${ROOT_DIR}console:/app \

diff --git a/scripts/e2e_test/start_test.sh b/scripts/e2e_test/start_test.sh
@@ -38,6 +38,8 @@ declare_env() {
   export REPO_NAME_DOCKER="${REPO_NAME_DOCKER:=docker-hosted}"
   export REPO_NAME_PYPI="${REPO_NAME_PYPI:=pypi-hosted}"
   export PYTHON_VERSION="${PYTHON_VERSION:=3.9}"
+  export SWNAME="${SWNAME:=starwhale-e2e}"
+  export SWNS="${SWNS:=starwhale-e2e}"
 }
 
 start_minikube() {
@@ -67,8 +69,9 @@ build_swcli() {
 }
 
 build_console() {
-  pushd ../../docker
-  make build-console
+  pushd ../../console
+  mkdir build
+  echo 'hi' > build/index.html
   popd
 }
 
@@ -145,33 +148,49 @@ upload_pypi_to_nexus() {
 
 buid_runtime_image() {
   pushd ../../docker
-  docker build -t starwhale -f Dockerfile.starwhale --build-arg ENABLE_E2E_TEST_PYPI_REPO=1 --build-arg PORT_NEXUS=$PORT_NEXUS --build-arg LOCAL_PYPI_HOSTNAME=$IP_DOCKER_BRIDGE --build-arg SW_VERSION=$PYPI_RELEASE_VERSION .
+  docker build -t starwhale -f Dockerfile.starwhale --build-arg ENABLE_E2E_TEST_PYPI_REPO=1 --build-arg PORT_NEXUS=$PORT_NEXUS --build-arg LOCAL_PYPI_HOSTNAME=$IP_MINIKUBE_BRIDGE --build-arg SW_VERSION=$PYPI_RELEASE_VERSION .
   docker tag starwhale $NEXUS_HOSTNAME:$PORT_NEXUS_DOCKER/star-whale/starwhale:$PYPI_RELEASE_VERSION
+  docker tag starwhale $NEXUS_HOSTNAME:$PORT_NEXUS_DOCKER/starwhale:$PYPI_RELEASE_VERSION
   popd
 }
 
 push_images_to_nexus() {
   docker login http://$NEXUS_HOSTNAME:$PORT_NEXUS_DOCKER -u $NEXUS_USER_NAME -p $NEXUS_USER_PWD
   docker push $NEXUS_HOSTNAME:$PORT_NEXUS_DOCKER/star-whale/server:$PYPI_RELEASE_VERSION
   docker push $NEXUS_HOSTNAME:$PORT_NEXUS_DOCKER/star-whale/starwhale:$PYPI_RELEASE_VERSION
+  docker push $NEXUS_HOSTNAME:$PORT_NEXUS_DOCKER/starwhale:$PYPI_RELEASE_VERSION
 }
 
 start_starwhale() {
   pushd ../../docker/charts
-  helm upgrade --install starwhale ./ --namespace starwhale --create-namespace --set "resources.controller.requests.memory=4G,resources.controller.requests.cpu=1000m,resources.controller.limits.cpu=1000m,minio.resources.requests.cpu=1000m,minio.resources.limits.cpu=2000m,controller.taskSplitSize=1,minikube.enabled=true,image.registry=$NEXUS_HOSTNAME:$PORT_NEXUS_DOCKER,image.tag=$PYPI_RELEASE_VERSION,mirror.pypi.indexUrl= http://$NEXUS_HOSTNAME:$PORT_NEXUS/repository/$REPO_NAME_PYPI/simple,mirror.pypi.extraIndexUrl=$SW_PYPI_EXTRA_INDEX_URL,mirror.pypi.trustedHost=$NEXUS_HOSTNAME"
+  helm upgrade --install $SWNAME --namespace $SWNS --create-namespace \
+  --set resources.controller.requests.cpu=700m \
+  --set mysql.resources.primary.requests.cpu=300m \
+  --set mysql.primary.persistence.storageClass=$SWNAME-mysql \
+  --set minio.resources.requests.cpu=200m \
+  --set minio.persistence.storageClass=$SWNAME-minio \
+  --set controller.taskSplitSize=1 \
+  --set minikube.enabled=true \
+  --set image.registry=$NEXUS_HOSTNAME:$PORT_NEXUS_DOCKER \
+  --set image.tag=$PYPI_RELEASE_VERSION \
+  --set mirror.pypi.indexUrl=http://$NEXUS_HOSTNAME:$PORT_NEXUS/repository/$REPO_NAME_PYPI/simple \
+  --set mirror.pypi.extraIndexUrl=$SW_PYPI_EXTRA_INDEX_URL \
+  --set mirror.pypi.trustedHost=$NEXUS_HOSTNAME \
+   .
   popd
 }
 
 check_controller_service() {
     while true
     do
-      started=`kubectl get pod -l starwhale.ai/role=controller -n starwhale -o json| jq -r '.items[0].status.containerStatuses[0].started'`
+      started=`kubectl get pod -l starwhale.ai/role=controller -n $SWNS -o json| jq -r '.items[0].status.containerStatuses[0].started'`
             if [[ "$started" == "true" ]]; then
                     echo "controller started"
                     break
             else
               echo "controller is starting"
-              kubectl get pods --namespace starwhale
+              kubectl get pods --namespace $SWNS
+              kubectl get svc --namespace $SWNS
   #            kubectl get pod -l starwhale.ai/role=controller -n starwhale -o json| jq -r '.items[0].status'
   #            ready=`kubectl get pod -l starwhale.ai/role=controller -n starwhale -o json| jq -r '.items[0].status.phase'`
   #            if [[ "$ready" == "Running" ]]; then
@@ -181,7 +200,7 @@ check_controller_service() {
             fi
             sleep 15
     done
-    nohup kubectl port-forward --namespace starwhale svc/starwhale-controller 8082:8082 &
+    nohup kubectl port-forward --namespace $SWNS svc/$SWNAME-controller 8082:8082 &
 }
 
 standalone_test() {

diff --git a/scripts/e2e_test/test_job_run.sh b/scripts/e2e_test/test_job_run.sh
@@ -28,7 +28,7 @@ job_id=`curl -X 'POST' \
   "datasetVersionUrls": "1",
   "runtimeVersionUrl": "1",
   "device": "1",
-  "deviceAmount": 1,
+  "deviceAmount": 0.4,
   "comment": "string"
 }' | jq -r '.data'`
 
@@ -42,7 +42,7 @@ do
   if curl -X 'GET' \
     "http://$1/api/v1/project/1/job/$job_id" \
     -H 'accept: application/json' \
-    -H "$auth_header" | jq -r '.data.jobStatus' > jobStatus ; then echo "8082 well"; else  kubectl logs --tail=10 -l starwhale.ai/role=controller -n starwhale; continue; fi
+    -H "$auth_header" | jq -r '.data.jobStatus' > jobStatus ; then echo "8082 well"; else  kubectl logs --tail=10 -l starwhale.ai/role=controller -n $SWNS; continue; fi
   job_status=`cat jobStatus`
   if [ "$job_status" == "null" ] ; then
     echo "Error! job_status id is null"  1>&2
@@ -54,6 +54,10 @@ do
   elif [[ "$job_status" = "FAIL" ]] ; then
         echo "job FAIL"
         break
+  elif [[ -z "$job_status" ]] ; then
+    if kill -9 `ps -ef|grep port-forward | grep -v grep | awk '{print $2}'` ; then echo "kill success"; fi
+    nohup kubectl port-forward --namespace $SWNS svc/$SWNAME-controller 8082:8082 &
+    sleep 20
   else
     echo "job status for " "$job_id" "is" "$job_status"
 #    kubectl logs --tail=10 -l job-name=1 -n starwhale -c data-provider