diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index c81c422f18..04469a3060 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -1,6 +1,13 @@ name: End to End test for starwhale on: + workflow_dispatch: + inputs: + debug_enabled: + type: boolean + description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' + required: false + default: false schedule: - cron: "34 10,13 * * *" @@ -61,6 +68,9 @@ jobs: cache: "maven" server-id: starwhale # Value of the distributionManagement/repository/id field of the pom.xml + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} - name: run e2e working-directory: ./scripts/e2e_test env: diff --git a/docker/Makefile b/docker/Makefile index 0df4ba87d9..6b6348ef2a 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -128,7 +128,7 @@ release-nodejs: build-console: docker volume create --name ${YARN_VOLUME} && \ docker run --rm -v ${YARN_VOLUME}:/app ${DH_NODEJS_IMAGE} /bin/sh -c "cp -r /root/.npmrc /app/ && chown $(shell id -u):$(shell id -g) -R /app" && \ - docker run --rm -it \ + docker run --rm \ -u $(shell id -u):$(shell id -g) \ -v ${YARN_VOLUME}:/var/yarn-cache \ -v ${ROOT_DIR}console:/app \ diff --git a/scripts/e2e_test/start_test.sh b/scripts/e2e_test/start_test.sh index e56fd57bca..4cfb4d7df2 100755 --- a/scripts/e2e_test/start_test.sh +++ b/scripts/e2e_test/start_test.sh @@ -38,6 +38,8 @@ declare_env() { export REPO_NAME_DOCKER="${REPO_NAME_DOCKER:=docker-hosted}" export REPO_NAME_PYPI="${REPO_NAME_PYPI:=pypi-hosted}" export PYTHON_VERSION="${PYTHON_VERSION:=3.9}" + export SWNAME="${SWNAME:=starwhale-e2e}" + export SWNS="${SWNS:=starwhale-e2e}" } start_minikube() { @@ -67,8 +69,9 @@ build_swcli() { } build_console() { - pushd ../../docker - make build-console + pushd ../../console + mkdir build + echo 'hi' > build/index.html popd } @@ -145,8 +148,9 @@ upload_pypi_to_nexus() { buid_runtime_image() { pushd ../../docker - docker build -t starwhale -f Dockerfile.starwhale --build-arg ENABLE_E2E_TEST_PYPI_REPO=1 --build-arg PORT_NEXUS=$PORT_NEXUS --build-arg LOCAL_PYPI_HOSTNAME=$IP_DOCKER_BRIDGE --build-arg SW_VERSION=$PYPI_RELEASE_VERSION . + docker build -t starwhale -f Dockerfile.starwhale --build-arg ENABLE_E2E_TEST_PYPI_REPO=1 --build-arg PORT_NEXUS=$PORT_NEXUS --build-arg LOCAL_PYPI_HOSTNAME=$IP_MINIKUBE_BRIDGE --build-arg SW_VERSION=$PYPI_RELEASE_VERSION . docker tag starwhale $NEXUS_HOSTNAME:$PORT_NEXUS_DOCKER/star-whale/starwhale:$PYPI_RELEASE_VERSION + docker tag starwhale $NEXUS_HOSTNAME:$PORT_NEXUS_DOCKER/starwhale:$PYPI_RELEASE_VERSION popd } @@ -154,24 +158,39 @@ push_images_to_nexus() { docker login http://$NEXUS_HOSTNAME:$PORT_NEXUS_DOCKER -u $NEXUS_USER_NAME -p $NEXUS_USER_PWD docker push $NEXUS_HOSTNAME:$PORT_NEXUS_DOCKER/star-whale/server:$PYPI_RELEASE_VERSION docker push $NEXUS_HOSTNAME:$PORT_NEXUS_DOCKER/star-whale/starwhale:$PYPI_RELEASE_VERSION + docker push $NEXUS_HOSTNAME:$PORT_NEXUS_DOCKER/starwhale:$PYPI_RELEASE_VERSION } start_starwhale() { pushd ../../docker/charts - helm upgrade --install starwhale ./ --namespace starwhale --create-namespace --set "resources.controller.requests.memory=4G,resources.controller.requests.cpu=1000m,resources.controller.limits.cpu=1000m,minio.resources.requests.cpu=1000m,minio.resources.limits.cpu=2000m,controller.taskSplitSize=1,minikube.enabled=true,image.registry=$NEXUS_HOSTNAME:$PORT_NEXUS_DOCKER,image.tag=$PYPI_RELEASE_VERSION,mirror.pypi.indexUrl= http://$NEXUS_HOSTNAME:$PORT_NEXUS/repository/$REPO_NAME_PYPI/simple,mirror.pypi.extraIndexUrl=$SW_PYPI_EXTRA_INDEX_URL,mirror.pypi.trustedHost=$NEXUS_HOSTNAME" + helm upgrade --install $SWNAME --namespace $SWNS --create-namespace \ + --set resources.controller.requests.cpu=700m \ + --set mysql.resources.primary.requests.cpu=300m \ + --set mysql.primary.persistence.storageClass=$SWNAME-mysql \ + --set minio.resources.requests.cpu=200m \ + --set minio.persistence.storageClass=$SWNAME-minio \ + --set controller.taskSplitSize=1 \ + --set minikube.enabled=true \ + --set image.registry=$NEXUS_HOSTNAME:$PORT_NEXUS_DOCKER \ + --set image.tag=$PYPI_RELEASE_VERSION \ + --set mirror.pypi.indexUrl=http://$NEXUS_HOSTNAME:$PORT_NEXUS/repository/$REPO_NAME_PYPI/simple \ + --set mirror.pypi.extraIndexUrl=$SW_PYPI_EXTRA_INDEX_URL \ + --set mirror.pypi.trustedHost=$NEXUS_HOSTNAME \ + . popd } check_controller_service() { while true do - started=`kubectl get pod -l starwhale.ai/role=controller -n starwhale -o json| jq -r '.items[0].status.containerStatuses[0].started'` + started=`kubectl get pod -l starwhale.ai/role=controller -n $SWNS -o json| jq -r '.items[0].status.containerStatuses[0].started'` if [[ "$started" == "true" ]]; then echo "controller started" break else echo "controller is starting" - kubectl get pods --namespace starwhale + kubectl get pods --namespace $SWNS + kubectl get svc --namespace $SWNS # kubectl get pod -l starwhale.ai/role=controller -n starwhale -o json| jq -r '.items[0].status' # ready=`kubectl get pod -l starwhale.ai/role=controller -n starwhale -o json| jq -r '.items[0].status.phase'` # if [[ "$ready" == "Running" ]]; then @@ -181,7 +200,7 @@ check_controller_service() { fi sleep 15 done - nohup kubectl port-forward --namespace starwhale svc/starwhale-controller 8082:8082 & + nohup kubectl port-forward --namespace $SWNS svc/$SWNAME-controller 8082:8082 & } standalone_test() { diff --git a/scripts/e2e_test/test_job_run.sh b/scripts/e2e_test/test_job_run.sh index fa930a36a7..e6a6c5ea0b 100755 --- a/scripts/e2e_test/test_job_run.sh +++ b/scripts/e2e_test/test_job_run.sh @@ -28,7 +28,7 @@ job_id=`curl -X 'POST' \ "datasetVersionUrls": "1", "runtimeVersionUrl": "1", "device": "1", - "deviceAmount": 1, + "deviceAmount": 0.4, "comment": "string" }' | jq -r '.data'` @@ -42,7 +42,7 @@ do if curl -X 'GET' \ "http://$1/api/v1/project/1/job/$job_id" \ -H 'accept: application/json' \ - -H "$auth_header" | jq -r '.data.jobStatus' > jobStatus ; then echo "8082 well"; else kubectl logs --tail=10 -l starwhale.ai/role=controller -n starwhale; continue; fi + -H "$auth_header" | jq -r '.data.jobStatus' > jobStatus ; then echo "8082 well"; else kubectl logs --tail=10 -l starwhale.ai/role=controller -n $SWNS; continue; fi job_status=`cat jobStatus` if [ "$job_status" == "null" ] ; then echo "Error! job_status id is null" 1>&2 @@ -54,6 +54,10 @@ do elif [[ "$job_status" = "FAIL" ]] ; then echo "job FAIL" break + elif [[ -z "$job_status" ]] ; then + if kill -9 `ps -ef|grep port-forward | grep -v grep | awk '{print $2}'` ; then echo "kill success"; fi + nohup kubectl port-forward --namespace $SWNS svc/$SWNAME-controller 8082:8082 & + sleep 20 else echo "job status for " "$job_id" "is" "$job_status" # kubectl logs --tail=10 -l job-name=1 -n starwhale -c data-provider