From fdeee654ad9b7a0855f97c25d88b52b62f24d446 Mon Sep 17 00:00:00 2001 From: "Ricardo M. Oliveira" Date: Tue, 16 Apr 2024 18:10:17 -0300 Subject: [PATCH] Add kfp-tekton integration tests and manifests Signed-off-by: Ricardo M. Oliveira --- .github/workflows/backend.yml | 83 ++++++ .github/workflows/codeql.yml | 72 +++++ .../kustomization.yaml | 27 ++ .../namespace.yaml | 4 + .../params.yaml | 4 + .../env/plain-multi-user/kustomization.yaml | 22 ++ .../kustomize/env/plain/kustomization.yaml | 19 ++ .../kustomization.yaml | 81 ++++++ .../tekton-config.yaml | 10 + .../kustomization.yaml | 81 ++++++ .../kustomization.yaml | 78 ++++++ scripts/deploy/github/build-images.sh | 33 +++ scripts/deploy/github/deploy-kfp.sh | 55 ++++ scripts/deploy/github/e2e-test.sh | 46 ++++ scripts/deploy/github/helper-functions.sh | 250 ++++++++++++++++++ .../github/manifests/kustomization.yaml | 117 ++++++++ scripts/deploy/github/re-tag.sh | 37 +++ scripts/deploy/github/test-dynamic-loop.sh | 27 ++ scripts/deploy/github/test-env.sh | 33 +++ scripts/deploy/github/test-flip-coin.sh | 27 ++ scripts/deploy/github/test-pipeline.sh | 97 +++++++ scripts/deploy/github/test-secret-as-env.sh | 39 +++ .../deploy/github/test-secret-as-volume.sh | 39 +++ scripts/deploy/github/test-static-loop.sh | 27 ++ scripts/deploy/github/test-volume.sh | 33 +++ 25 files changed, 1341 insertions(+) create mode 100644 .github/workflows/backend.yml create mode 100644 .github/workflows/codeql.yml create mode 100644 manifests/kustomize/cluster-scoped-resources-tekton/kustomization.yaml create mode 100644 manifests/kustomize/cluster-scoped-resources-tekton/namespace.yaml create mode 100644 manifests/kustomize/cluster-scoped-resources-tekton/params.yaml create mode 100644 manifests/kustomize/env/plain-multi-user/kustomization.yaml create mode 100644 manifests/kustomize/env/plain/kustomization.yaml create mode 100644 manifests/kustomize/env/platform-agnostic-tekton-multi-user/kustomization.yaml create mode 100644 manifests/kustomize/env/platform-agnostic-tekton-multi-user/tekton-config.yaml create mode 100644 manifests/kustomize/env/platform-agnostic-tekton/kustomization.yaml create mode 100644 manifests/kustomize/env/platform-openshift-pipelines/kustomization.yaml create mode 100755 scripts/deploy/github/build-images.sh create mode 100755 scripts/deploy/github/deploy-kfp.sh create mode 100755 scripts/deploy/github/e2e-test.sh create mode 100644 scripts/deploy/github/helper-functions.sh create mode 100644 scripts/deploy/github/manifests/kustomization.yaml create mode 100755 scripts/deploy/github/re-tag.sh create mode 100755 scripts/deploy/github/test-dynamic-loop.sh create mode 100755 scripts/deploy/github/test-env.sh create mode 100755 scripts/deploy/github/test-flip-coin.sh create mode 100755 scripts/deploy/github/test-pipeline.sh create mode 100755 scripts/deploy/github/test-secret-as-env.sh create mode 100755 scripts/deploy/github/test-secret-as-volume.sh create mode 100755 scripts/deploy/github/test-static-loop.sh create mode 100755 scripts/deploy/github/test-volume.sh diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml new file mode 100644 index 000000000000..759769b65652 --- /dev/null +++ b/.github/workflows/backend.yml @@ -0,0 +1,83 @@ +name: KFP Tekton backend unit tests + +on: + push: + branches: [master] + + # Run tests for any PRs which change the backend code + pull_request: + paths: + - 'go.mod' + - 'backend/**' + - 'scripts/deploy/github/**' + - 'manifests/kustomize/**' + +env: + GITHUB_ACTION: "true" + SETUPTOOLS_USE_DISTUTILS: "stdlib" + +jobs: + run-go-unittests: + runs-on: ubuntu-latest + steps: + - name: Install Go + uses: actions/setup-go@v4 + with: + go-version: 1.20.x + - name: Checkout code + uses: actions/checkout@v4 + - name: "run go unit tests" + run: go test -v -cover ./backend/... + backend-integration: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Create k8s Kind Cluster + uses: container-tools/kind-action@v2 + with: + cluster_name: kfp-tekton + kubectl_version: v1.26.3 + version: v0.17.0 + node_image: kindest/node:v1.26.3 + - name: build images + run: ./scripts/deploy/github/build-images.sh + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: "deploy kfp-tekton" + run: ./scripts/deploy/github/deploy-kfp.sh + - name: Install sdk + run: | + python3 -m venv .venv + . .venv/bin/activate + pip install -e sdk/python + - name: "flip coin test" + run: | + . .venv/bin/activate + TEST_SCRIPT="test-flip-coin.sh" ./scripts/deploy/github/e2e-test.sh + - name: "static loop test" + run: | + . .venv/bin/activate + TEST_SCRIPT="test-static-loop.sh" ./scripts/deploy/github/e2e-test.sh + - name: "dynamic loop test" + run: | + . .venv/bin/activate + TEST_SCRIPT="test-dynamic-loop.sh" ./scripts/deploy/github/e2e-test.sh + - name: "secret as env" + run: | + . .venv/bin/activate + TEST_SCRIPT="test-secret-as-env.sh" ./scripts/deploy/github/e2e-test.sh + - name: "secret as volume" + run: | + . .venv/bin/activate + TEST_SCRIPT="test-secret-as-volume.sh" ./scripts/deploy/github/e2e-test.sh + - name: "use env" + run: | + . .venv/bin/activate + TEST_SCRIPT="test-env.sh" ./scripts/deploy/github/e2e-test.sh + - name: "use volume" + run: | + . .venv/bin/activate + TEST_SCRIPT="test-volume.sh" ./scripts/deploy/github/e2e-test.sh diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000000..2f0b0b472e01 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,72 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + schedule: + # Every Friday at 19:39 + - cron: '39 19 * * 5' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'go', 'javascript', 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Use only 'java' to analyze code written in Java, Kotlin or both + # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{matrix.language}}" diff --git a/manifests/kustomize/cluster-scoped-resources-tekton/kustomization.yaml b/manifests/kustomize/cluster-scoped-resources-tekton/kustomization.yaml new file mode 100644 index 000000000000..ea1a04a385cf --- /dev/null +++ b/manifests/kustomize/cluster-scoped-resources-tekton/kustomization.yaml @@ -0,0 +1,27 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: kubeflow + +resources: +- namespace.yaml +bases: +- ../third-party/application/cluster-scoped +# - ../third-party/argo/installs/namespace/cluster-scoped +- ../base/pipeline/cluster-scoped +- ../base/cache-deployer/cluster-scoped +vars: +# NOTE: var name must be unique globally to allow composition of multiple kustomize +# packages. Therefore, we added prefix `kfp-cluster-scoped-` to distinguish it from +# others. +- name: kfp-cluster-scoped-namespace + objref: + # cache deployer sa's metadata.namespace will be first transformed by namespace field in kustomization.yaml + # so that we only need to change kustomization.yaml's namespace field for namespace customization. + kind: ServiceAccount + name: kubeflow-pipelines-cache-deployer-sa + apiVersion: v1 + fieldref: + fieldpath: metadata.namespace +configurations: +- params.yaml diff --git a/manifests/kustomize/cluster-scoped-resources-tekton/namespace.yaml b/manifests/kustomize/cluster-scoped-resources-tekton/namespace.yaml new file mode 100644 index 000000000000..3c65856e7b73 --- /dev/null +++ b/manifests/kustomize/cluster-scoped-resources-tekton/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: '$(kfp-cluster-scoped-namespace)' diff --git a/manifests/kustomize/cluster-scoped-resources-tekton/params.yaml b/manifests/kustomize/cluster-scoped-resources-tekton/params.yaml new file mode 100644 index 000000000000..cc253fe26603 --- /dev/null +++ b/manifests/kustomize/cluster-scoped-resources-tekton/params.yaml @@ -0,0 +1,4 @@ +# Allow Kustomize var to replace following fields. +varReference: +- path: metadata/name + kind: Namespace diff --git a/manifests/kustomize/env/plain-multi-user/kustomization.yaml b/manifests/kustomize/env/plain-multi-user/kustomization.yaml new file mode 100644 index 000000000000..47c8daea9cf7 --- /dev/null +++ b/manifests/kustomize/env/plain-multi-user/kustomization.yaml @@ -0,0 +1,22 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base/installs/multi-user + - ../../base/metadata/base + - ../../base/metadata/options/istio + - ../../third-party/mysql/base + - ../../third-party/mysql/options/istio + - ../../third-party/minio/base + - ../../third-party/minio/options/istio + - ../../third-party/metacontroller/base + +# Identifier for application manager to apply ownerReference. +# The ownerReference ensures the resources get garbage collected +# when application is deleted. +commonLabels: + application-crd-id: kubeflow-pipelines + +# !!! If you want to customize the namespace, +# please also update base/cache-deployer/cluster-scoped/cache-deployer-clusterrolebinding.yaml +namespace: kubeflow diff --git a/manifests/kustomize/env/plain/kustomization.yaml b/manifests/kustomize/env/plain/kustomization.yaml new file mode 100644 index 000000000000..7bf943cab335 --- /dev/null +++ b/manifests/kustomize/env/plain/kustomization.yaml @@ -0,0 +1,19 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../cluster-scoped-resources-tekton + - ../../base/installs/generic + - ../../base/metadata/base + - ../../third-party/minio/base + - ../../third-party/mysql/base + +# Identifier for application manager to apply ownerReference. +# The ownerReference ensures the resources get garbage collected +# when application is deleted. +commonLabels: + application-crd-id: kubeflow-pipelines + +# !!! If you want to customize the namespace, +# please also update base/cache-deployer/cluster-scoped/cache-deployer-clusterrolebinding.yaml +namespace: kubeflow diff --git a/manifests/kustomize/env/platform-agnostic-tekton-multi-user/kustomization.yaml b/manifests/kustomize/env/platform-agnostic-tekton-multi-user/kustomization.yaml new file mode 100644 index 000000000000..807b318cf14d --- /dev/null +++ b/manifests/kustomize/env/platform-agnostic-tekton-multi-user/kustomization.yaml @@ -0,0 +1,81 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../third-party/tekton/installs/cluster + - ../../third-party/tekton-custom-task + - ../plain-multi-user + +# Identifier for application manager to apply ownerReference. +# The ownerReference ensures the resources get garbage collected +# when application is deleted. +commonLabels: + application-crd-id: kubeflow-pipelines + +images: +- name: gcr.io/ml-pipeline/api-server + newName: quay.io/aipipeline/apiserver + newTag: 2.0.5 +- name: gcr.io/ml-pipeline/persistenceagent + newName: quay.io/aipipeline/persistenceagent + newTag: 2.0.5 +- name: gcr.io/ml-pipeline/scheduledworkflow + newName: quay.io/aipipeline/scheduledworkflow + newTag: 2.0.5 + +patchesStrategicMerge: +- tekton-config.yaml + +patches: +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ml-pipeline + spec: + template: + spec: + containers: + - name: ml-pipeline-api-server + env: + - name: EXECUTIONTYPE + value: PipelineRun +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ml-pipeline-persistenceagent + spec: + template: + spec: + containers: + - name: ml-pipeline-persistenceagent + env: + - name: EXECUTIONTYPE + value: PipelineRun +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ml-pipeline-scheduledworkflow + spec: + template: + spec: + containers: + - name: ml-pipeline-scheduledworkflow + env: + - name: EXECUTIONTYPE + value: PipelineRun +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ml-pipeline-ui + spec: + template: + spec: + containers: + - name: ml-pipeline-ui + env: + - name: POD_LOG_CONTAINER_NAME + value: step-user-main \ No newline at end of file diff --git a/manifests/kustomize/env/platform-agnostic-tekton-multi-user/tekton-config.yaml b/manifests/kustomize/env/platform-agnostic-tekton-multi-user/tekton-config.yaml new file mode 100644 index 000000000000..5707255acdd9 --- /dev/null +++ b/manifests/kustomize/env/platform-agnostic-tekton-multi-user/tekton-config.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: feature-flags + namespace: tekton-pipelines + labels: + app.kubernetes.io/instance: default + app.kubernetes.io/part-of: tekton-pipelines +data: + running-in-environment-with-injected-sidecars: "true" diff --git a/manifests/kustomize/env/platform-agnostic-tekton/kustomization.yaml b/manifests/kustomize/env/platform-agnostic-tekton/kustomization.yaml new file mode 100644 index 000000000000..cb9814e88d3e --- /dev/null +++ b/manifests/kustomize/env/platform-agnostic-tekton/kustomization.yaml @@ -0,0 +1,81 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../third-party/tekton/installs/cluster +- ../../third-party/tekton-custom-task +- ../plain + +# Identifier for application manager to apply ownerReference. +# The ownerReference ensures the resources get garbage collected +# when application is deleted. + +images: +- name: gcr.io/ml-pipeline/api-server + newName: quay.io/aipipeline/apiserver + newTag: 2.0.5 +- name: gcr.io/ml-pipeline/persistenceagent + newName: quay.io/aipipeline/persistenceagent + newTag: 2.0.5 +- name: gcr.io/ml-pipeline/scheduledworkflow + newName: quay.io/aipipeline/scheduledworkflow + newTag: 2.0.5 + +labels: +- includeSelectors: true + pairs: + application-crd-id: kubeflow-pipelines + +patches: +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ml-pipeline + spec: + template: + spec: + containers: + - name: ml-pipeline-api-server + env: + - name: EXECUTIONTYPE + value: PipelineRun +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ml-pipeline-persistenceagent + spec: + template: + spec: + containers: + - name: ml-pipeline-persistenceagent + env: + - name: EXECUTIONTYPE + value: PipelineRun +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ml-pipeline-scheduledworkflow + spec: + template: + spec: + containers: + - name: ml-pipeline-scheduledworkflow + env: + - name: EXECUTIONTYPE + value: PipelineRun +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ml-pipeline-ui + spec: + template: + spec: + containers: + - name: ml-pipeline-ui + env: + - name: POD_LOG_CONTAINER_NAME + value: step-user-main diff --git a/manifests/kustomize/env/platform-openshift-pipelines/kustomization.yaml b/manifests/kustomize/env/platform-openshift-pipelines/kustomization.yaml new file mode 100644 index 000000000000..342bbfe848df --- /dev/null +++ b/manifests/kustomize/env/platform-openshift-pipelines/kustomization.yaml @@ -0,0 +1,78 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../plain +- ../../third-party/openshift-pipelines-custom-task +- ../../third-party/openshift/standalone + +patches: +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ml-pipeline + spec: + template: + spec: + containers: + - name: ml-pipeline-api-server + env: + - name: EXECUTIONTYPE + value: PipelineRun +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ml-pipeline-persistenceagent + spec: + template: + spec: + containers: + - name: ml-pipeline-persistenceagent + env: + - name: EXECUTIONTYPE + value: PipelineRun +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ml-pipeline-scheduledworkflow + spec: + template: + spec: + containers: + - name: ml-pipeline-scheduledworkflow + env: + - name: EXECUTIONTYPE + value: PipelineRun +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ml-pipeline-ui + spec: + template: + spec: + containers: + - name: ml-pipeline-ui + env: + - name: POD_LOG_CONTAINER_NAME + value: step-user-main + + +images: +- name: gcr.io/ml-pipeline/api-server + newName: quay.io/aipipeline/apiserver + newTag: 2.0.0 +- name: gcr.io/ml-pipeline/persistenceagent + newName: quay.io/aipipeline/persistenceagent + newTag: 2.0.0 +- name: gcr.io/ml-pipeline/scheduledworkflow + newName: quay.io/aipipeline/scheduledworkflow + newTag: 2.0.0 + +labels: +- includeSelectors: true + pairs: + application-crd-id: kubeflow-pipelines diff --git a/scripts/deploy/github/build-images.sh b/scripts/deploy/github/build-images.sh new file mode 100755 index 000000000000..ffa6db2e216f --- /dev/null +++ b/scripts/deploy/github/build-images.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Copyright 2023 kubeflow.org +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# source: https://raw.githubusercontent.com/open-toolchain/commons/master/scripts/check_registry.sh + +# Remove the x if you need no print out of each command +set -e + +REGISTRY="${REGISTRY:-kind-registry:5000}" +TAG="${TAG:-latest}" + +docker system prune -a -f + +docker build -q -t "${REGISTRY}/apiserver:${TAG}" -f backend/Dockerfile . && docker push "${REGISTRY}/apiserver:${TAG}" & +docker build -q -t "${REGISTRY}/persistenceagent:${TAG}" -f backend/Dockerfile.persistenceagent . && docker push "${REGISTRY}/persistenceagent:${TAG}" & +docker build -q -t "${REGISTRY}/scheduledworkflow:${TAG}" -f backend/Dockerfile.scheduledworkflow . && docker push "${REGISTRY}/scheduledworkflow:${TAG}" & + +wait + +# clean up intermittent build caches to free up disk space +docker system prune -a -f diff --git a/scripts/deploy/github/deploy-kfp.sh b/scripts/deploy/github/deploy-kfp.sh new file mode 100755 index 000000000000..55b45e43bbc7 --- /dev/null +++ b/scripts/deploy/github/deploy-kfp.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# +# Copyright 2023 kubeflow.org +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Remove the x if you need no print out of each command +set -e + +REGISTRY="${REGISTRY:-kind-registry:5000}" +EXIT_CODE=0 + +C_DIR="${BASH_SOURCE%/*}" +if [[ ! -d "$C_DIR" ]]; then C_DIR="$PWD"; fi +source "${C_DIR}/helper-functions.sh" + +kubectl apply -k "manifests/kustomize/cluster-scoped-resources/" +kubectl wait crd/applications.app.k8s.io --for condition=established --timeout=60s || EXIT_CODE=$? +if [[ $EXIT_CODE -ne 0 ]] +then + echo "Failed to deploy cluster-scoped resources." + exit $EXIT_CODE +fi + +# Deploy manifest +kubectl apply -k "scripts/deploy/github/manifests" || EXIT_CODE=$? +if [[ $EXIT_CODE -ne 0 ]] +then + echo "Deploy unsuccessful. Failure applying $KUSTOMIZE_DIR." + exit 1 +fi + +# Check if all pods are running - allow 20 retries (10 minutes) +wait_for_pods kubeflow 40 30 || EXIT_CODE=$? +if [[ $EXIT_CODE -ne 0 ]] +then + echo "Deploy unsuccessful. Not all pods running." + exit 1 +fi + +echo "List Tekton control plane: " +kubectl get pod -n tekton-pipelines + +echo "Finished kfp-tekton deployment." + diff --git a/scripts/deploy/github/e2e-test.sh b/scripts/deploy/github/e2e-test.sh new file mode 100755 index 000000000000..a1a01a211b38 --- /dev/null +++ b/scripts/deploy/github/e2e-test.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# +# Copyright 2023 kubeflow.org +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Remove the x if you need no print out of each command +set -e + +# Need the following env +# - KUBEFLOW_NS: kubeflow namespace + +KUBEFLOW_NS="${KUBEFLOW_NS:-kubeflow}" +TEST_SCRIPT="${TEST_SCRIPT:="test-flip-coin.sh"}" + +C_DIR="${BASH_SOURCE%/*}" +if [[ ! -d "$C_DIR" ]]; then C_DIR="$PWD"; fi + +POD_NAME=$(kubectl get pod -n kubeflow -l app=ml-pipeline -o json | jq -r '.items[] | .metadata.name ') +kubectl port-forward -n "$KUBEFLOW_NS" "$POD_NAME" 8888:8888 2>&1 > /dev/null & +# wait for the port-forward +sleep 5 + +if [ -n "$TEST_SCRIPT" ]; then + source "${C_DIR}/${TEST_SCRIPT}" +fi + +kill %1 + +if [[ "$RESULT" -ne 0 ]]; then + echo "e2e test ${STATUS_MSG}" + exit 1 +fi + +echo "e2e test ${STATUS_MSG}" + diff --git a/scripts/deploy/github/helper-functions.sh b/scripts/deploy/github/helper-functions.sh new file mode 100644 index 000000000000..d22e2d781a08 --- /dev/null +++ b/scripts/deploy/github/helper-functions.sh @@ -0,0 +1,250 @@ +#!/bin/bash +# +# Copyright 2023 kubeflow.org +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +retry() { + local max=$1; shift + local interval=$1; shift + + until "$@"; do + echo "trying.." + max=$((max-1)) + if [[ "$max" -eq 0 ]]; then + return 1 + fi + sleep "$interval" + done +} + +wait_for_namespace () { + if [[ $# -ne 3 ]] + then + echo "Usage: wait_for_namespace namespace max_retries sleep_time" + return 1 + fi + + local namespace=$1 + local max_retries=$2 + local sleep_time=$3 + + local i=0 + + while [[ $i -lt $max_retries ]] + do + if kubectl get ns | grep -qow "$namespace" + then + return 0 + fi + echo "$namespace not found. Checking again in ${sleep_time}s." + sleep "$sleep_time" + i=$((i+1)) + done + + return 1 +} + +wait_for_pods () { + if [[ $# -ne 3 ]] + then + echo "Usage: wait_for_pods namespace max_retries sleep_time" + return 1 + fi + + local namespace=$1 + local max_retries=$2 + local sleep_time=$3 + + local i=0 + + while [[ $i -lt $max_retries ]] + do + local pods + local statuses + local num_pods + local num_running + pods=$(kubectl get pod -n "$namespace") + # echo "$pods" + # kubectl get pvc -n "$namespace" + + if [[ -z $pods ]] + then + echo "no pod is up yet" + else + # Using quotations around variables to keep column format in echo + # Remove 1st line (header line) -> trim whitespace -> cut statuses column (3rd column) + # Might be overkill to parse down to specific columns :). + statuses=$(echo "$pods" | tail -n +2 | tr -s ' ' | cut -d ' ' -f 3) + num_pods=$(echo "$statuses" | wc -l | xargs) + num_running=$(echo "$statuses" | grep -ow "Running\|Completed" | wc -l | xargs) + + local msg="${num_running}/${num_pods} pods running in \"${namespace}\"." + + if [[ $num_running -ne $num_pods ]] + then + # for debugging + # kubectl get pod -n "$namespace" | grep '0/1' | awk '{print $1}' | xargs kubectl describe pod -n "$namespace" + echo "$msg Checking again in ${sleep_time}s." + else + echo "$msg" + return 0 + fi + fi + + sleep "$sleep_time" + i=$((i+1)) + done + + return 1 +} + +deploy_with_retries () { + if [[ $# -ne 4 ]] + then + echo "Usage: deploy_with_retries (-f FILENAME | -k DIRECTORY) manifest max_retries sleep_time" + return 1 + fi + + local flag="$1" + local manifest="$2" + local max_retries="$3" + local sleep_time="$4" + + local i=0 + + while [[ $i -lt $max_retries ]] + do + local exit_code=0 + + kubectl apply "$flag" "$manifest" || exit_code=$? + + if [[ $exit_code -eq 0 ]] + then + return 0 + fi + + echo "Deploy unsuccessful with error code $exit_code. Trying again in ${sleep_time}s." + sleep "$sleep_time" + i=$((i+1)) + done + + return 1 +} + +wait_for_pod () { + local namespace=$1 + local pod_name=$2 + local max_tries=$3 + local sleep_time=$4 + + until pod_is_running "$namespace" "$pod_name"; do + max_tries=$((max_tries-1)) + if [[ "$max_tries" -eq 0 ]]; then + return 1 + fi + echo "Checking again in $sleep_time" + sleep "$sleep_time" + done + + return 0 +} + +pod_is_running () { + local namespace=$1 + local pod_name=$2 + + local pod_status + + # May have unexpected results if pod_name has multiple matches + pod_status=$(kubectl get pod -n "$namespace" | grep "$pod_name*" | head -1 | awk '{print $3}') + + if [ "$pod_status" = "Running" ]; then + return 0 + fi + + return 1 +} + +wait_for_pipeline_run () { + local run_name=$1 + local max_tries=$2 + local sleep_time=$3 + + until pipeline_run_is_success "$run_name"; do + max_tries=$((max_tries-1)) + if [[ "$max_tries" -eq 0 ]]; then + return 1 + fi + echo "Checking pipeline run again in $sleep_time" + sleep "$sleep_time" + done + + return 0 +} + +wait_for_pipeline_run_rev () { + local run_name=$1 + local max_tries=$2 + local sleep_time=$3 + + until [ "$(pipeline_run_is_success_rev "$run_name")" = "0" ]; do + max_tries=$((max_tries-1)) + if [[ "$max_tries" -eq 0 ]]; then + echo "1" + return + fi + sleep "$sleep_time" + done + + echo "0" + return +} + +pipeline_run_is_success () { + local run_name=$1 + + local run_status + + # May have unexpected results if run_status has multiple matches + run_status=$(kubectl get pipelineruns "$run_name" | tail -1 | awk '{print $2}') + + if [ "$run_status" = "True" ]; then + return 0 + elif [ "$run_status" = "False" ]; then + echo "Run Failed" + exit 1 + fi + + return 1 +} + +pipeline_run_is_success_rev () { + local run_name=$1 + + local run_status + + # May have unexpected results if run_status has multiple matches + run_status=$(kubectl get pipelineruns "$run_name" | tail -1 | awk '{print $2}') + + if [ "$run_status" = "True" ]; then + echo "0" + return + elif [ "$run_status" = "False" ]; then + echo "1" + return + fi + + echo "1" + return +} diff --git a/scripts/deploy/github/manifests/kustomization.yaml b/scripts/deploy/github/manifests/kustomization.yaml new file mode 100644 index 000000000000..b4fc2b581773 --- /dev/null +++ b/scripts/deploy/github/manifests/kustomization.yaml @@ -0,0 +1,117 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../../../manifests/kustomize/base/installs/generic +- ../../../../manifests/kustomize/base/metadata/base +- ../../../../manifests/kustomize/third-party/tekton/installs/cluster +- ../../../../manifests/kustomize/third-party/tekton-custom-task +- ../../../../manifests/kustomize/third-party/minio/base +- ../../../../manifests/kustomize/third-party/mysql/base + +# Identifier for application manager to apply ownerReference. +# The ownerReference ensures the resources get garbage collected +# when application is deleted. + +images: +- name: gcr.io/ml-pipeline/api-server + newName: kind-registry:5000/apiserver + newTag: latest +- name: gcr.io/ml-pipeline/persistenceagent + newName: kind-registry:5000/persistenceagent + newTag: latest +- name: gcr.io/ml-pipeline/scheduledworkflow + newName: kind-registry:5000/scheduledworkflow + newTag: latest +- name: '*/aipipeline/tekton-exithandler-controller' + newTag: latest +- name: '*/aipipeline/tekton-exithandler-webhook' + newTag: latest +- name: '*/aipipeline/tekton-kfptask-controller' + newTag: latest +- name: '*/aipipeline/tekton-kfptask-webhook' + newTag: latest +# For Staging image build only +# - name: '*/aipipeline/tekton-driver' +# newName: kind-registry:5000/tekton-driver +# newTag: latest +# - name: '*/aipipeline/tekton-driver' +# newName: kind-registry:5000/tekton-driver +# newTag: latest + +labels: +- includeSelectors: true + pairs: + application-crd-id: kubeflow-pipelines + +patches: +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ml-pipeline + spec: + template: + spec: + containers: + - name: ml-pipeline-api-server + env: + - name: EXECUTIONTYPE + value: PipelineRun +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ml-pipeline-persistenceagent + spec: + template: + spec: + containers: + - name: ml-pipeline-persistenceagent + env: + - name: EXECUTIONTYPE + value: PipelineRun +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ml-pipeline-scheduledworkflow + spec: + template: + spec: + containers: + - name: ml-pipeline-scheduledworkflow + env: + - name: EXECUTIONTYPE + value: PipelineRun +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ml-pipeline-ui + spec: + template: + spec: + containers: + - name: ml-pipeline-ui + env: + - name: POD_LOG_CONTAINER_NAME + value: step-user-main +- patch: |- + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: mysql-pv-claim + spec: + resources: + requests: + storage: 5Gi +- patch: |- + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: minio-pvc + spec: + resources: + requests: + storage: 5Gi diff --git a/scripts/deploy/github/re-tag.sh b/scripts/deploy/github/re-tag.sh new file mode 100755 index 000000000000..ffb6c9a4c3a0 --- /dev/null +++ b/scripts/deploy/github/re-tag.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# +# Copyright 2023 kubeflow.org +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# source: https://raw.githubusercontent.com/open-toolchain/commons/master/scripts/check_registry.sh + +# Remove the x if you need no print out of each command +set -e + +REGISTRY1="${REGISTRY1:-docker.io/aipipeline}" +REGISTRY2="${REGISTRY2:-quay.io/aipipeline}" +TAG1="${TAG1:-latest}" +TAG2="${TAG2:-latest}" + +docker system prune -a -f + +declare -a IMAGES=(apiserver persistenceagent scheduledworkflow tekton-driver) + +for IMAGE in "${IMAGES[@]}"; do + docker pull "${REGISTRY1}/${IMAGE}:${TAG1}" + docker tag "${REGISTRY1}/${IMAGE}:${TAG1}" "${REGISTRY2}/${IMAGE}:${TAG2}" + docker push "${REGISTRY2}/${IMAGE}:${TAG2}" +done + +# clean up intermittent build caches to free up disk space +docker system prune -a -f diff --git a/scripts/deploy/github/test-dynamic-loop.sh b/scripts/deploy/github/test-dynamic-loop.sh new file mode 100755 index 000000000000..016a9b7cfbac --- /dev/null +++ b/scripts/deploy/github/test-dynamic-loop.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Copyright 2023 kubeflow.org +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +C_DIR="${BASH_SOURCE%/*}" +if [[ ! -d "$C_DIR" ]]; then C_DIR="$PWD"; fi +source "${C_DIR}/test-pipeline.sh" + +RESULT=0 +run_test_case "loop_output" "samples/core/loop_output/loop_output.py" "SUCCEEDED" 20 || RESULT=$? + +STATUS_MSG=PASSED +if [[ "$RESULT" -ne 0 ]]; then + STATUS_MSG=FAILED +fi diff --git a/scripts/deploy/github/test-env.sh b/scripts/deploy/github/test-env.sh new file mode 100755 index 000000000000..20197505fe90 --- /dev/null +++ b/scripts/deploy/github/test-env.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Copyright 2023 kubeflow.org +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +KUBEFLOW_NS="${KUBEFLOW_NS:-kubeflow}" + +C_DIR="${BASH_SOURCE%/*}" +if [[ ! -d "$C_DIR" ]]; then C_DIR="$PWD"; fi +source "${C_DIR}/test-pipeline.sh" + +# need kfp-kubernetes for this test case +# unfortunately, we can't install it from kubernetes_platform/python +pip install kfp-kubernetes + +RESULT=0 +run_test_case "use-env" "samples/v2/pipeline_with_env.py" "SUCCEEDED" 5 || RESULT=$? + +STATUS_MSG=PASSED +if [[ "$RESULT" -ne 0 ]]; then + STATUS_MSG=FAILED +fi diff --git a/scripts/deploy/github/test-flip-coin.sh b/scripts/deploy/github/test-flip-coin.sh new file mode 100755 index 000000000000..471dde917d13 --- /dev/null +++ b/scripts/deploy/github/test-flip-coin.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Copyright 2023 kubeflow.org +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +C_DIR="${BASH_SOURCE%/*}" +if [[ ! -d "$C_DIR" ]]; then C_DIR="$PWD"; fi +source "${C_DIR}/test-pipeline.sh" + +RESULT=0 +run_test_case "flip-coin" "samples/core/condition/condition_v2.py" "SUCCEEDED" 20 || RESULT=$? + +STATUS_MSG=PASSED +if [[ "$RESULT" -ne 0 ]]; then + STATUS_MSG=FAILED +fi diff --git a/scripts/deploy/github/test-pipeline.sh b/scripts/deploy/github/test-pipeline.sh new file mode 100755 index 000000000000..8a09ab99a5e8 --- /dev/null +++ b/scripts/deploy/github/test-pipeline.sh @@ -0,0 +1,97 @@ +#!/bin/bash +# +# Copyright 2023 kubeflow.org +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +retry() { + local max=$1; shift + local interval=$1; shift + + until "$@"; do + echo "trying.." + max=$((max-1)) + if [[ "$max" -eq 0 ]]; then + return 1 + fi + sleep "$interval" + done +} + +# compile the python to a pipeline yaml, upload the pipeline, create a run, +# and wait until the run finishes. +run_test_case() { + if [[ $# -ne 4 ]] + then + echo "Usage: run_test_case test-case-name python-file condition-string wait-time" + return 1 + fi + local REV=1 + local TEST_CASE=$1 + shift + local PY_FILE=$1 + shift + local F_STATUS=$1 + shift + local DURATION=$1 + shift + local PIPELINE_ID + local RUN_ID + local KFP_COMMAND="kfp" + local PIPELINE_NAME="${TEST_CASE}-$((RANDOM%10000+1))" + local YAML_FILE=$(echo "${PY_FILE}" | sed "s/\.py$/\.yaml/") + + echo " ===== ${TEST_CASE} =====" + $KFP_COMMAND dsl compile --py "${PY_FILE}" --output "${YAML_FILE}" + retry 3 3 $KFP_COMMAND --endpoint http://localhost:8888 pipeline create -p "$PIPELINE_NAME" "${YAML_FILE}" 2>&1 || : + PIPELINE_ID=$($KFP_COMMAND --endpoint http://localhost:8888 pipeline list 2>&1| grep "$PIPELINE_NAME" | awk '{print $1}') + if [[ -z "$PIPELINE_ID" ]]; then + echo "Failed to upload pipeline" + return "$REV" + fi + VERSION_ID=$($KFP_COMMAND --endpoint http://localhost:8888 pipeline list-versions "${PIPELINE_ID}" 2>&1| grep "$PIPELINE_NAME" | awk '{print $1}') + + local RUN_NAME="${PIPELINE_NAME}-run" + retry 3 3 $KFP_COMMAND --endpoint http://localhost:8888 run create -e "exp-${TEST_CASE}" -r "$RUN_NAME" -p "$PIPELINE_ID" -v "$VERSION_ID" 2>&1 || : + RUN_ID=$($KFP_COMMAND --endpoint http://localhost:8888 run list 2>&1| grep "$RUN_NAME" | awk '{print $1}') + if [[ -z "$RUN_ID" ]]; then + echo "Failed to submit a run for ${TEST_CASE} pipeline" + return "$REV" + fi + + local RUN_STATUS + ENDTIME=$(date -ud "$DURATION minute" +%s) + while [[ "$(date -u +%s)" -le "$ENDTIME" ]]; do + RUN_STATUS=$($KFP_COMMAND --endpoint http://localhost:8888 run list 2>&1| grep "$RUN_NAME" | awk '{print $4}') + if [[ "$RUN_STATUS" == "$F_STATUS" ]]; then + REV=0 + break; + fi + echo " Status of ${TEST_CASE} run: $RUN_STATUS" + if [[ "$RUN_STATUS" == "FAILED" ]]; then + REV=1 + break; + fi + sleep 10 + done + + if [[ "$REV" -eq 0 ]]; then + echo " ===== ${TEST_CASE} PASSED =====" + else + echo " ===== ${TEST_CASE} FAILED =====" + fi + + echo 'y' | $KFP_COMMAND --endpoint http://localhost:8888 run delete "$RUN_ID" || : + + return "$REV" +} diff --git a/scripts/deploy/github/test-secret-as-env.sh b/scripts/deploy/github/test-secret-as-env.sh new file mode 100755 index 000000000000..b203182c8aeb --- /dev/null +++ b/scripts/deploy/github/test-secret-as-env.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# +# Copyright 2023 kubeflow.org +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +KUBEFLOW_NS="${KUBEFLOW_NS:-kubeflow}" + +C_DIR="${BASH_SOURCE%/*}" +if [[ ! -d "$C_DIR" ]]; then C_DIR="$PWD"; fi +source "${C_DIR}/test-pipeline.sh" + +# need kfp-kubernetes for this test case +# unfortunately, we can't install it from kubernetes_platform/python +pip install kfp-kubernetes + +# create the secret +kubectl create secret -n "$KUBEFLOW_NS" generic "user-gcp-sa" --from-literal="type=service_account" || true + +RESULT=0 +run_test_case "secret-env" "samples/v2/pipeline_with_secret_as_env.py" "SUCCEEDED" 5 || RESULT=$? + +# remove secret after the test finishes +kubectl delete secret -n "$KUBEFLOW_NS" "user-gcp-sa" + +STATUS_MSG=PASSED +if [[ "$RESULT" -ne 0 ]]; then + STATUS_MSG=FAILED +fi diff --git a/scripts/deploy/github/test-secret-as-volume.sh b/scripts/deploy/github/test-secret-as-volume.sh new file mode 100755 index 000000000000..f65c3e262629 --- /dev/null +++ b/scripts/deploy/github/test-secret-as-volume.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# +# Copyright 2023 kubeflow.org +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +KUBEFLOW_NS="${KUBEFLOW_NS:-kubeflow}" + +C_DIR="${BASH_SOURCE%/*}" +if [[ ! -d "$C_DIR" ]]; then C_DIR="$PWD"; fi +source "${C_DIR}/test-pipeline.sh" + +# need kfp-kubernetes for this test case +# unfortunately, we can't install it from kubernetes_platform/python +pip install kfp-kubernetes + +# create the secret +kubectl create secret -n "$KUBEFLOW_NS" generic "user-gcp-sa" --from-literal="type=service_account" || true + +RESULT=0 +run_test_case "secret-volume" "samples/v2/pipeline_with_secret_as_volume.py" "SUCCEEDED" 5 || RESULT=$? + +# remove secret after the test finishes +kubectl delete secret -n "$KUBEFLOW_NS" "user-gcp-sa" + +STATUS_MSG=PASSED +if [[ "$RESULT" -ne 0 ]]; then + STATUS_MSG=FAILED +fi diff --git a/scripts/deploy/github/test-static-loop.sh b/scripts/deploy/github/test-static-loop.sh new file mode 100755 index 000000000000..84fea57e5ab1 --- /dev/null +++ b/scripts/deploy/github/test-static-loop.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Copyright 2023 kubeflow.org +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +C_DIR="${BASH_SOURCE%/*}" +if [[ ! -d "$C_DIR" ]]; then C_DIR="$PWD"; fi +source "${C_DIR}/test-pipeline.sh" + +RESULT=0 +run_test_case "static-loop" "samples/core/loop_static/loop_static.py" "SUCCEEDED" 20 || RESULT=$? + +STATUS_MSG=PASSED +if [[ "$RESULT" -ne 0 ]]; then + STATUS_MSG=FAILED +fi diff --git a/scripts/deploy/github/test-volume.sh b/scripts/deploy/github/test-volume.sh new file mode 100755 index 000000000000..87fe2f0356f7 --- /dev/null +++ b/scripts/deploy/github/test-volume.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Copyright 2023 kubeflow.org +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +KUBEFLOW_NS="${KUBEFLOW_NS:-kubeflow}" + +C_DIR="${BASH_SOURCE%/*}" +if [[ ! -d "$C_DIR" ]]; then C_DIR="$PWD"; fi +source "${C_DIR}/test-pipeline.sh" + +# need kfp-kubernetes for this test case +# unfortunately, we can't install it from kubernetes_platform/python +pip install kfp-kubernetes + +RESULT=0 +run_test_case "use-volume" "samples/v2/pipeline_with_volume.py" "SUCCEEDED" 10 || RESULT=$? + +STATUS_MSG=PASSED +if [[ "$RESULT" -ne 0 ]]; then + STATUS_MSG=FAILED +fi