From 91287367a23d13ec8b5fb7ac70b7808806dd5c1e Mon Sep 17 00:00:00 2001 From: Revital Sur Date: Tue, 18 Jun 2024 00:12:58 +0300 Subject: [PATCH 1/6] Move hack dirs to scripts dir. Signed-off-by: Revital Sur --- kind/Makefile | 5 ++-- kind/README.md | 22 +++++++------- kind/hack/populate_minio.sh | 29 ------------------- {kind/hack => scripts/kind}/common.sh | 2 +- .../hack => scripts/kind}/helper-functions.sh | 0 {kind/hack => scripts/kind}/kfp_ingress.yaml | 0 .../kind}/kind-cluster-config.yaml | 0 .../hack => scripts/kind}/minio_ingress.yaml | 0 {kind/hack => scripts/kind}/nginx_deploy.yaml | 0 .../kind}/nginx_deploy_minio.yaml | 0 scripts/kind/populate_minio.sh | 29 +++++++++++++++++++ .../kind}/ray_api_server_ingress.yaml | 0 .../kind}/ray_api_server_values.yaml | 0 {kind/hack => scripts/kind}/s3_secret.yaml | 0 {kind/hack => scripts/kind}/tools/ingress.sh | 10 +++---- .../kind}/tools/install_kubeflow.sh | 0 .../kind}/tools/install_kuberay.sh | 4 +-- .../kind}/tools/install_minio.sh | 8 ++--- .../kind}/tools/install_nginx.sh | 6 ++-- .../kind}/tools/kind_management.sh | 2 +- .../transforms}/update_workflow_tags.sh | 0 transforms/.make.workflows | 2 +- 22 files changed, 59 insertions(+), 60 deletions(-) delete mode 100755 kind/hack/populate_minio.sh rename {kind/hack => scripts/kind}/common.sh (91%) rename {kind/hack => scripts/kind}/helper-functions.sh (100%) rename {kind/hack => scripts/kind}/kfp_ingress.yaml (100%) rename {kind/hack => scripts/kind}/kind-cluster-config.yaml (100%) rename {kind/hack => scripts/kind}/minio_ingress.yaml (100%) rename {kind/hack => scripts/kind}/nginx_deploy.yaml (100%) rename {kind/hack => scripts/kind}/nginx_deploy_minio.yaml (100%) create mode 100755 scripts/kind/populate_minio.sh rename {kind/hack => scripts/kind}/ray_api_server_ingress.yaml (100%) rename {kind/hack => scripts/kind}/ray_api_server_values.yaml (100%) rename {kind/hack => scripts/kind}/s3_secret.yaml (100%) rename {kind/hack => scripts/kind}/tools/ingress.sh (55%) rename {kind/hack => scripts/kind}/tools/install_kubeflow.sh (100%) rename {kind/hack => scripts/kind}/tools/install_kuberay.sh (78%) rename {kind/hack => scripts/kind}/tools/install_minio.sh (75%) rename {kind/hack => scripts/kind}/tools/install_nginx.sh (82%) rename {kind/hack => scripts/kind}/tools/kind_management.sh (93%) rename {transforms/hack => scripts/transforms}/update_workflow_tags.sh (100%) diff --git a/kind/Makefile b/kind/Makefile index da22e24f3..6da5104b9 100644 --- a/kind/Makefile +++ b/kind/Makefile @@ -12,12 +12,11 @@ IGNORE := $(shell bash -c "sed -n /=/p ${REPOROOT}/kind/requirements.env | sed include makeenv -export ROOT_DIR=${CURDIR} # Include the common rules. # Use "make help" to see them. include ../.make.defaults -export TOOLS_DIR=${ROOT_DIR}/hack/tools +export TOOLS_DIR=${REPOROOT}/scripts/kind/tools export EXTERNAL_CLUSTER ?= 0 export DEPLOY_KUBEFLOW ?= 1 @@ -35,7 +34,7 @@ endif populate-data:: @# Help: Populate test data in Minio - cd hack && ./populate_minio.sh + cd ${REPOROOT}/scripts/kind && ./populate_minio.sh cluster-deploy:: @# Help: Deploy all required tools on existing cluster diff --git a/kind/README.md b/kind/README.md index 60fae6841..cc261d785 100644 --- a/kind/README.md +++ b/kind/README.md @@ -16,12 +16,12 @@ Run the following command to create the cluster: cd /tmp git clone https://github.com/IBM/data-prep-kit.git cd data-prep-kit -ROOT_DIR=$PWD/kind/ -kind create cluster --name dataprep --config ${ROOT_DIR}/hack/kind-cluster-config.yaml +export REPOROOT=$PWD +kind create cluster --name dataprep --config ${REPOROOT}/scripts/kind/kind-cluster-config.yaml ``` Note that by default this will create a kind cluster with 2 worker nodes. If you would like a different -amount of node, modify [cluster configuration](hack/kind-cluster-config.yaml) +amount of node, modify [cluster configuration](../scripts/kind/kind-cluster-config.yaml) ### Install KFP @@ -31,7 +31,7 @@ Install [Kubeflow Pipelines](https://www.kubeflow.org/docs/components/pipelines/ # Set required KFP version. You can reference to the latest supported version in the [requirements.env](./requirements.env) file. # Currently, we support 1.8.5 for KFPv1 and 2.2.0 for KFP v2 export PIPELINE_VERSION=1.8.5 -cd $ROOT_DIR/hack/tools/ && ./install_kubeflow.sh deploy && cd - +cd $REPOROOT/scripts/kind/tools/ && ./install_kubeflow.sh deploy && cd - kubectl wait --for=condition=ready --all pod -n kubeflow --timeout=300s ``` @@ -40,7 +40,7 @@ kubectl wait --for=condition=ready --all pod -n kubeflow --timeout=300s Install Kuberay: ```shell -cd $ROOT_DIR/hack/tools && KUBERAY_APISERVER=1.1.0 KUBERAY_OPERATOR=1.0.0 ./install_kuberay.sh deploy && cd - +cd $REPOROOT/scripts/kind/tools/ && KUBERAY_APISERVER=1.1.0 KUBERAY_OPERATOR=1.0.0 ./install_kuberay.sh deploy && cd - kubectl wait --for=condition=ready --all pod -n kuberay --timeout=300s ``` @@ -52,7 +52,7 @@ To access the API server and Kubeflow pipeline UI externally, we make use NGINX Install [Ingress NGNIX](https://kind.sigs.k8s.io/docs/user/ingress/#ingress-nginx) for KFP, RAY and MinIO and wait for it to be ready: ```shell -${ROOT_DIR}/hack/tools/install_nginx.sh deploy +${REPOROOT}/scripts/kind/tools/install_nginx.sh deploy kubectl wait --namespace ingress-nginx \ --for=condition=ready pod \ --selector=app.kubernetes.io/component=controller \ @@ -61,9 +61,9 @@ kubectl wait --namespace ingress-nginx \ To deploy the ingress for Ray API Server, KFP and MinIO execute the following: ```shell -kubectl apply -f $ROOT_DIR/hack/ray_api_server_ingress.yaml -kubectl apply -f $ROOT_DIR/hack/kfp_ingress.yaml -kubectl apply -f $ROOT_DIR/hack/minio_ingress.yaml +kubectl apply -f $REPOROOT/scripts/kind/ray_api_server_ingress.yaml +kubectl apply -f $REPOROOT/scripts/kind/kfp_ingress.yaml +kubectl apply -f $REPOROOT/scripts/kind/minio_ingress.yaml ``` Open the Kubeflow Pipelines UI at http://localhost:8080/ @@ -79,7 +79,7 @@ as the secret key. A secret needs to be created for accessing MinIO using the following command: ```shell -kubectl apply -f $ROOT_DIR/hack/s3_secret.yaml +kubectl apply -f $REPOROOT/scripts/kind/s3_secret.yaml ``` #### Copy test data @@ -87,7 +87,7 @@ kubectl apply -f $ROOT_DIR/hack/s3_secret.yaml Populating Minio server with test data can be done using `mc`. Use the following command: ```shell -$ROOT_DIR/hack/populate_minio.sh +$REPOROOT/scripts/kind/populate_minio.sh ``` This file creates an mc alias, creates the test bucket and copies the local test data into MinIO. If you need diff --git a/kind/hack/populate_minio.sh b/kind/hack/populate_minio.sh deleted file mode 100755 index 348447587..000000000 --- a/kind/hack/populate_minio.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash - -if [ "$MINIO_SERVER" == "" ]; then - MINIO_SERVER="http://localhost:8090" -fi - -echo "creating minio alias to $MINIO_SERVER" -mc alias set kfp $MINIO_SERVER minio minio123 - -echo "creating test bucket" -mc mb kfp/test -echo "copying data" -# code modules -mc cp --recursive ${ROOT_DIR}/../transforms/code/code_quality/ray/test-data/input/ kfp/test/code_quality/input -mc cp --recursive ${ROOT_DIR}/../transforms/code/ingest_2_parquet/ray/test-data/input/data-processing-lib.zip kfp/test/ingest_2_parquet/input -mc cp --recursive ${ROOT_DIR}/../transforms/code/ingest_2_parquet/ray/test-data/languages/ kfp/test/ingest_2_parquet/languages -mc cp --recursive ${ROOT_DIR}/../transforms/code/proglang_select/ray/test-data/input/ kfp/test/proglang_select/input -mc cp --recursive ${ROOT_DIR}/../transforms/code/proglang_select/ray/test-data/languages/ kfp/test/proglang_select/languages -mc cp --recursive ${ROOT_DIR}/../transforms/code/malware/ray/test-data/input/ kfp/test/malware/input -# language -mc cp --recursive ${ROOT_DIR}/../transforms/language/lang_id/ray/test-data/input/ kfp/test/lang_id/input -# universal -mc cp --recursive ${ROOT_DIR}/../transforms/universal/doc_id/ray/test-data/input/ kfp/test/doc_id/input -mc cp --recursive ${ROOT_DIR}/../transforms/universal/ededup/ray/test-data/input/ kfp/test/ededup/input -mc cp --recursive ${ROOT_DIR}/../transforms/universal/fdedup/ray/test-data/input/ kfp/test/fdedup/input -mc cp --recursive ${ROOT_DIR}/../transforms/universal/filter/ray/test-data/input/ kfp/test/filter/input -mc cp --recursive ${ROOT_DIR}/../transforms/universal/noop/ray/test-data/input/ kfp/test/noop/input -mc cp --recursive ${ROOT_DIR}/../transforms/universal/tokenization/ray/test-data/ds01/input/ kfp/test/tokenization/ds01/input - diff --git a/kind/hack/common.sh b/scripts/kind/common.sh similarity index 91% rename from kind/hack/common.sh rename to scripts/kind/common.sh index 5a86ce51c..b567e0982 100755 --- a/kind/hack/common.sh +++ b/scripts/kind/common.sh @@ -17,7 +17,7 @@ if [[ "$os" == "unknown" ]]; then exit 1 fi -source ${ROOT_DIR}/hack/helper-functions.sh +source ${REPOROOT}/scripts/kind/helper-functions.sh # Turn colors in this script off by setting the NO_COLOR variable in your # environment to any value: diff --git a/kind/hack/helper-functions.sh b/scripts/kind/helper-functions.sh similarity index 100% rename from kind/hack/helper-functions.sh rename to scripts/kind/helper-functions.sh diff --git a/kind/hack/kfp_ingress.yaml b/scripts/kind/kfp_ingress.yaml similarity index 100% rename from kind/hack/kfp_ingress.yaml rename to scripts/kind/kfp_ingress.yaml diff --git a/kind/hack/kind-cluster-config.yaml b/scripts/kind/kind-cluster-config.yaml similarity index 100% rename from kind/hack/kind-cluster-config.yaml rename to scripts/kind/kind-cluster-config.yaml diff --git a/kind/hack/minio_ingress.yaml b/scripts/kind/minio_ingress.yaml similarity index 100% rename from kind/hack/minio_ingress.yaml rename to scripts/kind/minio_ingress.yaml diff --git a/kind/hack/nginx_deploy.yaml b/scripts/kind/nginx_deploy.yaml similarity index 100% rename from kind/hack/nginx_deploy.yaml rename to scripts/kind/nginx_deploy.yaml diff --git a/kind/hack/nginx_deploy_minio.yaml b/scripts/kind/nginx_deploy_minio.yaml similarity index 100% rename from kind/hack/nginx_deploy_minio.yaml rename to scripts/kind/nginx_deploy_minio.yaml diff --git a/scripts/kind/populate_minio.sh b/scripts/kind/populate_minio.sh new file mode 100755 index 000000000..074470b0f --- /dev/null +++ b/scripts/kind/populate_minio.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +if [ "$MINIO_SERVER" == "" ]; then + MINIO_SERVER="http://localhost:8090" +fi + +echo "creating minio alias to $MINIO_SERVER" +mc alias set kfp $MINIO_SERVER minio minio123 + +echo "creating test bucket" +mc mb kfp/test +echo "copying data" +# code modules +mc cp --recursive ${REPOROOT}/transforms/code/code_quality/ray/test-data/input/ kfp/test/code_quality/input +mc cp --recursive ${REPOROOT}/transforms/code/ingest_2_parquet/ray/test-data/input/data-processing-lib.zip kfp/test/ingest_2_parquet/input +mc cp --recursive ${REPOROOT}/transforms/code/ingest_2_parquet/ray/test-data/languages/ kfp/test/ingest_2_parquet/languages +mc cp --recursive ${REPOROOT}/transforms/code/proglang_select/ray/test-data/input/ kfp/test/proglang_select/input +mc cp --recursive ${REPOROOT}/transforms/code/proglang_select/ray/test-data/languages/ kfp/test/proglang_select/languages +mc cp --recursive ${REPOROOT}/transforms/code/malware/ray/test-data/input/ kfp/test/malware/input +# language +mc cp --recursive ${REPOROOT}/transforms/language/lang_id/ray/test-data/input/ kfp/test/lang_id/input +# universal +mc cp --recursive ${REPOROOT}/transforms/universal/doc_id/ray/test-data/input/ kfp/test/doc_id/input +mc cp --recursive ${REPOROOT}/transforms/universal/ededup/ray/test-data/input/ kfp/test/ededup/input +mc cp --recursive ${REPOROOT}/transforms/universal/fdedup/ray/test-data/input/ kfp/test/fdedup/input +mc cp --recursive ${REPOROOT}/transforms/universal/filter/ray/test-data/input/ kfp/test/filter/input +mc cp --recursive ${REPOROOT}/transforms/universal/noop/ray/test-data/input/ kfp/test/noop/input +mc cp --recursive ${REPOROOT}/transforms/universal/tokenization/ray/test-data/ds01/input/ kfp/test/tokenization/ds01/input + diff --git a/kind/hack/ray_api_server_ingress.yaml b/scripts/kind/ray_api_server_ingress.yaml similarity index 100% rename from kind/hack/ray_api_server_ingress.yaml rename to scripts/kind/ray_api_server_ingress.yaml diff --git a/kind/hack/ray_api_server_values.yaml b/scripts/kind/ray_api_server_values.yaml similarity index 100% rename from kind/hack/ray_api_server_values.yaml rename to scripts/kind/ray_api_server_values.yaml diff --git a/kind/hack/s3_secret.yaml b/scripts/kind/s3_secret.yaml similarity index 100% rename from kind/hack/s3_secret.yaml rename to scripts/kind/s3_secret.yaml diff --git a/kind/hack/tools/ingress.sh b/scripts/kind/tools/ingress.sh similarity index 55% rename from kind/hack/tools/ingress.sh rename to scripts/kind/tools/ingress.sh index 2bdf416ea..19ab91fe2 100755 --- a/kind/hack/tools/ingress.sh +++ b/scripts/kind/tools/ingress.sh @@ -2,19 +2,19 @@ op=$1 -source ${ROOT_DIR}/hack/common.sh +source ${REPOROOT}/scripts/kind/common.sh deploy() { - kubectl apply -f ${ROOT_DIR}/hack/ray_api_server_ingress.yaml + kubectl apply -f ${REPOROOT}/scripts/kind/ray_api_server_ingress.yaml if [[ "${DEPLOY_KUBEFLOW}" -eq 1 ]]; then - kubectl apply -f ${ROOT_DIR}/hack/kfp_ingress.yaml + kubectl apply -f ${REPOROOT}/scripts/kind/kfp_ingress.yaml fi } delete(){ - kubectl delete -f ${ROOT_DIR}/hack/ray_api_server_ingress.yaml + kubectl delete -f ${REPOROOT}/scripts/kind/ray_api_server_ingress.yaml if [[ "${DEPLOY_KUBEFLOW}" -eq 1 ]]; then - kubectl delete -f ${ROOT_DIR}/hack/kfp_ingress.yaml + kubectl delete -f ${REPOROOT}/scripts/kind/kfp_ingress.yaml fi } diff --git a/kind/hack/tools/install_kubeflow.sh b/scripts/kind/tools/install_kubeflow.sh similarity index 100% rename from kind/hack/tools/install_kubeflow.sh rename to scripts/kind/tools/install_kubeflow.sh diff --git a/kind/hack/tools/install_kuberay.sh b/scripts/kind/tools/install_kuberay.sh similarity index 78% rename from kind/hack/tools/install_kuberay.sh rename to scripts/kind/tools/install_kuberay.sh index 32c42c3d1..1705eba24 100755 --- a/kind/hack/tools/install_kuberay.sh +++ b/scripts/kind/tools/install_kuberay.sh @@ -9,11 +9,11 @@ MAX_RETRIES="${MAX_RETRIES:-5}" EXIT_CODE=0 deploy() { - sed -i.back "s/tag: v[0-9].*/tag: v${KUBERAY_APISERVER}/" ${ROOT_DIR}/hack/ray_api_server_values.yaml + sed -i.back "s/tag: v[0-9].*/tag: v${KUBERAY_APISERVER}/" ${REPOROOT}/scripts/kind/ray_api_server_values.yaml helm repo add kuberay https://ray-project.github.io/kuberay-helm/ helm repo update kuberay helm install kuberay-operator kuberay/kuberay-operator -n kuberay --version ${KUBERAY_OPERATOR} --set image.pullPolicy=IfNotPresent --create-namespace - helm install -f ${ROOT_DIR}/hack/ray_api_server_values.yaml kuberay-apiserver kuberay/kuberay-apiserver -n kuberay --version ${KUBERAY_APISERVER} --set image.pullPolicy=IfNotPresent + helm install -f ${REPOROOT}/scripts/kind/ray_api_server_values.yaml kuberay-apiserver kuberay/kuberay-apiserver -n kuberay --version ${KUBERAY_APISERVER} --set image.pullPolicy=IfNotPresent echo "Finished KubeRay deployment." } diff --git a/kind/hack/tools/install_minio.sh b/scripts/kind/tools/install_minio.sh similarity index 75% rename from kind/hack/tools/install_minio.sh rename to scripts/kind/tools/install_minio.sh index 0a7ed6271..2d7493090 100755 --- a/kind/hack/tools/install_minio.sh +++ b/scripts/kind/tools/install_minio.sh @@ -9,8 +9,8 @@ MAX_RETRIES="${MAX_RETRIES:-20}" EXIT_CODE=0 deploy() { - kubectl apply -f ${ROOT_DIR}/hack/s3_secret.yaml - kubectl apply -f ${ROOT_DIR}/hack/minio_ingress.yaml + kubectl apply -f ${REPOROOT}/scripts/kind/s3_secret.yaml + kubectl apply -f ${REPOROOT}/scripts/kind/minio_ingress.yaml } wait(){ @@ -24,8 +24,8 @@ echo "ingress minio is ready" } delete(){ - kubectl delete -f ${ROOT_DIR}/hack/s3_secret.yaml - kubectl delete -f ${ROOT_DIR}/hack/minio_ingress.yaml + kubectl delete -f ${REPOROOT}/scripts/kind/s3_secret.yaml + kubectl delete -f ${REPOROOT}/scripts/kind/minio_ingress.yaml } usage(){ diff --git a/kind/hack/tools/install_nginx.sh b/scripts/kind/tools/install_nginx.sh similarity index 82% rename from kind/hack/tools/install_nginx.sh rename to scripts/kind/tools/install_nginx.sh index 2003091d2..01339b7fb 100755 --- a/kind/hack/tools/install_nginx.sh +++ b/scripts/kind/tools/install_nginx.sh @@ -5,10 +5,10 @@ op=$1 SLEEP_TIME="${SLEEP_TIME:-30}" MAX_RETRIES="${MAX_RETRIES:-20}" EXIT_CODE=0 -NGINX_INSTALLATION_FILE="${ROOT_DIR}/hack/nginx_deploy.yaml" -NGINX_MINIO_INSTALLATION_FILE="${ROOT_DIR}/hack/nginx_deploy_minio.yaml" +NGINX_INSTALLATION_FILE="${REPOROOT}/scripts/kind/nginx_deploy.yaml" +NGINX_MINIO_INSTALLATION_FILE="${REPOROOT}/scripts/kind/nginx_deploy_minio.yaml" -source ${ROOT_DIR}/hack/common.sh +source ${REPOROOT}/scripts/kind/common.sh deploy() { kubectl apply -f "$NGINX_INSTALLATION_FILE" diff --git a/kind/hack/tools/kind_management.sh b/scripts/kind/tools/kind_management.sh similarity index 93% rename from kind/hack/tools/kind_management.sh rename to scripts/kind/tools/kind_management.sh index dd49ca501..e1044fb88 100755 --- a/kind/hack/tools/kind_management.sh +++ b/scripts/kind/tools/kind_management.sh @@ -38,7 +38,7 @@ kind_create() { echo "port 8090 is in use, please clear the port and try again" exit 1 fi - kind create cluster --name $cluster_name --config ${ROOT_DIR}/hack/kind-cluster-config.yaml + kind create cluster --name $cluster_name --config ${REPOROOT}/scripts/kind/kind-cluster-config.yaml } usage(){ diff --git a/transforms/hack/update_workflow_tags.sh b/scripts/transforms/update_workflow_tags.sh similarity index 100% rename from transforms/hack/update_workflow_tags.sh rename to scripts/transforms/update_workflow_tags.sh diff --git a/transforms/.make.workflows b/transforms/.make.workflows index 9beb51c55..b59272bbd 100644 --- a/transforms/.make.workflows +++ b/transforms/.make.workflows @@ -17,7 +17,7 @@ endef .workflows.set-versions: cd ${REPOROOT}/kfp/kfp_ray_components && $(MAKE) set-versions cd ${REPOROOT}/kfp/kfp_ray_components && $(MAKE) .reconcile-requirements FILE=${CURDIR}/${PIPELINE_FILE} - ${REPOROOT}/transforms/hack/update_workflow_tags.sh ${REPOROOT}/.make.versions ${CURDIR}/${PIPELINE_FILE} + ${REPOROOT}/scripts/transforms/update_workflow_tags.sh ${REPOROOT}/.make.versions ${CURDIR}/${PIPELINE_FILE} .PHONY: .workflows.compile-pipeline .workflows.compile-pipeline: From ff5d3704034d7c7799bb65c3e285b5176b2c2987 Mon Sep 17 00:00:00 2001 From: Revital Sur Date: Tue, 18 Jun 2024 00:16:10 +0300 Subject: [PATCH 2/6] Adjust test.yml gh workflow. Signed-off-by: Revital Sur --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0c706d7c5..0a2926b8c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -101,8 +101,8 @@ jobs: dir=("code" "universal") && index=$(($RANDOM % 2)) && subdirs=${dir[$index]} && transforms=($(find transforms/$subdirs/ -type d -maxdepth 1 )) # First element is not really a subdir but rather the current dir so remove it and randomly choose a transform to run set -- "${transforms[@]}" && shift && transforms=("$@") && size=${#transforms[@]} && index=$(($RANDOM % $size)) - export ROOT_DIR=kind - source kind/hack/common.sh + export REPOROOT=$PWD + source scripts/kind/common.sh header_text "Running ${transforms[$index]} workflow test" make -C ${transforms[$index]} workflow-test header_text "Run ${transforms[$index]} completed" From f9889d018d6e535ee56d04dff230fed246117c85 Mon Sep 17 00:00:00 2001 From: Revital Sur Date: Tue, 18 Jun 2024 05:26:27 +0300 Subject: [PATCH 3/6] Fix to list_pipelines. Signed-off-by: Revital Sur --- .../src/workflow_support/pipeline_utils/pipeline_utils.py | 2 +- .../src/workflow_support/pipeline_utils/pipeline_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kfp/kfp_support_lib/kfp_v1_workflow_support/src/workflow_support/pipeline_utils/pipeline_utils.py b/kfp/kfp_support_lib/kfp_v1_workflow_support/src/workflow_support/pipeline_utils/pipeline_utils.py index 7566f6b2e..3bc660c19 100644 --- a/kfp/kfp_support_lib/kfp_v1_workflow_support/src/workflow_support/pipeline_utils/pipeline_utils.py +++ b/kfp/kfp_support_lib/kfp_v1_workflow_support/src/workflow_support/pipeline_utils/pipeline_utils.py @@ -130,7 +130,7 @@ def get_pipeline_by_name(self, name: str, np: int = 100) -> models.api_pipeline. """ try: # Get all pipelines - pipelines = self.kfp_client.list_pipelines(page_size=np).pipelines + pipelines = self.kfp_client.list_pipelines(page_size=np, sort_by="created_at desc").pipelines required = list(filter(lambda p: name in p.name, pipelines)) if len(required) != 1: logger.warning(f"Failure to get pipeline. Number of pipelines with name {name} is {len(required)}") diff --git a/kfp/kfp_support_lib/kfp_v2_workflow_support/src/workflow_support/pipeline_utils/pipeline_utils.py b/kfp/kfp_support_lib/kfp_v2_workflow_support/src/workflow_support/pipeline_utils/pipeline_utils.py index da16ccb09..7403d01a6 100644 --- a/kfp/kfp_support_lib/kfp_v2_workflow_support/src/workflow_support/pipeline_utils/pipeline_utils.py +++ b/kfp/kfp_support_lib/kfp_v2_workflow_support/src/workflow_support/pipeline_utils/pipeline_utils.py @@ -149,7 +149,7 @@ def get_pipeline_by_name(self, name: str, np: int = 100) -> kfp_server_api.V2bet ] } ) - result = self.kfp_client.list_pipelines(filter=pipeline_filter) + result = self.kfp_client.list_pipelines(filter=pipeline_filter, page_size=np, sort_by="created_at desc") if result.pipelines is None: return None if len(result.pipelines) == 1: From 15f03d2e4984eaf5489d241e120b77aeb5354ed8 Mon Sep 17 00:00:00 2001 From: Revital Sur Date: Tue, 18 Jun 2024 05:46:18 +0300 Subject: [PATCH 4/6] Address review comments. Signed-off-by: Revital Sur --- kind/Makefile | 3 ++- scripts/kind/common.sh | 2 +- scripts/kind/tools/ingress.sh | 11 ++++++----- scripts/kind/tools/install_kuberay.sh | 4 ++-- scripts/kind/tools/install_minio.sh | 8 ++++---- scripts/kind/tools/install_nginx.sh | 6 +++--- scripts/kind/tools/kind_management.sh | 2 +- 7 files changed, 19 insertions(+), 17 deletions(-) diff --git a/kind/Makefile b/kind/Makefile index 6da5104b9..a80417333 100644 --- a/kind/Makefile +++ b/kind/Makefile @@ -17,6 +17,7 @@ include makeenv include ../.make.defaults export TOOLS_DIR=${REPOROOT}/scripts/kind/tools +export KIND_SCRIPTS=${REPOROOT}/scripts/kind export EXTERNAL_CLUSTER ?= 0 export DEPLOY_KUBEFLOW ?= 1 @@ -34,7 +35,7 @@ endif populate-data:: @# Help: Populate test data in Minio - cd ${REPOROOT}/scripts/kind && ./populate_minio.sh + cd ${KIND_SCRIPTS} && ./populate_minio.sh cluster-deploy:: @# Help: Deploy all required tools on existing cluster diff --git a/scripts/kind/common.sh b/scripts/kind/common.sh index b567e0982..8ebf23c70 100755 --- a/scripts/kind/common.sh +++ b/scripts/kind/common.sh @@ -17,7 +17,7 @@ if [[ "$os" == "unknown" ]]; then exit 1 fi -source ${REPOROOT}/scripts/kind/helper-functions.sh +source ${KIND_SCRIPTS}/helper-functions.sh # Turn colors in this script off by setting the NO_COLOR variable in your # environment to any value: diff --git a/scripts/kind/tools/ingress.sh b/scripts/kind/tools/ingress.sh index 19ab91fe2..a96a6a2ca 100755 --- a/scripts/kind/tools/ingress.sh +++ b/scripts/kind/tools/ingress.sh @@ -2,19 +2,20 @@ op=$1 -source ${REPOROOT}/scripts/kind/common.sh +source ${KIND_SCRIPTS}/common.sh deploy() { - kubectl apply -f ${REPOROOT}/scripts/kind/ray_api_server_ingress.yaml + sleep 10 + kubectl apply -f ${KIND_SCRIPTS}/ray_api_server_ingress.yaml if [[ "${DEPLOY_KUBEFLOW}" -eq 1 ]]; then - kubectl apply -f ${REPOROOT}/scripts/kind/kfp_ingress.yaml + kubectl apply -f ${KIND_SCRIPTS}/kfp_ingress.yaml fi } delete(){ - kubectl delete -f ${REPOROOT}/scripts/kind/ray_api_server_ingress.yaml + kubectl delete -f ${KIND_SCRIPTS}/ray_api_server_ingress.yaml if [[ "${DEPLOY_KUBEFLOW}" -eq 1 ]]; then - kubectl delete -f ${REPOROOT}/scripts/kind/kfp_ingress.yaml + kubectl delete -f ${KIND_SCRIPTS}/kfp_ingress.yaml fi } diff --git a/scripts/kind/tools/install_kuberay.sh b/scripts/kind/tools/install_kuberay.sh index 1705eba24..0a18afc11 100755 --- a/scripts/kind/tools/install_kuberay.sh +++ b/scripts/kind/tools/install_kuberay.sh @@ -9,11 +9,11 @@ MAX_RETRIES="${MAX_RETRIES:-5}" EXIT_CODE=0 deploy() { - sed -i.back "s/tag: v[0-9].*/tag: v${KUBERAY_APISERVER}/" ${REPOROOT}/scripts/kind/ray_api_server_values.yaml + sed -i.back "s/tag: v[0-9].*/tag: v${KUBERAY_APISERVER}/" ${KIND_SCRIPTS}/ray_api_server_values.yaml helm repo add kuberay https://ray-project.github.io/kuberay-helm/ helm repo update kuberay helm install kuberay-operator kuberay/kuberay-operator -n kuberay --version ${KUBERAY_OPERATOR} --set image.pullPolicy=IfNotPresent --create-namespace - helm install -f ${REPOROOT}/scripts/kind/ray_api_server_values.yaml kuberay-apiserver kuberay/kuberay-apiserver -n kuberay --version ${KUBERAY_APISERVER} --set image.pullPolicy=IfNotPresent + helm install -f ${KIND_SCRIPTS}/ray_api_server_values.yaml kuberay-apiserver kuberay/kuberay-apiserver -n kuberay --version ${KUBERAY_APISERVER} --set image.pullPolicy=IfNotPresent echo "Finished KubeRay deployment." } diff --git a/scripts/kind/tools/install_minio.sh b/scripts/kind/tools/install_minio.sh index 2d7493090..2b38bfac4 100755 --- a/scripts/kind/tools/install_minio.sh +++ b/scripts/kind/tools/install_minio.sh @@ -9,8 +9,8 @@ MAX_RETRIES="${MAX_RETRIES:-20}" EXIT_CODE=0 deploy() { - kubectl apply -f ${REPOROOT}/scripts/kind/s3_secret.yaml - kubectl apply -f ${REPOROOT}/scripts/kind/minio_ingress.yaml + kubectl apply -f ${KIND_SCRIPTS}/s3_secret.yaml + kubectl apply -f ${KIND_SCRIPTS}/minio_ingress.yaml } wait(){ @@ -24,8 +24,8 @@ echo "ingress minio is ready" } delete(){ - kubectl delete -f ${REPOROOT}/scripts/kind/s3_secret.yaml - kubectl delete -f ${REPOROOT}/scripts/kind/minio_ingress.yaml + kubectl delete -f ${KIND_SCRIPTS}/s3_secret.yaml + kubectl delete -f ${KIND_SCRIPTS}/minio_ingress.yaml } usage(){ diff --git a/scripts/kind/tools/install_nginx.sh b/scripts/kind/tools/install_nginx.sh index 01339b7fb..0d882156b 100755 --- a/scripts/kind/tools/install_nginx.sh +++ b/scripts/kind/tools/install_nginx.sh @@ -5,10 +5,10 @@ op=$1 SLEEP_TIME="${SLEEP_TIME:-30}" MAX_RETRIES="${MAX_RETRIES:-20}" EXIT_CODE=0 -NGINX_INSTALLATION_FILE="${REPOROOT}/scripts/kind/nginx_deploy.yaml" -NGINX_MINIO_INSTALLATION_FILE="${REPOROOT}/scripts/kind/nginx_deploy_minio.yaml" +NGINX_INSTALLATION_FILE="${KIND_SCRIPTS}/nginx_deploy.yaml" +NGINX_MINIO_INSTALLATION_FILE="${KIND_SCRIPTS}/nginx_deploy_minio.yaml" -source ${REPOROOT}/scripts/kind/common.sh +source ${KIND_SCRIPTS}/common.sh deploy() { kubectl apply -f "$NGINX_INSTALLATION_FILE" diff --git a/scripts/kind/tools/kind_management.sh b/scripts/kind/tools/kind_management.sh index e1044fb88..1e0678aa7 100755 --- a/scripts/kind/tools/kind_management.sh +++ b/scripts/kind/tools/kind_management.sh @@ -38,7 +38,7 @@ kind_create() { echo "port 8090 is in use, please clear the port and try again" exit 1 fi - kind create cluster --name $cluster_name --config ${REPOROOT}/scripts/kind/kind-cluster-config.yaml + kind create cluster --name $cluster_name --config ${KIND_SCRIPTS}/kind-cluster-config.yaml } usage(){ From 88e2246b830616103261e11f32b7e8c418458b24 Mon Sep 17 00:00:00 2001 From: Revital Sur Date: Tue, 18 Jun 2024 06:02:55 +0300 Subject: [PATCH 5/6] Minor fix. Signed-off-by: Revital Sur --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0a2926b8c..3d896e137 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -102,6 +102,7 @@ jobs: # First element is not really a subdir but rather the current dir so remove it and randomly choose a transform to run set -- "${transforms[@]}" && shift && transforms=("$@") && size=${#transforms[@]} && index=$(($RANDOM % $size)) export REPOROOT=$PWD + export KIND_SCRIPTS=$PWD/scripts/kind source scripts/kind/common.sh header_text "Running ${transforms[$index]} workflow test" make -C ${transforms[$index]} workflow-test From 307de94e9dbe0951dda455bfeff7d6a434a23995 Mon Sep 17 00:00:00 2001 From: Revital Sur Date: Wed, 19 Jun 2024 09:01:30 +0300 Subject: [PATCH 6/6] Minor fix. Signed-off-by: Revital Sur --- transforms/code/ingest_2_parquet/kfp_ray/Makefile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/transforms/code/ingest_2_parquet/kfp_ray/Makefile b/transforms/code/ingest_2_parquet/kfp_ray/Makefile index d3e4a4254..d3b6603f9 100644 --- a/transforms/code/ingest_2_parquet/kfp_ray/Makefile +++ b/transforms/code/ingest_2_parquet/kfp_ray/Makefile @@ -9,6 +9,10 @@ YAML_WF := $(patsubst %.py, %.yaml, ${PYTHON_WF}) workflow-venv: .check_python_version ${WORKFLOW_VENV_ACTIVATE} +clean:: .defaults.clean + +setup:: + venv:: build:: @@ -17,12 +21,13 @@ test:: test-src:: -test-image:: +publish:: image:: -load-image:: +test-image:: +load-image:: .PHONY: workflow-build