diff --git a/.github/workflows/test-code-code2parquet-kfp.yml b/.github/workflows/test-code-code2parquet-kfp.yml index 710654571..882e5b65e 100644 --- a/.github/workflows/test-code-code2parquet-kfp.yml +++ b/.github/workflows/test-code-code2parquet-kfp.yml @@ -13,6 +13,7 @@ on: - "*" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/code/code2parquet/**" - "!kfp/**" # This is tested in separate workflow @@ -27,6 +28,7 @@ on: - "releases/**" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/code/code2parquet/**" - "!data-processing-lib/**" # This is tested in separate workflow diff --git a/.github/workflows/test-code-code_quality-kfp.yml b/.github/workflows/test-code-code_quality-kfp.yml index c07bc1d2d..f6ccb3a9e 100644 --- a/.github/workflows/test-code-code_quality-kfp.yml +++ b/.github/workflows/test-code-code_quality-kfp.yml @@ -13,6 +13,7 @@ on: - "*" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/code/code_quality/**" - "!kfp/**" # This is tested in separate workflow @@ -27,6 +28,7 @@ on: - "releases/**" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/code/code_quality/**" - "!data-processing-lib/**" # This is tested in separate workflow diff --git a/.github/workflows/test-code-header_cleanser-kfp.yml b/.github/workflows/test-code-header_cleanser-kfp.yml index 7c419fd14..755393e5d 100644 --- a/.github/workflows/test-code-header_cleanser-kfp.yml +++ b/.github/workflows/test-code-header_cleanser-kfp.yml @@ -13,6 +13,7 @@ on: - "*" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/code/header_cleanser/**" - "!kfp/**" # This is tested in separate workflow @@ -27,6 +28,7 @@ on: - "releases/**" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/code/header_cleanser/**" - "!data-processing-lib/**" # This is tested in separate workflow diff --git a/.github/workflows/test-code-license_select-kfp.yml b/.github/workflows/test-code-license_select-kfp.yml index d72a85dd4..7b6b69ef9 100644 --- a/.github/workflows/test-code-license_select-kfp.yml +++ b/.github/workflows/test-code-license_select-kfp.yml @@ -13,6 +13,7 @@ on: - "*" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/code/license_select/**" - "!kfp/**" # This is tested in separate workflow @@ -27,6 +28,7 @@ on: - "releases/**" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/code/license_select/**" - "!data-processing-lib/**" # This is tested in separate workflow diff --git a/.github/workflows/test-code-malware-kfp.yml b/.github/workflows/test-code-malware-kfp.yml index 89bf47239..e68eb175e 100644 --- a/.github/workflows/test-code-malware-kfp.yml +++ b/.github/workflows/test-code-malware-kfp.yml @@ -13,6 +13,7 @@ on: - "*" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/code/malware/**" - "!kfp/**" # This is tested in separate workflow @@ -27,6 +28,7 @@ on: - "releases/**" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/code/malware/**" - "!data-processing-lib/**" # This is tested in separate workflow diff --git a/.github/workflows/test-code-proglang_select-kfp.yml b/.github/workflows/test-code-proglang_select-kfp.yml index 31328f3d5..edbf09ea1 100644 --- a/.github/workflows/test-code-proglang_select-kfp.yml +++ b/.github/workflows/test-code-proglang_select-kfp.yml @@ -13,6 +13,7 @@ on: - "*" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/code/proglang_select/**" - "!kfp/**" # This is tested in separate workflow @@ -27,6 +28,7 @@ on: - "releases/**" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/code/proglang_select/**" - "!data-processing-lib/**" # This is tested in separate workflow diff --git a/.github/workflows/test-code-repo_level_ordering-kfp.yml b/.github/workflows/test-code-repo_level_ordering-kfp.yml index 4e328f53e..26374677a 100644 --- a/.github/workflows/test-code-repo_level_ordering-kfp.yml +++ b/.github/workflows/test-code-repo_level_ordering-kfp.yml @@ -13,6 +13,7 @@ on: - "*" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/code/repo_level_ordering/**" - "!kfp/**" # This is tested in separate workflow @@ -27,6 +28,7 @@ on: - "releases/**" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/code/repo_level_ordering/**" - "!data-processing-lib/**" # This is tested in separate workflow diff --git a/.github/workflows/test-kfp-transform.template b/.github/workflows/test-kfp-transform.template index f12511118..bed645bed 100644 --- a/.github/workflows/test-kfp-transform.template +++ b/.github/workflows/test-kfp-transform.template @@ -13,6 +13,7 @@ on: - "*" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "@TARGET_TRANSFORM_DIR@/**" - "!kfp/**" # This is tested in separate workflow @@ -27,6 +28,7 @@ on: - "releases/**" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "@TARGET_TRANSFORM_DIR@/**" - "!data-processing-lib/**" # This is tested in separate workflow diff --git a/.github/workflows/test-language-doc_chunk-kfp.yml b/.github/workflows/test-language-doc_chunk-kfp.yml index 985f79b97..fe347dc61 100644 --- a/.github/workflows/test-language-doc_chunk-kfp.yml +++ b/.github/workflows/test-language-doc_chunk-kfp.yml @@ -13,6 +13,7 @@ on: - "*" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/language/doc_chunk/**" - "!kfp/**" # This is tested in separate workflow @@ -27,6 +28,7 @@ on: - "releases/**" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/language/doc_chunk/**" - "!data-processing-lib/**" # This is tested in separate workflow diff --git a/.github/workflows/test-language-doc_quality-kfp.yml b/.github/workflows/test-language-doc_quality-kfp.yml index 6842a1859..33910778a 100644 --- a/.github/workflows/test-language-doc_quality-kfp.yml +++ b/.github/workflows/test-language-doc_quality-kfp.yml @@ -13,6 +13,7 @@ on: - "*" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/language/doc_quality/**" - "!kfp/**" # This is tested in separate workflow @@ -27,6 +28,7 @@ on: - "releases/**" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/language/doc_quality/**" - "!data-processing-lib/**" # This is tested in separate workflow diff --git a/.github/workflows/test-language-html2parquet-kfp.yml b/.github/workflows/test-language-html2parquet-kfp.yml index c7a1cecc5..5da046347 100644 --- a/.github/workflows/test-language-html2parquet-kfp.yml +++ b/.github/workflows/test-language-html2parquet-kfp.yml @@ -13,6 +13,7 @@ on: - "*" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/language/html2parquet/**" - "!kfp/**" # This is tested in separate workflow @@ -27,6 +28,7 @@ on: - "releases/**" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/language/html2parquet/**" - "!data-processing-lib/**" # This is tested in separate workflow diff --git a/.github/workflows/test-language-lang_id-kfp.yml b/.github/workflows/test-language-lang_id-kfp.yml index 936ba8e45..562c38362 100644 --- a/.github/workflows/test-language-lang_id-kfp.yml +++ b/.github/workflows/test-language-lang_id-kfp.yml @@ -13,6 +13,7 @@ on: - "*" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/language/lang_id/**" - "!kfp/**" # This is tested in separate workflow @@ -27,6 +28,7 @@ on: - "releases/**" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/language/lang_id/**" - "!data-processing-lib/**" # This is tested in separate workflow diff --git a/.github/workflows/test-language-pdf2parquet-kfp.yml b/.github/workflows/test-language-pdf2parquet-kfp.yml index f232d78e4..a304e22ec 100644 --- a/.github/workflows/test-language-pdf2parquet-kfp.yml +++ b/.github/workflows/test-language-pdf2parquet-kfp.yml @@ -13,6 +13,7 @@ on: - "*" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/language/pdf2parquet/**" - "!kfp/**" # This is tested in separate workflow @@ -27,6 +28,7 @@ on: - "releases/**" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/language/pdf2parquet/**" - "!data-processing-lib/**" # This is tested in separate workflow diff --git a/.github/workflows/test-language-pii_redactor-kfp.yml b/.github/workflows/test-language-pii_redactor-kfp.yml index 451ac7961..3982c2f67 100644 --- a/.github/workflows/test-language-pii_redactor-kfp.yml +++ b/.github/workflows/test-language-pii_redactor-kfp.yml @@ -13,6 +13,7 @@ on: - "*" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/language/pii_redactor/**" - "!kfp/**" # This is tested in separate workflow @@ -27,6 +28,7 @@ on: - "releases/**" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/language/pii_redactor/**" - "!data-processing-lib/**" # This is tested in separate workflow diff --git a/.github/workflows/test-language-text_encoder-kfp.yml b/.github/workflows/test-language-text_encoder-kfp.yml index 96b8308d1..fdc085058 100644 --- a/.github/workflows/test-language-text_encoder-kfp.yml +++ b/.github/workflows/test-language-text_encoder-kfp.yml @@ -13,6 +13,7 @@ on: - "*" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/language/text_encoder/**" - "!kfp/**" # This is tested in separate workflow @@ -27,6 +28,7 @@ on: - "releases/**" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/language/text_encoder/**" - "!data-processing-lib/**" # This is tested in separate workflow diff --git a/.github/workflows/test-universal-doc_id-kfp.yml b/.github/workflows/test-universal-doc_id-kfp.yml index fadcc6403..194b5b65f 100644 --- a/.github/workflows/test-universal-doc_id-kfp.yml +++ b/.github/workflows/test-universal-doc_id-kfp.yml @@ -13,6 +13,7 @@ on: - "*" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/universal/doc_id/**" - "!kfp/**" # This is tested in separate workflow @@ -27,6 +28,7 @@ on: - "releases/**" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/universal/doc_id/**" - "!data-processing-lib/**" # This is tested in separate workflow diff --git a/.github/workflows/test-universal-ededup-kfp.yml b/.github/workflows/test-universal-ededup-kfp.yml index 225d7539e..48d3a1469 100644 --- a/.github/workflows/test-universal-ededup-kfp.yml +++ b/.github/workflows/test-universal-ededup-kfp.yml @@ -13,6 +13,7 @@ on: - "*" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/universal/ededup/**" - "!kfp/**" # This is tested in separate workflow @@ -27,6 +28,7 @@ on: - "releases/**" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/universal/ededup/**" - "!data-processing-lib/**" # This is tested in separate workflow diff --git a/.github/workflows/test-universal-fdedup-kfp.yml b/.github/workflows/test-universal-fdedup-kfp.yml index b36df964d..cb2055bfc 100644 --- a/.github/workflows/test-universal-fdedup-kfp.yml +++ b/.github/workflows/test-universal-fdedup-kfp.yml @@ -13,6 +13,7 @@ on: - "*" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/universal/fdedup/**" - "!kfp/**" # This is tested in separate workflow @@ -27,6 +28,7 @@ on: - "releases/**" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/universal/fdedup/**" - "!data-processing-lib/**" # This is tested in separate workflow diff --git a/.github/workflows/test-universal-filter-kfp.yml b/.github/workflows/test-universal-filter-kfp.yml index cab769f11..2fe6d33f7 100644 --- a/.github/workflows/test-universal-filter-kfp.yml +++ b/.github/workflows/test-universal-filter-kfp.yml @@ -13,6 +13,7 @@ on: - "*" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/universal/filter/**" - "!kfp/**" # This is tested in separate workflow @@ -27,6 +28,7 @@ on: - "releases/**" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/universal/filter/**" - "!data-processing-lib/**" # This is tested in separate workflow diff --git a/.github/workflows/test-universal-hap-kfp.yml b/.github/workflows/test-universal-hap-kfp.yml index d5d6aa63c..c69077368 100644 --- a/.github/workflows/test-universal-hap-kfp.yml +++ b/.github/workflows/test-universal-hap-kfp.yml @@ -13,6 +13,7 @@ on: - "*" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/universal/hap/**" - "!kfp/**" # This is tested in separate workflow @@ -27,6 +28,7 @@ on: - "releases/**" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/universal/hap/**" - "!data-processing-lib/**" # This is tested in separate workflow diff --git a/.github/workflows/test-universal-noop-kfp.yml b/.github/workflows/test-universal-noop-kfp.yml index 9322ea9b6..2be2a6e87 100644 --- a/.github/workflows/test-universal-noop-kfp.yml +++ b/.github/workflows/test-universal-noop-kfp.yml @@ -13,6 +13,7 @@ on: - "*" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/universal/noop/**" - "!kfp/**" # This is tested in separate workflow @@ -27,6 +28,7 @@ on: - "releases/**" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/universal/noop/**" - "!data-processing-lib/**" # This is tested in separate workflow diff --git a/.github/workflows/test-universal-profiler-kfp.yml b/.github/workflows/test-universal-profiler-kfp.yml index f31543ad3..63631e4d6 100644 --- a/.github/workflows/test-universal-profiler-kfp.yml +++ b/.github/workflows/test-universal-profiler-kfp.yml @@ -13,6 +13,7 @@ on: - "*" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/universal/profiler/**" - "!kfp/**" # This is tested in separate workflow @@ -27,6 +28,7 @@ on: - "releases/**" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/universal/profiler/**" - "!data-processing-lib/**" # This is tested in separate workflow diff --git a/.github/workflows/test-universal-resize-kfp.yml b/.github/workflows/test-universal-resize-kfp.yml index 3e099bba5..c502cb4b9 100644 --- a/.github/workflows/test-universal-resize-kfp.yml +++ b/.github/workflows/test-universal-resize-kfp.yml @@ -13,6 +13,7 @@ on: - "*" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/universal/resize/**" - "!kfp/**" # This is tested in separate workflow @@ -27,6 +28,7 @@ on: - "releases/**" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/universal/resize/**" - "!data-processing-lib/**" # This is tested in separate workflow diff --git a/.github/workflows/test-universal-tokenization-kfp.yml b/.github/workflows/test-universal-tokenization-kfp.yml index e4a5b5693..887dd4eb3 100644 --- a/.github/workflows/test-universal-tokenization-kfp.yml +++ b/.github/workflows/test-universal-tokenization-kfp.yml @@ -13,6 +13,7 @@ on: - "*" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/universal/tokenization/**" - "!kfp/**" # This is tested in separate workflow @@ -27,6 +28,7 @@ on: - "releases/**" paths: - ".make.*" + - "scripts/k8s-setup/requirements.env" - "transforms/.make.workflows" - "transforms/universal/tokenization/**" - "!data-processing-lib/**" # This is tested in separate workflow diff --git a/scripts/k8s-setup/requirements.env b/scripts/k8s-setup/requirements.env index 756cd69e4..9fbc5f9de 100644 --- a/scripts/k8s-setup/requirements.env +++ b/scripts/k8s-setup/requirements.env @@ -1,5 +1,5 @@ KUBERAY_OPERATOR=1.0.0 -KUBERAY_APISERVER=1.1.0 +KUBERAY_APISERVER=1.2.2 KIND_VERSION=0.22.0 HELM_VERSION=3.10.0 diff --git a/scripts/k8s-setup/tools/install_kuberay.sh b/scripts/k8s-setup/tools/install_kuberay.sh index 341ad2c85..30563b931 100755 --- a/scripts/k8s-setup/tools/install_kuberay.sh +++ b/scripts/k8s-setup/tools/install_kuberay.sh @@ -9,10 +9,10 @@ MAX_RETRIES="${MAX_RETRIES:-5}" EXIT_CODE=0 deploy() { - sed -i.back "s/tag: v[0-9].*/tag: v${KUBERAY_APISERVER}/" ${K8S_SETUP_SCRIPTS}/ray_api_server_values.yaml helm repo add kuberay https://ray-project.github.io/kuberay-helm/ helm repo update kuberay helm install kuberay-operator kuberay/kuberay-operator -n kuberay --version ${KUBERAY_OPERATOR} --set image.pullPolicy=IfNotPresent --create-namespace + sed -i.back "s/tag: v[0-9].*/tag: v${KUBERAY_APISERVER}/" ${K8S_SETUP_SCRIPTS}/ray_api_server_values.yaml helm install -f ${K8S_SETUP_SCRIPTS}/ray_api_server_values.yaml kuberay-apiserver kuberay/kuberay-apiserver -n kuberay --version ${KUBERAY_APISERVER} --set image.pullPolicy=IfNotPresent echo "Finished KubeRay deployment." }