set python versions

IBM · Sep 25, 2024 · 683197a · 683197a
1 parent c4707a2
commit 683197a
Show file tree

Hide file tree

Showing 50 changed files with 69 additions and 49 deletions.
diff --git a/.make.defaults b/.make.defaults
@@ -170,8 +170,8 @@ __check_defined = \
 	@version=$$(echo $(PYTHON_VERSION) | sed -e 's/Python[ ]*//');  \
 	major=$$(echo $$version | awk -F. '{print $$1}');	       \
 	minor=$$(echo $$version | awk -F. '{print $$2}');	       \
-	if [ $$major -lt 3 -o $$minor -lt 10 -o $$minor -gt 11 ]; then  \
-		echo Python 3.10 or 3.11 is required, but got $$version;	\
+	if [ $$major -lt 3 -o $$minor -lt 10 -o $$minor -gt 12 ]; then  \
+		echo Python 3.10 - 3.12 is required, but got $$version;	\
 		echo Try overriding PYTHON=$(PYTHON).  For example, "'"make PYTHON=python3.10" ...'";   \
 		exit 1;						 \
 	fi
@@ -553,12 +553,15 @@ MINIO_ADMIN_PWD= localminiosecretkey
 # Changes the version field of the pyproject.toml file to the given version
 # and update the referenced library versions as defined in .make.versions.
 # Expects TOML_VERSION
+
+
 .PHONY: .defaults.update-toml
 .defaults.update-toml:
 	$(call check_defined, TOML_VERSION)
 	if [ -e pyproject.toml ]; then						\
 	    $(MAKE) TOML_VERSION=$(TOML_VERSION) .defaults.__set-toml-version;	\
 	    $(MAKE) .defaults.__update-toml-lib-dep-versions;			\
+	    $(MAKE) .defaults.__update-toml-python-versions;    \
 	fi
 
 # Changes the version field of the pyproject.toml file to the given version
@@ -573,6 +576,18 @@ MINIO_ADMIN_PWD= localminiosecretkey
 	    mv tt.toml pyproject.toml;					\
 	fi
 
+# Updates the Python supported versions field of the pyproject.toml file to the given versions
+# Expects  REQUIRED_PYTHON_VERSIONS
+.PHONY: .defaults.__update-toml-python-versions
+.defaults.__update-toml-python-versions:
+	@# Help: Set the version= field of pyproject.toml
+	if [ -e pyproject.toml ]; then					\
+	    cat pyproject.toml | sed -e 				\
+	    's/^requires-python[ ]*=.*"/requires-python = "'${REQUIRED_PYTHON_VERSIONS}'"/' \
+		> tt.toml;						\
+	    mv tt.toml pyproject.toml;					\
+	fi
+
 # Updates the versions references to our repo source as defined in .make.versions
 .PHONY: .defaults.__update-toml-lib-dep-versions
 .defaults.__update-toml-lib-dep-versions:

diff --git a/.make.versions b/.make.versions
@@ -114,6 +114,8 @@ KFP_v1=1.8.5
 KFP_v1_SDK=1.8.22
 RAY=2.36.1
 
+REQUIRED_PYTHON_VERSIONS=">=3.10,<3.13"
+
 ifeq ($(KFPv2), 1)
         WORKFLOW_SUPPORT_LIB=kfp_v2_workflow_support
 else

diff --git a/data-processing-lib/python/pyproject.toml b/data-processing-lib/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "data_prep_toolkit"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 keywords = ["data", "data preprocessing", "data preparation", "llm", "generative", "ai", "fine-tuning", "llmapps" ]
 description = "Data Preparation Toolkit Library"
 license = {text = "Apache-2.0"}

diff --git a/data-processing-lib/ray/Makefile b/data-processing-lib/ray/Makefile
@@ -17,6 +17,9 @@ setup::
 
 set-versions: .check-env
 	$(MAKE) TOML_VERSION=$(DPK_LIB_VERSION) .defaults.update-toml
+	cat pyproject.toml | sed -e 's/"ray[default]==.*",/"ray[default]==$(RAY)",/' > tt.toml
+	mv tt.toml pyproject.toml
+
 
 build:: build-dist 
 

diff --git a/data-processing-lib/ray/pyproject.toml b/data-processing-lib/ray/pyproject.toml
@@ -2,7 +2,7 @@
 name = "data_prep_toolkit_ray"
 version = "0.2.2.dev0"
 keywords = ["data", "data preprocessing", "data preparation", "llm", "generative", "ai", "fine-tuning", "llmapps" ]
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Data Preparation Toolkit Library for Ray"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/data-processing-lib/spark/pyproject.toml b/data-processing-lib/spark/pyproject.toml
@@ -2,7 +2,7 @@
 name = "data_prep_toolkit_spark"
 version = "0.2.2.dev0"
 keywords = ["data", "data preprocessing", "data preparation", "llm", "generative", "ai", "fine-tuning", "llmapps" ]
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Data Preparation Toolkit Library for Spark"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/kfp/kfp_ray_components/Dockerfile b/kfp/kfp_ray_components/Dockerfile
@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=docker.io/rayproject/ray:2.24.0-py310
+ARG BASE_IMAGE=docker.io/rayproject/ray:2.36.1-py312
 
 FROM ${BASE_IMAGE}
 

diff --git a/transforms/code/code2parquet/python/pyproject.toml b/transforms/code/code2parquet/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_code2parquet_transform_python"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "code2parquet Python Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/code/code2parquet/ray/pyproject.toml b/transforms/code/code2parquet/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_code2parquet_transform_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "code2parquet Ray Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/code/code_quality/python/pyproject.toml b/transforms/code/code_quality/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_code_quality_transform_python"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Code Quality Python Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/code/code_quality/ray/pyproject.toml b/transforms/code/code_quality/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_code_quality_transform_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Code Quality Ray Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/code/header_cleanser/python/pyproject.toml b/transforms/code/header_cleanser/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_header_cleanser_transform_python"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "License and Copyright Removal Transform for Python"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/code/header_cleanser/ray/pyproject.toml b/transforms/code/header_cleanser/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_header_cleanser_transform_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "License and copyright removal Transform for Ray"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/code/malware/python/pyproject.toml b/transforms/code/malware/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_malware_transform_python"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Malware Python Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/code/malware/ray/pyproject.toml b/transforms/code/malware/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_malware_transform_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Malware Ray Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/code/proglang_select/python/pyproject.toml b/transforms/code/proglang_select/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_proglang_select_transform_python"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Programming Language Selection Python Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/code/proglang_select/ray/pyproject.toml b/transforms/code/proglang_select/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_proglang_select_transform_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Programming Language Selection Ray Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/code/repo_level_ordering/ray/pyproject.toml b/transforms/code/repo_level_ordering/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_repo_level_order_transform_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "repo_level_order Ray Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/language/doc_chunk/python/pyproject.toml b/transforms/language/doc_chunk/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_doc_chunk_transform_python"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "chunk documents Python Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/language/doc_chunk/ray/pyproject.toml b/transforms/language/doc_chunk/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_doc_chunk_transform_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "chunk documents Ray Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/language/doc_quality/python/pyproject.toml b/transforms/language/doc_quality/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_doc_quality_transform_python"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Document Quality Python Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/language/doc_quality/ray/pyproject.toml b/transforms/language/doc_quality/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_doc_quality_transform_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Document Quality Ray Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/language/lang_id/python/pyproject.toml b/transforms/language/lang_id/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_lang_id_transform_python"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Language Identification Python Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/language/lang_id/ray/pyproject.toml b/transforms/language/lang_id/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_lang_id_transform_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Language Identification Ray Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/language/pdf2parquet/python/pyproject.toml b/transforms/language/pdf2parquet/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_pdf2parquet_transform_python"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "PDF2PARQUET Python Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/language/pdf2parquet/ray/pyproject.toml b/transforms/language/pdf2parquet/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_pdf2parquet_transform_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "PDF2PARQUET Ray Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/language/pii_redactor/python/pyproject.toml b/transforms/language/pii_redactor/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_pii_redactor_transform_python"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "PII redactor Transform for Python"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/language/pii_redactor/ray/pyproject.toml b/transforms/language/pii_redactor/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_pii_redactor_transform_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "PII Redactor Ray Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/language/text_encoder/python/pyproject.toml b/transforms/language/text_encoder/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_text_encoder_transform_python"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Text Encoder Python Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/language/text_encoder/ray/pyproject.toml b/transforms/language/text_encoder/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_text_encoder_transform_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Text Encoder Ray Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/packaging/python/pyproject.toml b/transforms/packaging/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "data_prep_toolkit_transforms"
 version = "0.2.2.dev0"
-requires-python = ">=3.10,<3.12"
+requires-python = ">=3.10,<3.13"
 keywords = ["transforms", "data preprocessing", "data preparation", "llm", "generative", "ai", "fine-tuning", "llmapps" ]
 description = "Data Preparation Toolkit Transforms"
 license = {text = "Apache-2.0"}

diff --git a/transforms/packaging/ray/pyproject.toml b/transforms/packaging/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "data_prep_toolkit_transforms_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10,<3.12"
+requires-python = ">=3.10,<3.13"
 keywords = ["transforms", "data preprocessing", "data preparation", "llm", "generative", "ai", "fine-tuning", "llmapps" ]
 description = "Data Preparation Toolkit Transforms using Ray"
 license = {text = "Apache-2.0"}

diff --git a/transforms/universal/doc_id/python/pyproject.toml b/transforms/universal/doc_id/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_doc_id_transform_python"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "ededup Python Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/universal/doc_id/ray/pyproject.toml b/transforms/universal/doc_id/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_doc_id_transform_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "docid Ray Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/universal/doc_id/spark/pyproject.toml b/transforms/universal/doc_id/spark/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_doc_id_transform_spark"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Doc ID Spark Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/universal/ededup/python/pyproject.toml b/transforms/universal/ededup/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_ededup_transform_python"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "ededup Python Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/universal/ededup/ray/pyproject.toml b/transforms/universal/ededup/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_ededup_transform_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "ededup Ray Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/universal/fdedup/ray/pyproject.toml b/transforms/universal/fdedup/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_fdedup_transform_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "fdedup Ray Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/universal/filter/python/pyproject.toml b/transforms/universal/filter/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_filter_transform_python"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Filter Transform for Python"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/universal/filter/ray/pyproject.toml b/transforms/universal/filter/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_filter_transform_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Filter Transform for Ray"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}