Some new thoughts on cutting a release, especially scripts/release.sh

.github/workflows/test.yml

@@ -9,6 +9,14 @@ on:
         branches:
             - "dev"
 jobs:
+    test-make:
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+            - name: Test top-level recursive make targets.
+              run: |
+                  make -n clean test build publish set-versions
     test-python-lib:
         runs-on: ubuntu-latest
         steps:

.make.defaults

@@ -255,8 +255,6 @@ __check_defined = \
 # PYTHON_PROJECT_DIR is expected to have src and pyproject.toml
 .PHONY: .defaults.install-src-venv
 .defaults.install-src-venv:: 
-	@# Help: Install Ray and Python data processing library source into existing venv 
-	@echo Installing Ray and Python data processing library source to existing venv
 	@source venv/bin/activate;				      	\
 	if [ ! -z "$(EXTRA_INDEX_URL)" ]; then				\
 		extra_url='--extra-index-url $(EXTRA_INDEX_URL)';	\
@@ -305,9 +303,10 @@ __check_defined = \
 	@# Help: Install Ray and Python data processing library source into existing venv
 	@echo Installing Ray and Python data processing library source to existing venv
 	@source venv/bin/activate;				      	\
+	pip install pytest;						\
 	pip install pytest pytest-cov;					\
-	$(MAKE) PIP_TARGET=data-prep-toolkit .defaults.pip-uninstall;	\
 	$(MAKE) PIP_TARGET=data-prep-toolkit-ray .defaults.pip-uninstall;	\
+	$(MAKE) PIP_TARGET=data-prep-toolkit .defaults.pip-uninstall;	\
 	$(MAKE) PYTHON_PROJECT_DIR=$(DPK_PYTHON_LIB_DIR) .defaults.install-src-venv;		\
 	$(MAKE) PYTHON_PROJECT_DIR=$(DPK_RAY_LIB_DIR) .defaults.install-src-venv;		\
 	echo Installed source from Ray data processing library for `which $(PYTHON)`;		\
@@ -327,8 +326,8 @@ __check_defined = \
 	@echo Installing Spark and Python data processing library source to existing venv
 	@source venv/bin/activate;				      	\
 	pip install pytest;						\
-	$(MAKE) PIP_TARGET=data-prep-toolkit .defaults.pip-uninstall;	\
 	$(MAKE) PIP_TARGET=data-prep-toolkit-spark .defaults.pip-uninstall;	\
+	$(MAKE) PIP_TARGET=data-prep-toolkit .defaults.pip-uninstall;	\
 	$(MAKE) PYTHON_PROJECT_DIR=$(DPK_PYTHON_LIB_DIR) .defaults.install-src-venv;	\
 	$(MAKE) PYTHON_PROJECT_DIR=$(DPK_SPARK_LIB_DIR) .defaults.install-src-venv;	\
 	if [ -d ../python ]; then                                               \
@@ -495,32 +494,38 @@ MINIO_ADMIN_PWD= localminiosecretkey
 # Expects TOML_VERSION
 .PHONY: .defaults.update-toml
 .defaults.update-toml:
-	$(MAKE) TOML_VERSION=$(TOML_VERSION) .defaults.set-toml-version
-	$(MAKE) .defaults.update-toml-lib-dep-versions
+	if [ -e pyproject.toml ]; then						\
+	    $(MAKE) TOML_VERSION=$(TOML_VERSION) .defaults.__set-toml-version;	\
+	    $(MAKE) .defaults.__update-toml-lib-dep-versions;			\
+	fi
 
 # Changes the version field of the pyproject.toml file to the given version
 # Expects TOML_VERSION
-.PHONY: .defaults.set-toml-version
-.defaults.set-toml-version:
+.PHONY: .defaults.__set-toml-version
+.defaults.__set-toml-version:
 	@# Help: Set the version= field of pyproject.toml 
-	@cat pyproject.toml | sed -e \
-		's/^version[ ]*=.*/version = "'${TOML_VERSION}'"/' \
-		> tt.toml
-	@mv tt.toml pyproject.toml
+	if [ -e pyproject.toml ]; then					\
+	    cat pyproject.toml | sed -e 				\
+		's/^version[ ]*=.*/version = "'${TOML_VERSION}'"/' 	\
+		> tt.toml;						\
+	    mv tt.toml pyproject.toml;					\
+	fi
 
 # Updates the versions references to our repo source as defined in .make.versions
-.PHONY: .defaults.update-toml-lib-dep-versions
-.defaults.update-toml-lib-dep-versions:
+.PHONY: .defaults.__update-toml-lib-dep-versions
+.defaults.__update-toml-lib-dep-versions:
 	@# Help: Update pyproject.toml to depend on lib versions defined in .make.versions 
-	@cat pyproject.toml | sed 									\
+	@if [ -e pyproject.toml ]; then					\
+	    cat pyproject.toml | sed 									\
 		-e 's/"data-prep-toolkit-ray\(..\).*",/"data-prep-toolkit-ray\1$(DPK_LIB_VERSION)",/' 	\
 		-e 's/"data-prep-toolkit-spark\(..\).*",/"data-prep-toolkit-spark\1$(DPK_LIB_VERSION)",/' 	\
 		-e 's/"data-prep-toolkit-kfp\([=><][=><]\).*",/"data-prep-toolkit-kfp\1$(DPK_LIB_KFP_VERSION)",/' 	\
 		-e 's/"data-prep-toolkit\([=><][=><]\).*",/"data-prep-toolkit\1$(DPK_LIB_VERSION)",/' 		\
 		-e 's/"ray\[default\]\([=><][=><]\).*",/"ray\[default\]\1$(RAY)",/'				\
 		-e 's/"data-prep-toolkit-kfp-shared\(..\).*",/"data-prep-toolkit-kfp-shared\1$(DPK_LIB_KFP_VERSION)",/'     \
-		> tt.toml
-	mv tt.toml pyproject.toml
+		> tt.toml;				\
+	    mv tt.toml pyproject.toml;			\
+	 fi
 
 # Build the distribution, usually in preparation for publishing using ith the .defaults.publish-dist  target
 .PHONY: .defaults.build-dist 
@@ -545,6 +550,3 @@ MINIO_ADMIN_PWD= localminiosecretkey
 	fi
 	${PYTHON} -m twine check dist/*
 	${PYTHON} -m twine upload --verbose --non-interactive dist/*
-	#@echo "create a git tag to reference published version"
-	#@git tag ${TAG}
-	#@git push origin ${TAG}

.make.versions

@@ -2,69 +2,89 @@
 # Here we attempt to capture/define all the version numbers used across the 
 # repository in Makefile format.  These are generally considered the version
 # numbers TO BE published on the NEXT release/publishing of artifacts.
+#
+# NOTE: If you modify any of the version numbers, you MUST run "make set-versions"
+# from the top of the repo to have the new versions applied throughout the repo.
+#
 ################################################################################
 
-# do 
-# % make RELEASE_VERSION_SUFFIX= set-version 
-# % git push, tag, etc.
-# % make build, publish, etc.
-RELEASE_VERSION_SUFFIX=.dev6
+# See below for versions the repo depends on
 
-# Data prep lab wheel version
-DPK_LIB_VERSION=0.2.0$(RELEASE_VERSION_SUFFIX)
-DPK_LIB_KFP_VERSION=0.2.0$(RELEASE_VERSION_SUFFIX)
 
-KFP_DOCKER_VERSION=0.2.0$(RELEASE_VERSION_SUFFIX)
-KFP_DOCKER_VERSION_v2=0.2.0$(RELEASE_VERSION_SUFFIX)
+# This major versions is generally changed manually when a breaking change to apis is made in the libraries, for example.
+DPK_MAJOR_VERSION=0
+# The minor version is incremented manually when significant features have been added that are backward compatible with the previous major.minor release.
+DPK_MINOR_VERSION=2
+# The minor version is incremented AUTOMATICALLY by the release.sh script when a new release is set.
+DPK_MICRO_VERSION=0
+# The suffix is generally always set in the main/development branch and only nulled out when creating release branches.
+# It can be manually incremented, for example, to allow publishing a new intermediate version wheel to pypi. 
+DPK_VERSION_SUFFIX=.dev6
 
-KFP_v2=2.7.0
-KFP_v1=1.8.22
-RAY=2.24.0
+DPK_VERSION=$(DPK_MAJOR_VERSION).$(DPK_MINOR_VERSION).$(DPK_MICRO_VERSION)$(DPK_VERSION_SUFFIX)
 
-ifeq ($(KFPv2), 1)
-        WORKFLOW_SUPPORT_LIB=kfp_v2_workflow_support
-else
-        WORKFLOW_SUPPORT_LIB=kfp_v1_workflow_support
-endif
+# Data prep lab wheel version
+DPK_LIB_VERSION=$(DPK_VERSION)
+DPK_LIB_KFP_VERSION=$(DPK_VERSION)
+DPK_LIB_KFP_VERSION_v2=$(DPK_VERSION)
+DPK_LIB_KFP_SHARED=$(DPK_VERSION)
+
+KFP_DOCKER_VERSION=$(DPK_VERSION)
+KFP_DOCKER_VERSION_v2=$(DPK_VERSION)
 
 # Begin transform versions/tags
-BLOCKLIST_VERSION=0.4.2$(RELEASE_VERSION_SUFFIX)
+BLOCKLIST_VERSION=$(DPK_VERSION)
 
-DOC_ID_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
-DOC_ID_SPARK_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+DOC_ID_RAY_VERSION=$(DPK_VERSION)
+DOC_ID_SPARK_VERSION=$(DPK_VERSION)
 
-EDEDUP_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+EDEDUP_RAY_VERSION=$(DPK_VERSION)
 
-FDEDUP_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+FDEDUP_RAY_VERSION=$(DPK_VERSION)
 
-FILTER_PYTHON_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
-FILTER_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
-FILTER_SPARK_VERSION=0.2.0$(RELEASE_VERSION_SUFFIX)
+FILTER_PYTHON_VERSION=$(DPK_VERSION)
+FILTER_RAY_VERSION=$(DPK_VERSION)
+FILTER_SPARK_VERSION=$(DPK_VERSION)
 
-NOOP_PYTHON_VERSION=0.9.0$(RELEASE_VERSION_SUFFIX)
-NOOP_RAY_VERSION=0.9.0$(RELEASE_VERSION_SUFFIX)
-NOOP_SPARK_VERSION=0.2.0$(RELEASE_VERSION_SUFFIX)
-PROFILER_RAY_VERSION=0.2.0$(RELEASE_VERSION_SUFFIX)
+NOOP_PYTHON_VERSION=$(DPK_VERSION)
+NOOP_RAY_VERSION=$(DPK_VERSION)
+NOOP_SPARK_VERSION=$(DPK_VERSION)
+PROFILER_VERSION=$(DPK_VERSION)
+PROFILER_RAY_VERSION=$(DPK_VERSION)
 
-RESIZE_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+LANG_ID_PYTHON_VERSION=$(DPK_VERSION)
+LANG_ID_RAY_VERSION=$(DPK_VERSION)
 
-LANG_ID_PYTHON_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
-LANG_ID_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+TOKENIZATION_RAY_VERSION=$(DPK_VERSION)
+TOKENIZATION_PYTHON_VERSION=$(DPK_VERSION)
 
-TOKENIZATION_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
-TOKENIZATION_PYTHON_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+MALWARE_RAY_VERSION=$(DPK_VERSION)
+MALWARE_PYTHON_VERSION=$(DPK_VERSION)
 
-MALWARE_PYTHON_VERSION=0.5.0$(RELEASE_VERSION_SUFFIX)
-MALWARE_RAY_VERSION=0.5.0$(RELEASE_VERSION_SUFFIX)
+PROGLANG_SELECT_PYTHON_VERSION=$(DPK_VERSION)
+PROGLANG_SELECT_RAY_VERSION=$(DPK_VERSION)
 
-PROGLANG_SELECT_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
-PROGLANG_SELECT_PYTHON_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+CODE_QUALITY_RAY_VERSION=$(DPK_VERSION)
+CODE_QUALITY_PYTHON_VERSION=$(DPK_VERSION)
 
-CODE_QUALITY_PYTHON_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
-CODE_QUALITY_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+CODE2PARQUET_PYTHON_VERSION=$(DPK_VERSION)
+CODE2PARQUET_RAY_VERSION=$(DPK_VERSION)
+INGEST_TO_PARQUET_VERSION=$(DPK_VERSION)
 
-CODE2PARQUET_PYTHON_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
-CODE2PARQUET_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+KFP_DOCKER_VERSION=$(DPK_VERSION)
+KFP_DOCKER_VERSION_v2=$(DPK_VERSION)
+
+################## ################## ################## ################## ################## ##################
+# Begin versions that the repo depends on. 
+
+KFP_v2=2.7.0
+KFP_v1=1.8.22
+RAY=2.24.0
+
+ifeq ($(KFPv2), 1)
+        WORKFLOW_SUPPORT_LIB=kfp_v2_workflow_support
+else
+        WORKFLOW_SUPPORT_LIB=kfp_v1_workflow_support
+endif
 
-INGEST_TO_PARQUET_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
 

.secrets.baseline

@@ -132,3 +132,4 @@
     "hash": null
   }
 }
+

Makefile

@@ -51,15 +51,27 @@ set-versions:
 	@# Help: Recursively $@ in all subdirs 
 	@$(MAKE) RULE=$@ .recurse
 
-lib-release:
-	@# Help: Publish data-prep-kit $(DPK_LIB_VERSION) and data-prep-kit-kfp $(DPK_LIB_KFP_VERSION) libraries to pypi 
-	@$(MAKE) -C $(DPK_PYTHON_LIB_DIR) build publish
-	@$(MAKE) -C $(DPK_RAY_LIB_DIR) build publish
-	@$(MAKE) -C $(DPK_SPARK_LIB_DIR) build publish
-	@$(MAKE) -C kfp/kfp_support_lib build publish
-	@echo ""
-	@echo "This modified files in the repo. Please be sure to commit/push back to the repository."
-	@echo ""
+#set-release-verions:
+#	@# Help: Update all internally used versions to not include the release suffix. 
+#	@$(MAKE) DPK_VERSION_SUFFIX= set-versions
+
+#lib-release:
+#	@# Help: Set versions to be unsuffixed and publish libraries 
+#	@$(MAKE) set-release-versions 
+#	@$(MAKE) publish-lib 
+
+show-version:
+	@echo $(DPK_VERSION)
+
+#publish-lib:
+#	@# Help: Publish data-prep-kit $(DPK_LIB_VERSION) and data-prep-kit-kfp $(DPK_LIB_KFP_VERSION) libraries to pypi 
+#	@$(MAKE) -C $(DPK_PYTHON_LIB_DIR) build publish
+#	@$(MAKE) -C $(DPK_RAY_LIB_DIR) build publish
+#	@$(MAKE) -C $(DPK_SPARK_LIB_DIR) build publish
+#	@$(MAKE) -C kfp/kfp_support_lib build publish
+#	@echo ""
+#	@echo "This modified files in the repo. Please be sure to commit/push back to the repository."
+#	@echo ""
 
 
 

RELEASE.md

@@ -0,0 +1,57 @@
+# Release Management
+
+Release are created from the main repository branch using the version
+numbers, including an intermediate version suffix, 
+defined in `.make.versions`.
+The following points are important:
+
+1. `.make.versions` contains the version to be used when publishing the **next** release. 
+1. The main branch always has the version suffix set to .dev<N>, which
+allows intermediate publishing from the dev branch using version X.Y.Z.dev<N>.
+2. In general, common version number is used for all published pypi wheels and docker images.
+3. The `scripts/release.sh` script automates the following:
+   1. Creating a `release/vX.Y.Z` branch and `vX.Y.Z` tag
+   2. Nulling out the version suffix in the new branch's `.make.version` file. 
+   3. Applying the unsuffixed versions to the artifacts published from the repo.
+   4. Building and publishing the wheels to pypi and images to a docker registry. 
+   5. Incrementing the minor version and resetting the suffix in the main branch.
+
+# Cutting the release
+Creating the release requires running the `release.sh` script and optionally
+generating a release on github.  The latter can be performed manually
+once the `release.sh` script has done its work.
+
+## release.sh
+Running `release.sh` requires credentials to publish to the various cloud locations.
+
+For docker registry publishing, the following environment variables/credentials are needed:
+
+* DPK_DOCKER_REGISTRY_USER - user used with the registry defined in DOCKER_HOST in `.make.defaults`
+* DPK_DOCKER_REGISTRY_KEY - key/password for docker registry user.
+
+To publish to pypi, the credentials in `~/.pypirc` file (let us know if there is a way to do
+this with environment variables).
+See [pypi](https://packaging.python.org/en/latest/specifications/pypirc/) for details.
+
+To see the version that will be published,
+```
+make DPK_VERSION_SUFFIX= show-version
+```
+This will print for example, 1.2.3. 
+
+To generate the release :
+```shell
+bash scripts/release.sh
+```
+
+## Github release
+After running the `release.sh` script, to create tag `vX.Y.Z` and branch `releases/vX.Y.Z`
+1. Go to the [releases page](https://github.com/IBM/data-prep-kit/releases). 
+2. Select `Draft a new release`
+3. Select `Choose a tag -> vX.Y.Z`
+4. Press `Generate release notes` 
+5. Add a title (e.g., Release X.Y.Z) 
+6. Add any additional relese notes.
+7. Press `Publish release`
+
+