Some new thoughts on cutting a release, especially scripts/release.sh

.make.versions

@@ -8,50 +8,55 @@
 # % make RELEASE_VERSION_SUFFIX= set-version 
 # % git push, tag, etc.
 # % make build, publish, etc.
-RELEASE_VERSION_SUFFIX=.dev6
+DPK_MAJOR_VERSION=0
+DPK_MINOR_VERSION=2
+DPK_MICRO_VERSION=0
+DPK_VERSION_SUFFIX=.dev6
+
+DPK_VERSION=$(DPK_MAJOR_VERSION).$(DPK_MINOR_VERSION).$(DPK_MICRO_VERSION)$(DPK_VERSION_SUFFIX)
 
 # Data prep lab wheel version
-DPK_LIB_VERSION=0.2.0$(RELEASE_VERSION_SUFFIX)
-DPK_LIB_KFP_VERSION=0.2.0$(RELEASE_VERSION_SUFFIX)
-DPK_LIB_KFP_VERSION_v2=0.2.0$(RELEASE_VERSION_SUFFIX)
-DPK_LIB_KFP_SHARED=0.2.0$(RELEASE_VERSION_SUFFIX)
+DPK_LIB_VERSION=$(DPK_VERSION)
+DPK_LIB_KFP_VERSION=$(DPK_VERSION)
+DPK_LIB_KFP_VERSION_v2=$(DPK_VERSION)
+DPK_LIB_KFP_SHARED=$(DPK_VERSION)
 
 # Begin transform versions/tags
-BLOCKLIST_VERSION=0.4.2$(RELEASE_VERSION_SUFFIX)
+BLOCKLIST_VERSION=$(DPK_VERSION)
 
-DOC_ID_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
-DOC_ID_SPARK_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+DOC_ID_RAY_VERSION=$(DPK_VERSION)
+DOC_ID_SPARK_VERSION=$(DPK_VERSION)
 
-EDEDUP_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+EDEDUP_RAY_VERSION=$(DPK_VERSION)
 
-FDEDUP_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+FDEDUP_RAY_VERSION=$(DPK_VERSION)
 
-FILTER_PYTHON_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
-FILTER_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
-FILTER_SPARK_VERSION=0.2.0$(RELEASE_VERSION_SUFFIX)
+FILTER_PYTHON_VERSION=$(DPK_VERSION)
+FILTER_RAY_VERSION=$(DPK_VERSION)
+FILTER_SPARK_VERSION=$(DPK_VERSION)
 
-NOOP_PYTHON_VERSION=0.9.0$(RELEASE_VERSION_SUFFIX)
-NOOP_RAY_VERSION=0.9.0$(RELEASE_VERSION_SUFFIX)
-NOOP_SPARK_VERSION=0.2.0$(RELEASE_VERSION_SUFFIX)
-PROFILER_VERSION=0.2.0$(RELEASE_VERSION_SUFFIX)
+NOOP_PYTHON_VERSION=$(DPK_VERSION)
+NOOP_RAY_VERSION=$(DPK_VERSION)
+NOOP_SPARK_VERSION=$(DPK_VERSION)
+PROFILER_VERSION=$(DPK_VERSION)
 
-RESIZE_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+RESIZE_VERSION=$(DPK_VERSION)
 
-LANG_ID_PYTHON_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
-LANG_ID_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+LANG_ID_PYTHON_VERSION=$(DPK_VERSION)
+LANG_ID_RAY_VERSION=$(DPK_VERSION)
 
-TOKENIZATION_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
-TOKENIZATION_PYTHON_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+TOKENIZATION_RAY_VERSION=$(DPK_VERSION)
+TOKENIZATION_PYTHON_VERSION=$(DPK_VERSION)
 
-MALWARE_RAY_VERSION=0.5.0$(RELEASE_VERSION_SUFFIX)
+MALWARE_RAY_VERSION=$(DPK_VERSION)
 
-PROGLANG_SELECT_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+PROGLANG_SELECT_RAY_VERSION=$(DPK_VERSION)
 
-CODE_QUALITY_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+CODE_QUALITY_RAY_VERSION=$(DPK_VERSION)
 
-INGEST_TO_PARQUET_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
-INGEST_TO_PARQUET_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+INGEST_TO_PARQUET_RAY_VERSION=$(DPK_VERSION)
+INGEST_TO_PARQUET_VERSION=$(DPK_VERSION)
 
 
-KFP_DOCKER_VERSION=0.2.0$(RELEASE_VERSION_SUFFIX)
-KFP_DOCKER_VERSION_v2=0.2.0$(RELEASE_VERSION_SUFFIX)
+KFP_DOCKER_VERSION=$(DPK_VERSION)
+KFP_DOCKER_VERSION_v2=$(DPK_VERSION)

Makefile

@@ -51,15 +51,27 @@ set-versions:
 	@# Help: Recursively $@ in all subdirs 
 	@$(MAKE) RULE=$@ .recurse
 
-lib-release:
-	@# Help: Publish data-prep-kit $(DPK_LIB_VERSION) and data-prep-kit-kfp $(DPK_LIB_KFP_VERSION) libraries to pypi 
-	@$(MAKE) -C $(DPK_PYTHON_LIB_DIR) build publish
-	@$(MAKE) -C $(DPK_RAY_LIB_DIR) build publish
-	@$(MAKE) -C $(DPK_SPARK_LIB_DIR) build publish
-	@$(MAKE) -C kfp/kfp_support_lib build publish
-	@echo ""
-	@echo "This modified files in the repo. Please be sure to commit/push back to the repository."
-	@echo ""
+#set-release-verions:
+#	@# Help: Update all internally used versions to not include the release suffix. 
+#	@$(MAKE) DPK_VERSION_SUFFIX= set-versions
+
+#lib-release:
+#	@# Help: Set versions to be unsuffixed and publish libraries 
+#	@$(MAKE) set-release-versions 
+#	@$(MAKE) publish-lib 
+
+show-version:
+	@echo $(DPK_VERSION)
+
+#publish-lib:
+#	@# Help: Publish data-prep-kit $(DPK_LIB_VERSION) and data-prep-kit-kfp $(DPK_LIB_KFP_VERSION) libraries to pypi 
+#	@$(MAKE) -C $(DPK_PYTHON_LIB_DIR) build publish
+#	@$(MAKE) -C $(DPK_RAY_LIB_DIR) build publish
+#	@$(MAKE) -C $(DPK_SPARK_LIB_DIR) build publish
+#	@$(MAKE) -C kfp/kfp_support_lib build publish
+#	@echo ""
+#	@echo "This modified files in the repo. Please be sure to commit/push back to the repository."
+#	@echo ""
 
 
 

scripts/release.sh

@@ -0,0 +1,44 @@
+DEFAULT_BRANCH=dev
+# Assume this file is in the reporoot/scripts directory
+reporoot=$(dirname $0)/..
+cd $reporoot
+
+# Make sure we're starting from the base branch
+# git checkout $DEFAULT_BRANCH 
+
+# Get the currently defined version w/o any suffix.  This is the next release version
+# version=$(make DPK_VERSION_SUFFIX= show-version)
+# tag=v$version
+
+# Create a new branch for this version and switch to it
+# git branch -r release/$tag
+
+# Remove the release suffix in this branch
+# cat .make.versions | sed -e 's/^DPK_VERSION_SUFFIX.*/DPK_VERSION_SUFFIX=/' > tt
+# mv tt .make.version
+
+# Apply the unsuffixed version to the repo and check it into this release branch
+# make set-versions
+# git add -A
+# git commit -s -m "Cut release $version"
+# git push origin
+# git tag -a -s -m "Cut release $version" $tag 
+# git push origin $tag 
+
+# Now build with the updated version
+# make build publish
+
+# Now go back to the default branch so we can bump the minor version number and reset the version suffix
+# git branch $DEFAULT_BRANCH
+
+# Change to the next development version (bumped minor version with suffix).
+minor=$(cat .make.versions | grep '^DPK_MINOR_VERSION=' | sed -e 's/DPK_MINOR_VERSION=\([0-9]*\).*/\1/') 
+minor=$(($minor + 1))
+#cat .make.versions | sed -e "s/^DPK_MINOR_VERSION=.*/DPK_MINOR_VERSION=$minor/"  \
+# 			  -e "s/^DPK_VERSION_SUFFIX=.*/DPK_VERSION_SUFFIX=.dev0/"  > tt
+#mv tt .make.versions
+
+# Push the version change back to the origin
+# git add -A
+# git commit -s -m "Bump minor version to $minor after cutting release $version"
+# git push origin

transforms/universal/profiler/kfp_ray/Makefile

@@ -29,8 +29,6 @@ image::
 
 load-image::
 
-set-versions: workflow-reconcile-requirements
-
 .PHONY: workflow-build
 workflow-build: workflow-venv
 	$(MAKE) $(YAML_WF)
@@ -45,8 +43,9 @@ workflow-upload: workflow-build
 		$(MAKE) .transforms_workflows.upload-pipeline PIPELINE_FILE=$$file; \
 	done
 
-.PHONY: workflow-reconcile-requirements
-workflow-reconcile-requirements:
+.PHONY: set-versions
+set-versions:
 	@for file in $(PYTHON_WF); do \
-		$(MAKE) .transforms_workflows.reconcile-requirements PIPELINE_FILE=$$file; \
+		$(MAKE) .workflows.set-versions PIPELINE_FILE=$$file; \
 	done
+