Fix tasks tags in kfp workflows.

.make.versions

@@ -13,29 +13,45 @@ RELEASE_VERSION_SUFFIX=.dev6
 # Data prep lab wheel version
 DPK_LIB_VERSION=0.2.0$(RELEASE_VERSION_SUFFIX)
 DPK_LIB_KFP_VERSION=0.2.0$(RELEASE_VERSION_SUFFIX)
-DPK_LIB_KFP_VERSION_v2=0.2.0$(RELEASE_VERSION_SUFFIX)
 
 # Begin transform versions/tags
 BLOCKLIST_VERSION=0.4.2$(RELEASE_VERSION_SUFFIX)
-DOC_ID_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
-DOC_ID_SPARK_VERSION=0.2.0$(RELEASE_VERSION_SUFFIX)
-EDEDUP_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
-FDEDUP_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+
+DOC_ID_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+DOC_ID_SPARK_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+
+EDEDUP_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+
+FDEDUP_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+
 FILTER_PYTHON_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
 FILTER_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
-FILTER_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
 FILTER_SPARK_VERSION=0.2.0$(RELEASE_VERSION_SUFFIX)
+
 NOOP_PYTHON_VERSION=0.9.0$(RELEASE_VERSION_SUFFIX)
 NOOP_RAY_VERSION=0.9.0$(RELEASE_VERSION_SUFFIX)
 NOOP_SPARK_VERSION=0.2.0$(RELEASE_VERSION_SUFFIX)
+
 RESIZE_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
-LANG_ID_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
-TOKENIZATION_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
-MALWARE_VERSION=0.5.0$(RELEASE_VERSION_SUFFIX)
-PROGLANG_SELECT_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
-CODE_QUALITY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
-DOC_QUALITY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+
+LANG_ID_PYTHON_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+LANG_ID_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+LANG_ID_SPARK_VERSION=0.2.0$(RELEASE_VERSION_SUFFIX)
+
+TOKENIZATION_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+
+MALWARE_RAY_VERSION=0.5.0$(RELEASE_VERSION_SUFFIX)
+
+PROGLANG_SELECT_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+
+CODE_QUALITY_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+
+DOC_QUALITY_PYTHON_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+DOC_QUALITY_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+DOC_QUALITY_SPARK_VERSION=0.2.0$(RELEASE_VERSION_SUFFIX)
+
 INGEST_TO_PARQUET_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+INGEST_TO_PARQUET_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
 
 
 KFP_DOCKER_VERSION=0.2.0$(RELEASE_VERSION_SUFFIX)

data-processing-lib/doc/spark-runtime.md

@@ -4,7 +4,7 @@ The Spark runtime extends the base framework with the following set of component
 ## Transforms
 
 * [AbstractSparkTransform](../spark/src/data_processing_spark/runtime/spark/spark_transform.py) - this
-  is the base class for all spark-based tranforms over spark DataFrames.
+  is the base class for all spark-based transforms over spark DataFrames.
 * [SparkTransformConfiguration](../spark/src/data_processing_spark/runtime/spark/spark_transform_config.py) - this
   is simple extension of the base  TransformConfiguration class to hold the transformation class
   (an extension of AbstractSpartTransform).

kfp/doc/simple_transform_pipeline.md

@@ -56,7 +56,7 @@ Ray cluster. For each step we have to define a component that will execute them:
 ```python
     # components
     base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.0.2"
-    # compute execution parameters. Here different tranforms might need different implementations. As
+    # compute execution parameters. Here different transforms might need different implementations. As
     # a result, instead of creating a component we are creating it in place here.
     compute_exec_params_op = comp.func_to_container_op(
       func=ComponentUtils.default_compute_execution_params, base_image=base_kfp_image

kfp/pipeline_generator/pipeline.ptmpl

@@ -22,7 +22,7 @@ base_kfp_image = "__kfp_base_image__"
 # path to kfp component specifications files
 component_spec_path = "__component_spec_path__"
 
-# compute execution parameters. Here different tranforms might need different implementations. As
+# compute execution parameters. Here different transforms might need different implementations. As
 # a result, instead of creating a component we are creating it in place here.
 compute_exec_params_op = comp.func_to_container_op(
     func=__compute_func_name__, base_image=base_kfp_image

transforms/.make.workflows

@@ -1,4 +1,6 @@
-include ${REPOROOT}/.make.versions
+IGNORE := $(shell bash -c "sed -nr /^[^\#]*=/p ${REPOROOT}/.make.versions | sed 's/=/:=/' | sed 's/^/export /' > makeenv")
+
+include makeenv
 include ${REPOROOT}/kfp/requirements.env
 
 # Include the common rules.
@@ -14,7 +16,7 @@ endef
 # FIXME
 .PHONY: .transforms_workflows.reconcile-requirements
 .transforms_workflows.reconcile-requirements:
-
+	${REPOROOT}/transforms/hack/update_workflow_tags.sh ${REPOROOT}/.make.versions ${PIPELINE_FILE}
 
 .PHONY: .transforms_workflows.compile-pipeline
 .transforms_workflows.compile-pipeline:

transforms/code/code_quality/kfp_ray/code_quality_wf.py

@@ -29,7 +29,7 @@
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"
 
-# compute execution parameters. Here different tranforms might need different implementations. As
+# compute execution parameters. Here different transforms might need different implementations. As
 # a result, instead of creating a component we are creating it in place here.
 def compute_exec_params_func(
     worker_options: str,

transforms/code/code_quality/ray/Makefile

@@ -6,7 +6,7 @@ include $(REPOROOT)/transforms/.make.transforms
 
 TRANSFORM_NAME=code_quality
 # $(REPOROOT)/.make.versions file contains the versions
-DOCKER_IMAGE_VERSION=${CODE_QUALITY_VERSION}
+DOCKER_IMAGE_VERSION=${CODE_QUALITY_RAY_VERSION}
 
 # Use default rule inherited from makefile.common
 clean:: .transforms.clean

transforms/code/ingest_2_parquet/kfp_ray/v1/ingest_2_parquet_wf.py

@@ -32,7 +32,7 @@
 # path to kfp component specifications files
 component_spec_path = "../../../../../kfp/kfp_ray_components/"
 
-# compute execution parameters. Here different tranforms might need different implementations. As
+# compute execution parameters. Here different transforms might need different implementations. As
 # a result, instead of creating a component we are creating it in place here.
 compute_exec_params_op = comp.func_to_container_op(
     func=ComponentUtils.default_compute_execution_params, base_image=base_kfp_image

transforms/code/ingest_2_parquet/ray/Makefile

@@ -8,7 +8,7 @@ include $(REPOROOT)/transforms/.make.transforms
 
 TRANSFORM_NAME=ingest_2_parquet
 # $(REPOROOT)/.make.versions file contains the versions
-DOCKER_IMAGE_VERSION=${INGEST_TO_PARQUET_VERSION}
+DOCKER_IMAGE_VERSION=${INGEST_TO_PARQUET_RAY_VERSION}
 
 venv::	.transforms.ray-venv
 

transforms/code/malware/kfp_ray/malware_wf.py

@@ -30,7 +30,7 @@
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"
 
-# compute execution parameters. Here different tranforms might need different implementations. As
+# compute execution parameters. Here different transforms might need different implementations. As
 # a result, instead of creating a component we are creating it in place here.
 def compute_exec_params_func(
     worker_options: str,

transforms/code/malware/ray/Makefile

@@ -8,7 +8,7 @@ include $(REPOROOT)/transforms/.make.transforms
 
 TRANSFORM_NAME=malware
 # $(REPOROOT)/.make.versions file contains the versions
-DOCKER_IMAGE_VERSION=${MALWARE_VERSION}
+DOCKER_IMAGE_VERSION=${MALWARE_RAY_VERSION}
 
 OS := $(shell uname -s)
 ifeq ($(OS),Darwin)

transforms/code/proglang_select/kfp_ray/proglang_select_wf.py

@@ -31,7 +31,7 @@
 component_spec_path = "../../../../kfp/kfp_ray_components/"
 
 
-# compute execution parameters. Here different tranforms might need different implementations. As
+# compute execution parameters. Here different transforms might need different implementations. As
 # a result, instead of creating a component we are creating it in place here.
 def compute_exec_params_func(
     worker_options: str,

transforms/code/proglang_select/ray/Makefile

@@ -8,7 +8,7 @@ include $(REPOROOT)/transforms/.make.transforms
 
 TRANSFORM_NAME=proglang_select
 # $(REPOROOT)/.make.versions file contains the versions
-DOCKER_IMAGE_VERSION=${PROGLANG_SELECT_VERSION}
+DOCKER_IMAGE_VERSION=${PROGLANG_SELECT_RAY_VERSION}
 
 venv::	.transforms.ray-venv
 

transforms/hack/update_workflow_tags.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+if [[ $# != 2 ]]; then
+  cat << EOF
+	"Incorrect number of parameters provided. The required parameters are versions_file and pipeline_path. 
+EOF
+  exit 1
+fi
+
+versions_file=$1
+pipeline_path=$2
+
+# Modify the tasks tags as defined in the versions file
+while IFS= read -r line; do 
+	[ -z "$line" ] && continue
+	[[ $line == *#* ]] && continue
+	VERSION_NAME=$(echo $line |cut -d "=" -f 1)
+	DOCKER_IMAGE_NAME=$(echo $line |cut -d "=" -f 1 |sed "s/_VERSION//" |tr '[:upper:]' '[:lower:]')
+	DOCKER_IMAGE_NAME=$(echo $DOCKER_IMAGE_NAME |sed "s/_ray$/\-ray/" | sed "s/_spark$/\-spark/" | sed "s/_parquet$/\-parquet/")
+	DOCKER_IMAGE_VERSION=$(eval echo ${!VERSION_NAME})
+	sed -i.back "s/data-prep-kit\/$DOCKER_IMAGE_NAME:.*/data-prep-kit\/$DOCKER_IMAGE_NAME:$DOCKER_IMAGE_VERSION\"/" $pipeline_path
+done < $versions_file
+# Update kfp component image tag 
+sed -i.back "s/kfp-data-processing:.*/kfp-data-processing:$KFP_DOCKER_VERSION\"/" $pipeline_path

transforms/universal/doc_id/kfp_ray/doc_id_wf.py

@@ -28,7 +28,7 @@
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"
 
-# compute execution parameters. Here different tranforms might need different implementations. As
+# compute execution parameters. Here different transforms might need different implementations. As
 # a result, instead of creating a component we are creating it in place here.
 def compute_exec_params_func(
     worker_options: str,

transforms/universal/doc_id/ray/Makefile

@@ -8,7 +8,7 @@ include $(REPOROOT)/transforms/.make.transforms
 
 TRANSFORM_NAME=doc_id
 # $(REPOROOT)/.make.versions file contains the versions
-DOCKER_IMAGE_VERSION=${DOC_ID_VERSION}
+DOCKER_IMAGE_VERSION=${DOC_ID_RAY_VERSION}
 
 venv::	.transforms.ray-venv
 

transforms/universal/ededup/Makefile

@@ -62,4 +62,5 @@ workflow-upload:
 	$(MAKE) -C kfp_ray workflow-upload
 
 .PHONY: workflow-reconcile-requirements
+workflow-reconcile-requirements:
 	$(MAKE) -C kfp_ray workflow-reconcile-requirements

transforms/universal/ededup/ray/Makefile

@@ -8,7 +8,7 @@ include $(REPOROOT)/transforms/.make.transforms
 
 TRANSFORM_NAME=ededup
 # $(REPOROOT)/.make.versions file contains the versions
-DOCKER_IMAGE_VERSION=${EDEDUP_VERSION}
+DOCKER_IMAGE_VERSION=${EDEDUP_RAY_VERSION}
 
 venv::	.transforms.ray-venv
 

transforms/universal/fdedup/kfp_ray/Makefile

@@ -19,6 +19,10 @@ test-src::
 
 test-image::
 
+image::
+
+load-image::
+
 .PHONY: workflow-build
 workflow-build: workflow-venv
 	$(MAKE) $(YAML_WF)

transforms/universal/fdedup/ray/Makefile

@@ -8,7 +8,7 @@ include $(REPOROOT)/transforms/.make.transforms
 
 TRANSFORM_NAME=fdedup
 # $(REPOROOT)/.make.versions file contains the versions
-DOCKER_IMAGE_VERSION=${FDEDUP_VERSION}
+DOCKER_IMAGE_VERSION=${FDEDUP_RAY_VERSION}
 
 venv::	.transforms.ray-venv
 

transforms/universal/filter/kfp_ray/filter_wf.py

@@ -30,7 +30,7 @@
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"
 
-# compute execution parameters. Here different tranforms might need different implementations. As
+# compute execution parameters. Here different transforms might need different implementations. As
 # a result, instead of creating a component we are creating it in place here.
 def compute_exec_params_func(
     worker_options: str,

transforms/universal/filter/ray/Makefile

@@ -9,7 +9,7 @@ include $(REPOROOT)/transforms/.make.transforms
 
 TRANSFORM_NAME=filter
 # $(REPOROOT)/.make.versions file contains the versions
-DOCKER_IMAGE_VERSION=${FILTER_VERSION}
+DOCKER_IMAGE_VERSION=${FILTER_RAY_VERSION}
 
 venv::	.transforms.ray-venv
 

transforms/universal/noop/kfp_ray/noop_multiple_wf.py

@@ -29,7 +29,7 @@
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"
 
-# compute execution parameters. Here different tranforms might need different implementations. As
+# compute execution parameters. Here different transforms might need different implementations. As
 # a result, instead of creating a component we are creating it in place here.
 def compute_exec_params_func(
     worker_options: str,

transforms/universal/noop/kfp_ray/noop_wf.py

@@ -29,7 +29,7 @@
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"
 
-# compute execution parameters. Here different tranforms might need different implementations. As
+# compute execution parameters. Here different transforms might need different implementations. As
 # a result, instead of creating a component we are creating it in place here.
 def compute_exec_params_func(
     worker_options: str,

transforms/universal/tokenization/kfp_ray/tokenization_wf.py

@@ -30,7 +30,7 @@
 component_spec_path = "../../../../kfp/kfp_ray_components/"
 
 
-# compute execution parameters. Here different tranforms might need different implementations. As
+# compute execution parameters. Here different transforms might need different implementations. As
 # a result, instead of creating a component we are creating it in place here.
 def compute_exec_params_func(
     worker_options: str,

transforms/universal/tokenization/ray/Makefile

@@ -8,7 +8,7 @@ include $(REPOROOT)/transforms/.make.transforms
 
 TRANSFORM_NAME=tokenization
 # $(REPOROOT)/.make.versions file contains the versions
-DOCKER_IMAGE_VERSION=${TOKENIZATION_VERSION}
+DOCKER_IMAGE_VERSION=${TOKENIZATION_RAY_VERSION}
 
 venv::	.transforms.ray-venv