Define spark version in .make.versions (#632)

* set spark base image tag to that from .make.versions

Signed-off-by: David Wood <dawood@us.ibm.com>

* have spark transform images all use latest by default

Signed-off-by: David Wood <dawood@us.ibm.com>

* restore default spark FROM image to include quay.io

Signed-off-by: David Wood <dawood@us.ibm.com>

* Set base spark version in .make.versions

Signed-off-by: David Wood <dawood@us.ibm.com>

---------

Signed-off-by: David Wood <dawood@us.ibm.com>

.make.defaults

@@ -49,7 +49,7 @@ DOCKER_REGISTRY_KEY?=$(DPK_DOCKER_REGISTRY_KEY)
 DOCKER_REGISTRY_ENDPOINT?=$(DOCKER_HOSTNAME)/$(DOCKER_NAMESPACE)
 DOCKER_LOCAL_IMAGE=$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION)
 DOCKER_REMOTE_IMAGE=$(DOCKER_REGISTRY_ENDPOINT)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION)
-DOCKER_SPARK_BASE_IMAGE_NAME=data-prep-kit-spark-3.5.2
+DOCKER_SPARK_BASE_IMAGE_NAME=data-prep-kit-spark-$(SPARK_VERSION)
 DOCKER_SPARK_BASE_IMAGE=$(DOCKER_SPARK_BASE_IMAGE_NAME):$(DOCKER_IMAGE_VERSION)
 RAY_BASE_IMAGE?=docker.io/rayproject/ray:${RAY}-py310
 # Deprecated in favor of DOCKER_REMOTE_IMAGE

.make.versions

@@ -113,6 +113,7 @@ KFP_v2_SDK=2.8.0
 KFP_v1=1.8.5
 KFP_v1_SDK=1.8.22
 RAY=2.24.0
+SPARK_VERSION=3.5.2
 
 ifeq ($(KFPv2), 1)
         WORKFLOW_SUPPORT_LIB=kfp_v2_workflow_support

data-processing-lib/spark/Makefile

@@ -1,9 +1,7 @@
 # Use make help, to see the available rules
 REPOROOT=../..
 include $(REPOROOT)/.make.defaults
-SPARK_VERSION=3.5.2
-DOCKER_IMAGE_NAME=data-prep-kit-spark-$(SPARK_VERSION)
-DOCKER_IMAGE_LIB_NAME=data-prep-kit-spark
+DOCKER_IMAGE_NAME=$(DOCKER_SPARK_BASE_IMAGE_NAME)
 
 
 .check-env::