From 03ee01db982ceffa3ba2d560cec67a30a26a4e4a Mon Sep 17 00:00:00 2001 From: David Wood Date: Wed, 1 May 2024 17:10:00 -0400 Subject: [PATCH 1/2] add EXTRA_INDEX_URL support to makefiles and fix issue with DOCKER_IMAGE value Signed-off-by: David Wood --- .make.defaults | 22 +++++++++++++++------- transforms/.make.transforms | 9 ++++----- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/.make.defaults b/.make.defaults index 516c820cc..49680f7c5 100644 --- a/.make.defaults +++ b/.make.defaults @@ -5,7 +5,10 @@ # Before including, the following must be defined: # REPOROOT points to the top of the git repository. # For example REPOROOT=../../.. - +# DOCKER_NAME - defines the name of the docker image +# For example DOCKER_NAME=noop +# EXTRA_INDEX_URL - can define an additional pipy index for use +# in creating venv and images. # Targets defined here use double colon so can be overriden # # Reusable rules begin with '.'. To reuse without modification, for example, @@ -25,13 +28,14 @@ PYTEST=pytest -s PYTHON_VERSION=$(shell $(PYTHON) --version) ABS_REPOROOT=$(shell (cd $(REPOROOT); pwd)) DOCKER_FILE?=Dockerfile -DOCKER_NAME?=data-prep-lab +#DOCKER_NAME?=xyzzy # Must be defined by the includeing makefile DOCKER?=docker DOCKER_HOSTNAME?=quay.io DOCKER_NAMESPACE ?= dataprep1 DOCKER_REGISTRY_USER?=$(DPL_DOCKER_REGISTRY_USER) DOCKER_REGISTRY_KEY?=$(DPL_DOCKER_REGISTRY_KEY) -DOCKER_REGISTRY_ENDPOINT?=$(DOCKER_HOSTNAME)/$(DOCKER_NAMESPACE)/$(DOCKER_NAME) +DOCKER_REGISTRY_ENDPOINT?=$(DOCKER_HOSTNAME)/$(DOCKER_NAMESPACE) +DOCKER_IMAGE?=${DOCKER_REGISTRY_ENDPOINT}/$(DOCKER_NAME):$(DOCKER_IMAGE_VERSION) include $(REPOROOT)/.make.versions ####################################################################################### @@ -151,8 +155,9 @@ __check_defined = \ .PHONY: .defaults.image .defaults.image:: # Must be called with a DOCKER_IMAGE= settings. @# Help: Create the docker image $(DOCKER_IMAGE) - $(DOCKER) build -t $(DOCKER_IMAGE) \ - --build-arg BUILD_DATE=$(shell date -u +'%Y-%m-%dT%H:%M:%SZ') \ + $(DOCKER) build -t $(DOCKER_IMAGE) \ + --build-arg EXTRA_INDEX_URL=$(EXTRA_INDEX_URL) \ + --build-arg BUILD_DATE=$(shell date -u +'%Y-%m-%dT%H:%M:%SZ') \ --build-arg GIT_COMMIT=$(shell git log -1 --format=%h) . .PHONY: .defaults.lib-src-image @@ -175,7 +180,10 @@ __check_defined = \ source venv/bin/activate; \ pip install pytest; \ pip uninstall -y data-prep-lab; \ - pip install -e $(REPOROOT)/data-processing-lib/; \ + if [ ! -z "$(EXTRA_INDEX_URL)" ]; then \ + extra_url='--extra-index-url $(EXTRA_INDEX_URL)'; \ + fi; \ + pip install $${extra_url} -e $(REPOROOT)/data-processing-lib/; \ if [ $$? -eq 0 ]; then \ echo Installed source from data processing library for `which $(PYTHON)`; \ else \ @@ -232,7 +240,7 @@ __check_defined = \ @source venv/bin/activate; \ pip install --upgrade pip; \ pip install wheel; \ - pip install -r requirements.txt; \ + pip install --extra-index-url "$(EXTRA_INDEX_URL)" -r requirements.txt; .PHONY: .defaults.check.installed .defaults.check.installed:: diff --git a/transforms/.make.transforms b/transforms/.make.transforms index d7d69522d..71fd68fdc 100644 --- a/transforms/.make.transforms +++ b/transforms/.make.transforms @@ -5,7 +5,7 @@ # REPOROOT points to the top of the git repository. # For example REPOROOT=../../.. # TRANSFORM_NAME defines the name of the transform and is used to derive -# DOCKER_IMAGE_NAME and TRANSFORM_SRC_FILE if not already defined. +# DOCKER_NAME and TRANSFORM_SRC_FILE if not already defined. # Optional inputs (that have defaults) # TRANSFORM_SRC_FILE is the base name of the python source file containing the main() @@ -13,7 +13,7 @@ # By convention this name defaults to $(TRANSFORM_NAME)_transform.py. # This file is then assumed to be in the home dir of the docker image and # can be run with "python $(TRANSFORM_SRC_FILE) --help" -# DOCKER_IMAGE_NAME is the name of the docker image. +# DOCKER_NAME is the name of the docker image. # By default its value is $(TRANSFORM_NAME). # # Targets defined here use double colon so can be overriden @@ -28,8 +28,8 @@ ####################################################################################### include $(REPOROOT)/.make.defaults -DOCKER_IMAGE_NAME?=$(TRANSFORM_NAME) -DOCKER_IMAGE=${DOCKER_REGISTRY_ENDPOINT}/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION) +DOCKER_NAME?=$(TRANSFORM_NAME) +#DOCKER_IMAGE=${DOCKER_REGISTRY_ENDPOINT}/$(DOCKER_NAME):$(DOCKER_IMAGE_VERSION) # By convention the name of the python file is as follows and contains a main to start the ray-based transform # This file is used when starting the transformers in the Docker image. TRANSFORM_SRC_FILE?=$(TRANSFORM_NAME)_transform.py @@ -51,7 +51,6 @@ extra-help: @echo DOCKER_REGISTRY_USER - the docker user to use. DOCKER_REGISTRY_USER=$(DOCKER_REGISTRY_USER) @echo DOCKER_REGISTRY_KEY - the docker user to use. DOCKER_REGISTRY_KEY=secret @echo PYTHON - the python executable to use. PYTHON=$(PYTHON) - @echo DOCKER_IMAGE_NAME - the name of the docker image to produce. DOCKER_IMAGE_NAME=$(DOCKER_IMAGE_NAME) @echo "TRANSFORM_SRC_FILE is the base name of the python source file containing the main() (e.g. noop_local_ray.py)" @echo "" From e9384a8b8f9ea550bd6064409264fefc7d840957 Mon Sep 17 00:00:00 2001 From: David Wood Date: Fri, 3 May 2024 12:14:23 -0400 Subject: [PATCH 2/2] Doc on EXTRA_INDEX_URL Signed-off-by: David Wood --- .make.defaults | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.make.defaults b/.make.defaults index 49680f7c5..603fbe4c7 100644 --- a/.make.defaults +++ b/.make.defaults @@ -8,7 +8,12 @@ # DOCKER_NAME - defines the name of the docker image # For example DOCKER_NAME=noop # EXTRA_INDEX_URL - can define an additional pipy index for use -# in creating venv and images. +# in creating venv and images. This will be used on pip installs +# when building the venv and will be made available as +# a docker build argument when building the docker image. +# This is useful when the repo is used in conjunction with an +# inhouse or private pypi. +# # Targets defined here use double colon so can be overriden # # Reusable rules begin with '.'. To reuse without modification, for example, @@ -240,7 +245,10 @@ __check_defined = \ @source venv/bin/activate; \ pip install --upgrade pip; \ pip install wheel; \ - pip install --extra-index-url "$(EXTRA_INDEX_URL)" -r requirements.txt; + if [ ! -z "$(EXTRA_INDEX_URL)" ]; then \ + extra_url='--extra-index-url $(EXTRA_INDEX_URL)'; \ + fi; \ + pip install $$extra_url -r requirements.txt; .PHONY: .defaults.check.installed .defaults.check.installed::