From 0de60d34416192d795136fa42b694f5dff472599 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Thu, 8 Feb 2024 11:49:55 +0000 Subject: [PATCH 01/15] Pin psutil Signed-off-by: Ankita Katiyar --- kedro-docker/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kedro-docker/pyproject.toml b/kedro-docker/pyproject.toml index e83f04bed..4ccfa8594 100644 --- a/kedro-docker/pyproject.toml +++ b/kedro-docker/pyproject.toml @@ -29,7 +29,7 @@ test = [ "black~=22.0", "docker", "pre-commit>=2.9.2", - "psutil", + "psutil<5.9.8", "pytest", "pytest-cov", "pytest-mock", From 3cb35911cdc55edadd901c7e436b441caacf7011 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Thu, 8 Feb 2024 13:48:49 +0000 Subject: [PATCH 02/15] Add no capture to test Signed-off-by: Ankita Katiyar --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0ccf288a6..84355b933 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ test-sequential: cd $(plugin) && pytest tests --cov-config pyproject.toml e2e-tests: - cd $(plugin) && behave + cd $(plugin) && behave --no-capture secret-scan: trufflehog --max_depth 1 --exclude_paths trufflehog-ignore.txt . From eabe503a8a1f585dbdaade1c580cca4c16c4bf30 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Thu, 8 Feb 2024 14:05:50 +0000 Subject: [PATCH 03/15] Update pip version Signed-off-by: Ankita Katiyar --- Makefile | 2 +- kedro-docker/features/steps/cli_steps.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 84355b933..0ccf288a6 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ test-sequential: cd $(plugin) && pytest tests --cov-config pyproject.toml e2e-tests: - cd $(plugin) && behave --no-capture + cd $(plugin) && behave secret-scan: trufflehog --max_depth 1 --exclude_paths trufflehog-ignore.txt . diff --git a/kedro-docker/features/steps/cli_steps.py b/kedro-docker/features/steps/cli_steps.py index e4165c6eb..ea7c5716c 100644 --- a/kedro-docker/features/steps/cli_steps.py +++ b/kedro-docker/features/steps/cli_steps.py @@ -196,6 +196,19 @@ def exec_kedro_command(context, command): @given("I have installed the project dependencies") def pip_install_dependencies(context): """Install project dependencies using pip.""" + call( + [ + "python", + "-m", + "pip", + "install", + "-U", + # pip==23.2 breaks pip-tools<7.0, and pip-tools>=7.0 does not support Python 3.7 + # pip==23.3 breaks dependency resolution + "pip>=21.2,<23.2", + ], + env=context.env, + ) reqs_path = Path("requirements.txt") res = run( [context.pip, "install", "-r", str(reqs_path)], From f54205594edcbf9ce0dd0f422a099c2d69906dd2 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Thu, 8 Feb 2024 14:08:03 +0000 Subject: [PATCH 04/15] Update call Signed-off-by: Ankita Katiyar --- kedro-docker/features/steps/cli_steps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kedro-docker/features/steps/cli_steps.py b/kedro-docker/features/steps/cli_steps.py index ea7c5716c..e2318e71d 100644 --- a/kedro-docker/features/steps/cli_steps.py +++ b/kedro-docker/features/steps/cli_steps.py @@ -196,7 +196,7 @@ def exec_kedro_command(context, command): @given("I have installed the project dependencies") def pip_install_dependencies(context): """Install project dependencies using pip.""" - call( + run( [ "python", "-m", From 426d4ef8c7b1f44f8ec00abd3e159613a6ba3c60 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Thu, 8 Feb 2024 14:15:31 +0000 Subject: [PATCH 05/15] Update pip Signed-off-by: Ankita Katiyar --- kedro-docker/features/steps/cli_steps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kedro-docker/features/steps/cli_steps.py b/kedro-docker/features/steps/cli_steps.py index e2318e71d..174dfe2c4 100644 --- a/kedro-docker/features/steps/cli_steps.py +++ b/kedro-docker/features/steps/cli_steps.py @@ -205,7 +205,7 @@ def pip_install_dependencies(context): "-U", # pip==23.2 breaks pip-tools<7.0, and pip-tools>=7.0 does not support Python 3.7 # pip==23.3 breaks dependency resolution - "pip>=21.2,<23.2", + "pip=~24.0", ], env=context.env, ) From f1ab112e5fcbcd43542d85bbc0be45ae6a8e7033 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Thu, 8 Feb 2024 16:24:08 +0000 Subject: [PATCH 06/15] pip ruamel Signed-off-by: Ankita Katiyar --- kedro-docker/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/kedro-docker/pyproject.toml b/kedro-docker/pyproject.toml index 4ccfa8594..a0ff988bf 100644 --- a/kedro-docker/pyproject.toml +++ b/kedro-docker/pyproject.toml @@ -36,6 +36,7 @@ test = [ "pytest-xdist[psutil]~=2.2.1", "PyYAML>=5.1, <7.0", "trufflehog>=2.0.99, <3.0", + "ruamel.yaml<0.18.6", "ruff~=0.0.290", "wheel==0.32.2" ] From 619dabef126eec3499dd9b17cc68bd785f961ca4 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Fri, 9 Feb 2024 10:06:36 +0000 Subject: [PATCH 07/15] change pip v Signed-off-by: Ankita Katiyar --- kedro-docker/features/environment.py | 2 +- kedro-docker/features/steps/cli_steps.py | 13 ------------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/kedro-docker/features/environment.py b/kedro-docker/features/environment.py index e006227ee..3cefa0131 100644 --- a/kedro-docker/features/environment.py +++ b/kedro-docker/features/environment.py @@ -42,7 +42,7 @@ def before_all(context): # Temporarily pin pip to fix https://github.com/jazzband/pip-tools/issues/1503 # This can be removed when Kedro 0.17.6 is released, because pip-tools is upgraded # for that version. - "pip>=20.0,<21.3", + "pip>=21.2,<23.2", "setuptools>=38.0", "wheel", ".", diff --git a/kedro-docker/features/steps/cli_steps.py b/kedro-docker/features/steps/cli_steps.py index 174dfe2c4..e4165c6eb 100644 --- a/kedro-docker/features/steps/cli_steps.py +++ b/kedro-docker/features/steps/cli_steps.py @@ -196,19 +196,6 @@ def exec_kedro_command(context, command): @given("I have installed the project dependencies") def pip_install_dependencies(context): """Install project dependencies using pip.""" - run( - [ - "python", - "-m", - "pip", - "install", - "-U", - # pip==23.2 breaks pip-tools<7.0, and pip-tools>=7.0 does not support Python 3.7 - # pip==23.3 breaks dependency resolution - "pip=~24.0", - ], - env=context.env, - ) reqs_path = Path("requirements.txt") res = run( [context.pip, "install", "-r", str(reqs_path)], From ec782009c018f0827ca14d9479ce5cb0af5b3c7e Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Fri, 9 Feb 2024 10:16:18 +0000 Subject: [PATCH 08/15] change pip v Signed-off-by: Ankita Katiyar --- kedro-docker/features/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kedro-docker/features/environment.py b/kedro-docker/features/environment.py index 3cefa0131..f9d069d1a 100644 --- a/kedro-docker/features/environment.py +++ b/kedro-docker/features/environment.py @@ -42,7 +42,7 @@ def before_all(context): # Temporarily pin pip to fix https://github.com/jazzband/pip-tools/issues/1503 # This can be removed when Kedro 0.17.6 is released, because pip-tools is upgraded # for that version. - "pip>=21.2,<23.2", + "pip>=21.2", "setuptools>=38.0", "wheel", ".", From 0c3fa6d2438f9f87ca4885908adae3bd87b11f67 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Fri, 9 Feb 2024 11:16:55 +0000 Subject: [PATCH 09/15] show stdout Signed-off-by: Ankita Katiyar --- kedro-docker/features/steps/cli_steps.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/kedro-docker/features/steps/cli_steps.py b/kedro-docker/features/steps/cli_steps.py index e4165c6eb..061122f11 100644 --- a/kedro-docker/features/steps/cli_steps.py +++ b/kedro-docker/features/steps/cli_steps.py @@ -203,10 +203,9 @@ def pip_install_dependencies(context): cwd=str(context.root_project_dir), ) - if res.returncode != OK_EXIT_CODE: - print(res.stdout) - print(res.stderr) - assert False + print(res.stdout) + print(res.stderr) + # assert False @given("I have removed old docker image of test project") From 94b7ef7ce16b9db0b549d341242d7d341da41719 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Fri, 9 Feb 2024 11:34:42 +0000 Subject: [PATCH 10/15] use no cache dir Signed-off-by: Ankita Katiyar --- kedro-docker/features/steps/cli_steps.py | 2 +- kedro-docker/kedro_docker/template/Dockerfile.simple | 2 +- kedro-docker/kedro_docker/template/Dockerfile.spark | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kedro-docker/features/steps/cli_steps.py b/kedro-docker/features/steps/cli_steps.py index 061122f11..28c7a71d4 100644 --- a/kedro-docker/features/steps/cli_steps.py +++ b/kedro-docker/features/steps/cli_steps.py @@ -198,7 +198,7 @@ def pip_install_dependencies(context): """Install project dependencies using pip.""" reqs_path = Path("requirements.txt") res = run( - [context.pip, "install", "-r", str(reqs_path)], + [context.pip, "install", "-r", str(reqs_path), "--no-cache"], env=context.env, cwd=str(context.root_project_dir), ) diff --git a/kedro-docker/kedro_docker/template/Dockerfile.simple b/kedro-docker/kedro_docker/template/Dockerfile.simple index da9e8a591..24c9cabf5 100644 --- a/kedro-docker/kedro_docker/template/Dockerfile.simple +++ b/kedro-docker/kedro_docker/template/Dockerfile.simple @@ -3,7 +3,7 @@ FROM $BASE_IMAGE as runtime-environment # install project requirements COPY requirements.txt /tmp/requirements.txt -RUN pip install --no-cache -r /tmp/requirements.txt && rm -f /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt && rm -f /tmp/requirements.txt # add kedro user ARG KEDRO_UID=999 diff --git a/kedro-docker/kedro_docker/template/Dockerfile.spark b/kedro-docker/kedro_docker/template/Dockerfile.spark index 5f1205580..158db439e 100644 --- a/kedro-docker/kedro_docker/template/Dockerfile.spark +++ b/kedro-docker/kedro_docker/template/Dockerfile.spark @@ -7,7 +7,7 @@ apt-get install -y procps default-jre-headless && rm -rf /var/lib/apt/lists/* # install project requirements COPY requirements.txt /tmp/requirements.txt -RUN pip install --no-cache -r /tmp/requirements.txt && rm -f /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt && rm -f /tmp/requirements.txt # add kedro user ARG KEDRO_UID=999 From a33afc0ed456cff7ce92615e06619cb04074e707 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Fri, 9 Feb 2024 11:48:03 +0000 Subject: [PATCH 11/15] revert extra changes Signed-off-by: Ankita Katiyar --- kedro-docker/features/steps/cli_steps.py | 9 +++++---- kedro-docker/pyproject.toml | 3 +-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/kedro-docker/features/steps/cli_steps.py b/kedro-docker/features/steps/cli_steps.py index 28c7a71d4..e4165c6eb 100644 --- a/kedro-docker/features/steps/cli_steps.py +++ b/kedro-docker/features/steps/cli_steps.py @@ -198,14 +198,15 @@ def pip_install_dependencies(context): """Install project dependencies using pip.""" reqs_path = Path("requirements.txt") res = run( - [context.pip, "install", "-r", str(reqs_path), "--no-cache"], + [context.pip, "install", "-r", str(reqs_path)], env=context.env, cwd=str(context.root_project_dir), ) - print(res.stdout) - print(res.stderr) - # assert False + if res.returncode != OK_EXIT_CODE: + print(res.stdout) + print(res.stderr) + assert False @given("I have removed old docker image of test project") diff --git a/kedro-docker/pyproject.toml b/kedro-docker/pyproject.toml index a0ff988bf..e83f04bed 100644 --- a/kedro-docker/pyproject.toml +++ b/kedro-docker/pyproject.toml @@ -29,14 +29,13 @@ test = [ "black~=22.0", "docker", "pre-commit>=2.9.2", - "psutil<5.9.8", + "psutil", "pytest", "pytest-cov", "pytest-mock", "pytest-xdist[psutil]~=2.2.1", "PyYAML>=5.1, <7.0", "trufflehog>=2.0.99, <3.0", - "ruamel.yaml<0.18.6", "ruff~=0.0.290", "wheel==0.32.2" ] From f7f66ce517297306fc751cc03ce5318c84ed41b0 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Fri, 9 Feb 2024 13:43:33 +0000 Subject: [PATCH 12/15] pin pip Signed-off-by: Ankita Katiyar --- kedro-docker/features/docker.feature | 4 ++-- kedro-docker/features/environment.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kedro-docker/features/docker.feature b/kedro-docker/features/docker.feature index 9c32a0a58..41b0f1d46 100644 --- a/kedro-docker/features/docker.feature +++ b/kedro-docker/features/docker.feature @@ -52,12 +52,12 @@ Feature: Docker commands in new projects Scenario: Execute docker jupyter notebook target Given I have executed the kedro command "docker build" When I execute the kedro command "docker jupyter notebook" - Then Jupyter Notebook should run on port 8888 + Then Jupyter Server should run on port 8888 Scenario: Execute docker jupyter notebook target on custom port Given I have executed the kedro command "docker build" When I execute the kedro command "docker jupyter notebook --port 8899" - Then Jupyter Notebook should run on port 8899 + Then Jupyter Server should run on port 8899 Scenario: Execute docker jupyter lab target Given I have executed the kedro command "docker build" diff --git a/kedro-docker/features/environment.py b/kedro-docker/features/environment.py index f9d069d1a..3cefa0131 100644 --- a/kedro-docker/features/environment.py +++ b/kedro-docker/features/environment.py @@ -42,7 +42,7 @@ def before_all(context): # Temporarily pin pip to fix https://github.com/jazzband/pip-tools/issues/1503 # This can be removed when Kedro 0.17.6 is released, because pip-tools is upgraded # for that version. - "pip>=21.2", + "pip>=21.2,<23.2", "setuptools>=38.0", "wheel", ".", From a7f7bcaeda1ef98001a94f7d1eab3be844662cbc Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Fri, 9 Feb 2024 14:25:07 +0000 Subject: [PATCH 13/15] gitpod Signed-off-by: Ankita Katiyar --- .gitpod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index 70738f4c0..5f4d2cb3b 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,5 +1,5 @@ # Learn more from ready-to-use templates: https://www.gitpod.io/docs/introduction/getting-started/quickstart -image: gitpod/workspace-python-3.10:2023-04-20-16-32-37 +image: gitpod/workspace-python-3.11:2023-04-20-16-32-37 tasks: From 4ca7979a2fe9cb16710fe5d4d70b1d63ebe6ea87 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Fri, 9 Feb 2024 15:18:52 +0000 Subject: [PATCH 14/15] pip inside dockerfile Signed-off-by: Ankita Katiyar --- kedro-docker/kedro_docker/template/Dockerfile.simple | 1 + kedro-docker/kedro_docker/template/Dockerfile.spark | 1 + 2 files changed, 2 insertions(+) diff --git a/kedro-docker/kedro_docker/template/Dockerfile.simple b/kedro-docker/kedro_docker/template/Dockerfile.simple index 24c9cabf5..34b6cbcfd 100644 --- a/kedro-docker/kedro_docker/template/Dockerfile.simple +++ b/kedro-docker/kedro_docker/template/Dockerfile.simple @@ -3,6 +3,7 @@ FROM $BASE_IMAGE as runtime-environment # install project requirements COPY requirements.txt /tmp/requirements.txt +RUN python -m pip install -U pip RUN pip install --no-cache-dir -r /tmp/requirements.txt && rm -f /tmp/requirements.txt # add kedro user diff --git a/kedro-docker/kedro_docker/template/Dockerfile.spark b/kedro-docker/kedro_docker/template/Dockerfile.spark index 158db439e..8b5384da4 100644 --- a/kedro-docker/kedro_docker/template/Dockerfile.spark +++ b/kedro-docker/kedro_docker/template/Dockerfile.spark @@ -7,6 +7,7 @@ apt-get install -y procps default-jre-headless && rm -rf /var/lib/apt/lists/* # install project requirements COPY requirements.txt /tmp/requirements.txt +RUN python -m pip install -U pip RUN pip install --no-cache-dir -r /tmp/requirements.txt && rm -f /tmp/requirements.txt # add kedro user From 3e59dfcaaede38cf762477cf8d50052c51239367 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Mon, 12 Feb 2024 16:46:49 +0000 Subject: [PATCH 15/15] pip pip inside dockerfile Signed-off-by: Ankita Katiyar --- .gitpod.yml | 2 +- kedro-docker/kedro_docker/template/Dockerfile.simple | 2 +- kedro-docker/kedro_docker/template/Dockerfile.spark | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index 5f4d2cb3b..70738f4c0 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,5 +1,5 @@ # Learn more from ready-to-use templates: https://www.gitpod.io/docs/introduction/getting-started/quickstart -image: gitpod/workspace-python-3.11:2023-04-20-16-32-37 +image: gitpod/workspace-python-3.10:2023-04-20-16-32-37 tasks: diff --git a/kedro-docker/kedro_docker/template/Dockerfile.simple b/kedro-docker/kedro_docker/template/Dockerfile.simple index 34b6cbcfd..57a71e222 100644 --- a/kedro-docker/kedro_docker/template/Dockerfile.simple +++ b/kedro-docker/kedro_docker/template/Dockerfile.simple @@ -3,7 +3,7 @@ FROM $BASE_IMAGE as runtime-environment # install project requirements COPY requirements.txt /tmp/requirements.txt -RUN python -m pip install -U pip +RUN python -m pip install -U "pip>=21.2,<23.2" RUN pip install --no-cache-dir -r /tmp/requirements.txt && rm -f /tmp/requirements.txt # add kedro user diff --git a/kedro-docker/kedro_docker/template/Dockerfile.spark b/kedro-docker/kedro_docker/template/Dockerfile.spark index 8b5384da4..3dadc9a38 100644 --- a/kedro-docker/kedro_docker/template/Dockerfile.spark +++ b/kedro-docker/kedro_docker/template/Dockerfile.spark @@ -7,7 +7,7 @@ apt-get install -y procps default-jre-headless && rm -rf /var/lib/apt/lists/* # install project requirements COPY requirements.txt /tmp/requirements.txt -RUN python -m pip install -U pip +RUN python -m pip install -U "pip>=21.2,<23.2" RUN pip install --no-cache-dir -r /tmp/requirements.txt && rm -f /tmp/requirements.txt # add kedro user