From fdc3a4ebfc1c264bb8058a2d62f0b92a9bfc81df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Mon, 8 May 2023 16:22:29 +0200 Subject: [PATCH] Migrate default project template to static project metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Includes consolidated dev requirements in project template. Fix gh-2280. Fix gh-2519. Signed-off-by: Juan Luis Cano Rodríguez --- features/environment.py | 39 ++++++++++++--- .../{{ cookiecutter.repo_name }}/README.md | 22 +++----- .../pyproject.toml | 50 +++++++++++++++++++ .../requirements.txt | 14 ------ .../{{ cookiecutter.repo_name }}/setup.py | 39 --------------- test_requirements.txt | 1 + tests/framework/cli/test_starters.py | 6 +-- 7 files changed, 92 insertions(+), 79 deletions(-) delete mode 100644 kedro/templates/project/{{ cookiecutter.repo_name }}/requirements.txt delete mode 100644 kedro/templates/project/{{ cookiecutter.repo_name }}/setup.py diff --git a/features/environment.py b/features/environment.py index c98246dc85..b7a9e738aa 100644 --- a/features/environment.py +++ b/features/environment.py @@ -8,6 +8,11 @@ import venv from pathlib import Path +try: + import tomllib +except ImportError: + import tomli as tomllib # type: ignore + from features.steps.sh_run import run _PATHS_TO_REMOVE: set[Path] = set() @@ -115,13 +120,33 @@ def _setup_minimal_env(context): def _install_project_requirements(context): - install_reqs = ( - Path( - "kedro/templates/project/{{ cookiecutter.repo_name }}/src/requirements.txt" - ) - .read_text(encoding="utf-8") - .splitlines() - ) + # HACK: Dev requirements of the project are stored in pyproject.toml, + # but the file cannot be directly loaded because it's a cookicutter template, + # so we read only the part that we need + with open( + "kedro/templates/project/{{ cookiecutter.repo_name }}/pyproject.toml", + encoding="utf-8", + ) as pyproject_fh: + _buffer = [] + while True: + try: + line = next(pyproject_fh) + except StopIteration as exc: + raise RuntimeError("Error while reading project dependencies") from exc + + if line.startswith("[project.optional-dependencies]"): + _buffer.append(line) + for line in pyproject_fh: + if line.startswith("["): + break + + _buffer.append(line) + break + + _meta = tomllib.loads("\n".join(_buffer)) + install_reqs = _meta["project"]["optional-dependencies"]["dev"] + # EOH + install_reqs = [req for req in install_reqs if "{" not in req] install_reqs.append(".[pandas.CSVDataSet]") call([context.pip, "install", *install_reqs], env=context.env) diff --git a/kedro/templates/project/{{ cookiecutter.repo_name }}/README.md b/kedro/templates/project/{{ cookiecutter.repo_name }}/README.md index 9ece71ad8e..c78aa077a8 100644 --- a/kedro/templates/project/{{ cookiecutter.repo_name }}/README.md +++ b/kedro/templates/project/{{ cookiecutter.repo_name }}/README.md @@ -17,12 +17,12 @@ In order to get the best out of the template: ## How to install dependencies -Declare any dependencies in `requirements.txt` for `pip` installation and `environment.yml` for `conda` installation. +Declare any dependencies in `pyproject.toml` for `pip` installation and `environment.yml` for `conda` installation. To install them, run: ``` -pip install -r requirements.txt +pip install -e . ``` ## How to run your Kedro pipeline @@ -51,25 +51,23 @@ To generate or update the dependency requirements for your project: kedro build-reqs ``` -This will `pip-compile` the contents of `requirements.txt` into a new file `requirements.lock`. You can see the output of the resolution by opening `requirements.lock`. +This will `pip-compile` the contents of `pyproject.toml` into a new file `requirements.lock`. You can see the output of the resolution by opening `requirements.lock`. -After this, if you'd like to update your project requirements, please update `requirements.txt` and re-run `kedro build-reqs`. +After this, if you'd like to update your project requirements, please update `pyproject.toml` and re-run `kedro build-reqs`. [Further information about project dependencies](https://docs.kedro.org/en/stable/kedro_project_setup/dependencies.html#project-specific-dependencies) ## How to work with Kedro and notebooks > Note: Using `kedro jupyter` or `kedro ipython` to run your notebook provides these variables in scope: `context`, `catalog`, and `startup_error`. -> -> Jupyter, JupyterLab, and IPython are already included in the project requirements by default, so once you have run `pip install -r requirements.txt` you will not need to take any extra steps before you use them. -### Jupyter -To use Jupyter notebooks in your Kedro project, you need to install Jupyter: +Jupyter, JupyterLab, and IPython are not included in the project requirements by default. To install them, run the following command: ``` -pip install jupyter +pip install -e .[dev] ``` +### Jupyter After installing Jupyter, you can start a local notebook server: ``` @@ -77,12 +75,6 @@ kedro jupyter notebook ``` ### JupyterLab -To use JupyterLab, you need to install it: - -``` -pip install jupyterlab -``` - You can also start JupyterLab: ``` diff --git a/kedro/templates/project/{{ cookiecutter.repo_name }}/pyproject.toml b/kedro/templates/project/{{ cookiecutter.repo_name }}/pyproject.toml index 7ae06368bd..e24b66fce4 100644 --- a/kedro/templates/project/{{ cookiecutter.repo_name }}/pyproject.toml +++ b/kedro/templates/project/{{ cookiecutter.repo_name }}/pyproject.toml @@ -1,3 +1,53 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "{{ cookiecutter.python_package }}" +dependencies = [ + "kedro~={{ cookiecutter.kedro_version }}", + "kedro-telemetry~=0.2.0", +] +dynamic = ["version"] + +[project.scripts] +{{ cookiecutter.repo_name }} = "{{ cookiecutter.python_package }}.__main__:main" + +[project.optional-dependencies] +docs = [ + "docutils<0.18.0", + "sphinx~=3.4.3", + "sphinx_rtd_theme==0.5.1", + "nbsphinx==0.8.1", + "nbstripout~=0.4", + "sphinx-autodoc-typehints==1.11.1", + "sphinx_copybutton==0.3.1", + "ipykernel>=5.3, <7.0", + "Jinja2<3.1.0", + "myst-parser~=0.17.2", +] +dev = [ + "black~=22.0", + "flake8>=3.7.9, <5.0", + "ipython>=7.31.1, <8.0; python_version < '3.8'", + "ipython~=8.10; python_version >= '3.8'", + "isort~=5.0", + "jupyter~=1.0", + "jupyterlab_server>=2.11.1, <2.16.0", + "jupyterlab~=3.0, <3.6.0", + "nbstripout~=0.4", + "pytest-cov~=3.0", + "pytest-mock>=1.7.1, <2.0", + "pytest~=7.2", +] + +[tool.setuptools.dynamic] +version = {attr = "{{ cookiecutter.python_package }}.__version__"} + +[tool.setuptools.packages.find] +where = ["src"] +namespaces = false + [tool.kedro] package_name = "{{ cookiecutter.python_package }}" project_name = "{{ cookiecutter.project_name }}" diff --git a/kedro/templates/project/{{ cookiecutter.repo_name }}/requirements.txt b/kedro/templates/project/{{ cookiecutter.repo_name }}/requirements.txt deleted file mode 100644 index aa7ee32014..0000000000 --- a/kedro/templates/project/{{ cookiecutter.repo_name }}/requirements.txt +++ /dev/null @@ -1,14 +0,0 @@ -black~=22.0 -flake8>=3.7.9, <5.0 -ipython>=7.31.1, <8.0; python_version < '3.8' -ipython~=8.10; python_version >= '3.8' -isort~=5.0 -jupyter~=1.0 -jupyterlab_server>=2.11.1, <2.16.0 -jupyterlab~=3.0, <3.6.0 -kedro~={{ cookiecutter.kedro_version }} -kedro-telemetry~=0.2.0 -nbstripout~=0.4 -pytest-cov~=3.0 -pytest-mock>=1.7.1, <2.0 -pytest~=7.2 diff --git a/kedro/templates/project/{{ cookiecutter.repo_name }}/setup.py b/kedro/templates/project/{{ cookiecutter.repo_name }}/setup.py deleted file mode 100644 index 8e62d661f8..0000000000 --- a/kedro/templates/project/{{ cookiecutter.repo_name }}/setup.py +++ /dev/null @@ -1,39 +0,0 @@ -from setuptools import find_packages, setup - -entry_point = ( - "{{ cookiecutter.repo_name }} = {{ cookiecutter.python_package }}.__main__:main" -) - - -# get the dependencies and installs -with open("requirements.txt", encoding="utf-8") as f: - # Make sure we strip all comments and options (e.g "--extra-index-url") - # that arise from a modified pip.conf file that configure global options - # when running kedro build-reqs - requires = [] - for line in f: - req = line.split("#", 1)[0].strip() - if req and not req.startswith("--"): - requires.append(req) - -setup( - name="{{ cookiecutter.python_package }}", - version="0.1", - packages=find_packages(exclude=["tests"]), - entry_points={"console_scripts": [entry_point]}, - install_requires=requires, - extras_require={ - "docs": [ - "docutils<0.18.0", - "sphinx~=3.4.3", - "sphinx_rtd_theme==0.5.1", - "nbsphinx==0.8.1", - "nbstripout~=0.4", - "sphinx-autodoc-typehints==1.11.1", - "sphinx_copybutton==0.3.1", - "ipykernel>=5.3, <7.0", - "Jinja2<3.1.0", - "myst-parser~=0.17.2", - ] - }, -) diff --git a/test_requirements.txt b/test_requirements.txt index cc448d41f7..4fc4d73783 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -59,6 +59,7 @@ SQLAlchemy~=1.2 tables~=3.6.0; platform_system == "Windows" and python_version<'3.9' tables~=3.6; platform_system != "Windows" tensorflow~=2.0 +tomli~=2.0.1; python_version < '3.11' triad>=0.6.7, <1.0 trufflehog~=2.1 xlsxwriter~=1.0 diff --git a/tests/framework/cli/test_starters.py b/tests/framework/cli/test_starters.py index 26fc6ac3e5..b1fc05433b 100644 --- a/tests/framework/cli/test_starters.py +++ b/tests/framework/cli/test_starters.py @@ -17,7 +17,7 @@ KedroStarterSpec, ) -FILES_IN_TEMPLATE = 31 +FILES_IN_TEMPLATE = 29 @pytest.fixture @@ -70,9 +70,7 @@ def _assert_template_ok( assert (full_path / ".gitignore").is_file() assert project_name in (full_path / "README.md").read_text(encoding="utf-8") assert "KEDRO" in (full_path / ".gitignore").read_text(encoding="utf-8") - assert kedro_version in (full_path / "src" / "requirements.txt").read_text( - encoding="utf-8" - ) + assert kedro_version in (full_path / "pyproject.toml").read_text(encoding="utf-8") assert (full_path / "src" / python_package / "__init__.py").is_file()