Merge branch 'main' into 3995-data-catalog-2.0

Signed-off-by: Elena Khaustova <ymax70rus@gmail.com>

RELEASE.md

@@ -6,6 +6,7 @@
   * Added get dataset by name feature;
   * `add_feed_dict()` was simplified and renamed to `add_data()`;
   * Datasets' initialisation was moved out from `from_config()` method to the constructor.
+* Moved development requirements from `requirements.txt` to the dedicated section in `pyproject.toml` for project template.
 * Implemented `Protocol` abstraction for the current `DataCatalog` and adding new catalog implementations.
 * Refactored `kedro run` and `kedro catalog` commands.
 * Moved pattern resolution logic from `DataCatalog` to a separate component - `CatalogConfigResolver`. Updated `DataCatalog` to use `CatalogConfigResolver` internally.

docs/source/development/automated_testing.md

@@ -19,21 +19,36 @@ There are many testing frameworks available for Python. One of the most popular
 
 Let's look at how you can start working with `pytest` in your Kedro project.
 
-### Prerequisite: Install your Kedro project
+### Install test requirements
+Before getting started with test requirements, it is important to ensure you have installed your project locally. This allows you to test different parts of your project by importing them into your test files.
+
+
+To install your project including all the project-specific dependencies and test requirements:
+1. Add the following section to the `pyproject.toml` file located in the project root:
+```toml
+[project.optional-dependencies]
+dev = [
+    "pytest-cov",
+    "pytest-mock",
+    "pytest",
+]
+```
+
+2. Navigate to the root directory of the project and run:
+```bash
+pip install ."[dev]"
+```
 
-Before getting started with `pytest`, it is important to ensure you have installed your project locally. This allows you to test different parts of your project by importing them into your test files.
+Alternatively, you can individually install test requirements as you would install other packages with `pip`, making sure you have installed your project locally and your [project's virtual environment is active](../get_started/install.md#create-a-virtual-environment-for-your-kedro-project).
 
-To install your project, navigate to your project root and run the following command:
+1. To install your project, navigate to your project root and run the following command:
 
 ```bash
 pip install -e .
 ```
-
 >**NOTE**: The option `-e` installs an editable version of your project, allowing you to make changes to the project files without needing to re-install them each time.
-### Install `pytest`
-
-Install `pytest` as you would install other packages with `pip`, making sure your [project's virtual environment is active](../get_started/install.md#create-a-virtual-environment-for-your-kedro-project).
 
+2. Install test requirements one by one:
 ```bash
 pip install pytest
 ```

docs/source/development/linting.md

@@ -18,17 +18,17 @@ There are a variety of Python tools available to use with your Kedro projects. T
 type.
 
 ### Install the tools
-Install `ruff` by adding the following lines to your project's `requirements.txt`
-file:
-```text
-ruff # Used for linting, formatting and sorting module imports
+To install `ruff` add the following section to the `pyproject.toml` file located in the project root:
+```toml
+[project.optional-dependencies]
+dev = ["ruff"]
 ```
 
-To install all the project-specific dependencies, including the linting tools, navigate to the root directory of the
+Then to install your project including all the project-specific dependencies and the linting tools, navigate to the root directory of the
 project and run:
 
 ```bash
-pip install -r requirements.txt
+pip install ."[dev]"
 ```
 
 Alternatively, you can individually install the linting tools using the following shell commands:

features/load_node.feature

@@ -5,5 +5,6 @@ Feature: load_node in new project
     And I have run a non-interactive kedro new with starter "default"
 
   Scenario: Execute ipython load_node magic
-    When I execute the load_node magic command
+    When I install project and its dev dependencies
+    And I execute the load_node magic command
     Then the logs should show that load_node executed successfully

features/steps/cli_steps.py

@@ -755,3 +755,13 @@ def exec_magic_command(context):
 def change_dir(context, dir):
     """Execute Kedro target."""
     util.chdir(dir)
+
+
+@when("I install project and its dev dependencies")
+def pip_install_project_and_dev_dependencies(context):
+    """Install project and its development dependencies using pip."""
+    _ = run(
+        [context.pip, "install", ".[dev]"],
+        env=context.env,
+        cwd=str(context.root_project_dir),
+    )

features/steps/test_starter/{{ cookiecutter.repo_name }}/pyproject.toml

@@ -12,15 +12,21 @@ dynamic = ["dependencies", "version"]
 
 [project.optional-dependencies]
 docs = [
-    "docutils<0.18.0",
-    "sphinx~=3.4.3",
-    "sphinx_rtd_theme==0.5.1",
+    "docutils<0.21",
+    "sphinx>=5.3,<7.3",
+    "sphinx_rtd_theme==2.0.0",
     "nbsphinx==0.8.1",
-    "sphinx-autodoc-typehints==1.11.1",
-    "sphinx_copybutton==0.3.1",
+    "sphinx-autodoc-typehints==1.20.2",
+    "sphinx_copybutton==0.5.2",
     "ipykernel>=5.3, <7.0",
-    "Jinja2<3.1.0",
-    "myst-parser~=0.17.2",
+    "Jinja2<3.2.0",
+    "myst-parser>=1.0,<2.1"
+]
+dev = [
+    "pytest-cov~=3.0",
+    "pytest-mock>=1.7.1, <2.0",
+    "pytest~=7.2",
+    "ruff~=0.1.8"
 ]
 
 [tool.setuptools.dynamic]

features/steps/test_starter/{{ cookiecutter.repo_name }}/requirements.txt

@@ -1,10 +1,6 @@
-ruff==0.1.8
 ipython>=8.10
 jupyterlab>=3.0
 notebook
 kedro~={{ cookiecutter.kedro_version}}
 kedro-datasets[pandas-csvdataset]; python_version >= "3.9"
 kedro-datasets[pandas.CSVDataset]<2.0.0; python_version < '3.9'
-pytest-cov~=3.0
-pytest-mock>=1.7.1, <2.0
-pytest~=7.2

kedro/templates/project/hooks/utils.py

@@ -20,7 +20,9 @@
 ]
 
 # Configuration key for documentation dependencies
-docs_pyproject_requirements = ["project.optional-dependencies"]  # For pyproject.toml
+docs_pyproject_requirements = ["project.optional-dependencies.docs"]  # For pyproject.toml
+# Configuration key for linting and testing dependencies
+dev_pyproject_requirements = ["project.optional-dependencies.dev"]  # For pyproject.toml
 
 # Requirements for example pipelines
 example_pipeline_requirements = "seaborn~=0.12.1\nscikit-learn~=1.0\n"
@@ -191,12 +193,14 @@ def setup_template_tools(
         python_package_name (str): The name of the python package.
         example_pipeline (str): 'True' if example pipeline was selected
     """
+
+    if "Linting" not in selected_tools_list and "Testing" not in selected_tools_list:
+        _remove_from_toml(pyproject_file_path, dev_pyproject_requirements)
+
     if "Linting" not in selected_tools_list:
-        _remove_from_file(requirements_file_path, lint_requirements)
         _remove_from_toml(pyproject_file_path, lint_pyproject_requirements)
 
     if "Testing" not in selected_tools_list:
-        _remove_from_file(requirements_file_path, test_requirements)
         _remove_from_toml(pyproject_file_path, test_pyproject_requirements)
         _remove_dir(current_dir / "tests")
 

kedro/templates/project/{{ cookiecutter.repo_name }}/pyproject.toml

@@ -24,6 +24,12 @@ docs = [
     "Jinja2<3.2.0",
     "myst-parser>=1.0,<2.1"
 ]
+dev = [
+    "pytest-cov~=3.0",
+    "pytest-mock>=1.7.1, <2.0",
+    "pytest~=7.2",
+    "ruff~=0.1.8"
+]
 
 
 [tool.setuptools.dynamic]

kedro/templates/project/{{ cookiecutter.repo_name }}/requirements.txt

@@ -2,7 +2,3 @@ ipython>=8.10
 jupyterlab>=3.0
 notebook
 kedro~={{ cookiecutter.kedro_version }}
-pytest-cov~=3.0
-pytest-mock>=1.7.1, <2.0
-pytest~=7.2
-ruff~=0.1.8

tests/framework/cli/test_starters.py

@@ -147,17 +147,11 @@ def _assert_requirements_ok(
     assert "Congratulations!" in result.output
     assert f"has been created in the directory \n{root_path}" in result.output
 
-    requirements_file_path = root_path / "requirements.txt"
     pyproject_file_path = root_path / "pyproject.toml"
 
     tools_list = _parse_tools_input(tools)
 
     if "1" in tools_list:
-        with open(requirements_file_path) as requirements_file:
-            requirements = requirements_file.read()
-
-        assert "ruff" in requirements
-
         pyproject_config = toml.load(pyproject_file_path)
         expected = {
             "tool": {
@@ -171,15 +165,11 @@ def _assert_requirements_ok(
             }
         }
         assert expected["tool"]["ruff"] == pyproject_config["tool"]["ruff"]
+        assert (
+            "ruff~=0.1.8" in pyproject_config["project"]["optional-dependencies"]["dev"]
+        )
 
     if "2" in tools_list:
-        with open(requirements_file_path) as requirements_file:
-            requirements = requirements_file.read()
-
-        assert "pytest-cov~=3.0" in requirements
-        assert "pytest-mock>=1.7.1, <2.0" in requirements
-        assert "pytest~=7.2" in requirements
-
         pyproject_config = toml.load(pyproject_file_path)
         expected = {
             "pytest": {
@@ -198,6 +188,18 @@ def _assert_requirements_ok(
         assert expected["pytest"] == pyproject_config["tool"]["pytest"]
         assert expected["coverage"] == pyproject_config["tool"]["coverage"]
 
+        assert (
+            "pytest-cov~=3.0"
+            in pyproject_config["project"]["optional-dependencies"]["dev"]
+        )
+        assert (
+            "pytest-mock>=1.7.1, <2.0"
+            in pyproject_config["project"]["optional-dependencies"]["dev"]
+        )
+        assert (
+            "pytest~=7.2" in pyproject_config["project"]["optional-dependencies"]["dev"]
+        )
+
     if "4" in tools_list:
         pyproject_config = toml.load(pyproject_file_path)
         expected = {