Skip to content

Commit

Permalink
fix: backwards compatibility for kedro-airflow (#381)
Browse files Browse the repository at this point in the history
Signed-off-by: Simon Brugman <sfbbrugman@gmail.com>
  • Loading branch information
sbrugman authored Oct 12, 2023
1 parent 6d1fca4 commit cc75b40
Show file tree
Hide file tree
Showing 3 changed files with 33 additions and 5 deletions.
1 change: 1 addition & 0 deletions kedro-airflow/RELEASE.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
* Added support for Python 3.11
* Added the `--all` CLI argument to `kedro-airflow` to convert registered all pipelines at once.
* Simplify the output of the `kedro airflow create` command.
* Fixed compatibility of `kedro-airflow` with older versions of the config loaders (`kedro<=0.18.2`).

## Community contributions
Many thanks to the following Kedroids for contributing PRs to this release:
Expand Down
13 changes: 9 additions & 4 deletions kedro-airflow/kedro_airflow/plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,17 +36,22 @@ def airflow_commands():


def _load_config(context: KedroContext) -> dict[str, Any]:
# Backwards compatibility for ConfigLoader that does not support `config_patterns`
config_loader = context.config_loader
if not hasattr(config_loader, "config_patterns"):
return config_loader.get("airflow*", "airflow/**")

# Set the default pattern for `airflow` if not provided in `settings.py`
if "airflow" not in context.config_loader.config_patterns.keys():
context.config_loader.config_patterns.update( # pragma: no cover
if "airflow" not in config_loader.config_patterns.keys():
config_loader.config_patterns.update( # pragma: no cover
{"airflow": ["airflow*", "airflow/**"]}
)

assert "airflow" in context.config_loader.config_patterns.keys()
assert "airflow" in config_loader.config_patterns.keys()

# Load the config
try:
return context.config_loader["airflow"]
return config_loader["airflow"]
except MissingConfigException:
# File does not exist
return {}
Expand Down
24 changes: 23 additions & 1 deletion kedro-airflow/tests/test_plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,11 @@

import pytest
import yaml
from kedro.config import ConfigLoader
from kedro.framework.context import KedroContext
from pluggy import PluginManager

from kedro_airflow.plugin import commands
from kedro_airflow.plugin import _load_config, commands


@pytest.mark.parametrize(
Expand Down Expand Up @@ -264,3 +267,22 @@ def test_create_airflow_all_and_pipeline(cli_runner, metadata):
"Error: Invalid value: The `--all` and `--pipeline` option are mutually exclusive."
in result.stdout
)


def test_config_loader_backwards_compatibility(cli_runner, metadata):
# Emulate ConfigLoader in kedro <= 0.18.2
conf_source = Path.cwd() / "conf"
config_loader = ConfigLoader(conf_source=conf_source)
del config_loader.config_patterns
context = KedroContext(
config_loader=config_loader,
hook_manager=PluginManager(project_name=metadata.project_name),
package_name=metadata.package_name,
project_path=metadata.project_path,
)

config = _load_config(context)
assert config == {
"default": {"owner": "again someone else"},
"ds": {"owner": "finally someone else"},
}

0 comments on commit cc75b40

Please sign in to comment.