Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Decouple warn error options from core and adapters #9338

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .changes/unreleased/Under the Hood-20240104-165248.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
kind: Under the Hood
body: Accept valid_error_names in WarnErrorOptions constructor, remove global usage
of event modules
time: 2024-01-04T16:52:48.173716-05:00
custom:
Author: michelleark
Issue: "9337"
5 changes: 4 additions & 1 deletion core/dbt/cli/flags.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from dbt.common.clients import jinja
from dbt.deprecations import renamed_env_var
from dbt.common.helper_types import WarnErrorOptions
from dbt.events import ALL_EVENT_NAMES

if os.name != "nt":
# https://bugs.python.org/issue41567
Expand Down Expand Up @@ -50,7 +51,9 @@ def convert_config(config_name, config_value):
ret = config_value
if config_name.lower() == "warn_error_options" and type(config_value) == dict:
ret = WarnErrorOptions(
include=config_value.get("include", []), exclude=config_value.get("exclude", [])
include=config_value.get("include", []),
exclude=config_value.get("exclude", []),
valid_error_names=ALL_EVENT_NAMES,
)
return ret

Expand Down
5 changes: 4 additions & 1 deletion core/dbt/cli/option_types.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from click import ParamType, Choice

from dbt.config.utils import parse_cli_yaml_string
from dbt.events import ALL_EVENT_NAMES
from dbt.exceptions import ValidationError, OptionNotYamlDictError
from dbt.common.exceptions import DbtValidationError

Expand Down Expand Up @@ -53,7 +54,9 @@ def convert(self, value, param, ctx):
include_exclude = super().convert(value, param, ctx)

return WarnErrorOptions(
include=include_exclude.get("include", []), exclude=include_exclude.get("exclude", [])
include=include_exclude.get("include", []),
exclude=include_exclude.get("exclude", []),
valid_error_names=ALL_EVENT_NAMES,
)


Expand Down
26 changes: 11 additions & 15 deletions core/dbt/common/helper_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,13 @@

from dataclasses import dataclass, field
from typing import Tuple, AbstractSet, Union
from typing import Callable, cast, Generic, Optional, TypeVar, List, NewType, Any, Dict
from typing import Callable, cast, Generic, Optional, TypeVar, List, NewType, Set

from dbt.common.dataclass_schema import (
dbtClassMixin,
ValidationError,
StrEnum,
)
import dbt.adapters.events.types as adapter_dbt_event_types
import dbt.common.events.types as dbt_event_types
import dbt.events.types as core_dbt_event_types


Port = NewType("Port", int)

Expand Down Expand Up @@ -68,18 +64,18 @@ def _validate_items(self, items: List[str]):


class WarnErrorOptions(IncludeExclude):
def _validate_items(self, items: List[str]):
all_event_types: Dict[str, Any] = {
**dbt_event_types.__dict__,
**core_dbt_event_types.__dict__,
**adapter_dbt_event_types.__dict__,
}
valid_exception_names = set(
[name for name, cls in all_event_types.items() if isinstance(cls, type)]
)
def __init__(
self,
include: Union[str, List[str]],
exclude: Optional[List[str]] = None,
valid_error_names: Optional[Set[str]] = None,
):
self._valid_error_names: Set[str] = valid_error_names or set()
super().__init__(include=include, exclude=(exclude or []))

def _validate_items(self, items: List[str]):
for item in items:
if item not in valid_exception_names:
if item not in self._valid_error_names:
raise ValidationError(f"{item} is not a valid dbt error name.")


Expand Down
15 changes: 15 additions & 0 deletions core/dbt/events/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
from typing import Dict, Any, Set

import dbt.adapters.events.types as adapter_dbt_event_types
import dbt.common.events.types as dbt_event_types
import dbt.events.types as core_dbt_event_types

ALL_EVENT_TYPES: Dict[str, Any] = {
**dbt_event_types.__dict__,
**core_dbt_event_types.__dict__,
**adapter_dbt_event_types.__dict__,
}

ALL_EVENT_NAMES: Set[str] = set(
[name for name, cls in ALL_EVENT_TYPES.items() if isinstance(cls, type)]
)
23 changes: 22 additions & 1 deletion tests/unit/test_functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@
from dbt.common.events.functions import msg_to_dict, warn_or_error
from dbt.events.logging import setup_event_logger
from dbt.common.events.types import InfoLevel
from dbt.events.types import NoNodesForSelectionCriteria
from dbt.common.exceptions import EventCompilationError
from dbt.events.types import NoNodesForSelectionCriteria
from dbt.adapters.events.types import AdapterDeprecationWarning
from dbt.common.events.types import RetryExternalCall


@pytest.mark.parametrize(
Expand All @@ -30,6 +32,25 @@ def test_warn_or_error_warn_error_options(warn_error_options, expect_compilation
warn_or_error(NoNodesForSelectionCriteria())


@pytest.mark.parametrize(
"error_cls",
[
NoNodesForSelectionCriteria, # core event
AdapterDeprecationWarning, # adapter event
RetryExternalCall, # common event
],
)
def test_warn_error_options_captures_all_events(error_cls):
args = Namespace(warn_error_options={"include": [error_cls.__name__]})
flags.set_from_args(args, {})
with pytest.raises(EventCompilationError):
warn_or_error(error_cls())

args = Namespace(warn_error_options={"include": "*", "exclude": [error_cls.__name__]})
flags.set_from_args(args, {})
warn_or_error(error_cls())


@pytest.mark.parametrize(
"warn_error,expect_compilation_exception",
[
Expand Down
31 changes: 18 additions & 13 deletions tests/unit/test_helper_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,25 +30,30 @@ def test_includes(self, include, exclude, expected_includes):

class TestWarnErrorOptions:
def test_init_invalid_error(self):
with pytest.raises(ValidationError):
WarnErrorOptions(include=["InvalidError"], valid_error_names=set(["ValidError"]))

with pytest.raises(ValidationError):
WarnErrorOptions(
include="*", exclude=["InvalidError"], valid_error_names=set(["ValidError"])
)

def test_init_invalid_error_default_valid_error_names(self):
with pytest.raises(ValidationError):
WarnErrorOptions(include=["InvalidError"])

with pytest.raises(ValidationError):
WarnErrorOptions(include="*", exclude=["InvalidError"])

@pytest.mark.parametrize(
"valid_error_name",
[
"NoNodesForSelectionCriteria", # core event
"AdapterDeprecationWarning", # adapter event
"RetryExternalCall", # common event
],
)
def test_init_valid_error(self, valid_error_name):
warn_error_options = WarnErrorOptions(include=[valid_error_name])
assert warn_error_options.include == [valid_error_name]
def test_init_valid_error(self):
warn_error_options = WarnErrorOptions(
include=["ValidError"], valid_error_names=set(["ValidError"])
)
assert warn_error_options.include == ["ValidError"]
assert warn_error_options.exclude == []

warn_error_options = WarnErrorOptions(include="*", exclude=[valid_error_name])
warn_error_options = WarnErrorOptions(
include="*", exclude=["ValidError"], valid_error_names=set(["ValidError"])
)
assert warn_error_options.include == "*"
assert warn_error_options.exclude == [valid_error_name]
assert warn_error_options.exclude == ["ValidError"]