Move events to common (#8676)

* Move events to common

* More Type Annotations (#8536)

* Extend use of type annotations in the events module.

* Add return type of None to more __init__ definitions.

* Still more type annotations adding -> None to __init__

* Tweak per review

* Allow adapters to include python package logging in dbt logs (#8643)

* add set_package_log_level functionality

* set package handler

* set package handler

* add logging about stting up logging

* test event log handler

* add event log handler

* add event log level

* rename package and add unit tests

* revert logfile config change

* cleanup and add code comments

* add changie

* swap function for dict

* add additional unit tests

* fix unit test

* update README and protos

* fix formatting

* update precommit

---------

Co-authored-by: Peter Webb <peter.webb@dbtlabs.com>

.changes/unreleased/Features-20230915-123733.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: 'Allow adapters to include package logs in dbt standard logging '
+time: 2023-09-15T12:37:33.862862-07:00
+custom:
+  Author: colin-rogers-dbt
+  Issue: "7859"

.changes/unreleased/Under the Hood-20230831-164435.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Added more type annotations.
+time: 2023-08-31T16:44:35.737954-04:00
+custom:
+  Author: peterallenwebb
+  Issue: "8537"

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 # Configuration for pre-commit hooks (see https://pre-commit.com/).
 # Eventually the hooks described here will be run as tests before merging each PR.
 
-exclude: ^(core/dbt/docs/build/|core/dbt/events/types_pb2.py)
+exclude: ^(core/dbt/docs/build/|core/dbt/common/events/types_pb2.py)
 
 # Force all unspecified python hooks to run python 3.8
 default_language_version:

Makefile

@@ -40,7 +40,7 @@ dev: dev_req ## Installs dbt-* packages in develop mode along with development d
 
 .PHONY: proto_types
 proto_types:  ## generates google protobuf python file from types.proto
-	protoc -I=./core/dbt/events --python_out=./core/dbt/events ./core/dbt/events/types.proto
+	protoc -I=./core/dbt/common/events --python_out=./core/dbt/common/events ./core/dbt/common/events/types.proto
 
 .PHONY: mypy
 mypy: .env ## Runs mypy against staged changes for static type checking.

core/dbt/adapters/base/connections.py

@@ -36,9 +36,9 @@
 from dbt.adapters.base.query_headers import (
     MacroQueryStringSetter,
 )
-from dbt.events import AdapterLogger
-from dbt.events.functions import fire_event
-from dbt.events.types import (
+from dbt.common.events import AdapterLogger
+from dbt.common.events.functions import fire_event
+from dbt.common.events.types import (
     NewConnection,
     ConnectionReused,
     ConnectionLeftOpenInCleanup,
@@ -48,7 +48,7 @@
     Rollback,
     RollbackFailed,
 )
-from dbt.events.contextvars import get_node_info
+from dbt.common.events.contextvars import get_node_info
 from dbt import flags
 from dbt.utils import cast_to_str
 
@@ -72,7 +72,7 @@ class BaseConnectionManager(metaclass=abc.ABCMeta):
 
     TYPE: str = NotImplemented
 
-    def __init__(self, profile: AdapterRequiredConfig):
+    def __init__(self, profile: AdapterRequiredConfig) -> None:
         self.profile = profile
         self.thread_connections: Dict[Hashable, Connection] = {}
         self.lock: RLock = flags.MP_CONTEXT.RLock()

core/dbt/adapters/base/impl.py

@@ -48,8 +48,8 @@
 from dbt.clients.jinja import MacroGenerator
 from dbt.contracts.graph.manifest import Manifest, MacroManifest
 from dbt.contracts.graph.nodes import ResultNode
-from dbt.events.functions import fire_event, warn_or_error
-from dbt.events.types import (
+from dbt.common.events.functions import fire_event, warn_or_error
+from dbt.common.events.types import (
     CacheMiss,
     ListRelations,
     CodeExecution,
@@ -222,7 +222,7 @@ class BaseAdapter(metaclass=AdapterMeta):
         ConstraintType.foreign_key: ConstraintSupport.ENFORCED,
     }
 
-    def __init__(self, config):
+    def __init__(self, config) -> None:
         self.config = config
         self.cache = RelationsCache()
         self.connections = self.ConnectionManager(config)

core/dbt/adapters/base/plugin.py

@@ -29,7 +29,7 @@ def __init__(
         credentials: Type[Credentials],
         include_path: str,
         dependencies: Optional[List[str]] = None,
-    ):
+    ) -> None:
 
         self.adapter: Type[AdapterProtocol] = adapter
         self.credentials: Type[Credentials] = credentials

core/dbt/adapters/base/query_headers.py

@@ -11,7 +11,7 @@
 
 
 class NodeWrapper:
-    def __init__(self, node):
+    def __init__(self, node) -> None:
         self._inner_node = node
 
     def __getattr__(self, name):
@@ -57,7 +57,7 @@ def set(self, comment: Optional[str], append: bool):
 
 
 class MacroQueryStringSetter:
-    def __init__(self, config: AdapterRequiredConfig, manifest: Manifest):
+    def __init__(self, config: AdapterRequiredConfig, manifest: Manifest) -> None:
         self.manifest = manifest
         self.config = config
 

core/dbt/adapters/cache.py

@@ -14,8 +14,8 @@
     ReferencedLinkNotCachedError,
     TruncatedModelNameCausedCollisionError,
 )
-from dbt.events.functions import fire_event, fire_event_if
-from dbt.events.types import CacheAction, CacheDumpGraph
+from dbt.common.events.functions import fire_event, fire_event_if
+from dbt.common.events.types import CacheAction, CacheDumpGraph
 from dbt.flags import get_flags
 from dbt.utils import lowercase
 
@@ -38,8 +38,8 @@ class _CachedRelation:
     :attr BaseRelation inner: The underlying dbt relation.
     """
 
-    def __init__(self, inner):
-        self.referenced_by = {}
+    def __init__(self, inner) -> None:
+        self.referenced_by: Dict[_ReferenceKey, _CachedRelation] = {}
         self.inner = inner
 
     def __str__(self) -> str:

core/dbt/adapters/contracts/connection.py

@@ -26,12 +26,12 @@
 from dbt.common.contracts.util import Replaceable
 from dbt.common.util import md5
 
-# TODO: dbt.events dependency
-from dbt.events.functions import fire_event
-from dbt.events.types import NewConnectionOpening
+# TODO: dbt.common.events dependency
+from dbt.common.events.functions import fire_event
+from dbt.common.events.types import NewConnectionOpening
 
 # TODO: this is a very bad dependency - shared global state
-from dbt.events.contextvars import get_node_info
+from dbt.common.events.contextvars import get_node_info
 
 
 class Identifier(ValidatedStringMixin):
@@ -111,7 +111,7 @@ class LazyHandle:
     connection, updating the handle on the Connection.
     """
 
-    def __init__(self, opener: Callable[[Connection], Connection]):
+    def __init__(self, opener: Callable[[Connection], Connection]) -> None:
         self.opener = opener
 
     def resolve(self, connection: Connection) -> Connection:

core/dbt/adapters/factory.py

@@ -8,8 +8,8 @@
 from dbt.adapters.base.plugin import AdapterPlugin
 from dbt.adapters.protocol import AdapterConfig, AdapterProtocol, RelationProtocol
 from dbt.adapters.contracts.connection import AdapterRequiredConfig, Credentials
-from dbt.events.functions import fire_event
-from dbt.events.types import AdapterImportError, PluginLoadError, AdapterRegistered
+from dbt.common.events.functions import fire_event
+from dbt.common.events.types import AdapterImportError, PluginLoadError, AdapterRegistered
 from dbt.exceptions import DbtInternalError, DbtRuntimeError
 from dbt.include.global_project import PACKAGE_PATH as GLOBAL_PROJECT_PATH
 from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME

core/dbt/adapters/protocol.py

@@ -90,7 +90,7 @@ class AdapterProtocol(  # type: ignore[misc]
     ConnectionManager: Type[ConnectionManager_T]
     connections: ConnectionManager_T
 
-    def __init__(self, config: AdapterRequiredConfig):
+    def __init__(self, config: AdapterRequiredConfig) -> None:
         ...
 
     @classmethod

core/dbt/adapters/sql/connections.py

@@ -8,9 +8,9 @@
 import dbt.exceptions
 from dbt.adapters.base import BaseConnectionManager
 from dbt.adapters.contracts.connection import Connection, ConnectionState, AdapterResponse
-from dbt.events.functions import fire_event
-from dbt.events.types import ConnectionUsed, SQLQuery, SQLCommit, SQLQueryStatus
-from dbt.events.contextvars import get_node_info
+from dbt.common.events.functions import fire_event
+from dbt.common.events.types import ConnectionUsed, SQLQuery, SQLCommit, SQLQueryStatus
+from dbt.common.events.contextvars import get_node_info
 from dbt.utils import cast_to_str
 
 

core/dbt/adapters/sql/impl.py

@@ -6,8 +6,8 @@
 from dbt.adapters.base import BaseAdapter, available
 from dbt.adapters.cache import _make_ref_key_dict
 from dbt.adapters.sql import SQLConnectionManager
-from dbt.events.functions import fire_event
-from dbt.events.types import ColTypeChange, SchemaCreation, SchemaDrop
+from dbt.common.events.functions import fire_event
+from dbt.common.events.types import ColTypeChange, SchemaCreation, SchemaDrop
 
 
 from dbt.adapters.base.relation import BaseRelation

core/dbt/cli/main.py

@@ -20,7 +20,7 @@
     CatalogArtifact,
     RunExecutionResult,
 )
-from dbt.events.base_types import EventMsg
+from dbt.common.events.base_types import EventMsg
 from dbt.task.build import BuildTask
 from dbt.task.clean import CleanTask
 from dbt.task.clone import CloneTask
@@ -64,7 +64,7 @@ def __init__(
         self,
         manifest: Optional[Manifest] = None,
         callbacks: Optional[List[Callable[[EventMsg], None]]] = None,
-    ):
+    ) -> None:
         self.manifest = manifest
 
         if callbacks is None:

core/dbt/cli/options.py

@@ -8,7 +8,7 @@
 # Implementation from: https://stackoverflow.com/a/48394004
 # Note MultiOption options must be specified with type=tuple or type=ChoiceTuple (https://github.com/pallets/click/issues/2012)
 class MultiOption(click.Option):
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args, **kwargs) -> None:
         self.save_other_options = kwargs.pop("save_other_options", True)
         nargs = kwargs.pop("nargs", -1)
         assert nargs == -1, "nargs, if set, must be -1 not {}".format(nargs)

core/dbt/cli/requires.py

@@ -9,15 +9,20 @@
 from dbt.cli.flags import Flags
 from dbt.config import RuntimeConfig
 from dbt.config.runtime import load_project, load_profile, UnsetProfile
-from dbt.events.functions import fire_event, LOG_VERSION, set_invocation_id, setup_event_logger
-from dbt.events.types import (
+from dbt.common.events.functions import (
+    fire_event,
+    LOG_VERSION,
+    set_invocation_id,
+    setup_event_logger,
+)
+from dbt.common.events.types import (
     CommandCompleted,
     MainReportVersion,
     MainReportArgs,
     MainTrackingUserState,
 )
-from dbt.events.helpers import get_json_string_utcnow
-from dbt.events.types import MainEncounteredError, MainStackTrace
+from dbt.common.events.helpers import get_json_string_utcnow
+from dbt.common.events.types import MainEncounteredError, MainStackTrace
 from dbt.exceptions import Exception as DbtException, DbtProjectError, FailFastError
 from dbt.parser.manifest import ManifestLoader, write_manifest
 from dbt.profiler import profiler

core/dbt/clients/agate_helper.py

@@ -165,7 +165,7 @@ class _NullMarker:
 
 
 class ColumnTypeBuilder(Dict[str, NullableAgateType]):
-    def __init__(self):
+    def __init__(self) -> None:
         super().__init__()
 
     def __setitem__(self, key, value):

core/dbt/clients/git.py

@@ -2,8 +2,8 @@
 import os.path
 
 from dbt.clients.system import run_cmd, rmdir
-from dbt.events.functions import fire_event
-from dbt.events.types import (
+from dbt.common.events.functions import fire_event
+from dbt.common.events.types import (
     GitSparseCheckoutSubdirectory,
     GitProgressCheckoutRevision,
     GitProgressUpdatingExistingDependency,

core/dbt/clients/registry.py

@@ -1,8 +1,8 @@
 import functools
 from typing import Any, Dict, List
 import requests
-from dbt.events.functions import fire_event
-from dbt.events.types import (
+from dbt.common.events.functions import fire_event
+from dbt.common.events.types import (
     RegistryProgressGETRequest,
     RegistryProgressGETResponse,
     RegistryIndexProgressGETRequest,

core/dbt/clients/system.py

@@ -15,8 +15,8 @@
 
 import dbt.exceptions
 import requests
-from dbt.events.functions import fire_event
-from dbt.events.types import (
+from dbt.common.events.functions import fire_event
+from dbt.common.events.types import (
     SystemCouldNotWrite,
     SystemExecutingCmd,
     SystemStdOut,

core/dbt/events/README.md → core/dbt/common/events/README.md

@@ -43,7 +43,7 @@ from dbt.logger import GLOBAL_LOGGER as logger
 
 Simply change it to these two lines with your adapter's database name, and all your existing call sites will now use the new system for v1.0:
 ```python
-from dbt.events import AdapterLogger
+from dbt.common.events import AdapterLogger
 logger = AdapterLogger("<database name>")
 # e.g. AdapterLogger("Snowflake")
 ```
@@ -52,4 +52,4 @@ logger = AdapterLogger("<database name>")
 
 After adding a new message in `types.proto`, either:
 - In the repository root directory: `make proto_types`
-- In the `core/dbt/events` directory: `protoc -I=. --python_out=. types.proto`
+- In the `core/dbt/common/events` directory: `protoc -I=. --python_out=. types.proto`

core/dbt/events/adapter_endpoint.py → core/dbt/common/events/adapter_endpoint.py

@@ -1,8 +1,10 @@
 import traceback
 from dataclasses import dataclass
-from dbt.events.functions import fire_event
-from dbt.events.contextvars import get_node_info
-from dbt.events.types import (
+
+from dbt.common.events.event_handler import set_package_logging
+from dbt.common.events.functions import fire_event, EVENT_MANAGER
+from dbt.common.events.contextvars import get_node_info
+from dbt.common.events.types import (
     AdapterEventDebug,
     AdapterEventInfo,
     AdapterEventWarning,
@@ -15,32 +17,32 @@
 class AdapterLogger:
     name: str
 
-    def debug(self, msg, *args):
+    def debug(self, msg, *args) -> None:
         event = AdapterEventDebug(
             name=self.name, base_msg=str(msg), args=list(args), node_info=get_node_info()
         )
         fire_event(event)
 
-    def info(self, msg, *args):
+    def info(self, msg, *args) -> None:
         event = AdapterEventInfo(
             name=self.name, base_msg=str(msg), args=list(args), node_info=get_node_info()
         )
         fire_event(event)
 
-    def warning(self, msg, *args):
+    def warning(self, msg, *args) -> None:
         event = AdapterEventWarning(
             name=self.name, base_msg=str(msg), args=list(args), node_info=get_node_info()
         )
         fire_event(event)
 
-    def error(self, msg, *args):
+    def error(self, msg, *args) -> None:
         event = AdapterEventError(
             name=self.name, base_msg=str(msg), args=list(args), node_info=get_node_info()
         )
         fire_event(event)
 
     # The default exc_info=True is what makes this method different
-    def exception(self, msg, *args):
+    def exception(self, msg, *args) -> None:
         exc_info = str(traceback.format_exc())
         event = AdapterEventError(
             name=self.name,
@@ -51,8 +53,15 @@ def exception(self, msg, *args):
         )
         fire_event(event)
 
-    def critical(self, msg, *args):
+    def critical(self, msg, *args) -> None:
         event = AdapterEventError(
             name=self.name, base_msg=str(msg), args=list(args), node_info=get_node_info()
         )
         fire_event(event)
+
+    @staticmethod
+    def set_adapter_dependency_log_level(package_name, level):
+        """By default, dbt suppresses non-dbt package logs. This method allows
+        you to set the log level for a specific package.
+        """
+        set_package_logging(package_name, level, EVENT_MANAGER)