Move events to common (#8676)

* Move events to common

* More Type Annotations (#8536)

* Extend use of type annotations in the events module.

* Add return type of None to more __init__ definitions.

* Still more type annotations adding -> None to __init__

* Tweak per review

* Allow adapters to include python package logging in dbt logs (#8643)

* add set_package_log_level functionality

* set package handler

* set package handler

* add logging about stting up logging

* test event log handler

* add event log handler

* add event log level

* rename package and add unit tests

* revert logfile config change

* cleanup and add code comments

* add changie

* swap function for dict

* add additional unit tests

* fix unit test

* update README and protos

* fix formatting

* update precommit

---------

Co-authored-by: Peter Webb <peter.webb@dbtlabs.com>

.changes/unreleased/Features-20230915-123733.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: 'Allow adapters to include package logs in dbt standard logging '
+time: 2023-09-15T12:37:33.862862-07:00
+custom:
+  Author: colin-rogers-dbt
+  Issue: "7859"

.changes/unreleased/Under the Hood-20230831-164435.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Added more type annotations.
+time: 2023-08-31T16:44:35.737954-04:00
+custom:
+  Author: peterallenwebb
+  Issue: "8537"

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 # Configuration for pre-commit hooks (see https://pre-commit.com/).
 # Eventually the hooks described here will be run as tests before merging each PR.
 
-exclude: ^(core/dbt/docs/build/|core/dbt/events/types_pb2.py)
+exclude: ^(core/dbt/docs/build/|core/dbt/common/events/types_pb2.py)
 
 # Force all unspecified python hooks to run python 3.8
 default_language_version:

Makefile

@@ -40,7 +40,7 @@ dev: dev_req ## Installs dbt-* packages in develop mode along with development d
 
 .PHONY: proto_types
 proto_types:  ## generates google protobuf python file from types.proto
-	protoc -I=./core/dbt/events --python_out=./core/dbt/events ./core/dbt/events/types.proto
+	protoc -I=./core/dbt/common/events --python_out=./core/dbt/common/events ./core/dbt/common/events/types.proto
 
 .PHONY: mypy
 mypy: .env ## Runs mypy against staged changes for static type checking.

core/dbt/adapters/base/connections.py

@@ -36,9 +36,9 @@
 from dbt.adapters.base.query_headers import (
     MacroQueryStringSetter,
 )
-from dbt.events import AdapterLogger
-from dbt.events.functions import fire_event
-from dbt.events.types import (
+from dbt.common.events import AdapterLogger
+from dbt.common.events.functions import fire_event
+from dbt.common.events.types import (
     NewConnection,
     ConnectionReused,
     ConnectionLeftOpenInCleanup,
@@ -48,7 +48,7 @@
     Rollback,
     RollbackFailed,
 )
-from dbt.events.contextvars import get_node_info
+from dbt.common.events.contextvars import get_node_info
 from dbt import flags
 from dbt.utils import cast_to_str
 

core/dbt/adapters/base/impl.py

@@ -55,8 +55,8 @@
 from dbt.clients.jinja import MacroGenerator
 from dbt.contracts.graph.manifest import Manifest, MacroManifest
 from dbt.contracts.graph.nodes import ResultNode
-from dbt.events.functions import fire_event, warn_or_error
-from dbt.events.types import (
+from dbt.common.events.functions import fire_event, warn_or_error
+from dbt.common.events.types import (
     CacheMiss,
     ListRelations,
     CodeExecution,

core/dbt/adapters/cache.py

@@ -14,8 +14,8 @@
     ReferencedLinkNotCachedError,
     TruncatedModelNameCausedCollisionError,
 )
-from dbt.events.functions import fire_event, fire_event_if
-from dbt.events.types import CacheAction, CacheDumpGraph
+from dbt.common.events.functions import fire_event, fire_event_if
+from dbt.common.events.types import CacheAction, CacheDumpGraph
 from dbt.flags import get_flags
 from dbt.utils import lowercase
 

core/dbt/adapters/contracts/connection.py

@@ -26,12 +26,12 @@
 from dbt.common.contracts.util import Replaceable
 from dbt.common.util import md5
 
-# TODO: dbt.events dependency
-from dbt.events.functions import fire_event
-from dbt.events.types import NewConnectionOpening
+# TODO: dbt.common.events dependency
+from dbt.common.events.functions import fire_event
+from dbt.common.events.types import NewConnectionOpening
 
 # TODO: this is a very bad dependency - shared global state
-from dbt.events.contextvars import get_node_info
+from dbt.common.events.contextvars import get_node_info
 
 
 class Identifier(ValidatedStringMixin):

core/dbt/adapters/factory.py

@@ -8,8 +8,8 @@
 from dbt.adapters.base.plugin import AdapterPlugin
 from dbt.adapters.protocol import AdapterConfig, AdapterProtocol, RelationProtocol
 from dbt.adapters.contracts.connection import AdapterRequiredConfig, Credentials
-from dbt.events.functions import fire_event
-from dbt.events.types import AdapterImportError, PluginLoadError, AdapterRegistered
+from dbt.common.events.functions import fire_event
+from dbt.common.events.types import AdapterImportError, PluginLoadError, AdapterRegistered
 from dbt.exceptions import DbtInternalError, DbtRuntimeError
 from dbt.include.global_project import PACKAGE_PATH as GLOBAL_PROJECT_PATH
 from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME

core/dbt/adapters/sql/connections.py

@@ -8,9 +8,9 @@
 import dbt.exceptions
 from dbt.adapters.base import BaseConnectionManager
 from dbt.adapters.contracts.connection import Connection, ConnectionState, AdapterResponse
-from dbt.events.functions import fire_event
-from dbt.events.types import ConnectionUsed, SQLQuery, SQLCommit, SQLQueryStatus
-from dbt.events.contextvars import get_node_info
+from dbt.common.events.functions import fire_event
+from dbt.common.events.types import ConnectionUsed, SQLQuery, SQLCommit, SQLQueryStatus
+from dbt.common.events.contextvars import get_node_info
 from dbt.utils import cast_to_str
 
 

core/dbt/adapters/sql/impl.py

@@ -6,8 +6,8 @@
 from dbt.adapters.base import BaseAdapter, available
 from dbt.adapters.cache import _make_ref_key_dict
 from dbt.adapters.sql import SQLConnectionManager
-from dbt.events.functions import fire_event
-from dbt.events.types import ColTypeChange, SchemaCreation, SchemaDrop
+from dbt.common.events.functions import fire_event
+from dbt.common.events.types import ColTypeChange, SchemaCreation, SchemaDrop
 
 
 from dbt.adapters.base.relation import BaseRelation

core/dbt/cli/main.py

@@ -21,7 +21,7 @@
     CatalogArtifact,
     RunExecutionResult,
 )
-from dbt.events.base_types import EventMsg
+from dbt.common.events.base_types import EventMsg
 from dbt.task.build import BuildTask
 from dbt.task.clean import CleanTask
 from dbt.task.clone import CloneTask

core/dbt/cli/requires.py

@@ -9,17 +9,23 @@
 from dbt.cli.flags import Flags
 from dbt.config import RuntimeConfig
 from dbt.config.runtime import load_project, load_profile, UnsetProfile
-from dbt.events.base_types import EventLevel
-from dbt.events.functions import fire_event, LOG_VERSION, set_invocation_id, setup_event_logger
-from dbt.events.types import (
+
+from dbt.common.events.base_types import EventLevel
+from dbt.common.events.functions import (
+    fire_event,
+    LOG_VERSION,
+    set_invocation_id,
+    setup_event_logger,
+)
+from dbt.common.events.types import (
     CommandCompleted,
     MainReportVersion,
     MainReportArgs,
     MainTrackingUserState,
     ResourceReport,
 )
-from dbt.events.helpers import get_json_string_utcnow
-from dbt.events.types import MainEncounteredError, MainStackTrace
+from dbt.common.events.helpers import get_json_string_utcnow
+from dbt.common.events.types import MainEncounteredError, MainStackTrace
 from dbt.exceptions import Exception as DbtException, DbtProjectError, FailFastError
 from dbt.parser.manifest import ManifestLoader, write_manifest
 from dbt.profiler import profiler

core/dbt/clients/git.py

@@ -2,8 +2,8 @@
 import os.path
 
 from dbt.clients.system import run_cmd, rmdir
-from dbt.events.functions import fire_event
-from dbt.events.types import (
+from dbt.common.events.functions import fire_event
+from dbt.common.events.types import (
     GitSparseCheckoutSubdirectory,
     GitProgressCheckoutRevision,
     GitProgressUpdatingExistingDependency,

core/dbt/clients/registry.py

@@ -1,8 +1,8 @@
 import functools
 from typing import Any, Dict, List
 import requests
-from dbt.events.functions import fire_event
-from dbt.events.types import (
+from dbt.common.events.functions import fire_event
+from dbt.common.events.types import (
     RegistryProgressGETRequest,
     RegistryProgressGETResponse,
     RegistryIndexProgressGETRequest,

core/dbt/clients/system.py

@@ -15,8 +15,8 @@
 
 import dbt.exceptions
 import requests
-from dbt.events.functions import fire_event
-from dbt.events.types import (
+from dbt.common.events.functions import fire_event
+from dbt.common.events.types import (
     SystemCouldNotWrite,
     SystemExecutingCmd,
     SystemStdOut,

core/dbt/events/README.md → core/dbt/common/events/README.md

@@ -43,7 +43,7 @@ from dbt.logger import GLOBAL_LOGGER as logger
 
 Simply change it to these two lines with your adapter's database name, and all your existing call sites will now use the new system for v1.0:
 ```python
-from dbt.events import AdapterLogger
+from dbt.common.events import AdapterLogger
 logger = AdapterLogger("<database name>")
 # e.g. AdapterLogger("Snowflake")
 ```
@@ -52,4 +52,4 @@ logger = AdapterLogger("<database name>")
 
 After adding a new message in `types.proto`, either:
 - In the repository root directory: `make proto_types`
-- In the `core/dbt/events` directory: `protoc -I=. --python_out=. types.proto`
+- In the `core/dbt/common/events` directory: `protoc -I=. --python_out=. types.proto`

core/dbt/events/adapter_endpoint.py → core/dbt/common/events/adapter_endpoint.py

@@ -1,9 +1,10 @@
 import traceback
 from dataclasses import dataclass
-from dbt.events.functions import fire_event, EVENT_MANAGER
-from dbt.events.contextvars import get_node_info
-from dbt.events.event_handler import set_package_logging
-from dbt.events.types import (
+
+from dbt.common.events.event_handler import set_package_logging
+from dbt.common.events.functions import fire_event, EVENT_MANAGER
+from dbt.common.events.contextvars import get_node_info
+from dbt.common.events.types import (
     AdapterEventDebug,
     AdapterEventInfo,
     AdapterEventWarning,

core/dbt/events/base_types.py → core/dbt/common/events/base_types.py

@@ -1,11 +1,11 @@
 from enum import Enum
 import os
 import threading
-from dbt.events import types_pb2
+from dbt.common.events import types_pb2
 import sys
 from google.protobuf.json_format import ParseDict, MessageToDict, MessageToJson
 from google.protobuf.message import Message
-from dbt.events.helpers import get_json_string_utcnow
+from dbt.common.events.helpers import get_json_string_utcnow
 from typing import Optional
 
 if sys.version_info >= (3, 8):
@@ -21,13 +21,13 @@
 
 
 def get_global_metadata_vars() -> dict:
-    from dbt.events.functions import get_metadata_vars
+    from dbt.common.events.functions import get_metadata_vars
 
     return get_metadata_vars()
 
 
 def get_invocation_id() -> str:
-    from dbt.events.functions import get_invocation_id
+    from dbt.common.events.functions import get_invocation_id
 
     return get_invocation_id()
 
@@ -72,8 +72,8 @@ def __init__(self, *args, **kwargs) -> None:
             self.pb_msg = ParseDict(kwargs, msg_cls())
         except Exception:
             # Imports need to be here to avoid circular imports
-            from dbt.events.types import Note
-            from dbt.events.functions import fire_event
+            from dbt.common.events.types import Note
+            from dbt.common.events.functions import fire_event
 
             error_msg = f"[{class_name}]: Unable to parse dict {kwargs}"
             # If we're testing throw an error so that we notice failures

core/dbt/events/event_handler.py → core/dbt/common/events/event_handler.py

@@ -1,10 +1,10 @@
 import logging
 from typing import Union
 
-from dbt.events.base_types import EventLevel
-from dbt.events.types import Note
+from dbt.common.events.base_types import EventLevel
+from dbt.common.events.types import Note
+from dbt.common.events.eventmgr import IEventManager
 
-from dbt.events.eventmgr import IEventManager
 
 _log_level_to_event_level_map = {
     logging.DEBUG: EventLevel.DEBUG,

core/dbt/events/eventmgr.py → core/dbt/common/events/eventmgr.py

@@ -3,8 +3,8 @@
 from typing import Callable, List, Optional, Protocol, Tuple
 from uuid import uuid4
 
-from dbt.events.base_types import BaseEvent, EventLevel, msg_from_base_event, EventMsg
-from dbt.events.logger import LoggerConfig, _Logger, _TextLogger, _JsonLogger, LineFormat
+from dbt.common.events.base_types import BaseEvent, EventLevel, msg_from_base_event, EventMsg
+from dbt.common.events.logger import LoggerConfig, _Logger, _TextLogger, _JsonLogger, LineFormat
 
 
 class EventManager:

core/dbt/events/functions.py → core/dbt/common/events/functions.py

@@ -1,9 +1,9 @@
 from dbt.constants import METADATA_ENV_PREFIX
-from dbt.events.base_types import BaseEvent, EventLevel, EventMsg
-from dbt.events.eventmgr import EventManager, IEventManager
-from dbt.events.logger import LoggerConfig, NoFilter, LineFormat
+from dbt.common.events.base_types import BaseEvent, EventLevel, EventMsg
+from dbt.common.events.eventmgr import EventManager, IEventManager
+from dbt.common.events.logger import LoggerConfig, NoFilter, LineFormat
 from dbt.exceptions import scrub_secrets, env_secrets
-from dbt.events.types import Note
+from dbt.common.events.types import Note
 from dbt.flags import get_flags, ENABLE_LEGACY_LOGGER
 from dbt.logger import GLOBAL_LOGGER, make_log_dir_if_missing
 from functools import partial

core/dbt/events/logger.py → core/dbt/common/events/logger.py

@@ -10,8 +10,8 @@
 from colorama import Style
 
 import dbt.utils
-from dbt.events.base_types import EventLevel, EventMsg
-from dbt.events.format import timestamp_to_datetime_string
+from dbt.common.events.base_types import EventLevel, EventMsg
+from dbt.common.events.format import timestamp_to_datetime_string
 
 # A Filter is a function which takes a BaseEvent and returns True if the event
 # should be logged, False otherwise.
@@ -172,7 +172,7 @@ def _get_thread_name(self) -> str:
 
 class _JsonLogger(_Logger):
     def create_line(self, msg: EventMsg) -> str:
-        from dbt.events.functions import msg_to_dict
+        from dbt.common.events.functions import msg_to_dict
 
         msg_dict = msg_to_dict(msg)
         raw_log_line = json.dumps(msg_dict, sort_keys=True, cls=dbt.utils.ForgivingJSONEncoder)

core/dbt/events/types.py → core/dbt/common/events/types.py

@@ -1,15 +1,19 @@
 import json
 
 from dbt.constants import MAXIMUM_SEED_SIZE_NAME, PIN_PACKAGE_URL
-from dbt.events.base_types import (
+from dbt.common.events.base_types import (
     DynamicLevel,
     DebugLevel,
     InfoLevel,
     WarnLevel,
     ErrorLevel,
     EventLevel,
 )
-from dbt.events.format import format_fancy_output_line, pluralize, timestamp_to_datetime_string
+from dbt.common.events.format import (
+    format_fancy_output_line,
+    pluralize,
+    timestamp_to_datetime_string,
+)
 from dbt.node_types import NodeType
 from dbt.ui import line_wrap_message, warning_tag, red, green, yellow
 

core/dbt/compilation.py

@@ -28,11 +28,11 @@
     DbtRuntimeError,
 )
 from dbt.graph import Graph
-from dbt.events.functions import fire_event, get_invocation_id
-from dbt.events.types import FoundStats, Note, WritingInjectedSQLForNode
-from dbt.events.contextvars import get_node_info
+from dbt.common.events.functions import fire_event, get_invocation_id
+from dbt.common.events.types import FoundStats, Note, WritingInjectedSQLForNode
+from dbt.common.events.contextvars import get_node_info
 from dbt.node_types import NodeType, ModelLanguage
-from dbt.events.format import pluralize
+from dbt.common.events.format import pluralize
 import dbt.tracking
 import dbt.task.list as list_task
 import sqlparse

core/dbt/config/profile.py

@@ -17,8 +17,8 @@
     DbtRuntimeError,
     ProfileConfigError,
 )
-from dbt.events.types import MissingProfileTarget
-from dbt.events.functions import fire_event
+from dbt.common.events.types import MissingProfileTarget
+from dbt.common.events.functions import fire_event
 from dbt.utils import coerce_dict_str
 
 from .renderer import ProfileRenderer

core/dbt/config/runtime.py

@@ -23,8 +23,8 @@
 from dbt.contracts.project import Configuration, UserConfig
 from dbt.contracts.relation import ComponentName
 from dbt.common.dataclass_schema import ValidationError
-from dbt.events.functions import warn_or_error
-from dbt.events.types import UnusedResourceConfigPath
+from dbt.common.events.functions import warn_or_error
+from dbt.common.events.types import UnusedResourceConfigPath
 from dbt.exceptions import (
     ConfigContractBrokenError,
     DbtProjectError,

core/dbt/config/utils.py

@@ -2,8 +2,8 @@
 
 
 from dbt.clients import yaml_helper
-from dbt.events.functions import fire_event
-from dbt.events.types import InvalidOptionYAML
+from dbt.common.events.functions import fire_event
+from dbt.common.events.types import InvalidOptionYAML
 from dbt.exceptions import DbtValidationError, OptionNotYamlDictError