Model Deprecation (#7562)

* CT-2461: Work toward model deprecation

* CT-2461: Remove unneeded conversions

* CT-2461: Fix up unit tests for new fields, correct a couple oversights

* CT-2461: Remaining implementation and tests for model/ref deprecation warnings

* CT-2461: Changelog entry for deprecation warnings

* CT-2461: Refine datetime handling and tests

* CT-2461: Fix up unit test data

* CT-2461: Fix some more unit test data.

* CT-2461: Fix merge issues

* CT-2461: Code review items.

* CT-2461: Improve version -> str conversion

.changes/unreleased/Features-20230509-233329.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Added warnings for model and ref deprecations
+time: 2023-05-09T23:33:29.679333-04:00
+custom:
+  Author: peterallenwebb
+  Issue: "7433"

core/dbt/config/renderer.py

@@ -1,6 +1,7 @@
 from typing import Dict, Any, Tuple, Optional, Union, Callable
 import re
 import os
+from datetime import date
 
 from dbt.clients.jinja import get_rendered, catch_jinja
 from dbt.constants import SECRET_ENV_PREFIX
@@ -33,10 +34,10 @@ def render_entry(self, value: Any, keypath: Keypath) -> Any:
         return self.render_value(value, keypath)
 
     def render_value(self, value: Any, keypath: Optional[Keypath] = None) -> Any:
-        # keypath is ignored.
-        # if it wasn't read as a string, ignore it
+        # keypath is ignored (and someone who knows should explain why here)
         if not isinstance(value, str):
-            return value
+            return value if not isinstance(value, date) else value.isoformat()
+
         try:
             with catch_jinja():
                 return get_rendered(value, self.context, native=True)

core/dbt/contracts/graph/manifest.py

@@ -965,6 +965,23 @@ def analysis_lookup(self) -> AnalysisLookup:
             self._analysis_lookup = AnalysisLookup(self)
         return self._analysis_lookup
 
+    def resolve_refs(
+        self, source_node: GraphMemberNode, current_project: str
+    ) -> List[MaybeNonSource]:
+        resolved_refs: List[MaybeNonSource] = []
+        for ref in source_node.refs:
+            resolved = self.resolve_ref(
+                source_node,
+                ref.name,
+                ref.package,
+                ref.version,
+                current_project,
+                source_node.package_name,
+            )
+            resolved_refs.append(resolved)
+
+        return resolved_refs
+
     # Called by dbt.parser.manifest._process_refs_for_exposure, _process_refs_for_metric,
     # and dbt.parser.manifest._process_refs_for_node
     def resolve_ref(

core/dbt/contracts/graph/nodes.py

@@ -1,4 +1,5 @@
 import os
+from datetime import datetime
 import time
 from dataclasses import dataclass, field
 from enum import Enum
@@ -568,6 +569,7 @@ class ModelNode(CompiledNode):
     constraints: List[ModelLevelConstraint] = field(default_factory=list)
     version: Optional[NodeVersion] = None
     latest_version: Optional[NodeVersion] = None
+    deprecation_date: Optional[datetime] = None
     state_relation: Optional[StateRelation] = None
 
     @property
@@ -1415,6 +1417,7 @@ class ParsedNodePatch(ParsedPatch):
     version: Optional[NodeVersion]
     latest_version: Optional[NodeVersion]
     constraints: List[Dict[str, Any]]
+    deprecation_date: Optional[datetime]
 
 
 @dataclass

core/dbt/contracts/graph/unparsed.py

@@ -1,3 +1,4 @@
+import datetime
 import re
 
 from dbt import deprecations
@@ -154,6 +155,7 @@ class UnparsedVersion(dbtClassMixin):
     columns: Sequence[Union[dbt.helper_types.IncludeExclude, UnparsedColumn]] = field(
         default_factory=list
     )
+    deprecation_date: Optional[datetime.datetime] = None
 
     def __lt__(self, other):
         try:
@@ -192,6 +194,8 @@ def __post_init__(self):
             else:
                 self._unparsed_columns.append(column)
 
+        self.deprecation_date = normalize_date(self.deprecation_date)
+
 
 @dataclass
 class UnparsedAnalysisUpdate(HasConfig, HasColumnDocs, HasColumnProps, HasYamlMetadata):
@@ -210,6 +214,7 @@ class UnparsedModelUpdate(UnparsedNodeUpdate):
     access: Optional[str] = None
     latest_version: Optional[NodeVersion] = None
     versions: Sequence[UnparsedVersion] = field(default_factory=list)
+    deprecation_date: Optional[datetime.datetime] = None
 
     def __post_init__(self):
         if self.latest_version:
@@ -229,6 +234,8 @@ def __post_init__(self):
 
         self._version_map = {version.v: version for version in self.versions}
 
+        self.deprecation_date = normalize_date(self.deprecation_date)
+
     def get_columns_for_version(self, version: NodeVersion) -> List[UnparsedColumn]:
         if version not in self._version_map:
             raise DbtInternalError(
@@ -652,3 +659,18 @@ def validate(cls, data):
         super(UnparsedGroup, cls).validate(data)
         if data["owner"].get("name") is None and data["owner"].get("email") is None:
             raise ValidationError("Group owner must have at least one of 'name' or 'email'.")
+
+
+def normalize_date(d: Optional[datetime.date]) -> Optional[datetime.datetime]:
+    """Convert date to datetime (at midnight), and add local time zone if naive"""
+    if d is None:
+        return None
+
+    # convert date to datetime
+    dt = d if type(d) == datetime.datetime else datetime.datetime(d.year, d.month, d.day)
+
+    if not dt.tzinfo:
+        # date is naive, re-interpret as system time zone
+        dt = dt.astimezone()
+
+    return dt

core/dbt/contracts/publication.py

@@ -45,6 +45,7 @@ class PublicModel(dbtClassMixin, ManifestOrPublicNode):
     # list of model unique_ids
     public_node_dependencies: List[str] = field(default_factory=list)
     generated_at: datetime = field(default_factory=datetime.utcnow)
+    deprecation_date: Optional[datetime] = None
 
     @property
     def is_latest_version(self) -> bool:

core/dbt/events/types.proto

@@ -1174,6 +1174,49 @@ message UnpinnedRefNewVersionAvailableMsg {
     UnpinnedRefNewVersionAvailable data = 2;
 }
 
+// I065
+message DeprecatedModel {
+    string model_name = 1;
+    string model_version = 2;
+    string deprecation_date = 3;
+}
+
+message DeprecatedModelMsg {
+    EventInfo info = 1;
+    DeprecatedModel data = 2;
+}
+
+// I066
+message UpcomingReferenceDeprecation {
+    string model_name = 1;
+    string ref_model_package = 2;
+    string ref_model_name = 3;
+    string ref_model_version = 4;
+    string ref_model_latest_version = 5;
+    string ref_model_deprecation_date = 6;
+}
+
+message UpcomingReferenceDeprecationMsg {
+    EventInfo info = 1;
+    UpcomingReferenceDeprecation data = 2;
+}
+
+// I067
+message DeprecatedReference {
+    string model_name = 1;
+    string ref_model_package = 2;
+    string ref_model_name = 3;
+    string ref_model_version = 4;
+    string ref_model_latest_version = 5;
+    string ref_model_deprecation_date = 6;
+}
+
+message DeprecatedReferenceMsg {
+    EventInfo info = 1;
+    DeprecatedReference data = 2;
+}
+
+
 // M - Deps generation
 
 // M001

core/dbt/events/types.py

@@ -1146,6 +1146,62 @@ def message(self) -> str:
         return msg
 
 
+class DeprecatedModel(WarnLevel):
+    def code(self):
+        return "I065"
+
+    def message(self) -> str:
+        version = ".v" + self.model_version if self.model_version else ""
+        return (
+            f"Model {self.model_name}{version} has passed its deprecation date of {self.deprecation_date}. "
+            "This model should be disabled or removed."
+        )
+
+
+class UpcomingReferenceDeprecation(WarnLevel):
+    def code(self):
+        return "I066"
+
+    def message(self) -> str:
+        ref_model_version = ".v" + self.ref_model_version if self.ref_model_version else ""
+        msg = (
+            f"While compiling '{self.model_name}': Found a reference to {self.ref_model_name}{ref_model_version}, "
+            f"which is slated for deprecation on '{self.ref_model_deprecation_date}'. "
+        )
+
+        if self.ref_model_version and self.ref_model_version != self.ref_model_latest_version:
+            coda = (
+                f"A new version of '{self.ref_model_name}' is available. Try it out: "
+                f"{{{{ ref('{self.ref_model_package}', '{self.ref_model_name}', "
+                f"v='{self.ref_model_latest_version}') }}}}."
+            )
+            msg = msg + coda
+
+        return msg
+
+
+class DeprecatedReference(WarnLevel):
+    def code(self):
+        return "I067"
+
+    def message(self) -> str:
+        ref_model_version = ".v" + self.ref_model_version if self.ref_model_version else ""
+        msg = (
+            f"While compiling '{self.model_name}': Found a reference to {self.ref_model_name}{ref_model_version}, "
+            f"which was deprecated on '{self.ref_model_deprecation_date}'. "
+        )
+
+        if self.ref_model_version and self.ref_model_version != self.ref_model_latest_version:
+            coda = (
+                f"A new version of '{self.ref_model_name}' is available. Migrate now: "
+                f"{{{{ ref('{self.ref_model_package}', '{self.ref_model_name}', "
+                f"v='{self.ref_model_latest_version}') }}}}."
+            )
+            msg = msg + coda
+
+        return msg
+
+
 # =======================================================
 # M - Deps generation
 # =======================================================