Support hierarchical config setting for SavedQueryExport configs

.changes/unreleased/Features-20231110-154255.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Support setting export configs hierarchically via saved query and project configs
+time: 2023-11-10T15:42:55.042317-08:00
+custom:
+  Author: QMalcolm
+  Issue: "8956"

core/dbt/contracts/graph/model_config.py

@@ -15,6 +15,7 @@
 from dbt.exceptions import DbtInternalError, CompilationError
 from dbt import hooks
 from dbt.node_types import NodeType, AccessType
+from dbt_semantic_interfaces.type_enums.export_destination_type import ExportDestinationType
 from mashumaro.jsonschema.annotations import Pattern
 
 
@@ -407,6 +408,8 @@ class SavedQueryConfig(BaseConfig):
         default_factory=dict,
         metadata=MergeBehavior.Update.meta(),
     )
+    export_as: Optional[ExportDestinationType] = None
+    schema: Optional[str] = None
 
 
 @dataclass

core/dbt/contracts/graph/unparsed.py

@@ -19,7 +19,6 @@
 from dbt.exceptions import CompilationError, ParsingError, DbtInternalError
 
 from dbt.dataclass_schema import dbtClassMixin, StrEnum, ExtensibleDbtClassMixin, ValidationError
-from dbt_semantic_interfaces.type_enums.export_destination_type import ExportDestinationType
 
 from dataclasses import dataclass, field
 from datetime import timedelta
@@ -729,21 +728,12 @@ class UnparsedQueryParams(dbtClassMixin):
     where: Optional[Union[str, List[str]]] = None
 
 
-@dataclass
-class UnparsedExportConfig(dbtClassMixin):
-    """Nested configuration attributes for exports."""
-
-    export_as: ExportDestinationType
-    schema: Optional[str] = None
-    alias: Optional[str] = None
-
-
 @dataclass
 class UnparsedExport(dbtClassMixin):
     """Configuration for writing query results to a table."""
 
     name: str
-    config: UnparsedExportConfig
+    config: Dict[str, Any] = field(default_factory=dict)
 
 
 @dataclass

core/dbt/parser/schema_yaml_readers.py

@@ -6,7 +6,6 @@
     UnparsedDimensionTypeParams,
     UnparsedEntity,
     UnparsedExport,
-    UnparsedExportConfig,
     UnparsedExposure,
     UnparsedGroup,
     UnparsedMeasure,
@@ -19,6 +18,7 @@
     UnparsedSavedQuery,
     UnparsedSemanticModel,
 )
+from dbt.contracts.graph.model_config import SavedQueryConfig
 from dbt.contracts.graph.nodes import (
     Exposure,
     Group,
@@ -57,7 +57,7 @@
     MetricType,
     TimeGranularity,
 )
-from typing import List, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 
 
 def parse_where_filter(
@@ -669,16 +669,25 @@
 
         return config
 
-    def _get_export_config(self, unparsed: UnparsedExportConfig) -> ExportConfig:
-        return ExportConfig(
-            export_as=unparsed.export_as,
-            schema_name=unparsed.schema,
-            alias=unparsed.alias,
+    def _get_export_config(
+        self, unparsed: Dict[str, Any], saved_query_config: SavedQueryConfig
+    ) -> ExportConfig:
+        # Combine the two dictionaries using dictionary unpacking
+        # the second dictionary is the one whose keys take priority
+        combined = {**saved_query_config.__dict__, **unparsed}
+        # `schema` is the user facing attribute, but for DSI protocol purposes we track it as `schema_name`
+        if combined.get("schema") is not combined.get("schema_name") is None:
+            combined["schema_name"] = combined["schema"]
+
+        return ExportConfig.from_dict(combined)
+
+    def _get_export(
+        self, unparsed: UnparsedExport, saved_query_config: SavedQueryConfig
+    ) -> Export:
+        return Export(
+            name=unparsed.name, config=self._get_export_config(unparsed.config, saved_query_config)
         )
 
-    def _get_export(self, unparsed: UnparsedExport) -> Export:
-        return Export(name=unparsed.name, config=self._get_export_config(unparsed.config))
-
     def _get_query_params(self, unparsed: UnparsedQueryParams) -> QueryParams:
         return QueryParams(
             group_by=unparsed.group_by,
@@ -721,7 +730,7 @@
             resource_type=NodeType.SavedQuery,
             unique_id=unique_id,
             query_params=self._get_query_params(unparsed.query_params),
-            exports=[self._get_export(export) for export in unparsed.exports],
+            exports=[self._get_export(export, config) for export in unparsed.exports],
             config=config,
             unrendered_config=unrendered_config,
             group=config.group,

tests/functional/saved_queries/fixtures.py

@@ -24,3 +24,70 @@
             export_as: table
             schema: my_export_schema_name
 """
+
+saved_query_with_extra_config_attributes_yml = """
+version: 2
+
+saved_queries:
+  - name: test_saved_query
+    description: "{{ doc('saved_query_description') }}"
+    label: Test Saved Query
+    query_params:
+        metrics:
+            - simple_metric
+        group_by:
+            - "Dimension('user__ds')"
+        where:
+            - "{{ Dimension('user__ds', 'DAY') }} <= now()"
+            - "{{ Dimension('user__ds', 'DAY') }} >= '2023-01-01'"
+    exports:
+        - name: my_export
+          config:
+            my_random_config: 'I have this for some reason'
+            export_as: table
+"""
+
+saved_query_with_export_configs_defined_at_saved_query_level_yml = """
+version: 2
+
+saved_queries:
+  - name: test_saved_query
+    description: "{{ doc('saved_query_description') }}"
+    label: Test Saved Query
+    config:
+      export_as: table
+      schema: my_default_export_schema
+    query_params:
+        metrics:
+            - simple_metric
+        group_by:
+            - "Dimension('user__ds')"
+        where:
+            - "{{ Dimension('user__ds', 'DAY') }} <= now()"
+            - "{{ Dimension('user__ds', 'DAY') }} >= '2023-01-01'"
+    exports:
+        - name: my_export
+          config:
+            export_as: view
+            schema: my_custom_export_schema
+        - name: my_export2
+"""
+
+saved_query_without_export_configs_defined_yml = """
+version: 2
+
+saved_queries:
+  - name: test_saved_query
+    description: "{{ doc('saved_query_description') }}"
+    label: Test Saved Query
+    query_params:
+        metrics:
+            - simple_metric
+        group_by:
+            - "Dimension('user__ds')"
+        where:
+            - "{{ Dimension('user__ds', 'DAY') }} <= now()"
+            - "{{ Dimension('user__ds', 'DAY') }} >= '2023-01-01'"
+    exports:
+        - name: my_export
+"""

tests/functional/saved_queries/test_configs.py

@@ -0,0 +1,186 @@
+import pytest
+
+from dbt.contracts.graph.manifest import Manifest
+from dbt.tests.util import update_config_file
+from dbt_semantic_interfaces.type_enums.export_destination_type import ExportDestinationType
+from tests.functional.assertions.test_runner import dbtTestRunner
+from tests.functional.configs.fixtures import BaseConfigProject
+from tests.functional.saved_queries.fixtures import (
+    saved_queries_yml,
+    saved_query_description,
+    saved_query_with_extra_config_attributes_yml,
+    saved_query_with_export_configs_defined_at_saved_query_level_yml,
+    saved_query_without_export_configs_defined_yml,
+)
+from tests.functional.semantic_models.fixtures import (
+    fct_revenue_sql,
+    metricflow_time_spine_sql,
+    schema_yml,
+)
+
+
+class TestSavedQueryConfigs(BaseConfigProject):
+    @pytest.fixture(scope="class")
+    def project_config_update(self):
+        return {
+            "saved-queries": {
+                "test": {
+                    "test_saved_query": {
+                        "+enabled": True,
+                        "+export_as": ExportDestinationType.VIEW.value,
+                        "+schema": "my_default_export_schema",
+                    }
+                },
+            },
+        }
+
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "saved_queries.yml": saved_query_with_extra_config_attributes_yml,
+            "schema.yml": schema_yml,
+            "fct_revenue.sql": fct_revenue_sql,
+            "metricflow_time_spine.sql": metricflow_time_spine_sql,
+            "docs.md": saved_query_description,
+        }
+
+    def test_basic_saved_query_config(
+        self,
+        project,
+    ):
+        runner = dbtTestRunner()
+
+        # parse with default fixture project config
+        result = runner.invoke(["parse"])
+        assert result.success
+        assert isinstance(result.result, Manifest)
+        assert len(result.result.saved_queries) == 1
+        saved_query = result.result.saved_queries["saved_query.test.test_saved_query"]
+        assert saved_query.config.export_as == ExportDestinationType.VIEW
+        assert saved_query.config.schema == "my_default_export_schema"
+
+        # disable the saved_query via project config and rerun
+        config_patch = {"saved-queries": {"test": {"test_saved_query": {"+enabled": False}}}}
+        update_config_file(config_patch, project.project_root, "dbt_project.yml")
+        result = runner.invoke(["parse"])
+        assert result.success
+        assert len(result.result.saved_queries) == 0
+
+
+class TestExportConfigsWithAdditionalProperties(BaseConfigProject):
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "saved_queries.yml": saved_queries_yml,
+            "schema.yml": schema_yml,
+            "fct_revenue.sql": fct_revenue_sql,
+            "metricflow_time_spine.sql": metricflow_time_spine_sql,
+            "docs.md": saved_query_description,
+        }
+
+    def test_extra_config_properties_dont_break_parsing(self, project):
+        runner = dbtTestRunner()
+
+        # parse with default fixture project config
+        result = runner.invoke(["parse"])
+        assert result.success
+        assert isinstance(result.result, Manifest)
+        assert len(result.result.saved_queries) == 1
+        saved_query = result.result.saved_queries["saved_query.test.test_saved_query"]
+        assert len(saved_query.exports) == 1
+        assert saved_query.exports[0].config.__dict__.get("my_random_config") is None
+
+
+class TestInheritingExportConfigFromSavedQueryConfig(BaseConfigProject):
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "saved_queries.yml": saved_query_with_export_configs_defined_at_saved_query_level_yml,
+            "schema.yml": schema_yml,
+            "fct_revenue.sql": fct_revenue_sql,
+            "metricflow_time_spine.sql": metricflow_time_spine_sql,
+            "docs.md": saved_query_description,
+        }
+
+    def test_export_config_inherits_from_saved_query(self, project):
+        runner = dbtTestRunner()
+
+        # parse with default fixture project config
+        result = runner.invoke(["parse"])
+        assert result.success
+        assert isinstance(result.result, Manifest)
+        assert len(result.result.saved_queries) == 1
+        saved_query = result.result.saved_queries["saved_query.test.test_saved_query"]
+        assert len(saved_query.exports) == 2
+
+        # assert Export `my_export` has its configs defined from itself because they should take priority
+        export1 = next(
+            (export for export in saved_query.exports if export.name == "my_export"), None
+        )
+        assert export1 is not None
+        assert export1.config.export_as == ExportDestinationType.VIEW
+        assert export1.config.export_as != saved_query.config.export_as
+        assert export1.config.schema_name == "my_custom_export_schema"
+        assert export1.config.schema_name != saved_query.config.schema
+
+        # assert Export `my_export` has its configs defined from the saved_query because they should take priority
+        export2 = next(
+            (export for export in saved_query.exports if export.name == "my_export2"), None
+        )
+        assert export2 is not None
+        assert export2.config.export_as == ExportDestinationType.TABLE
+        assert export2.config.export_as == saved_query.config.export_as
+        assert export2.config.schema_name == "my_default_export_schema"
+        assert export2.config.schema_name == saved_query.config.schema
+
+
+class TestInheritingExportConfigsFromProject(BaseConfigProject):
+    @pytest.fixture(scope="class")
+    def project_config_update(self):
+        return {
+            "saved-queries": {
+                "test": {
+                    "test_saved_query": {
+                        "+export_as": ExportDestinationType.VIEW.value,
+                    }
+                },
+            },
+        }
+
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "saved_queries.yml": saved_query_without_export_configs_defined_yml,
+            "schema.yml": schema_yml,
+            "fct_revenue.sql": fct_revenue_sql,
+            "metricflow_time_spine.sql": metricflow_time_spine_sql,
+            "docs.md": saved_query_description,
+        }
+
+    def test_export_config_inherits_from_project(
+        self,
+        project,
+    ):
+        runner = dbtTestRunner()
+
+        # parse with default fixture project config
+        result = runner.invoke(["parse"])
+        assert result.success
+        assert isinstance(result.result, Manifest)
+        assert len(result.result.saved_queries) == 1
+        saved_query = result.result.saved_queries["saved_query.test.test_saved_query"]
+        assert saved_query.config.export_as == ExportDestinationType.VIEW
+
+        # change export's `export_as` to `TABLE` via project config
+        config_patch = {
+            "saved-queries": {
+                "test": {"test_saved_query": {"+export_as": ExportDestinationType.TABLE.value}}
+            }
+        }
+        update_config_file(config_patch, project.project_root, "dbt_project.yml")
+        result = runner.invoke(["parse"])
+        assert result.success
+        assert isinstance(result.result, Manifest)
+        assert len(result.result.saved_queries) == 1
+        saved_query = result.result.saved_queries["saved_query.test.test_saved_query"]
+        assert saved_query.config.export_as == ExportDestinationType.TABLE