From fc1a14a0e34aa1466847d1d43b5c2cbe31f9f5d2 Mon Sep 17 00:00:00 2001 From: Peter Webb Date: Wed, 30 Aug 2023 17:28:49 -0400 Subject: [PATCH] Include Compiled Node Attributes in run_results.json (#8492) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add compiled node properties to run_results.json * Include compiled-node attributes in run_results.json * Fix typo * Bump schema version of run_results * Fix test assertions * Update expected run_results to reflect new attributes * Code review changes * Fix mypy warnings for ManifestLoader.load() (#8443) * revert python version for docker images (#8445) * revert python version for docker images * add comment to not update python version, update changelog * Bumping version to 1.7.0b1 and generate changelog * [CT-3013] Fix parsing of `window_groupings` (#8454) * Update semantic model parsing tests to check measure non_additive_dimension spec * Make `window_groupings` default to empty list if not specified on `non_additive_dimension` * Add changie doc for `window_groupings` parsing fix * update `Number` class to handle integer values (#8306) * add show test for json data * oh changie my changie * revert unecessary cahnge to fixture * keep decimal class for precision methods, but return __int__ value * jerco updates * update integer type * update other tests * Update .changes/unreleased/Fixes-20230803-093502.yaml --------- Co-authored-by: Emily Rockman * Improve docker image README (#8212) * Improve docker image README - Fix unnecessary/missing newline escapes - Remove double whitespace between parameters - 2-space indent for extra lines in image build commands * Add changelog entry for #8212 * ADAP-814: Refactor prep for MV updates (#8459) * apply reformatting changes only for #8449 * add logging back to get_create_materialized_view_as_sql * changie * swap trigger (#8463) * update the implementation template (#8466) * update the implementation template * add colon * Split tests into classes (#8474) * add flaky decorator * split up tests into classes * revert update agate for int (#8478) * updated typing and methods to meet mypy standards (#8485) * Convert error to conditional warning for unversioned contracted model, fix msg format (#8451) * first pass, tests need updates * update proto defn * fixing tests * more test fixes * finish fixing test file * reformat the message * formatting messages * changelog * add event to unit test * feedback on message structure * WIP * fix up event to take in all fields * fix test * Fix ambiguous reference error for duplicate model names across packages with tests (#8488) * Safely remove external nodes from manifest (#8495) * [CT-2840] Improved semantic layer protocol satisfaction tests (#8456) * Test `SemanticModel` satisfies protocol when none of it's `Optionals` are specified * Add tests ensuring SourceFileMetadata and FileSlice satisfiy DSI protocols * Add test asserting Defaults obj satisfies protocol * Add test asserting SemanticModel with optionals specified satisfies protocol * Split dimension protocol satisfaction tests into with and without optionals * Simplify DSI Protocol import strategy in protocol satisfaction tests * Add test asserting DimensionValidtyParams satisfies protocol * Add test asserting DimensionTypeParams satisfies protocol * Split entity protocol satisfaction tests into with and without optionals * Split measure protocol satisfication tests and add measure aggregation params satisficaition test * Split metric protocol satisfaction test into optional specified an unspecified Additionally, create where_filter pytest fixture * Improve protocol satisfaction tests for MetricTypeParams and sub protocols Specifically we added/improved protocol satisfaction tests for - MetricTypeParams - MetricInput - MetricInputMeasure - MetricTimeWindow * Convert to using mashumaro jsonschema with acceptable performance (#8437) * Regenerate run_results schema after merging in changes from main. --------- Co-authored-by: Gerda Shank Co-authored-by: Matthew McKnight <91097623+McKnight-42@users.noreply.github.com> Co-authored-by: Github Build Bot Co-authored-by: Quigley Malcolm Co-authored-by: dave-connors-3 <73915542+dave-connors-3@users.noreply.github.com> Co-authored-by: Emily Rockman Co-authored-by: Jaime Martínez Rincón Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Co-authored-by: Michelle Ark --- .../unreleased/Features-20230821-103357.yaml | 6 + core/dbt/contracts/results.py | 13 +- core/dbt/tests/util.py | 12 +- schemas/dbt/run-results/v5.json | 229 ++++++++++++++++++ .../artifacts/expected_run_results.py | 51 ++++ tests/functional/assertions/test_runner.py | 43 ++++ tests/functional/compile/test_compile.py | 8 +- .../test_semantic_model_parsing.py | 13 +- 8 files changed, 361 insertions(+), 14 deletions(-) create mode 100644 .changes/unreleased/Features-20230821-103357.yaml create mode 100644 schemas/dbt/run-results/v5.json create mode 100644 tests/functional/assertions/test_runner.py diff --git a/.changes/unreleased/Features-20230821-103357.yaml b/.changes/unreleased/Features-20230821-103357.yaml new file mode 100644 index 00000000000..24f165beee2 --- /dev/null +++ b/.changes/unreleased/Features-20230821-103357.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add node attributes related to compilation to run_results.json +time: 2023-08-21T10:33:57.200883-04:00 +custom: + Author: peterallenwebb + Issue: "7519" diff --git a/core/dbt/contracts/results.py b/core/dbt/contracts/results.py index aaa036e6a74..4d8a945a674 100644 --- a/core/dbt/contracts/results.py +++ b/core/dbt/contracts/results.py @@ -1,7 +1,7 @@ import threading from dbt.contracts.graph.unparsed import FreshnessThreshold -from dbt.contracts.graph.nodes import SourceDefinition, ResultNode +from dbt.contracts.graph.nodes import CompiledNode, SourceDefinition, ResultNode from dbt.contracts.util import ( BaseArtifactMetadata, ArtifactMixin, @@ -203,9 +203,15 @@ class RunResultsMetadata(BaseArtifactMetadata): @dataclass class RunResultOutput(BaseResult): unique_id: str + compiled: Optional[bool] + compiled_code: Optional[str] + relation_name: Optional[str] def process_run_result(result: RunResult) -> RunResultOutput: + + compiled = isinstance(result.node, CompiledNode) + return RunResultOutput( unique_id=result.node.unique_id, status=result.status, @@ -215,6 +221,9 @@ def process_run_result(result: RunResult) -> RunResultOutput: message=result.message, adapter_response=result.adapter_response, failures=result.failures, + compiled=result.node.compiled if compiled else None, # type:ignore + compiled_code=result.node.compiled_code if compiled else None, # type:ignore + relation_name=result.node.relation_name if compiled else None, # type:ignore ) @@ -237,7 +246,7 @@ def write(self, path: str): @dataclass -@schema_version("run-results", 4) +@schema_version("run-results", 5) class RunResultsArtifact(ExecutionResult, ArtifactMixin): results: Sequence[RunResultOutput] args: Dict[str, Any] = field(default_factory=dict) diff --git a/core/dbt/tests/util.py b/core/dbt/tests/util.py index 541f0f0f089..7c22a4df94f 100644 --- a/core/dbt/tests/util.py +++ b/core/dbt/tests/util.py @@ -5,7 +5,7 @@ import json import warnings from datetime import datetime -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional from contextlib import contextmanager from dbt.adapters.factory import Adapter @@ -160,6 +160,16 @@ def get_manifest(project_root) -> Optional[Manifest]: return None +# Used in test cases to get the run_results.json file. +def get_run_results(project_root) -> Any: + path = os.path.join(project_root, "target", "run_results.json") + if os.path.exists(path): + with open(path) as run_result_text: + return json.load(run_result_text) + else: + return None + + # Used in tests to copy a file, usually from a data directory to the project directory def copy_file(src_path, src, dest_path, dest) -> None: # dest is a list, so that we can provide nested directories, like 'models' etc. diff --git a/schemas/dbt/run-results/v5.json b/schemas/dbt/run-results/v5.json new file mode 100644 index 00000000000..4e400e5f18a --- /dev/null +++ b/schemas/dbt/run-results/v5.json @@ -0,0 +1,229 @@ +{ + "$ref": "#/$defs/RunResultsArtifact", + "$defs": { + "BaseArtifactMetadata": { + "type": "object", + "title": "BaseArtifactMetadata", + "properties": { + "dbt_schema_version": { + "type": "string" + }, + "dbt_version": { + "type": "string", + "default": "1.7.0b1" + }, + "generated_at": { + "type": "string" + }, + "invocation_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "propertyNames": { + "type": "string" + } + } + }, + "additionalProperties": false, + "required": [ + "dbt_schema_version" + ] + }, + "TimingInfo": { + "type": "object", + "title": "TimingInfo", + "properties": { + "name": { + "type": "string" + }, + "started_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null + }, + "completed_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "additionalProperties": false, + "required": [ + "name" + ] + }, + "RunResultOutput": { + "type": "object", + "title": "RunResultOutput", + "properties": { + "status": { + "anyOf": [ + { + "enum": [ + "success", + "error", + "skipped" + ] + }, + { + "enum": [ + "pass", + "error", + "fail", + "warn", + "skipped" + ] + }, + { + "enum": [ + "pass", + "warn", + "error", + "runtime error" + ] + } + ] + }, + "timing": { + "type": "array", + "items": { + "$ref": "#/$defs/TimingInfo" + } + }, + "thread_id": { + "type": "string" + }, + "execution_time": { + "type": "number" + }, + "adapter_response": { + "type": "object", + "propertyNames": { + "type": "string" + } + }, + "message": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "failures": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "unique_id": { + "type": "string" + }, + "compiled": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "compiled_code": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "relation_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "required": [ + "status", + "timing", + "thread_id", + "execution_time", + "adapter_response", + "message", + "failures", + "unique_id", + "compiled", + "compiled_code", + "relation_name" + ] + }, + "RunResultsArtifact": { + "type": "object", + "title": "RunResultsArtifact", + "properties": { + "metadata": { + "$ref": "#/$defs/BaseArtifactMetadata" + }, + "results": { + "type": "array", + "items": { + "$ref": "#/$defs/RunResultOutput" + } + }, + "elapsed_time": { + "type": "number" + }, + "args": { + "type": "object", + "propertyNames": { + "type": "string" + } + } + }, + "additionalProperties": false, + "required": [ + "metadata", + "results", + "elapsed_time" + ] + } + }, + "$id": "https://schemas.getdbt.com/dbt/run-results/v5.json" +} diff --git a/tests/functional/artifacts/expected_run_results.py b/tests/functional/artifacts/expected_run_results.py index c6187440ca1..889dcf6353d 100644 --- a/tests/functional/artifacts/expected_run_results.py +++ b/tests/functional/artifacts/expected_run_results.py @@ -17,6 +17,9 @@ def expected_run_results(): "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, + "compiled": True, + "compiled_code": ANY, + "relation_name": ANY, }, { "status": "success", @@ -27,6 +30,9 @@ def expected_run_results(): "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, + "compiled": True, + "compiled_code": ANY, + "relation_name": ANY, }, { "status": "success", @@ -37,6 +43,9 @@ def expected_run_results(): "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, + "compiled": None, + "compiled_code": ANY, + "relation_name": None, }, { "status": "success", @@ -47,6 +56,9 @@ def expected_run_results(): "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, + "compiled": True, + "compiled_code": ANY, + "relation_name": ANY, }, { "status": "success", @@ -57,6 +69,9 @@ def expected_run_results(): "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, + "compiled": True, + "compiled_code": ANY, + "relation_name": None, }, { "status": "success", @@ -67,6 +82,9 @@ def expected_run_results(): "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, + "compiled": True, + "compiled_code": ANY, + "relation_name": None, }, { "status": "success", @@ -77,6 +95,9 @@ def expected_run_results(): "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, + "compiled": True, + "compiled_code": ANY, + "relation_name": None, }, ] @@ -92,6 +113,9 @@ def expected_references_run_results(): "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, + "compiled": True, + "compiled_code": ANY, + "relation_name": ANY, }, { "status": "success", @@ -102,6 +126,9 @@ def expected_references_run_results(): "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, + "compiled": True, + "compiled_code": ANY, + "relation_name": ANY, }, { "status": "success", @@ -112,6 +139,9 @@ def expected_references_run_results(): "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, + "compiled": None, + "compiled_code": ANY, + "relation_name": ANY, }, { "status": "success", @@ -122,6 +152,9 @@ def expected_references_run_results(): "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, + "compiled": True, + "compiled_code": ANY, + "relation_name": ANY, }, ] @@ -137,6 +170,9 @@ def expected_versions_run_results(): "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, + "compiled": True, + "compiled_code": ANY, + "relation_name": ANY, }, { "status": "success", @@ -147,6 +183,9 @@ def expected_versions_run_results(): "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, + "compiled": True, + "compiled_code": ANY, + "relation_name": ANY, }, { "status": "success", @@ -157,6 +196,9 @@ def expected_versions_run_results(): "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, + "compiled": True, + "compiled_code": ANY, + "relation_name": ANY, }, { "status": "success", @@ -167,6 +209,9 @@ def expected_versions_run_results(): "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, + "compiled": True, + "compiled_code": ANY, + "relation_name": ANY, }, { "status": "success", @@ -177,6 +222,9 @@ def expected_versions_run_results(): "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, + "compiled": True, + "compiled_code": ANY, + "relation_name": ANY, }, { "status": "success", @@ -187,5 +235,8 @@ def expected_versions_run_results(): "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, + "compiled": True, + "compiled_code": ANY, + "relation_name": ANY, }, ] diff --git a/tests/functional/assertions/test_runner.py b/tests/functional/assertions/test_runner.py new file mode 100644 index 00000000000..f65a7619631 --- /dev/null +++ b/tests/functional/assertions/test_runner.py @@ -0,0 +1,43 @@ +import os +from typing import Callable, List, Optional + +from dbt.cli.main import dbtRunner, dbtRunnerResult +from dbt.contracts.graph.manifest import Manifest +from dbt.events.base_types import EventMsg +from dbt.tests.util import get_run_results + + +def assert_run_results_have_compiled_node_attributes( + args: List[str], result: dbtRunnerResult +) -> None: + commands_with_run_results = ["build", "compile", "docs", "run", "test"] + if not [a for a in args if a in commands_with_run_results] or not result.success: + return + + run_results = get_run_results(os.getcwd()) + for r in run_results["results"]: + if r["unique_id"].startswith("model") and r["status"] == "success": + assert "compiled_code" in r + assert "compiled" in r + + +_STANDARD_ASSERTIONS = [assert_run_results_have_compiled_node_attributes] + + +class dbtTestRunner(dbtRunner): + def __init__( + self, + manifest: Optional[Manifest] = None, + callbacks: Optional[List[Callable[[EventMsg], None]]] = None, + exit_assertions: Optional[List[Callable[[List[str], dbtRunnerResult], None]]] = None, + ): + self.exit_assertions = exit_assertions if exit_assertions else _STANDARD_ASSERTIONS + super().__init__(manifest, callbacks) + + def invoke(self, args: List[str], **kwargs) -> dbtRunnerResult: + result = super().invoke(args, **kwargs) + + for assertion in self.exit_assertions: + assertion(args, result) + + return result diff --git a/tests/functional/compile/test_compile.py b/tests/functional/compile/test_compile.py index 86c747cde8f..ca03904901f 100644 --- a/tests/functional/compile/test_compile.py +++ b/tests/functional/compile/test_compile.py @@ -3,7 +3,6 @@ import pytest import re -from dbt.cli.main import dbtRunner from dbt.exceptions import DbtRuntimeError, Exception as DbtException from dbt.tests.util import run_dbt, run_dbt_and_capture, read_file from tests.functional.compile.fixtures import ( @@ -16,6 +15,7 @@ schema_yml, model_multiline_jinja, ) +from tests.functional.assertions.test_runner import dbtTestRunner def norm_whitespace(string): @@ -189,11 +189,11 @@ def test_output_json_inline(self, project): assert '"compiled"' in log_output def test_compile_inline_not_add_node(self, project): - dbt = dbtRunner() + dbt = dbtTestRunner() parse_result = dbt.invoke(["parse"]) manifest = parse_result.result assert len(manifest.nodes) == 4 - dbt = dbtRunner(manifest=manifest) + dbt = dbtTestRunner(manifest=manifest) dbt.invoke( ["compile", "--inline", "select * from {{ ref('second_model') }}"], populate_cache=False, @@ -218,7 +218,7 @@ def test_graph_summary_output(self, project): """Ensure that the compile command generates a file named graph_summary.json in the target directory, that the file contains valid json, and that the json has the high level structure it should.""" - dbtRunner().invoke(["compile"]) + dbtTestRunner().invoke(["compile"]) summary_path = pathlib.Path(project.project_root, "target/graph_summary.json") with open(summary_path, "r") as summary_file: summary = json.load(summary_file) diff --git a/tests/functional/semantic_models/test_semantic_model_parsing.py b/tests/functional/semantic_models/test_semantic_model_parsing.py index 76add767035..5ee6798aeaa 100644 --- a/tests/functional/semantic_models/test_semantic_model_parsing.py +++ b/tests/functional/semantic_models/test_semantic_model_parsing.py @@ -4,11 +4,10 @@ from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity -from dbt.cli.main import dbtRunner from dbt.contracts.graph.manifest import Manifest from dbt.events.base_types import BaseEvent from dbt.tests.util import write_file - +from tests.functional.assertions.test_runner import dbtTestRunner schema_yml = """models: - name: fct_revenue @@ -119,7 +118,7 @@ def models(self): } def test_semantic_model_parsing(self, project): - runner = dbtRunner() + runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success assert isinstance(result.result, Manifest) @@ -142,7 +141,7 @@ def test_semantic_model_error(self, project): error_schema_yml = schema_yml.replace("sum_of_things", "has_revenue") write_file(error_schema_yml, project.project_root, "models", "schema.yml") events: List[BaseEvent] = [] - runner = dbtRunner(callbacks=[events.append]) + runner = dbtTestRunner(callbacks=[events.append]) result = runner.invoke(["parse"]) assert not result.success @@ -162,7 +161,7 @@ def models(self): def test_semantic_model_changed_partial_parsing(self, project): # First, use the default schema.yml to define our semantic model, and # run the dbt parse command - runner = dbtRunner() + runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success @@ -183,7 +182,7 @@ def test_semantic_model_changed_partial_parsing(self, project): def test_semantic_model_deleted_partial_parsing(self, project): # First, use the default schema.yml to define our semantic model, and # run the dbt parse command - runner = dbtRunner() + runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success assert "semantic_model.test.revenue" in result.result.semantic_models @@ -203,7 +202,7 @@ def test_semantic_model_flipping_create_metric_partial_parsing(self, project): # First, use the default schema.yml to define our semantic model, and # run the dbt parse command write_file(schema_yml, project.project_root, "models", "schema.yml") - runner = dbtRunner() + runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success