From e0d2b02d466502a32ca5a33293f4a67aaca7d33b Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Fri, 1 Oct 2021 09:27:32 -0400 Subject: [PATCH] [#3867] Turn on partial parsing by default --- CHANGELOG.md | 1 + core/dbt/contracts/util.py | 6 ++++++ core/dbt/flags.py | 2 +- core/dbt/parser/manifest.py | 8 ++++++++ test/rpc/test_run.py | 4 ++-- 5 files changed, 18 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f713644061e..e0a45fdaade 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Normalize global CLI arguments/flags ([#2990](https://github.com/dbt-labs/dbt/issues/2990), [#3839](https://github.com/dbt-labs/dbt/pull/3839)) - Turns on the static parser by default and adds the flag `--no-static-parser` to disable it. ([#3377](https://github.com/dbt-labs/dbt/issues/3377), [#3939](https://github.com/dbt-labs/dbt/pull/3939)) - Generic test FQNs have changed to include the relative path, resource, and column (if applicable) where they are defined. This makes it easier to configure them from the `tests` block in `dbt_project.yml` ([#3259](https://github.com/dbt-labs/dbt/pull/3259), [#3880](https://github.com/dbt-labs/dbt/pull/3880) +- Turn on partial parsing by default ([#3867](https://github.com/dbt-labs/dbt/issues/3867), [#3989](https://github.com/dbt-labs/dbt/issues/3989)) ### Fixes - Add generic tests defined on sources to the manifest once, not twice ([#3347](https://github.com/dbt-labs/dbt/issues/3347), [#3880](https://github.com/dbt-labs/dbt/pull/3880)) diff --git a/core/dbt/contracts/util.py b/core/dbt/contracts/util.py index ef4a8fca5a5..1c741ab9eb9 100644 --- a/core/dbt/contracts/util.py +++ b/core/dbt/contracts/util.py @@ -170,6 +170,12 @@ class BaseArtifactMetadata(dbtClassMixin): ) env: Dict[str, str] = dataclasses.field(default_factory=get_metadata_env) + def __post_serialize__(self, dct): + dct = super().__post_serialize__(dct) + if dct['generated_at'] and dct['generated_at'].endswith('+00:00'): + dct['generated_at'] = dct['generated_at'].replace('+00:00', '') + "Z" + return dct + def schema_version(name: str, version: int): def inner(cls: Type[VersionedSchema]): diff --git a/core/dbt/flags.py b/core/dbt/flags.py index 711d0b1debe..c6dd8d35217 100644 --- a/core/dbt/flags.py +++ b/core/dbt/flags.py @@ -41,7 +41,7 @@ "STATIC_PARSER": True, "WARN_ERROR": False, "WRITE_JSON": True, - "PARTIAL_PARSE": False, + "PARTIAL_PARSE": True, "USE_COLORS": True, "PROFILES_DIR": DEFAULT_PROFILES_DIR, "DEBUG": False, diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index e3c610f0e67..a4f778e551f 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -1,5 +1,6 @@ from dataclasses import dataclass from dataclasses import field +from datetime import datetime import os import traceback from typing import ( @@ -557,6 +558,13 @@ def read_manifest_for_partial_parse(self) -> Optional[Manifest]: # different version of dbt is_partial_parseable, reparse_reason = self.is_partial_parsable(manifest) if is_partial_parseable: + # We don't want to have stale generated_at dates + manifest.metadata.generated_at = datetime.utcnow() + # or invocation_ids + if dbt.tracking.active_user: + manifest.metadata.invocation_id = dbt.tracking.active_user.invocation_id + else: + manifest.metadata.invocation_id = None return manifest except Exception as exc: logger.debug( diff --git a/test/rpc/test_run.py b/test/rpc/test_run.py index eb57b106005..075ebfea215 100644 --- a/test/rpc/test_run.py +++ b/test/rpc/test_run.py @@ -73,12 +73,12 @@ def test_rpc_run_vars_compiled( test_kwargs={}, ) with querier_ctx as querier: - results = querier.async_wait_for_result(querier.cli_args('run --vars "{materialized_var: table}"')) + results = querier.async_wait_for_result(querier.cli_args('--no-partial-parse run --vars "{materialized_var: table}"')) assert len(results['results']) == 1 assert results['results'][0]['node']['config']['materialized'] == 'table' # make sure that `--vars` doesn't update global state - if it does, # this run() will result in a view! - results = querier.async_wait_for_result(querier.cli_args('run')) + results = querier.async_wait_for_result(querier.cli_args('--no-partial-parse run')) assert len(results['results']) == 1 assert results['results'][0]['node']['config']['materialized'] == 'view'