From 3ac3e5b2384b47f8cb732818f55c3e81797c0a04 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Mon, 14 Aug 2023 13:24:16 +0200 Subject: [PATCH] add --no-partial-parse-file-diff flag (#8361) * add --no-partial-parse-file-diff flag * changelog entry (cherry picked from commit 8350dfead3cf3a9257d1d4e7f76d0a7e1f04d9d3) --- .../Under the Hood-20230811-100902.yaml | 7 +++++ core/dbt/cli/main.py | 1 + core/dbt/cli/params.py | 8 +++++ core/dbt/parser/manifest.py | 11 ++++++- .../partial_parsing/test_file_diff.py | 30 +++++++++++++++++-- 5 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20230811-100902.yaml diff --git a/.changes/unreleased/Under the Hood-20230811-100902.yaml b/.changes/unreleased/Under the Hood-20230811-100902.yaml new file mode 100644 index 00000000000..9fe64ab6442 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230811-100902.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: 'add internal flag: --no-partial-parse-file-diff to inform whether to compute + a file diff during partial parsing' +time: 2023-08-11T10:09:02.832241-04:00 +custom: + Author: michelleark + Issue: "8363" diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 390cde0adc4..b53ff3277ba 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -141,6 +141,7 @@ def invoke(self, args: List[str], **kwargs) -> dbtRunnerResult: @p.macro_debugging @p.partial_parse @p.partial_parse_file_path +@p.partial_parse_file_diff @p.populate_cache @p.print @p.printer_width diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index e75f5f6419b..3c016958fb4 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -257,6 +257,14 @@ type=click.Path(exists=True, dir_okay=False, resolve_path=True), ) +partial_parse_file_diff = click.option( + "--partial-parse-file-diff/--no-partial-parse-file-diff", + envvar="DBT_PARTIAL_PARSE_FILE_DIFF", + help="Internal flag for whether to compute a file diff during partial parsing.", + hidden=True, + default=True, +) + populate_cache = click.option( "--populate-cache/--no-populate-cache", envvar="DBT_POPULATE_CACHE", diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index c7decc1e482..a4bcef28f01 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -284,8 +284,17 @@ def get_full_manifest( adapter.clear_macro_manifest() macro_hook = adapter.connections.set_query_header + flags = get_flags() + if not flags.PARTIAL_PARSE_FILE_DIFF: + file_diff = FileDiff.from_dict( + { + "deleted": [], + "changed": [], + "added": [], + } + ) # Hack to test file_diffs - if os.environ.get("DBT_PP_FILE_DIFF_TEST"): + elif os.environ.get("DBT_PP_FILE_DIFF_TEST"): file_diff_path = "file_diff.json" if path_exists(file_diff_path): file_diff_dct = read_json(file_diff_path) diff --git a/tests/functional/partial_parsing/test_file_diff.py b/tests/functional/partial_parsing/test_file_diff.py index a9f4c8fdd09..f2493de19df 100644 --- a/tests/functional/partial_parsing/test_file_diff.py +++ b/tests/functional/partial_parsing/test_file_diff.py @@ -1,6 +1,8 @@ import os +import pytest -from dbt.tests.util import run_dbt, write_artifact +from dbt.tests.util import run_dbt, write_artifact, write_file +from tests.functional.partial_parsing.fixtures import model_one_sql, model_two_sql first_file_diff = { @@ -17,7 +19,7 @@ } -class TestFileDiffs: +class TestFileDiffPaths: def test_file_diffs(self, project): os.environ["DBT_PP_FILE_DIFF_TEST"] = "true" @@ -35,3 +37,27 @@ def test_file_diffs(self, project): write_artifact(second_file_diff, "file_diff.json") results = run_dbt() assert len(results) == 2 + + +class TestFileDiffs: + @pytest.fixture(scope="class") + def models(self): + return { + "model_one.sql": model_one_sql, + } + + def test_no_file_diffs(self, project): + # We start with a project with one model + manifest = run_dbt(["parse"]) + assert len(manifest.nodes) == 1 + + # add a model file + write_file(model_two_sql, project.project_root, "models", "model_two.sql") + + # parse without computing a file diff + manifest = run_dbt(["--partial-parse", "--no-partial-parse-file-diff", "parse"]) + assert len(manifest.nodes) == 1 + + # default behaviour - parse with computing a file diff + manifest = run_dbt(["--partial-parse", "parse"]) + assert len(manifest.nodes) == 2