From 57660c9f5d8de00093275849483ad32883919a2e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 27 Jul 2023 12:18:45 -0400 Subject: [PATCH] Initialize sqlparse lexer and tweak order of setting compilation fields (#8215) (#8220) (cherry picked from commit fdeccfaf248e6421017f7cfe911b549b93f86a32) Co-authored-by: Gerda Shank --- .changes/unreleased/Fixes-20230726-104448.yaml | 6 ++++++ core/dbt/compilation.py | 18 ++++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 .changes/unreleased/Fixes-20230726-104448.yaml diff --git a/.changes/unreleased/Fixes-20230726-104448.yaml b/.changes/unreleased/Fixes-20230726-104448.yaml new file mode 100644 index 00000000000..0c8e55dd6b9 --- /dev/null +++ b/.changes/unreleased/Fixes-20230726-104448.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Improve handling of CTE injection with ephemeral models +time: 2023-07-26T10:44:48.888451-04:00 +custom: + Author: gshank + Issue: "8213" diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index 41fd3e0da22..959ad2516d1 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -4,7 +4,6 @@ import networkx as nx # type: ignore import os import pickle -import sqlparse from collections import defaultdict from typing import List, Dict, Any, Tuple, Optional @@ -36,6 +35,7 @@ from dbt.events.format import pluralize import dbt.tracking import dbt.task.list as list_task +import sqlparse graph_file_name = "graph.gpickle" @@ -378,16 +378,16 @@ def _recursively_prepend_ctes( _add_prepended_cte(prepended_ctes, InjectedCTE(id=cte.id, sql=sql)) - injected_sql = inject_ctes_into_sql( - model.compiled_code, - prepended_ctes, - ) # Check again before updating for multi-threading if not model.extra_ctes_injected: + injected_sql = inject_ctes_into_sql( + model.compiled_code, + prepended_ctes, + ) + model.extra_ctes_injected = True model._pre_injected_sql = model.compiled_code model.compiled_code = injected_sql model.extra_ctes = prepended_ctes - model.extra_ctes_injected = True # if model.extra_ctes is not set to prepended ctes, something went wrong return model, model.extra_ctes @@ -523,6 +523,12 @@ def compile_node( the node's raw_code into compiled_code, and then calls the recursive method to "prepend" the ctes. """ + # Make sure Lexer for sqlparse 0.4.4 is initialized + from sqlparse.lexer import Lexer # type: ignore + + if hasattr(Lexer, "get_default_instance"): + Lexer.get_default_instance() + node = self._compile_code(node, manifest, extra_context) node, _ = self._recursively_prepend_ctes(node, manifest, extra_context)