From 8443ff3434457201130453639280a48b2cc12e81 Mon Sep 17 00:00:00 2001 From: brunofaustino Date: Wed, 6 Jul 2022 22:20:49 -0300 Subject: [PATCH 1/2] bigquery partition pruning otimization --- .../dbt-project-template/macros/incremental.sql | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/incremental.sql b/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/incremental.sql index 86750a85ebcb..36c6e7ea3124 100644 --- a/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/incremental.sql +++ b/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/incremental.sql @@ -28,6 +28,16 @@ and cast({{ col_emitted_at }} as {{ type_timestamp_with_timezone() }}) >= {% endif %} {%- endmacro -%} +{# -- see https://cloud.google.com/bigquery/docs/querying-partitioned-tables#best_practices_for_partition_pruning #} +{%- macro bigquery__incremental_clause(col_emitted_at, tablename) -%} +{% if is_incremental() %} + {% if get_max_normalized_cursor(col_emitted_at, tablename) %} +and cast({{ col_emitted_at }} as {{ type_timestamp_with_timezone() }}) >= + cast('{{ get_max_normalized_cursor(col_emitted_at, tablename) }}' as {{ type_timestamp_with_timezone() }}) + {% endif %} +{% endif %} +{%- endmacro -%} + {%- macro sqlserver__incremental_clause(col_emitted_at, tablename) -%} {% if is_incremental() %} and ((select max(cast({{ col_emitted_at }} as {{ type_timestamp_with_timezone() }})) from {{ tablename }}) is null From 3b593957f66cae06b5783a204f93def5ed61f2dd Mon Sep 17 00:00:00 2001 From: brunofaustino Date: Wed, 6 Jul 2022 22:27:47 -0300 Subject: [PATCH 2/2] bump version and add changelog --- airbyte-integrations/bases/base-normalization/Dockerfile | 2 +- .../workers/normalization/NormalizationRunnerFactory.java | 2 +- docs/understanding-airbyte/basic-normalization.md | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/bases/base-normalization/Dockerfile b/airbyte-integrations/bases/base-normalization/Dockerfile index a0d6d3a80faf..2cbda1fe6f59 100644 --- a/airbyte-integrations/bases/base-normalization/Dockerfile +++ b/airbyte-integrations/bases/base-normalization/Dockerfile @@ -28,5 +28,5 @@ WORKDIR /airbyte ENV AIRBYTE_ENTRYPOINT "/airbyte/entrypoint.sh" ENTRYPOINT ["/airbyte/entrypoint.sh"] -LABEL io.airbyte.version=0.2.6 +LABEL io.airbyte.version=0.2.7 LABEL io.airbyte.name=airbyte/normalization diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java index 9a0246a2ae44..33d1f0f4c4b7 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java @@ -14,7 +14,7 @@ public class NormalizationRunnerFactory { public static final String BASE_NORMALIZATION_IMAGE_NAME = "airbyte/normalization"; - public static final String NORMALIZATION_VERSION = "0.2.6"; + public static final String NORMALIZATION_VERSION = "0.2.7"; static final Map> NORMALIZATION_MAPPING = ImmutableMap.>builder() diff --git a/docs/understanding-airbyte/basic-normalization.md b/docs/understanding-airbyte/basic-normalization.md index 8a739b119834..7dbd9d2e6a99 100644 --- a/docs/understanding-airbyte/basic-normalization.md +++ b/docs/understanding-airbyte/basic-normalization.md @@ -353,6 +353,7 @@ Therefore, in order to "upgrade" to the desired normalization version, you need | Airbyte Version | Normalization Version | Date | Pull Request | Subject | |:----------------| :--- | :--- | :--- | :--- | +| | 0.2.7 | 2022-07-06 | [\#14485](https://github.com/airbytehq/airbyte/pull/14485/) | BigQuery partition pruning otimization | | | 0.2.6 | 2022-06-16 | [\#13894](https://github.com/airbytehq/airbyte/pull/13894) | Fix incorrect jinja2 macro `json_extract_array` call | | | 0.2.5 | 2022-06-15 | [\#11470](https://github.com/airbytehq/airbyte/pull/11470) | Upgrade MySQL to dbt 1.0.0 | | | 0.2.4 | 2022-06-14 | [\#12846](https://github.com/airbytehq/airbyte/pull/12846) | CDC correctly deletes propagates deletions to final tables |