diff --git a/airbyte-integrations/bases/base-normalization/Dockerfile b/airbyte-integrations/bases/base-normalization/Dockerfile index 1de510acc0ee..d7d9c552ced7 100644 --- a/airbyte-integrations/bases/base-normalization/Dockerfile +++ b/airbyte-integrations/bases/base-normalization/Dockerfile @@ -28,5 +28,5 @@ WORKDIR /airbyte ENV AIRBYTE_ENTRYPOINT "/airbyte/entrypoint.sh" ENTRYPOINT ["/airbyte/entrypoint.sh"] -LABEL io.airbyte.version=0.2.8 -LABEL io.airbyte.name=airbyte/normalization +LABEL io.airbyte.version=0.2.9 +LABEL io.airbyte.name=airbyte/normalization \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/incremental.sql b/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/incremental.sql index 86750a85ebcb..36c6e7ea3124 100644 --- a/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/incremental.sql +++ b/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/incremental.sql @@ -28,6 +28,16 @@ and cast({{ col_emitted_at }} as {{ type_timestamp_with_timezone() }}) >= {% endif %} {%- endmacro -%} +{# -- see https://cloud.google.com/bigquery/docs/querying-partitioned-tables#best_practices_for_partition_pruning #} +{%- macro bigquery__incremental_clause(col_emitted_at, tablename) -%} +{% if is_incremental() %} + {% if get_max_normalized_cursor(col_emitted_at, tablename) %} +and cast({{ col_emitted_at }} as {{ type_timestamp_with_timezone() }}) >= + cast('{{ get_max_normalized_cursor(col_emitted_at, tablename) }}' as {{ type_timestamp_with_timezone() }}) + {% endif %} +{% endif %} +{%- endmacro -%} + {%- macro sqlserver__incremental_clause(col_emitted_at, tablename) -%} {% if is_incremental() %} and ((select max(cast({{ col_emitted_at }} as {{ type_timestamp_with_timezone() }})) from {{ tablename }}) is null diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java index a7af7eff4871..1bb56b3a28ef 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java @@ -14,7 +14,7 @@ public class NormalizationRunnerFactory { public static final String BASE_NORMALIZATION_IMAGE_NAME = "airbyte/normalization"; - public static final String NORMALIZATION_VERSION = "0.2.8"; + public static final String NORMALIZATION_VERSION = "0.2.9"; static final Map> NORMALIZATION_MAPPING = ImmutableMap.>builder() diff --git a/docs/understanding-airbyte/basic-normalization.md b/docs/understanding-airbyte/basic-normalization.md index 7ead51689783..817f94be688b 100644 --- a/docs/understanding-airbyte/basic-normalization.md +++ b/docs/understanding-airbyte/basic-normalization.md @@ -353,6 +353,7 @@ Therefore, in order to "upgrade" to the desired normalization version, you need | Airbyte Version | Normalization Version | Date | Pull Request | Subject | |:----------------| :--- | :--- | :--- | :--- | +| | 0.2.9 | 2022-07-06 | [\#14485](https://github.com/airbytehq/airbyte/pull/14485/) | BigQuery partition pruning otimization | | | 0.2.8 | 2022-07-13 | [\#14522](https://github.com/airbytehq/airbyte/pull/14522) | BigQuery replaces `NULL` array entries with the string value `"NULL"` | | | 0.2.7 | 2022-07-05 | [\#11694](https://github.com/airbytehq/airbyte/pull/11694) | Do not return NULL for MySQL column values > 512 chars | | | 0.2.6 | 2022-06-16 | [\#13894](https://github.com/airbytehq/airbyte/pull/13894) | Fix incorrect jinja2 macro `json_extract_array` call |