diff --git a/.env b/.env index 5860902a3235..c5a9122db653 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -FLINK_VERSION=1.19.0 +FLINK_VERSION=1.19.1 CLOUDSDK_ACTIVE_CONFIG_NAME=ibis-gbq GOOGLE_CLOUD_PROJECT="$CLOUDSDK_ACTIVE_CONFIG_NAME" PGPASSWORD="postgres" diff --git a/.github/workflows/ibis-backends.yml b/.github/workflows/ibis-backends.yml index d85adf947d95..8300d1326e39 100644 --- a/.github/workflows/ibis-backends.yml +++ b/.github/workflows/ibis-backends.yml @@ -227,8 +227,8 @@ jobs: extras: - flink additional_deps: - - "'apache-flink < 1.20.0'" - - "'pandas < 2.2'" + - "'apache-flink@1.19.1'" + - "'pandas<2.2'" - setuptools services: - flink @@ -249,8 +249,8 @@ jobs: extras: - flink additional_deps: - - "'apache-flink < 1.20.0'" - - "'pandas < 2.2'" + - "'apache-flink@1.19.1'" + - "'pandas<2.2'" - setuptools services: - flink @@ -400,8 +400,8 @@ jobs: extras: - flink additional_deps: - - "'apache-flink < 1.20.0'" - - "'pandas < 2.2'" + - "'apache-flink@1.19.1'" + - "'pandas<2.2'" - setuptools services: - flink @@ -413,8 +413,8 @@ jobs: extras: - flink additional_deps: - - "'apache-flink < 1.20.0'" - - "'pandas < 2.2'" + - "'apache-flink@1.19.1'" + - "'pandas<2.2'" - setuptools services: - flink diff --git a/conda/environment-arm64-flink.yml b/conda/environment-arm64-flink.yml index 746679da7bd0..0c76ae81a161 100644 --- a/conda/environment-arm64-flink.yml +++ b/conda/environment-arm64-flink.yml @@ -96,4 +96,4 @@ dependencies: - py4j =0.10.9.7 - pip - pip: - - apache-flink >=1.19.1 + - apache-flink =1.19.1 diff --git a/docker/flink/Dockerfile b/docker/flink/Dockerfile index 6260bc8fa99d..515d1ef66145 100644 --- a/docker/flink/Dockerfile +++ b/docker/flink/Dockerfile @@ -1,4 +1,4 @@ -ARG FLINK_VERSION=1.20.0 +ARG FLINK_VERSION=1.19.1 FROM flink:${FLINK_VERSION} # ibis-flink requires PyFlink dependency diff --git a/ibis/backends/sql/dialects.py b/ibis/backends/sql/dialects.py index a4920a4b5a96..499a76e75daa 100644 --- a/ibis/backends/sql/dialects.py +++ b/ibis/backends/sql/dialects.py @@ -5,6 +5,7 @@ from copy import deepcopy import sqlglot.expressions as sge +import sqlglot.generator as sgn from sqlglot import transforms from sqlglot.dialects import ( TSQL, @@ -219,6 +220,16 @@ class Generator(Hive.Generator): sge.Interval: _interval_with_precision, } + # Flink is like Hive except where it might actually be convenient + # + # UNNEST works like the SQL standard, and not like Hive, so we have to + # override sqlglot here and convince it that flink is not like Hive + # when it comes to unnesting + TRANSFORMS.pop(sge.Unnest, None) + + def unnest_sql(self, expression: sge.Unnest) -> str: + return sgn.Generator.unnest_sql(self, expression) + def struct_sql(self, expression: sge.Struct) -> str: from sqlglot.optimizer.annotate_types import annotate_types diff --git a/poetry.lock b/poetry.lock index 8e8ba15d5d2a..b933ea8cf712 100644 --- a/poetry.lock +++ b/poetry.lock @@ -6799,13 +6799,13 @@ sqlcipher = ["sqlcipher3_binary"] [[package]] name = "sqlglot" -version = "25.8.1" +version = "25.9.0" description = "An easily customizable SQL parser and transpiler" optional = false python-versions = ">=3.7" files = [ - {file = "sqlglot-25.8.1-py3-none-any.whl", hash = "sha256:9e79b211ecc15b23440be3fc82f292615ee2c49d85571ca861cc0398120d7baf"}, - {file = "sqlglot-25.8.1.tar.gz", hash = "sha256:afc4df057ab11be23278655ec8c31b21118077d0c68c001a9764767b72799a99"}, + {file = "sqlglot-25.9.0-py3-none-any.whl", hash = "sha256:6299c246435f680edc0c1be9af1faa55176625f69b23c49229f3208960e04927"}, + {file = "sqlglot-25.9.0.tar.gz", hash = "sha256:6bdc047414066d3730b95e655392d71b12894a22de15a145c984b53e5c5d91aa"}, ] [package.extras] @@ -7789,4 +7789,4 @@ visualization = ["graphviz"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "03717a8b1d7b3944ba9e69f6ce2d3b023f406674f7d1b5dca610d85754a4c6ba" +content-hash = "e7d5e50095adedffcb009420bd275ad64d37ded89f5270fa0cf612f01f1df266" diff --git a/pyproject.toml b/pyproject.toml index 843b9bb8be31..6b0d70b2273f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ pyarrow-hotfix = { version = ">=0.4,<1", optional = true } python-dateutil = ">=2.8.2,<3" pytz = ">=2022.7" rich = ">=12.4.4,<14" -sqlglot = ">=23.4,<25.9" +sqlglot = ">=23.4,<25.10" toolz = ">=0.11,<1" typing-extensions = ">=4.3.0,<5" black = { version = ">=22.1.0,<25", optional = true } diff --git a/requirements-dev.txt b/requirements-dev.txt index e3ccaf290bd3..6b87afbd8366 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -256,7 +256,7 @@ sortedcontainers==2.4.0 ; python_version >= "3.10" and python_version < "4.0" soupsieve==2.5 ; python_version >= "3.10" and python_version < "3.13" sphobjinv==2.3.1.1 ; python_version >= "3.10" and python_version < "3.13" sqlalchemy==2.0.31 ; python_version >= "3.10" and python_version < "3.13" -sqlglot==25.8.1 ; python_version >= "3.10" and python_version < "4.0" +sqlglot==25.9.0 ; python_version >= "3.10" and python_version < "4.0" stack-data==0.6.3 ; python_version >= "3.10" and python_version < "4.0" statsmodels==0.14.2 ; python_version >= "3.10" and python_version < "3.13" tabulate==0.9.0 ; python_version >= "3.10" and python_version < "3.13"