From 295903dcd7aacfe6e8dca5d08675c34c8ba88125 Mon Sep 17 00:00:00 2001 From: Nicola Coretti Date: Wed, 16 Aug 2023 10:00:50 +0200 Subject: [PATCH] feat(exasol): add exasol backend --- .github/workflows/ibis-backends.yml | 18 +- ci/schema/exasol.sql | 75 +++++++ compose.yaml | 19 ++ docs/backends/exasol.qmd | 96 +++++++++ ibis/backends/base/__init__.py | 6 +- ibis/backends/base/sql/alchemy/__init__.py | 2 +- ibis/backends/conftest.py | 1 + ibis/backends/exasol/__init__.py | 234 +++++++++++++++++++++ ibis/backends/exasol/compiler.py | 24 +++ ibis/backends/exasol/datatypes.py | 26 +++ ibis/backends/exasol/registry.py | 46 ++++ ibis/backends/exasol/tests/__init__.py | 0 ibis/backends/exasol/tests/conftest.py | 120 +++++++++++ ibis/backends/tests/test_aggregation.py | 72 ++++++- ibis/backends/tests/test_api.py | 2 +- ibis/backends/tests/test_array.py | 4 +- ibis/backends/tests/test_binary.py | 5 + ibis/backends/tests/test_client.py | 21 +- ibis/backends/tests/test_column.py | 1 + ibis/backends/tests/test_dot_sql.py | 5 +- ibis/backends/tests/test_examples.py | 2 +- ibis/backends/tests/test_export.py | 27 ++- ibis/backends/tests/test_generic.py | 99 +++++++-- ibis/backends/tests/test_join.py | 18 +- ibis/backends/tests/test_json.py | 2 +- ibis/backends/tests/test_map.py | 2 +- ibis/backends/tests/test_network.py | 6 + ibis/backends/tests/test_numeric.py | 125 +++++++++-- ibis/backends/tests/test_param.py | 20 +- ibis/backends/tests/test_register.py | 2 +- ibis/backends/tests/test_set_ops.py | 27 ++- ibis/backends/tests/test_sql.py | 2 +- ibis/backends/tests/test_string.py | 47 +++-- ibis/backends/tests/test_struct.py | 2 +- ibis/backends/tests/test_temporal.py | 89 +++++++- ibis/backends/tests/test_timecontext.py | 1 + ibis/backends/tests/test_udf.py | 1 + ibis/backends/tests/test_uuid.py | 2 + ibis/backends/tests/test_window.py | 2 +- ibis/util.py | 4 +- poetry-overrides.nix | 3 + pyproject.toml | 7 + requirements-dev.txt | 4 + 43 files changed, 1175 insertions(+), 96 deletions(-) create mode 100644 ci/schema/exasol.sql create mode 100644 docs/backends/exasol.qmd create mode 100644 ibis/backends/exasol/__init__.py create mode 100644 ibis/backends/exasol/compiler.py create mode 100644 ibis/backends/exasol/datatypes.py create mode 100644 ibis/backends/exasol/registry.py create mode 100644 ibis/backends/exasol/tests/__init__.py create mode 100644 ibis/backends/exasol/tests/conftest.py diff --git a/.github/workflows/ibis-backends.yml b/.github/workflows/ibis-backends.yml index 2017fd831bb3..5148c052d991 100644 --- a/.github/workflows/ibis-backends.yml +++ b/.github/workflows/ibis-backends.yml @@ -159,6 +159,13 @@ jobs: - oracle services: - oracle + - name: exasol + title: Exasol + serial: true + extras: + - exasol + services: + - exasol - name: flink title: Flink serial: true @@ -283,6 +290,15 @@ jobs: - flink services: - flink + - os: windows-latest + backend: + name: exasol + title: Exasol + serial: true + extras: + - exasol + services: + - exasol steps: - name: update and install system dependencies if: matrix.os == 'ubuntu-latest' && matrix.backend.sys-deps != null @@ -604,7 +620,7 @@ jobs: - run: python -m pip install --upgrade pip 'poetry==1.7.1' - name: remove deps that are not compatible with sqlalchemy 2 - run: poetry remove snowflake-sqlalchemy + run: poetry remove snowflake-sqlalchemy sqlalchemy-exasol - name: add sqlalchemy 2 run: poetry add --lock --optional 'sqlalchemy>=2,<3' diff --git a/ci/schema/exasol.sql b/ci/schema/exasol.sql new file mode 100644 index 000000000000..856b059e7407 --- /dev/null +++ b/ci/schema/exasol.sql @@ -0,0 +1,75 @@ +DROP SCHEMA IF EXISTS EXASOL CASCADE; +CREATE SCHEMA EXASOL; + +CREATE OR REPLACE TABLE EXASOL.diamonds +( + "carat" DOUBLE, + "cut" VARCHAR(256), + "color" VARCHAR(256), + "clarity" VARCHAR(256), + "depth" DOUBLE, + "table" DOUBLE, + "price" BIGINT, + "x" DOUBLE, + "y" DOUBLE, + "z" DOUBLE +); + +CREATE OR REPLACE TABLE EXASOL.batting +( + "playerID" VARCHAR(256), + "yearID" BIGINT, + "stint" BIGINT, + "teamID" VARCHAR(256), + "logID" VARCHAR(256), + "G" BIGINT, + "AB" BIGINT, + "R" BIGINT, + "H" BIGINT, + "X2B" BIGINT, + "X3B" BIGINT, + "HR" BIGINT, + "RBI" BIGINT, + "SB" BIGINT, + "CS" BIGINT, + "BB" BIGINT, + "SO" BIGINT, + "IBB" BIGINT, + "HBP" BIGINT, + "SH" BIGINT, + "SF" BIGINT, + "GIDP" BIGINT +); + +CREATE OR REPLACE TABLE EXASOL.awards_players +( + "playerId" VARCHAR(256), + "awardID" VARCHAR(256), + "yearID" VARCHAR(256), + "logID" VARCHAR(256), + "tie" VARCHAR(256), + "notest" VARCHAR(256) +); + +CREATE OR REPLACE TABLE EXASOL.functional_alltypes +( + "id" INTEGER, + "bool_col" BOOLEAN, + "tinyint_col" SHORTINT, + "small_int" SMALLINT, + "int_col" INTEGER, + "bigint_col" BIGINT, + "float_col" FLOAT, + "double_col" DOUBLE PRECISION, + "date_string_col" VARCHAR(256), + "string_col" VARCHAR(256), + "timestamp_col" TIMESTAMP, + "year" INTEGER, + "month" INTEGER +); + + +IMPORT INTO EXASOL.diamonds FROM LOCAL CSV FILE '/data/diamonds.csv' COLUMN SEPARATOR = ',' SKIP = 1; +IMPORT INTO EXASOL.batting FROM LOCAL CSV FILE '/data/batting.csv' COLUMN SEPARATOR = ',' SKIP = 1; +IMPORT INTO EXASOL.awards_players FROM LOCAL CSV FILE '/data/awards_players.csv' COLUMN SEPARATOR = ',' SKIP = 1; +IMPORT INTO EXASOL.functional_alltypes FROM LOCAL CSV FILE '/data/functional_alltypes.csv' COLUMN SEPARATOR = ',' SKIP = 1; diff --git a/compose.yaml b/compose.yaml index 7aa6d01eedba..26f92da1fe39 100644 --- a/compose.yaml +++ b/compose.yaml @@ -412,6 +412,23 @@ services: volumes: - oracle:/opt/oracle/data + exasol: + image: exasol/docker-db:7.1.23 + privileged: true + ports: + - 8563:8563 + healthcheck: + interval: 10s + retries: 9 + timeout: 90s + test: + - CMD-SHELL + - /usr/opt/EXASuite-7/EXASolution-7.1.23/bin/Console/exaplus -c 127.0.0.1:8563 -u sys -p exasol -encryption OFF <<< 'SELECT 1' + networks: + - exasol + volumes: + - exasol:/data + flink-jobmanager: build: ./docker/flink image: ibis-flink @@ -452,6 +469,7 @@ networks: trino: druid: oracle: + exasol: flink: volumes: @@ -469,3 +487,4 @@ volumes: oracle: postgres: minio: + exasol: diff --git a/docs/backends/exasol.qmd b/docs/backends/exasol.qmd new file mode 100644 index 000000000000..2b8c81c27e21 --- /dev/null +++ b/docs/backends/exasol.qmd @@ -0,0 +1,96 @@ +# Exasol + +[https://www.exasol.com](https://www.exasol.com) + +## Install + +Install Ibis and dependencies for the Exasol backend: + +::: {.panel-tabset} + +## `pip` + +Install with the `exasol` extra: + +```{.bash} +pip install 'ibis-framework[exasol]' +``` + +And connect: + +```{.python} +import ibis + +con = ibis.exasol.connect(...) # <1> +``` + +1. Adjust connection parameters as needed. + +## `conda` + +Install for Exasol: + +```{.bash} +conda install -c conda-forge ibis-exasol +``` + +And connect: + +```{.python} +import ibis + +con = ibis.exasol.connect(...) # <1> +``` + +1. Adjust connection parameters as needed. + +## `mamba` + +Install for Exasol: + +```{.bash} +mamba install -c conda-forge ibis-exasol +``` + +And connect: + +```{.python} +import ibis + +con = ibis.exasol.connect(...) # <1> +``` + +1. Adjust connection parameters as needed. + +::: + +## Connect + +### `ibis.exasol.connect` + +```python +con = ibis.exasol.connect( + user = "username", + password = "password", + host = "localhost", + port = 8563, + schema = None, + encryption = True, + certificate_validation = True, + encoding = "en_US.UTF-8" +) +``` + +::: {.callout-note} +`ibis.exasol.connect` is a thin wrapper around [`ibis.backends.exasol.Backend.do_connect`](#ibis.backends.exasol.Backend.do_connect). +::: + +### Connection Parameters + +```{python} +#| echo: false +#| output: asis +from _utils import render_do_connect + +render_do_connect("exasol") +``` diff --git a/ibis/backends/base/__init__.py b/ibis/backends/base/__init__.py index eeeb6d1c1113..8b9904049f49 100644 --- a/ibis/backends/base/__init__.py +++ b/ibis/backends/base/__init__.py @@ -33,16 +33,16 @@ __all__ = ("BaseBackend", "Database", "connect") - _IBIS_TO_SQLGLOT_DIALECT = { "mssql": "tsql", "impala": "hive", "pyspark": "spark", "polars": "postgres", "datafusion": "postgres", + # closest match see https://github.com/ibis-project/ibis/pull/7303#discussion_r1350223901 + "exa.websocket": "oracle", } - _SQLALCHEMY_TO_SQLGLOT_DIALECT = { # sqlalchemy dialects of backends not listed here match the sqlglot dialect # name @@ -52,6 +52,8 @@ # druid allows double quotes for identifiers, like postgres: # https://druid.apache.org/docs/latest/querying/sql#identifiers-and-literals "druid": "postgres", + # closest match see https://github.com/ibis-project/ibis/pull/7303#discussion_r1350223901 + "exa.websocket": "oracle", } diff --git a/ibis/backends/base/sql/alchemy/__init__.py b/ibis/backends/base/sql/alchemy/__init__.py index e43963c7595d..7f1f03d961d7 100644 --- a/ibis/backends/base/sql/alchemy/__init__.py +++ b/ibis/backends/base/sql/alchemy/__init__.py @@ -554,7 +554,7 @@ def _schema_from_sqla_table( dtype = schema[name] else: dtype = cls.compiler.translator_class.get_ibis_type( - column.type, nullable=column.nullable + column.type, nullable=column.nullable or column.nullable is None ) pairs.append((name, dtype)) return sch.schema(pairs) diff --git a/ibis/backends/conftest.py b/ibis/backends/conftest.py index e6a87364e76d..dd9e5474d58b 100644 --- a/ibis/backends/conftest.py +++ b/ibis/backends/conftest.py @@ -539,6 +539,7 @@ def ddl_con(ddl_backend): params=_get_backends_to_test( keep=( "duckdb", + "exasol", "mssql", "mysql", "oracle", diff --git a/ibis/backends/exasol/__init__.py b/ibis/backends/exasol/__init__.py new file mode 100644 index 000000000000..d00f9b7f9c96 --- /dev/null +++ b/ibis/backends/exasol/__init__.py @@ -0,0 +1,234 @@ +from __future__ import annotations + +import re +import warnings +from collections import ChainMap +from contextlib import contextmanager +from typing import TYPE_CHECKING, Any + +import sqlalchemy as sa +import sqlglot as sg + +from ibis import util +from ibis.backends.base.sql.alchemy import AlchemyCanCreateSchema, BaseAlchemyBackend +from ibis.backends.base.sqlglot.datatypes import PostgresType +from ibis.backends.exasol.compiler import ExasolCompiler + +if TYPE_CHECKING: + from collections.abc import Iterable, MutableMapping + + from ibis.backends.base import BaseBackend + from ibis.expr import datatypes as dt + + +class Backend(BaseAlchemyBackend, AlchemyCanCreateSchema): + name = "exasol" + compiler = ExasolCompiler + supports_temporary_tables = False + supports_create_or_replace = False + supports_in_memory_tables = False + supports_python_udfs = False + + def do_connect( + self, + user: str, + password: str, + host: str = "localhost", + port: int = 8563, + schema: str | None = None, + encryption: bool = True, + certificate_validation: bool = True, + encoding: str = "en_US.UTF-8", + ) -> None: + """Create an Ibis client connected to an Exasol database. + + Parameters + ---------- + user + Username used for authentication. + password + Password used for authentication. + host + Hostname to connect to (default: "localhost"). + port + Port number to connect to (default: 8563) + schema + Database schema to open, if `None`, no schema will be opened. + encryption + Enables/disables transport layer encryption (default: True). + certificate_validation + Enables/disables certificate validation (default: True). + encoding + The encoding format (default: "en_US.UTF-8"). + """ + options = [ + "SSLCertificate=SSL_VERIFY_NONE" if not certificate_validation else "", + f"ENCRYPTION={'yes' if encryption else 'no'}", + f"CONNECTIONCALL={encoding}", + ] + url_template = ( + "exa+websocket://{user}:{password}@{host}:{port}/{schema}?{options}" + ) + url = sa.engine.url.make_url( + url_template.format( + user=user, + password=password, + host=host, + port=port, + schema=schema, + options="&".join(options), + ) + ) + engine = sa.create_engine(url, poolclass=sa.pool.StaticPool) + super().do_connect(engine) + + def _convert_kwargs(self, kwargs: MutableMapping) -> None: + def convert_sqla_to_ibis(keyword_arguments): + sqla_to_ibis = {"tls": "encryption", "username": "user"} + for sqla_kwarg, ibis_kwarg in sqla_to_ibis.items(): + if sqla_kwarg in keyword_arguments: + keyword_arguments[ibis_kwarg] = keyword_arguments.pop(sqla_kwarg) + + def filter_kwargs(keyword_arguments): + allowed_parameters = [ + "user", + "password", + "host", + "port", + "schema", + "encryption", + "certificate", + "encoding", + ] + to_be_removed = [ + key for key in keyword_arguments if key not in allowed_parameters + ] + for parameter_name in to_be_removed: + del keyword_arguments[parameter_name] + + convert_sqla_to_ibis(kwargs) + filter_kwargs(kwargs) + + def _from_url(self, url: str, **kwargs) -> BaseBackend: + """Construct an ibis backend from a SQLAlchemy-conforming URL.""" + kwargs = ChainMap(kwargs) + _, new_kwargs = self.inspector.dialect.create_connect_args(url) + kwargs = kwargs.new_child(new_kwargs) + kwargs = dict(kwargs) + self._convert_kwargs(kwargs) + + return self.connect(**kwargs) + + @property + def inspector(self): + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=sa.exc.RemovedIn20Warning) + return super().inspector + + @contextmanager + def begin(self): + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=sa.exc.RemovedIn20Warning) + with super().begin() as con: + yield con + + def list_tables(self, like=None, database=None): + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=sa.exc.RemovedIn20Warning) + return super().list_tables(like=like, database=database) + + def _get_sqla_table( + self, + name: str, + autoload: bool = True, + **kwargs: Any, + ) -> sa.Table: + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=sa.exc.RemovedIn20Warning) + return super()._get_sqla_table(name=name, autoload=autoload, **kwargs) + + def _metadata(self, query: str) -> Iterable[tuple[str, dt.DataType]]: + table = sg.table(util.gen_name("exasol_metadata")) + create_view = sg.exp.Create( + kind="VIEW", this=table, expression=sg.parse_one(query, dialect="postgres") + ) + drop_view = sg.exp.Drop(kind="VIEW", this=table) + describe = sg.exp.Describe(this=table).sql(dialect="postgres") + # strip trailing encodings e.g., UTF8 + varchar_regex = re.compile(r"^(VARCHAR(?:\(\d+\)))?(?:\s+.+)?$") + with self.begin() as con: + con.exec_driver_sql(create_view.sql(dialect="postgres")) + try: + yield from ( + ( + name, + PostgresType.from_string(varchar_regex.sub(r"\1", typ)), + ) + for name, typ, *_ in con.exec_driver_sql(describe) + ) + finally: + con.exec_driver_sql(drop_view.sql(dialect="postgres")) + + @property + def current_schema(self) -> str: + return self._scalar_query(sa.select(sa.text("CURRENT_SCHEMA"))) + + @property + def current_database(self) -> str: + return None + + def drop_schema( + self, name: str, database: str | None = None, force: bool = False + ) -> None: + if database is not None: + raise NotImplementedError( + "`database` argument is not supported for the Exasol backend" + ) + drop_schema = sg.exp.Drop( + kind="SCHEMA", this=sg.to_identifier(name), exists=force + ) + with self.begin() as con: + con.exec_driver_sql(drop_schema.sql(dialect="postgres")) + + def create_schema( + self, name: str, database: str | None = None, force: bool = False + ) -> None: + if database is not None: + raise NotImplementedError( + "`database` argument is not supported for the Exasol backend" + ) + create_schema = sg.exp.Create( + kind="SCHEMA", this=sg.to_identifier(name), exists=force + ) + with self.begin() as con: + open_schema = self.current_schema + con.exec_driver_sql(create_schema.sql(dialect="postgres")) + # Exasol implicitly opens the created schema, therefore we need to restore + # the previous context. + action = ( + sa.text(f"OPEN SCHEMA {open_schema}") + if open_schema + else sa.text(f"CLOSE SCHEMA {name}") + ) + con.exec_driver_sql(action) + + def list_schemas( + self, like: str | None = None, database: str | None = None + ) -> list[str]: + if database is not None: + raise NotImplementedError( + "`database` argument is not supported for the Exasol backend" + ) + + schema, table = "SYS", "EXA_SCHEMAS" + sch = sa.table( + table, + sa.column("schema_name", sa.TEXT()), + schema=schema, + ) + + query = sa.select(sch.c.schema_name) + + with self.begin() as con: + schemas = list(con.execute(query).scalars()) + return self._filter_with_like(schemas, like=like) diff --git a/ibis/backends/exasol/compiler.py b/ibis/backends/exasol/compiler.py new file mode 100644 index 000000000000..d4e5fcc6d114 --- /dev/null +++ b/ibis/backends/exasol/compiler.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +import sqlalchemy as sa + +from ibis.backends.base.sql.alchemy import AlchemyCompiler, AlchemyExprTranslator +from ibis.backends.exasol import registry +from ibis.backends.exasol.datatypes import ExasolSQLType + + +class ExasolExprTranslator(AlchemyExprTranslator): + _registry = registry.create() + _rewrites = AlchemyExprTranslator._rewrites.copy() + _integer_to_timestamp = sa.func.from_unixtime + _dialect_name = "exa.websocket" + native_json_type = False + type_mapper = ExasolSQLType + + +rewrites = ExasolExprTranslator.rewrites + + +class ExasolCompiler(AlchemyCompiler): + translator_class = ExasolExprTranslator + support_values_syntax_in_select = False diff --git a/ibis/backends/exasol/datatypes.py b/ibis/backends/exasol/datatypes.py new file mode 100644 index 000000000000..afc13c9d7896 --- /dev/null +++ b/ibis/backends/exasol/datatypes.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import sqlalchemy.types as sa_types + +from ibis.backends.base.sql.alchemy.datatypes import AlchemyType + +if TYPE_CHECKING: + import ibis.expr.datatypes as dt + + +class ExasolSQLType(AlchemyType): + dialect = "exa.websocket" + + @classmethod + def from_ibis(cls, dtype: dt.DataType) -> sa_types.TypeEngine: + if dtype.is_string(): + # see also: https://docs.exasol.com/db/latest/sql_references/data_types/datatypesoverview.htm + MAX_VARCHAR_SIZE = 2_000_000 + return sa_types.VARCHAR(MAX_VARCHAR_SIZE) + return super().from_ibis(dtype) + + @classmethod + def to_ibis(cls, typ: sa_types.TypeEngine, nullable: bool = True) -> dt.DataType: + return super().to_ibis(typ, nullable=nullable) diff --git a/ibis/backends/exasol/registry.py b/ibis/backends/exasol/registry.py new file mode 100644 index 000000000000..5c23f3996662 --- /dev/null +++ b/ibis/backends/exasol/registry.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import sqlalchemy as sa + +import ibis.expr.operations as ops + +# used for literal translate +from ibis.backends.base.sql.alchemy import ( + fixed_arity, + sqlalchemy_operation_registry, +) + + +class _String: + @staticmethod + def find(t, op): + args = [t.translate(op.substr), t.translate(op.arg)] + if (start := op.start) is not None: + args.append(t.translate(start) + 1) + return sa.func.locate(*args) - 1 + + @staticmethod + def translate(t, op): + func = fixed_arity(sa.func.translate, 3) + return func(t, op) + + +class _Registry: + _unsupported = {ops.StringJoin} + + _supported = { + ops.Translate: _String.translate, + ops.StringFind: _String.find, + } + + @classmethod + def create(cls): + registry = sqlalchemy_operation_registry.copy() + registry = {k: v for k, v in registry.items() if k not in cls._unsupported} + registry.update(cls._supported) + return registry + + +def create(): + """Create an operation registry for an Exasol backend.""" + return _Registry.create() diff --git a/ibis/backends/exasol/tests/__init__.py b/ibis/backends/exasol/tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/ibis/backends/exasol/tests/conftest.py b/ibis/backends/exasol/tests/conftest.py new file mode 100644 index 000000000000..e5ac62566efb --- /dev/null +++ b/ibis/backends/exasol/tests/conftest.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +import os +import subprocess +from typing import TYPE_CHECKING + +import ibis +from ibis.backends.tests.base import ( + ServiceBackendTest, +) + +if TYPE_CHECKING: + from collections.abc import Iterable + from pathlib import Path + from typing import Any + +EXASOL_USER = os.environ.get("IBIS_TEST_EXASOL_USER", "sys") +EXASOL_PASS = os.environ.get("IBIS_TEST_EXASOL_PASSWORD", "exasol") +EXASOL_HOST = os.environ.get("IBIS_TEST_EXASOL_HOST", "localhost") +EXASOL_PORT = int(os.environ.get("IBIS_TEST_EXASOL_PORT", 8563)) +IBIS_TEST_EXASOL_DB = os.environ.get("IBIS_TEST_EXASOL_DATABASE", "EXASOL") + + +class TestConf(ServiceBackendTest): + check_dtype = False + check_names = False + supports_arrays = False + supports_arrays_outside_of_select = supports_arrays + supports_window_operations = True + supports_divide_by_zero = False + returned_timestamp_unit = "us" + supported_to_timestamp_units = {"s", "ms", "us"} + supports_floating_modulus = True + native_bool = True + supports_structs = False + supports_json = False + supports_map = False + reduction_tolerance = 1e-7 + stateful = True + service_name = "exasol" + supports_tpch = False + force_sort = True + deps = "sqlalchemy", "sqlalchemy_exasol", "pyexasol" + + @staticmethod + def connect(*, tmpdir, worker_id, **kw: Any): + kwargs = { + "user": EXASOL_USER, + "password": EXASOL_PASS, + "host": EXASOL_HOST, + "port": EXASOL_PORT, + "schema": IBIS_TEST_EXASOL_DB, + "certificate_validation": False, + } + return ibis.exasol.connect(**kwargs) + + @property + def test_files(self) -> Iterable[Path]: + return self.data_dir.joinpath("csv").glob("*.csv") + + def _exaplus(self) -> str: + find_exaplus = [ + "docker", + "compose", + "exec", + self.service_name, + "find", + "/", + "-name", + "exaplus", + "-type", + "f", # only files + "-executable", # only executable files + "-print", # -print -quit will stop after the result is found + "-quit", + ] + result = subprocess.run( + find_exaplus, capture_output=True, check=True, text=True + ) + return result.stdout.strip() + + def _load_data(self, **_: Any) -> None: + """Load test data into a backend.""" + ddl_file = f"{self.data_volume}/exasol.sql" + execute_ddl_file = [ + "docker", + "compose", + "exec", + self.service_name, + self._exaplus(), + "-c", + f"{EXASOL_HOST}:{EXASOL_PORT}", + "-u", + EXASOL_USER, + "-p", + EXASOL_PASS, + "-f", + ddl_file, + "--jdbcparam", + "validateservercertificate=0", + ] + subprocess.check_call(execute_ddl_file) + + def preload(self): + # copy data files + super().preload() + + service = self.service_name + data_volume = self.data_volume + path = self.script_dir / f"{self.name()}.sql" + + subprocess.check_call( + [ + "docker", + "compose", + "cp", + f"{path}", + f"{service}:{data_volume}/{path.name}", + ] + ) diff --git a/ibis/backends/tests/test_aggregation.py b/ibis/backends/tests/test_aggregation.py index 0f70f8627d3e..942567316d89 100644 --- a/ibis/backends/tests/test_aggregation.py +++ b/ibis/backends/tests/test_aggregation.py @@ -41,6 +41,12 @@ Py4JError = None +try: + from pyexasol.exceptions import ExaQueryError +except ImportError: + ExaQueryError = None + + @reduction(input_type=[dt.double], output_type=dt.double) def mean_udf(s): return s.mean() @@ -68,6 +74,7 @@ def mean_udf(s): "druid", "oracle", "flink", + "exasol", ], raises=com.OperationNotDefinedError, ), @@ -99,6 +106,7 @@ def mean_udf(s): "druid", "oracle", "flink", + "exasol", ], raises=com.OperationNotDefinedError, ), @@ -131,6 +139,7 @@ def mean_udf(s): "druid", "oracle", "flink", + "exasol", ] argidx_grouped_marks = ["dask"] + argidx_not_grouped_marks @@ -221,6 +230,7 @@ def test_aggregate_grouped(backend, alltypes, df, result_fn, expected_fn): "druid", "oracle", "flink", + "exasol", ], raises=com.OperationNotDefinedError, ) @@ -309,6 +319,9 @@ def mean_and_std(v): raises=sa.exc.DatabaseError, reason="ORA-02000: missing AS keyword", ), + pytest.mark.notimpl( + ["exasol"], raises=(sa.exc.DBAPIError, ExaQueryError) + ), ], ), param( @@ -326,6 +339,9 @@ def mean_and_std(v): raises=sa.exc.DatabaseError, reason="ORA-02000: missing AS keyword", ), + pytest.mark.notimpl( + ["exasol"], raises=(sa.exc.DBAPIError, ExaQueryError) + ), ], ), param( @@ -355,6 +371,9 @@ def mean_and_std(v): raises=sa.exc.DatabaseError, reason="ORA-02000: missing AS keyword", ), + pytest.mark.notimpl( + ["exasol"], raises=(sa.exc.DBAPIError, ExaQueryError) + ), ], ), param( @@ -372,6 +391,9 @@ def mean_and_std(v): raises=sa.exc.DatabaseError, reason="ORA-02000: missing AS keyword", ), + pytest.mark.notimpl( + ["exasol"], raises=(sa.exc.DBAPIError, ExaQueryError) + ), ], ), param( @@ -432,6 +454,7 @@ def mean_and_std(v): "trino", "druid", "oracle", + "exasol", "flink", ], raises=com.OperationNotDefinedError, @@ -451,6 +474,7 @@ def mean_and_std(v): "mssql", "druid", "oracle", + "exasol", "flink", ], raises=com.OperationNotDefinedError, @@ -470,6 +494,7 @@ def mean_and_std(v): "mssql", "druid", "oracle", + "exasol", "flink", ], raises=com.OperationNotDefinedError, @@ -545,6 +570,7 @@ def mean_and_std(v): "mssql", "druid", "oracle", + "exasol", "flink", ], raises=com.OperationNotDefinedError, @@ -563,6 +589,7 @@ def mean_and_std(v): "mssql", "druid", "oracle", + "exasol", "flink", ], raises=com.OperationNotDefinedError, @@ -582,6 +609,7 @@ def mean_and_std(v): "mssql", "druid", "oracle", + "exasol", "flink", ], raises=com.OperationNotDefinedError, @@ -611,6 +639,7 @@ def mean_and_std(v): "pandas", "polars", "sqlite", + "exasol", "flink", ], raises=com.OperationNotDefinedError, @@ -736,6 +765,7 @@ def mean_and_std(v): "mssql", "druid", "oracle", + "exasol", ], raises=com.OperationNotDefinedError, ), @@ -757,15 +787,34 @@ def mean_and_std(v): @pytest.mark.parametrize( ("ibis_cond", "pandas_cond"), [ - param(lambda _: None, lambda _: slice(None), id="no_cond"), + param( + lambda _: None, + lambda _: slice(None), + marks=pytest.mark.notimpl( + ["exasol"], + raises=(com.OperationNotDefinedError, ExaQueryError, sa.exc.DBAPIError), + strict=False, + ), + id="no_cond", + ), param( lambda t: t.string_col.isin(["1", "7"]), lambda t: t.string_col.isin(["1", "7"]), + marks=pytest.mark.notimpl( + ["exasol"], + raises=(com.OperationNotDefinedError, ExaQueryError), + strict=False, + ), id="is_in", ), param( lambda _: ibis._.string_col.isin(["1", "7"]), lambda t: t.string_col.isin(["1", "7"]), + marks=pytest.mark.notimpl( + ["exasol"], + raises=(com.OperationNotDefinedError, ExaQueryError), + strict=False, + ), id="is_in_deferred", ), ], @@ -823,6 +872,9 @@ def test_reduction_ops( raises=com.OperationNotDefinedError, reason="no one has attempted implementation yet", ) +@pytest.mark.notimpl( + ["exasol"], raises=(sa.exc.DBAPIError, com.UnsupportedOperationError) +) def test_count_distinct_star(alltypes, df, ibis_cond, pandas_cond): table = alltypes[["int_col", "double_col", "string_col"]] expr = table.nunique(where=ibis_cond(table)) @@ -851,6 +903,7 @@ def test_count_distinct_star(alltypes, df, ibis_cond, pandas_cond): "sqlite", "druid", "oracle", + "exasol", ], raises=com.OperationNotDefinedError, ), @@ -893,6 +946,7 @@ def test_count_distinct_star(alltypes, df, ibis_cond, pandas_cond): "sqlite", "druid", "oracle", + "exasol", ], raises=com.OperationNotDefinedError, ), @@ -1092,10 +1146,8 @@ def test_quantile( ), ], ) -@pytest.mark.notimpl( - ["mssql"], - raises=com.OperationNotDefinedError, -) +@pytest.mark.notimpl(["mssql"], raises=com.OperationNotDefinedError) +@pytest.mark.notimpl(["exasol"], raises=AttributeError) def test_corr_cov( con, batting, @@ -1123,7 +1175,7 @@ def test_corr_cov( @pytest.mark.notimpl( - ["mysql", "sqlite", "mssql", "druid"], + ["mysql", "sqlite", "mssql", "druid", "exasol"], raises=com.OperationNotDefinedError, ) @pytest.mark.broken( @@ -1139,7 +1191,7 @@ def test_approx_median(alltypes): @pytest.mark.notimpl( - ["bigquery", "druid", "sqlite"], raises=com.OperationNotDefinedError + ["bigquery", "druid", "sqlite", "exasol"], raises=com.OperationNotDefinedError ) @pytest.mark.notyet( ["impala", "mysql", "mssql", "druid", "pyspark", "trino"], @@ -1246,6 +1298,7 @@ def test_median(alltypes, df): raises=sa.exc.DatabaseError, reason="ORA-00904: 'GROUP_CONCAT': invalid identifier", ) +@pytest.mark.notimpl(["exasol"], raises=ExaQueryError) @pytest.mark.notyet( ["flink"], raises=Py4JError, @@ -1349,6 +1402,7 @@ def test_topk_filter_op(alltypes, df, result_fn, expected_fn): "trino", "druid", "oracle", + "exasol", "flink", ], raises=com.OperationNotDefinedError, @@ -1389,6 +1443,8 @@ def test_aggregate_list_like(backend, alltypes, df, agg_fn): "druid", "oracle", "flink", + "exasol", + "flink", ], raises=com.OperationNotDefinedError, ) @@ -1485,7 +1541,7 @@ def test_grouped_case(backend, con): @pytest.mark.notimpl( - ["datafusion", "mssql", "polars"], raises=com.OperationNotDefinedError + ["datafusion", "mssql", "polars", "exasol"], raises=com.OperationNotDefinedError ) @pytest.mark.broken( ["dask", "pandas"], diff --git a/ibis/backends/tests/test_api.py b/ibis/backends/tests/test_api.py index 473f030c3831..e8fe98f55f20 100644 --- a/ibis/backends/tests/test_api.py +++ b/ibis/backends/tests/test_api.py @@ -21,7 +21,7 @@ def test_version(backend): # 1. `current_database` returns '.', but isn't listed in list_databases() @pytest.mark.never( - ["polars", "dask", "pandas", "druid", "oracle"], + ["polars", "dask", "exasol", "pandas", "druid", "oracle"], reason="backend does not support databases", raises=AttributeError, ) diff --git a/ibis/backends/tests/test_array.py b/ibis/backends/tests/test_array.py index 432a1532c775..6f10d4889962 100644 --- a/ibis/backends/tests/test_array.py +++ b/ibis/backends/tests/test_array.py @@ -36,7 +36,9 @@ pytestmark = [ pytest.mark.never( - ["sqlite", "mysql", "mssql"], reason="No array support", raises=Exception + ["sqlite", "mysql", "mssql", "exasol"], + reason="No array support", + raises=Exception, ), pytest.mark.notyet(["impala"], reason="No array support", raises=Exception), pytest.mark.notimpl(["druid", "oracle"], raises=Exception), diff --git a/ibis/backends/tests/test_binary.py b/ibis/backends/tests/test_binary.py index fc0186747e9c..310c80bcca00 100644 --- a/ibis/backends/tests/test_binary.py +++ b/ibis/backends/tests/test_binary.py @@ -30,6 +30,11 @@ "Unsupported type: Binary(nullable=True)", raises=NotImplementedError, ) +@pytest.mark.notimpl( + ["exasol"], + "Exasol does not have native support for a binary data type.", + raises=sqlalchemy.exc.StatementError, +) def test_binary_literal(con, backend): expr = ibis.literal(b"A") result = con.execute(expr) diff --git a/ibis/backends/tests/test_client.py b/ibis/backends/tests/test_client.py index 77c98803122b..72e4fdf2c3d8 100644 --- a/ibis/backends/tests/test_client.py +++ b/ibis/backends/tests/test_client.py @@ -251,6 +251,7 @@ def tmpcon(alchemy_con): ["mssql"], reason="mssql supports support temporary tables through naming conventions", ) +@mark.notimpl(["exasol"], reason="Exasol does not support temporary tables") def test_create_temporary_table_from_schema(tmpcon, new_schema): temp_table = f"_{guid()}" table = tmpcon.create_table(temp_table, schema=new_schema, temp=True) @@ -276,6 +277,7 @@ def test_create_temporary_table_from_schema(tmpcon, new_schema): "datafusion", "druid", "duckdb", + "exasol", "flink", "mssql", "mysql", @@ -505,7 +507,7 @@ def test_insert_overwrite_from_expr( ["trino"], reason="memory connector doesn't allow writing to tables" ) @pytest.mark.notyet( - "oracle", + ["oracle", "exasol"], reason="No support for in-place multirow inserts", raises=sa.exc.CompileError, ) @@ -547,6 +549,11 @@ def test_insert_from_memtable(alchemy_con, alchemy_temp_table): raises=AttributeError, reason="oracle doesn't support the common notion of a database", ) +@pytest.mark.notyet( + ["exasol"], + raises=AttributeError, + reason="exasol doesn't support the common notion of a database", +) def test_list_databases(alchemy_con): # Every backend has its own databases test_databases = { @@ -558,6 +565,7 @@ def test_list_databases(alchemy_con): "snowflake": {"IBIS_TESTING"}, "trino": {"memory"}, "oracle": set(), + "exasol": set(), } assert test_databases[alchemy_con.name] <= set(alchemy_con.list_databases()) @@ -570,6 +578,7 @@ def test_list_databases(alchemy_con): @pytest.mark.notyet( ["trino"], reason="memory connector doesn't allow writing to tables" ) +@pytest.mark.notimpl(["exasol"]) def test_in_memory(alchemy_backend, alchemy_temp_table): con = getattr(ibis, alchemy_backend.name()).connect(":memory:") with con.begin() as c: @@ -884,7 +893,7 @@ def test_self_join_memory_table(backend, con): ], ) @pytest.mark.notimpl(["dask", "datafusion", "druid"]) -def test_create_from_in_memory_table(con, t, temp_table): +def test_create_from_in_memory_table(backend, con, t, temp_table): con.create_table(temp_table, t) assert temp_table in con.list_tables() @@ -1187,6 +1196,7 @@ def test_set_backend_url(url, monkeypatch): "dask", "datafusion", "duckdb", + "exasol", "impala", "mssql", "mysql", @@ -1229,6 +1239,7 @@ def test_create_table_timestamp(con, temp_table): ["mssql"], reason="mssql supports support temporary tables through naming conventions", ) +@mark.notimpl(["exasol"], reason="Exasol does not support temporary tables") def test_persist_expression_ref_count(backend, con, alltypes): non_persisted_table = alltypes.mutate(test_column="calculation") persisted_table = non_persisted_table.cache() @@ -1248,6 +1259,7 @@ def test_persist_expression_ref_count(backend, con, alltypes): ["mssql"], reason="mssql supports support temporary tables through naming conventions", ) +@mark.notimpl(["exasol"], reason="Exasol does not support temporary tables") def test_persist_expression(backend, alltypes): non_persisted_table = alltypes.mutate(test_column="calculation", other_calc="xyz") persisted_table = non_persisted_table.cache() @@ -1261,6 +1273,7 @@ def test_persist_expression(backend, alltypes): ["mssql"], reason="mssql supports support temporary tables through naming conventions", ) +@mark.notimpl(["exasol"], reason="Exasol does not support temporary tables") def test_persist_expression_contextmanager(backend, alltypes): non_cached_table = alltypes.mutate( test_column="calculation", other_column="big calc" @@ -1276,6 +1289,7 @@ def test_persist_expression_contextmanager(backend, alltypes): ["mssql"], reason="mssql supports support temporary tables through naming conventions", ) +@mark.notimpl(["exasol"], reason="Exasol does not support temporary tables") def test_persist_expression_contextmanager_ref_count(backend, con, alltypes): non_cached_table = alltypes.mutate( test_column="calculation", other_column="big calc 2" @@ -1294,6 +1308,7 @@ def test_persist_expression_contextmanager_ref_count(backend, con, alltypes): ["mssql"], reason="mssql supports support temporary tables through naming conventions", ) +@mark.notimpl(["exasol"], reason="Exasol does not support temporary tables") def test_persist_expression_multiple_refs(backend, con, alltypes): non_cached_table = alltypes.mutate( test_column="calculation", other_column="big calc 2" @@ -1329,6 +1344,7 @@ def test_persist_expression_multiple_refs(backend, con, alltypes): ["mssql"], reason="mssql supports support temporary tables through naming conventions", ) +@mark.notimpl(["exasol"], reason="Exasol does not support temporary tables") def test_persist_expression_repeated_cache(alltypes): non_cached_table = alltypes.mutate( test_column="calculation", other_column="big calc 2" @@ -1343,6 +1359,7 @@ def test_persist_expression_repeated_cache(alltypes): ["mssql"], reason="mssql supports support temporary tables through naming conventions", ) +@mark.notimpl(["exasol"], reason="Exasol does not support temporary tables") def test_persist_expression_release(con, alltypes): non_cached_table = alltypes.mutate( test_column="calculation", other_column="big calc 3" diff --git a/ibis/backends/tests/test_column.py b/ibis/backends/tests/test_column.py index 95ff88826c61..f26b2a876ded 100644 --- a/ibis/backends/tests/test_column.py +++ b/ibis/backends/tests/test_column.py @@ -11,6 +11,7 @@ "clickhouse", "dask", "datafusion", + "exasol", "impala", "mssql", "mysql", diff --git a/ibis/backends/tests/test_dot_sql.py b/ibis/backends/tests/test_dot_sql.py index 20029ba143b6..77f5c4b34a9f 100644 --- a/ibis/backends/tests/test_dot_sql.py +++ b/ibis/backends/tests/test_dot_sql.py @@ -15,7 +15,7 @@ PolarsComputeError = None table_dot_sql_notimpl = pytest.mark.notimpl(["bigquery", "impala", "druid"]) -dot_sql_notimpl = pytest.mark.notimpl(["datafusion", "flink"]) +dot_sql_notimpl = pytest.mark.notimpl(["datafusion", "exasol", "flink"]) dot_sql_notyet = pytest.mark.notyet( ["snowflake", "oracle"], reason="snowflake and oracle column names are case insensitive", @@ -203,6 +203,7 @@ def test_table_dot_sql_repr(con): @dot_sql_never @pytest.mark.notimpl(["oracle"]) @pytest.mark.notyet(["polars"], raises=PolarsComputeError) +@pytest.mark.notimpl(["exasol"], strict=False) def test_table_dot_sql_does_not_clobber_existing_tables(con, temp_table): t = con.create_table(temp_table, schema=ibis.schema(dict(a="string"))) expr = t.sql("SELECT 1 as x FROM functional_alltypes") @@ -235,7 +236,7 @@ def test_dot_sql_reuse_alias_with_different_types(backend, alltypes, df): backend.assert_series_equal(foo2.x.execute(), expected2) -_NO_SQLGLOT_DIALECT = {"pandas", "dask", "druid", "flink"} +_NO_SQLGLOT_DIALECT = {"pandas", "dask", "druid", "flink", "exasol"} no_sqlglot_dialect = sorted( param(backend, marks=pytest.mark.xfail) for backend in _NO_SQLGLOT_DIALECT ) diff --git a/ibis/backends/tests/test_examples.py b/ibis/backends/tests/test_examples.py index a339113421cb..34e6d4907f61 100644 --- a/ibis/backends/tests/test_examples.py +++ b/ibis/backends/tests/test_examples.py @@ -12,7 +12,7 @@ (LINUX or MACOS) and SANDBOXED, reason="nix on linux cannot download duckdb extensions or data due to sandboxing", ) -@pytest.mark.notimpl(["dask", "datafusion", "pyspark", "flink"]) +@pytest.mark.notimpl(["dask", "datafusion", "pyspark", "flink", "exasol"]) @pytest.mark.notyet(["clickhouse", "druid", "impala", "mssql", "trino"]) @pytest.mark.parametrize( ("example", "columns"), diff --git a/ibis/backends/tests/test_export.py b/ibis/backends/tests/test_export.py index 06d9f0a00f5b..12529246df7c 100644 --- a/ibis/backends/tests/test_export.py +++ b/ibis/backends/tests/test_export.py @@ -99,6 +99,7 @@ def test_empty_column_to_pyarrow(limit, awards_players): @pytest.mark.parametrize("limit", no_limit) +@pytest.mark.notimpl(["exasol"], raises=AttributeError) def test_empty_scalar_to_pyarrow(limit, awards_players): expr = awards_players.filter(awards_players.awardID == "DEADBEEF").yearID.sum() array = expr.to_pyarrow(limit=limit) @@ -106,12 +107,13 @@ def test_empty_scalar_to_pyarrow(limit, awards_players): @pytest.mark.parametrize("limit", no_limit) +@pytest.mark.notimpl(["exasol"], raises=AttributeError) def test_scalar_to_pyarrow_scalar(limit, awards_players): scalar = awards_players.yearID.sum().to_pyarrow(limit=limit) assert isinstance(scalar, pa.Scalar) -@pytest.mark.notimpl(["druid", "flink"]) +@pytest.mark.notimpl(["druid", "flink", "exasol"]) def test_table_to_pyarrow_table_schema(awards_players): table = awards_players.to_pyarrow() assert isinstance(table, pa.Table) @@ -260,6 +262,7 @@ def test_table_to_parquet_writer_kwargs(version, tmp_path, backend, awards_playe reason="no partitioning support", ) @pytest.mark.notimpl(["druid", "flink"], reason="No to_parquet support") +@pytest.mark.notimpl(["exasol"], raises=TypeError) def test_roundtrip_partitioned_parquet(tmp_path, con, backend, awards_players): outparquet = tmp_path / "outhive.parquet" awards_players.to_parquet(outparquet, partition_by="yearID") @@ -304,7 +307,7 @@ def test_memtable_to_file(tmp_path, con, ftype, monkeypatch): assert outfile.is_file() -@pytest.mark.notimpl(["flink"]) +@pytest.mark.notimpl(["flink", "exasol"]) def test_table_to_csv(tmp_path, backend, awards_players): outcsv = tmp_path / "out.csv" @@ -318,7 +321,7 @@ def test_table_to_csv(tmp_path, backend, awards_players): backend.assert_frame_equal(awards_players.to_pandas(), df) -@pytest.mark.notimpl(["flink"]) +@pytest.mark.notimpl(["flink", "exasol"]) @pytest.mark.notimpl( ["duckdb"], reason="cannot inline WriteOptions objects", @@ -345,6 +348,7 @@ def test_table_to_csv_writer_kwargs(delimiter, tmp_path, awards_players): marks=[ pytest.mark.notyet(["druid"], raises=sa.exc.ProgrammingError), pytest.mark.notyet(["flink"], raises=NotImplementedError), + pytest.mark.notyet(["exasol"], raises=sa.exc.DBAPIError), ], ), param( @@ -365,6 +369,7 @@ def test_table_to_csv_writer_kwargs(delimiter, tmp_path, awards_players): reason="precision is out of range", ), pytest.mark.notyet(["flink"], raises=NotImplementedError), + pytest.mark.notyet(["exasol"], raises=sa.exc.DBAPIError), ], ), ], @@ -392,6 +397,7 @@ def test_to_pyarrow_decimal(backend, dtype, pyarrow_dtype): "dask", "trino", "flink", + "exasol", ], raises=NotImplementedError, reason="read_delta not yet implemented", @@ -486,7 +492,10 @@ def test_to_pandas_batches_empty_table(backend, con): @pytest.mark.notimpl(["druid", "flink"]) -@pytest.mark.parametrize("n", [None, 1]) +@pytest.mark.parametrize( + "n", + [param(None, marks=pytest.mark.notimpl(["exasol"], raises=sa.exc.CompileError)), 1], +) def test_to_pandas_batches_nonempty_table(backend, con, n): t = backend.functional_alltypes.limit(n) n = t.count().execute() @@ -496,7 +505,15 @@ def test_to_pandas_batches_nonempty_table(backend, con, n): @pytest.mark.notimpl(["flink"]) -@pytest.mark.parametrize("n", [None, 0, 1, 2]) +@pytest.mark.parametrize( + "n", + [ + param(None, marks=pytest.mark.notimpl(["exasol"], raises=sa.exc.CompileError)), + 0, + 1, + 2, + ], +) def test_to_pandas_batches_column(backend, con, n): t = backend.functional_alltypes.limit(n).timestamp_col n = t.count().execute() diff --git a/ibis/backends/tests/test_generic.py b/ibis/backends/tests/test_generic.py index c508b696666f..1639ac8dab1f 100644 --- a/ibis/backends/tests/test_generic.py +++ b/ibis/backends/tests/test_generic.py @@ -33,18 +33,20 @@ except ImportError: ClickhouseDriverDatabaseError = None - try: from google.api_core.exceptions import BadRequest except ImportError: BadRequest = None - try: from impala.error import HiveServer2Error except ImportError: HiveServer2Error = None +try: + from pyexasol.exceptions import ExaQueryError +except ImportError: + ExaQueryError = None NULL_BACKEND_TYPES = { "bigquery": "NULL", @@ -84,6 +86,7 @@ def test_null_literal(con, backend): } +@pytest.mark.notimpl(["exasol"]) def test_boolean_literal(con, backend): expr = ibis.literal(False, type=dt.boolean) result = con.execute(expr) @@ -135,6 +138,7 @@ def test_scalar_fillna_nullif(con, expr, expected): ) @pytest.mark.notimpl(["mssql", "druid", "oracle"]) @pytest.mark.notyet(["flink"], "NaN is not supported in Flink SQL", raises=ValueError) +@pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError, strict=False) def test_isna(backend, alltypes, col, filt): table = alltypes.select( nan_col=ibis.literal(np.nan), none_col=ibis.NA.cast("float64") @@ -168,6 +172,7 @@ def test_isna(backend, alltypes, col, filt): "mssql", "druid", "oracle", + "exasol", ], reason="NaN != NULL for these backends", ), @@ -209,7 +214,9 @@ def test_coalesce(con, expr, expected): # TODO(dask) - identicalTo - #2553 -@pytest.mark.notimpl(["clickhouse", "datafusion", "dask", "pyspark", "mssql", "druid"]) +@pytest.mark.notimpl( + ["clickhouse", "datafusion", "dask", "pyspark", "mssql", "druid", "exasol"] +) def test_identical_to(backend, alltypes, sorted_df): sorted_alltypes = alltypes.order_by("id") df = sorted_df @@ -323,6 +330,7 @@ def test_filter(backend, alltypes, sorted_df, predicate_fn, expected_fn): "trino", "druid", "oracle", + "exasol", ] ) @pytest.mark.never( @@ -370,7 +378,7 @@ def test_case_where(backend, alltypes, df): # TODO: some of these are notimpl (datafusion) others are probably never -@pytest.mark.notimpl(["mysql", "sqlite", "mssql", "druid", "oracle"]) +@pytest.mark.notimpl(["mysql", "sqlite", "mssql", "druid", "oracle", "exasol"]) @pytest.mark.notyet(["flink"], "NaN is not supported in Flink SQL", raises=ValueError) def test_select_filter_mutate(backend, alltypes, df): """Test that select, filter and mutate are executed in right order. @@ -489,7 +497,17 @@ def test_dropna_invalid(alltypes): @pytest.mark.parametrize("how", ["any", "all"]) @pytest.mark.parametrize( - "subset", [None, [], "col_1", ["col_1", "col_2"], ["col_1", "col_3"]] + "subset", + [ + None, + param( + [], + marks=pytest.mark.notimpl(["exasol"], raises=ExaQueryError, strict=False), + ), + "col_1", + ["col_1", "col_2"], + ["col_1", "col_3"], + ], ) def test_dropna_table(backend, alltypes, how, subset): is_two = alltypes.int_col == 2 @@ -559,6 +577,10 @@ def test_order_by_random(alltypes): raises=sa.exc.ProgrammingError, reason="Druid only supports trivial unions", ) +@pytest.mark.notyet( + ["exasol"], + raises=AssertionError, +) def test_table_info(alltypes): expr = alltypes.info() df = expr.execute() @@ -581,11 +603,13 @@ def test_table_info(alltypes): param( _.string_col.isin([]), lambda df: df.string_col.isin([]), + marks=pytest.mark.notimpl(["exasol"], raises=ExaQueryError), id="isin", ), param( _.string_col.notin([]), lambda df: ~df.string_col.isin([]), + marks=pytest.mark.notimpl(["exasol"], raises=ExaQueryError), id="notin", ), param( @@ -656,6 +680,7 @@ def test_isin_notin_column_expr(backend, alltypes, df, ibis_op, pandas_op): param(False, True, neg, id="false_negate"), ], ) +@pytest.mark.notimpl(["exasol"]) def test_logical_negation_literal(con, expr, expected, op): assert con.execute(op(ibis.literal(expr)).name("tmp")) == expected @@ -795,7 +820,7 @@ def test_int_column(alltypes): assert result.dtype == np.int8 -@pytest.mark.notimpl(["druid", "oracle"]) +@pytest.mark.notimpl(["druid", "oracle", "exasol"]) @pytest.mark.never( ["bigquery", "sqlite", "snowflake"], reason="backend only implements int64" ) @@ -806,7 +831,7 @@ def test_int_scalar(alltypes): assert result.dtype == np.int16 -@pytest.mark.notimpl(["dask", "datafusion", "pandas", "polars", "druid"]) +@pytest.mark.notimpl(["dask", "datafusion", "pandas", "polars", "druid", "exasol"]) @pytest.mark.notyet( ["clickhouse"], reason="https://github.com/ClickHouse/ClickHouse/issues/6697" ) @@ -834,6 +859,7 @@ def test_exists(batting, awards_players, method_name): "polars", "druid", "oracle", + "exasol", ], raises=com.OperationNotDefinedError, ) @@ -849,6 +875,7 @@ def test_typeof(con): @pytest.mark.notyet(["impala"], reason="can't find table in subquery") @pytest.mark.notimpl(["datafusion", "pyspark", "druid"]) @pytest.mark.notyet(["dask", "mssql"], reason="not supported by the backend") +@pytest.mark.notimpl(["exasol"], raises=sa.exc.DBAPIError) def test_isin_uncorrelated( backend, batting, awards_players, batting_df, awards_players_df ): @@ -868,7 +895,7 @@ def test_isin_uncorrelated( @pytest.mark.broken(["polars"], reason="incorrect answer") -@pytest.mark.notimpl(["datafusion", "pyspark", "druid"]) +@pytest.mark.notimpl(["datafusion", "pyspark", "druid", "exasol"]) @pytest.mark.notyet(["dask"], reason="not supported by the backend") def test_isin_uncorrelated_filter( backend, batting, awards_players, batting_df, awards_players_df @@ -914,6 +941,7 @@ def test_literal_na(con, dtype): assert pd.isna(result) +@pytest.mark.notimpl(["exasol"]) def test_memtable_bool_column(backend, con): t = ibis.memtable({"a": [True, False, True]}) backend.assert_series_equal( @@ -989,6 +1017,7 @@ def test_memtable_column_naming_mismatch(backend, con, monkeypatch, df, columns) ["pyspark"], reason="pyspark doesn't generate SQL", raises=NotImplementedError ) @pytest.mark.notimpl(["druid", "flink"], reason="no sqlglot dialect", raises=ValueError) +@pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError) def test_many_subqueries(con, snapshot): def query(t, group_cols): t2 = t.mutate(key=ibis.row_number().over(ibis.window(order_by=group_cols))) @@ -1003,7 +1032,7 @@ def query(t, group_cols): @pytest.mark.notimpl( - ["dask", "pandas", "oracle", "flink"], raises=com.OperationNotDefinedError + ["dask", "pandas", "oracle", "flink", "exasol"], raises=com.OperationNotDefinedError ) @pytest.mark.notimpl(["druid"], raises=AssertionError) @pytest.mark.notyet( @@ -1117,6 +1146,10 @@ def test_pivot_wider(backend): raises=com.OperationNotDefinedError, reason="backend doesn't implement deduplication", ) +@pytest.mark.notimpl( + ["exasol"], + raises=com.OperationNotDefinedError, +) def test_distinct_on_keep(backend, on, keep): from ibis import _ @@ -1167,6 +1200,10 @@ def test_distinct_on_keep(backend, on, keep): raises=(NotImplementedError, sa.exc.ProgrammingError, com.OperationNotDefinedError), reason="arbitrary not implemented in the backend", ) +@pytest.mark.notimpl( + ["exasol"], + raises=com.OperationNotDefinedError, +) @pytest.mark.notimpl( ["dask", "datafusion"], raises=com.OperationNotDefinedError, @@ -1209,7 +1246,7 @@ def test_distinct_on_keep_is_none(backend, on): assert len(result) == len(expected) -@pytest.mark.notimpl(["dask", "pandas", "postgres", "flink"]) +@pytest.mark.notimpl(["dask", "pandas", "postgres", "flink", "exasol"]) @pytest.mark.notyet( [ "sqlite", @@ -1241,6 +1278,7 @@ def test_hash_consistent(backend, alltypes): "pyspark", "snowflake", "sqlite", + "exasol", ] ) @pytest.mark.parametrize( @@ -1307,6 +1345,7 @@ def test_try_cast_expected(con, from_val, to_type, expected): "pyspark", "snowflake", "sqlite", + "exasol", ] ) def test_try_cast_table(backend, con): @@ -1336,6 +1375,7 @@ def test_try_cast_table(backend, con): "pyspark", "snowflake", "sqlite", + "exasol", ] ) @pytest.mark.parametrize( @@ -1371,9 +1411,25 @@ def test_try_cast_func(con, from_val, to_type, func): ### NONE/ZERO start # no stop param(slice(None, 0), lambda _: 0, id="[:0]"), - param(slice(None, None), lambda t: t.count().to_pandas(), id="[:]"), + param( + slice(None, None), + lambda t: t.count().to_pandas(), + marks=pytest.mark.notyet( + ["exasol"], + raises=sa.exc.CompileError, + ), + id="[:]", + ), param(slice(0, 0), lambda _: 0, id="[0:0]"), - param(slice(0, None), lambda t: t.count().to_pandas(), id="[0:]"), + param( + slice(0, None), + lambda t: t.count().to_pandas(), + marks=pytest.mark.notyet( + ["exasol"], + raises=sa.exc.CompileError, + ), + id="[0:]", + ), # positive stop param(slice(None, 2), lambda _: 2, id="[:2]"), param(slice(0, 2), lambda _: 2, id="[0:2]"), @@ -1404,6 +1460,10 @@ def test_try_cast_func(con, from_val, to_type, func): raises=sa.exc.CompileError, reason="mssql doesn't support OFFSET without LIMIT", ), + pytest.mark.notyet( + ["exasol"], + raises=sa.exc.CompileError, + ), pytest.mark.never( ["impala"], raises=HiveServer2Error, @@ -1428,6 +1488,10 @@ def test_try_cast_func(con, from_val, to_type, func): raises=sa.exc.CompileError, reason="mssql doesn't support OFFSET without LIMIT", ), + pytest.mark.notyet( + ["exasol"], + raises=sa.exc.DBAPIError, + ), pytest.mark.notyet( ["impala"], raises=HiveServer2Error, @@ -1486,6 +1550,10 @@ def test_static_table_slice(backend, slc, expected_count_fn): raises=sa.exc.CompileError, reason="mssql doesn't support dynamic limit/offset without an ORDER BY", ) +@pytest.mark.notimpl( + ["exasol"], + raises=sa.exc.CompileError, +) @pytest.mark.notyet( ["clickhouse"], raises=ClickhouseDriverDatabaseError, @@ -1525,6 +1593,10 @@ def test_dynamic_table_slice(backend, slc, expected_count_fn): raises=sa.exc.ProgrammingError, reason="backend doesn't support dynamic limit/offset", ) +@pytest.mark.notimpl( + ["exasol"], + raises=sa.exc.CompileError, +) @pytest.mark.notyet( ["clickhouse"], raises=ClickhouseDriverDatabaseError, @@ -1579,6 +1651,7 @@ def test_dynamic_table_slice_with_computed_offset(backend): "flink", "polars", "snowflake", + "exasol", ] ) def test_sample(backend): @@ -1603,6 +1676,7 @@ def test_sample(backend): "flink", "polars", "snowflake", + "exasol", ] ) def test_sample_memtable(con, backend): @@ -1628,6 +1702,7 @@ def test_sample_memtable(con, backend): "snowflake", "sqlite", "trino", + "exasol", ] ) def test_sample_with_seed(backend): diff --git a/ibis/backends/tests/test_join.py b/ibis/backends/tests/test_join.py index a52d0f2c0650..502de33ef6b0 100644 --- a/ibis/backends/tests/test_join.py +++ b/ibis/backends/tests/test_join.py @@ -74,6 +74,7 @@ def check_eq(left, right, how, **kwargs): reason="https://github.com/pola-rs/polars/issues/9955", raises=ColumnNotFoundError, ) +@pytest.mark.notimpl(["exasol"], raises=AttributeError) def test_mutating_join(backend, batting, awards_players, how): left = batting[batting.yearID == 2015] right = awards_players[awards_players.lgID == "NL"].drop("yearID", "lgID") @@ -122,7 +123,7 @@ def test_mutating_join(backend, batting, awards_players, how): @pytest.mark.parametrize("how", ["semi", "anti"]) -@pytest.mark.notimpl(["dask", "druid"]) +@pytest.mark.notimpl(["dask", "druid", "exasol"]) @pytest.mark.notyet(["flink"], reason="Flink doesn't support semi joins or anti joins") def test_filtering_join(backend, batting, awards_players, how): left = batting[batting.yearID == 2015] @@ -157,6 +158,7 @@ def test_filtering_join(backend, batting, awards_players, how): raises=ValueError, reason="https://github.com/pola-rs/polars/issues/9335", ) +@pytest.mark.notimpl(["exasol"], raises=com.IbisTypeError) def test_join_then_filter_no_column_overlap(awards_players, batting): left = batting[batting.yearID == 2015] year = left.yearID.name("year") @@ -174,6 +176,7 @@ def test_join_then_filter_no_column_overlap(awards_players, batting): raises=ValueError, reason="https://github.com/pola-rs/polars/issues/9335", ) +@pytest.mark.notimpl(["exasol"], raises=com.IbisTypeError) def test_mutate_then_join_no_column_overlap(batting, awards_players): left = batting.mutate(year=batting.yearID).filter(lambda t: t.year == 2015) left = left["year", "RBI"] @@ -201,6 +204,7 @@ def test_mutate_then_join_no_column_overlap(batting, awards_players): param(lambda left, right: left.join(right, "year", how="semi"), id="how_semi"), ], ) +@pytest.mark.notimpl(["exasol"], raises=com.IbisTypeError) def test_semi_join_topk(batting, awards_players, func): batting = batting.mutate(year=batting.yearID) left = func(batting, batting.year.topk(5)).select("year", "RBI") @@ -208,7 +212,7 @@ def test_semi_join_topk(batting, awards_players, func): assert not expr.limit(5).execute().empty -@pytest.mark.notimpl(["dask", "druid"]) +@pytest.mark.notimpl(["dask", "druid", "exasol"]) def test_join_with_pandas(batting, awards_players): batting_filt = batting[lambda t: t.yearID < 1900] awards_players_filt = awards_players[lambda t: t.yearID < 1900].execute() @@ -218,7 +222,7 @@ def test_join_with_pandas(batting, awards_players): assert df.yearID.nunique() == 7 -@pytest.mark.notimpl(["dask"]) +@pytest.mark.notimpl(["dask", "exasol"]) def test_join_with_pandas_non_null_typed_columns(batting, awards_players): batting_filt = batting[lambda t: t.yearID < 1900][["yearID"]] awards_players_filt = awards_players[lambda t: t.yearID < 1900][ @@ -309,6 +313,10 @@ def test_join_with_pandas_non_null_typed_columns(batting, awards_players): raises=TypeError, reason="dask and pandas don't support join predicates", ) +@pytest.mark.notimpl( + ["exasol"], + raises=com.IbisTypeError, +) def test_join_with_trivial_predicate(awards_players, predicate, how, pandas_value): n = 5 @@ -329,7 +337,9 @@ def test_join_with_trivial_predicate(awards_players, predicate, how, pandas_valu @pytest.mark.notimpl( - ["druid"], raises=sa.exc.NoSuchTableError, reason="`win` table isn't loaded" + ["druid", "exasol"], + raises=sa.exc.NoSuchTableError, + reason="`win` table isn't loaded", ) @pytest.mark.notimpl(["flink"], reason="`win` table isn't loaded") @pytest.mark.parametrize( diff --git a/ibis/backends/tests/test_json.py b/ibis/backends/tests/test_json.py index eab13961c7bf..78d379ae0bde 100644 --- a/ibis/backends/tests/test_json.py +++ b/ibis/backends/tests/test_json.py @@ -12,7 +12,7 @@ pytestmark = [ pytest.mark.never(["impala"], reason="doesn't support JSON and never will"), pytest.mark.notyet(["clickhouse"], reason="upstream is broken"), - pytest.mark.notimpl(["datafusion", "mssql", "druid", "oracle"]), + pytest.mark.notimpl(["datafusion", "exasol", "mssql", "druid", "oracle"]), ] diff --git a/ibis/backends/tests/test_map.py b/ibis/backends/tests/test_map.py index e9ae99068801..0983f59b7ccb 100644 --- a/ibis/backends/tests/test_map.py +++ b/ibis/backends/tests/test_map.py @@ -17,7 +17,7 @@ ["bigquery", "impala"], reason="Backend doesn't yet implement map types" ), pytest.mark.notimpl( - ["datafusion", "pyspark", "polars", "druid", "oracle"], + ["datafusion", "exasol", "pyspark", "polars", "druid", "oracle"], reason="Not yet implemented in ibis", ), ] diff --git a/ibis/backends/tests/test_network.py b/ibis/backends/tests/test_network.py index 10ef3de896b6..7af7c66d671b 100644 --- a/ibis/backends/tests/test_network.py +++ b/ibis/backends/tests/test_network.py @@ -10,6 +10,11 @@ import ibis.common.exceptions as com import ibis.expr.datatypes as dt +try: + from pyexasol.exceptions import ExaQueryError +except ImportError: + ExaQueryError = None + MACADDR_BACKEND_TYPE = { "bigquery": "STRING", "clickhouse": "String", @@ -106,6 +111,7 @@ def test_macaddr_literal(con, backend): ) @pytest.mark.notimpl(["flink", "polars"], raises=NotImplementedError) @pytest.mark.notimpl(["druid", "oracle"], raises=KeyError) +@pytest.mark.notimpl(["exasol"], raises=(ExaQueryError, KeyError)) def test_inet_literal(con, backend, test_value, expected_values, expected_types): backend_name = backend.name() expr = ibis.literal(test_value, type=dt.inet) diff --git a/ibis/backends/tests/test_numeric.py b/ibis/backends/tests/test_numeric.py index 8fe6c8084b76..627c32b498f4 100644 --- a/ibis/backends/tests/test_numeric.py +++ b/ibis/backends/tests/test_numeric.py @@ -28,7 +28,6 @@ duckdb = None DuckDBConversionException = None - try: import clickhouse_connect as cc @@ -56,6 +55,11 @@ except ImportError: ImpalaHiveServer2Error = None +try: + from pyexasol.exceptions import ExaQueryError +except ImportError: + ExaQueryError = None + @pytest.mark.parametrize( ("expr", "expected_types"), @@ -204,6 +208,10 @@ "Expected np.float16 instance", raises=ArrowNotImplementedError, ), + pytest.mark.notimpl( + ["exasol"], + raises=ExaQueryError, + ), ], id="float16", ), @@ -220,6 +228,12 @@ "postgres": "numeric", "flink": "FLOAT NOT NULL", }, + marks=[ + pytest.mark.notimpl( + ["exasol"], + raises=ExaQueryError, + ), + ], id="float32", ), param( @@ -235,6 +249,12 @@ "postgres": "numeric", "flink": "DOUBLE NOT NULL", }, + marks=[ + pytest.mark.notimpl( + ["exasol"], + raises=ExaQueryError, + ), + ], id="float64", ), ], @@ -281,6 +301,10 @@ def test_numeric_literal(con, backend, expr, expected_types): "flink": "DECIMAL(38, 18) NOT NULL", }, marks=[ + pytest.mark.notimpl( + ["exasol"], + raises=ExaQueryError, + ), pytest.mark.notimpl( ["clickhouse"], "Unsupported precision. Supported values: [1 : 76]. Current value: None", @@ -331,6 +355,10 @@ def test_numeric_literal(con, backend, expr, expected_types): "flink": "DECIMAL(38, 9) NOT NULL", }, marks=[ + pytest.mark.notimpl( + ["exasol"], + raises=ExaQueryError, + ), pytest.mark.broken( ["impala"], "impala.error.HiveServer2Error: AnalysisException: Syntax error in line 1:" @@ -374,6 +402,10 @@ def test_numeric_literal(con, backend, expr, expected_types): "postgres": "numeric", }, marks=[ + pytest.mark.notimpl( + ["exasol"], + raises=ExaQueryError, + ), pytest.mark.broken( ["impala"], "impala.error.HiveServer2Error: AnalysisException: Syntax error in line 1:" @@ -409,6 +441,7 @@ def test_numeric_literal(con, backend, expr, expected_types): "pandas": decimal.Decimal("Infinity"), "dask": decimal.Decimal("Infinity"), "impala": float("inf"), + "exasol": float("inf"), }, { "bigquery": "FLOAT64", @@ -489,6 +522,7 @@ def test_numeric_literal(con, backend, expr, expected_types): "pandas": decimal.Decimal("-Infinity"), "dask": decimal.Decimal("-Infinity"), "impala": float("-inf"), + "exasol": float("-inf"), }, { "bigquery": "FLOAT64", @@ -569,6 +603,7 @@ def test_numeric_literal(con, backend, expr, expected_types): "pandas": decimal.Decimal("NaN"), "dask": decimal.Decimal("NaN"), "impala": float("nan"), + "exasol": float("nan"), }, { "bigquery": "FLOAT64", @@ -676,11 +711,15 @@ def test_decimal_literal(con, backend, expr, expected_types, expected_result): lambda t: t.float_col, id="float-column", marks=[ + pytest.mark.notimpl( + ["exasol"], + raises=com.OperationNotDefinedError, + ), pytest.mark.notimpl( ["druid"], raises=AttributeError, reason="AttributeError: 'DecimalColumn' object has no attribute 'isinf'", - ) + ), ], ), param( @@ -688,11 +727,15 @@ def test_decimal_literal(con, backend, expr, expected_types, expected_result): lambda t: t.double_col, id="double-column", marks=[ + pytest.mark.notimpl( + ["exasol"], + raises=com.OperationNotDefinedError, + ), pytest.mark.notimpl( ["druid"], raises=AttributeError, reason="AttributeError: 'DecimalColumn' object has no attribute 'isinf'", - ) + ), ], ), param( @@ -700,10 +743,14 @@ def test_decimal_literal(con, backend, expr, expected_types, expected_result): lambda t: 1.3, id="float-literal", marks=[ + pytest.mark.notimpl( + ["exasol"], + raises=com.OperationNotDefinedError, + ), pytest.mark.notimpl( ["druid"], raises=com.OperationNotDefinedError, - ) + ), ], ), param( @@ -722,10 +769,14 @@ def test_decimal_literal(con, backend, expr, expected_types, expected_result): lambda t: np.inf, id="inf-literal", marks=[ + pytest.mark.notimpl( + ["exasol"], + raises=com.OperationNotDefinedError, + ), pytest.mark.notimpl( ["druid"], raises=com.OperationNotDefinedError, - ) + ), ], ), param( @@ -733,10 +784,14 @@ def test_decimal_literal(con, backend, expr, expected_types, expected_result): lambda t: -np.inf, id="-inf-literal", marks=[ + pytest.mark.notimpl( + ["exasol"], + raises=com.OperationNotDefinedError, + ), pytest.mark.notimpl( ["druid"], raises=com.OperationNotDefinedError, - ) + ), ], ), ], @@ -747,6 +802,12 @@ def test_decimal_literal(con, backend, expr, expected_types, expected_result): param( operator.methodcaller("isnan"), np.isnan, + marks=[ + pytest.mark.notimpl( + ["exasol"], + raises=com.OperationNotDefinedError, + ), + ], id="isnan", ), param( @@ -754,10 +815,14 @@ def test_decimal_literal(con, backend, expr, expected_types, expected_result): np.isinf, id="isinf", marks=[ + pytest.mark.notimpl( + ["exasol"], + raises=com.OperationNotDefinedError, + ), pytest.mark.notimpl( ["datafusion"], raises=com.OperationNotDefinedError, - ) + ), ], ), ], @@ -858,7 +923,13 @@ def test_isnan_isinf( L(5.556).log(2), math.log(5.556, 2), id="log-base", - marks=pytest.mark.notimpl(["druid"], raises=com.OperationNotDefinedError), + marks=[ + pytest.mark.notimpl( + ["exasol"], + raises=com.OperationNotDefinedError, + ), + pytest.mark.notimpl(["druid"], raises=com.OperationNotDefinedError), + ], ), param( L(5.556).ln(), @@ -869,11 +940,18 @@ def test_isnan_isinf( L(5.556).log2(), math.log(5.556, 2), id="log2", - marks=pytest.mark.notimpl(["druid"], raises=com.OperationNotDefinedError), + marks=[ + pytest.mark.notimpl( + ["exasol"], + raises=com.OperationNotDefinedError, + ), + pytest.mark.notimpl(["druid"], raises=com.OperationNotDefinedError), + ], ), param( L(5.556).log10(), math.log10(5.556), + marks=pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError), id="log10", ), param( @@ -889,6 +967,7 @@ def test_isnan_isinf( param( L(11) % 3, 11 % 3, + marks=pytest.mark.notimpl(["exasol"], raises=ExaQueryError), id="mod", ), ], @@ -1022,6 +1101,7 @@ def test_simple_math_functions_columns( param( lambda t: t.double_col.add(1).log(2), lambda t: np.log2(t.double_col + 1), + marks=pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError), id="log2", ), param( @@ -1032,6 +1112,7 @@ def test_simple_math_functions_columns( param( lambda t: t.double_col.add(1).log10(), lambda t: np.log10(t.double_col + 1), + marks=pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError), id="log10", ), param( @@ -1046,6 +1127,7 @@ def test_simple_math_functions_columns( ), id="log_base_bigint", marks=[ + pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError), pytest.mark.notimpl( ["datafusion"], raises=com.OperationNotDefinedError ), @@ -1158,6 +1240,7 @@ def test_backend_specific_numerics(backend, con, df, alltypes, expr_fn, expected ], ids=lambda op: op.__name__, ) +@pytest.mark.notimpl(["exasol"], raises=AttributeError) def test_binary_arithmetic_operations(backend, alltypes, df, op): smallint_col = alltypes.smallint_col + 1 # make it nonzero smallint_series = df.smallint_col + 1 @@ -1175,6 +1258,7 @@ def test_binary_arithmetic_operations(backend, alltypes, df, op): backend.assert_series_equal(result, expected, check_exact=False) +@pytest.mark.notimpl(["exasol"], raises=AttributeError) def test_mod(backend, alltypes, df): expr = operator.mod(alltypes.smallint_col, alltypes.smallint_col + 1).name("tmp") @@ -1199,6 +1283,7 @@ def test_mod(backend, alltypes, df): "Cannot apply '%' to arguments of type ' % '. Supported form(s): ' % ", raises=Py4JError, ) +@pytest.mark.notimpl(["exasol"], raises=AttributeError) def test_floating_mod(backend, alltypes, df): expr = operator.mod(alltypes.double_col, alltypes.smallint_col + 1).name("tmp") @@ -1351,6 +1436,7 @@ def test_floating_mod(backend, alltypes, df): @pytest.mark.notyet(["mssql"], raises=sa.exc.OperationalError) @pytest.mark.notyet(["postgres"], raises=sa.exc.DataError) @pytest.mark.notyet(["snowflake"], raises=sa.exc.ProgrammingError) +@pytest.mark.notimpl(["exasol"], raises=(sa.exc.DBAPIError, com.IbisTypeError)) def test_divide_by_zero(backend, alltypes, df, column, denominator): expr = alltypes[column] / denominator result = expr.name("tmp").execute() @@ -1402,7 +1488,7 @@ def test_divide_by_zero(backend, alltypes, df, column, denominator): ], reason="Not SQLAlchemy backends", ) -@pytest.mark.notimpl(["druid"], raises=KeyError) +@pytest.mark.notimpl(["druid", "exasol"], raises=KeyError) def test_sa_default_numeric_precision_and_scale( con, backend, default_precisions, default_scales, temp_table ): @@ -1451,7 +1537,13 @@ def test_random(con): [ param(lambda x: x.clip(lower=0), lambda x: x.clip(lower=0), id="lower-int"), param( - lambda x: x.clip(lower=0.0), lambda x: x.clip(lower=0.0), id="lower-float" + lambda x: x.clip(lower=0.0), + lambda x: x.clip(lower=0.0), + marks=pytest.mark.notimpl( + "exasol", + raises=ExaQueryError, + ), + id="lower-float", ), param(lambda x: x.clip(upper=0), lambda x: x.clip(upper=0), id="upper-int"), param( @@ -1472,6 +1564,10 @@ def test_random(con): param( lambda x: x.clip(lower=0, upper=1.0), lambda x: x.clip(lower=0, upper=1.0), + marks=pytest.mark.notimpl( + "exasol", + raises=ExaQueryError, + ), id="lower-upper-float", ), param( @@ -1495,7 +1591,7 @@ def test_clip(backend, alltypes, df, ibis_func, pandas_func): backend.assert_series_equal(result, expected, check_names=False) -@pytest.mark.notimpl(["polars"], raises=com.OperationNotDefinedError) +@pytest.mark.notimpl(["polars", "exasol"], raises=com.OperationNotDefinedError) @pytest.mark.broken( ["druid"], raises=sa.exc.ProgrammingError, @@ -1543,6 +1639,7 @@ def test_constants(con, const): ], ) @pytest.mark.notimpl(["oracle"], raises=sa.exc.DatabaseError) +@pytest.mark.notimpl(["exasol"], raises=(sa.exc.DBAPIError, ExaQueryError)) @flink_no_bitwise def test_bitwise_columns(backend, con, alltypes, df, op, left_fn, right_fn): expr = op(left_fn(alltypes), right_fn(alltypes)).name("tmp") @@ -1579,6 +1676,7 @@ def test_bitwise_columns(backend, con, alltypes, df, op, left_fn, right_fn): ], ) @pytest.mark.notimpl(["oracle"], raises=sa.exc.DatabaseError) +@pytest.mark.notimpl(["exasol"], raises=(sa.exc.DBAPIError, ExaQueryError)) @pyspark_no_bitshift @flink_no_bitwise def test_bitwise_shift(backend, alltypes, df, op, left_fn, right_fn): @@ -1610,6 +1708,7 @@ def test_bitwise_shift(backend, alltypes, df, op, left_fn, right_fn): [param(4, L(2), id="int_col"), param(L(4), 2, id="col_int")], ) @pytest.mark.notimpl(["oracle"], raises=sa.exc.DatabaseError) +@pytest.mark.notimpl(["exasol"], raises=ExaQueryError) @flink_no_bitwise def test_bitwise_scalars(con, op, left, right): expr = op(left, right) @@ -1620,6 +1719,7 @@ def test_bitwise_scalars(con, op, left, right): @pytest.mark.notimpl(["datafusion"], raises=com.OperationNotDefinedError) @pytest.mark.notimpl(["oracle"], raises=sa.exc.DatabaseError) +@pytest.mark.notimpl(["exasol"], raises=ExaQueryError) @flink_no_bitwise def test_bitwise_not_scalar(con): expr = ~L(2) @@ -1630,6 +1730,7 @@ def test_bitwise_not_scalar(con): @pytest.mark.notimpl(["datafusion"], raises=com.OperationNotDefinedError) @pytest.mark.notimpl(["oracle"], raises=sa.exc.DatabaseError) +@pytest.mark.notimpl(["exasol"], raises=sa.exc.DBAPIError) @flink_no_bitwise def test_bitwise_not_col(backend, alltypes, df): expr = (~alltypes.int_col).name("tmp") diff --git a/ibis/backends/tests/test_param.py b/ibis/backends/tests/test_param.py index e8fd6ff6bda2..61b6c7ea17e7 100644 --- a/ibis/backends/tests/test_param.py +++ b/ibis/backends/tests/test_param.py @@ -69,7 +69,9 @@ def test_timestamp_accepts_date_literals(alltypes): assert expr.compile(params=params) is not None -@pytest.mark.notimpl(["dask", "impala", "pandas", "pyspark", "druid", "oracle"]) +@pytest.mark.notimpl( + ["dask", "impala", "pandas", "pyspark", "druid", "oracle", "exasol"] +) @pytest.mark.never( ["mysql", "sqlite", "mssql"], reason="backend will never implement array types" ) @@ -81,7 +83,16 @@ def test_scalar_param_array(con): @pytest.mark.notimpl( - ["datafusion", "impala", "flink", "postgres", "pyspark", "druid", "oracle"] + [ + "datafusion", + "impala", + "flink", + "postgres", + "pyspark", + "druid", + "oracle", + "exasol", + ] ) @pytest.mark.never( ["mysql", "sqlite", "mssql"], @@ -95,7 +106,9 @@ def test_scalar_param_struct(con): assert result == value["a"] -@pytest.mark.notimpl(["datafusion", "impala", "pyspark", "polars", "druid", "oracle"]) +@pytest.mark.notimpl( + ["datafusion", "impala", "pyspark", "polars", "druid", "oracle", "exasol"] +) @pytest.mark.never( ["mysql", "sqlite", "mssql"], reason="mysql and sqlite will never implement map types", @@ -235,6 +248,7 @@ def test_scalar_param_date(backend, alltypes, value): "pyspark", "mssql", "druid", + "exasol", ] ) @pytest.mark.notimpl(["flink"], "WIP") diff --git a/ibis/backends/tests/test_register.py b/ibis/backends/tests/test_register.py index 4ac1a216d3cd..75a121f578d8 100644 --- a/ibis/backends/tests/test_register.py +++ b/ibis/backends/tests/test_register.py @@ -18,7 +18,7 @@ import pyarrow as pa -pytestmark = pytest.mark.notimpl(["druid", "oracle"]) +pytestmark = pytest.mark.notimpl(["druid", "exasol", "oracle"]) @contextlib.contextmanager diff --git a/ibis/backends/tests/test_set_ops.py b/ibis/backends/tests/test_set_ops.py index 11e518791afc..25b9c49a0560 100644 --- a/ibis/backends/tests/test_set_ops.py +++ b/ibis/backends/tests/test_set_ops.py @@ -12,6 +12,11 @@ import ibis.expr.types as ir from ibis import _ +try: + from pyexasol.exceptions import ExaQueryError +except ImportError: + ExaQueryError = None + @pytest.fixture def union_subsets(alltypes, df): @@ -68,7 +73,15 @@ def test_union_mixed_distinct(backend, union_subsets): param( False, marks=pytest.mark.notyet( - ["bigquery", "dask", "pandas", "sqlite", "snowflake", "mssql"], + [ + "bigquery", + "dask", + "pandas", + "sqlite", + "snowflake", + "mssql", + "exasol", + ], reason="backend doesn't support INTERSECT ALL", ), id="all", @@ -107,7 +120,15 @@ def test_intersect(backend, alltypes, df, distinct): param( False, marks=pytest.mark.notyet( - ["bigquery", "dask", "pandas", "sqlite", "snowflake", "mssql"], + [ + "bigquery", + "dask", + "pandas", + "sqlite", + "snowflake", + "mssql", + "exasol", + ], reason="backend doesn't support EXCEPT ALL", ), id="all", @@ -178,7 +199,7 @@ def test_top_level_union(backend, con, alltypes, distinct): param( False, marks=pytest.mark.notimpl( - ["bigquery", "dask", "mssql", "pandas", "snowflake", "sqlite"] + ["bigquery", "dask", "mssql", "pandas", "snowflake", "sqlite", "exasol"] ), ), ], diff --git a/ibis/backends/tests/test_sql.py b/ibis/backends/tests/test_sql.py index fb18f18c87aa..31cfddc4f09a 100644 --- a/ibis/backends/tests/test_sql.py +++ b/ibis/backends/tests/test_sql.py @@ -13,7 +13,7 @@ sa = pytest.importorskip("sqlalchemy") sg = pytest.importorskip("sqlglot") -pytestmark = pytest.mark.notimpl(["druid", "flink"]) +pytestmark = pytest.mark.notimpl(["druid", "flink", "exasol"]) @pytest.mark.never( diff --git a/ibis/backends/tests/test_string.py b/ibis/backends/tests/test_string.py index 9850c277e0e3..d762025c43e8 100644 --- a/ibis/backends/tests/test_string.py +++ b/ibis/backends/tests/test_string.py @@ -201,7 +201,7 @@ def uses_java_re(t): id="rlike", marks=[ pytest.mark.notimpl( - ["mssql", "oracle"], + ["mssql", "oracle", "exasol"], raises=com.OperationNotDefinedError, ), ], @@ -212,7 +212,7 @@ def uses_java_re(t): id="re_search_substring", marks=[ pytest.mark.notimpl( - ["mssql", "oracle"], + ["mssql", "oracle", "exasol"], raises=com.OperationNotDefinedError, ), ], @@ -223,7 +223,7 @@ def uses_java_re(t): id="re_search", marks=[ pytest.mark.notimpl( - ["mssql", "oracle"], + ["mssql", "oracle", "exasol"], raises=com.OperationNotDefinedError, ), ], @@ -236,7 +236,7 @@ def uses_java_re(t): id="re_search_posix", marks=[ pytest.mark.notimpl( - ["mssql", "oracle"], + ["mssql", "oracle", "exasol"], raises=com.OperationNotDefinedError, ), pytest.mark.broken(["pyspark"], raises=PythonException), @@ -253,7 +253,7 @@ def uses_java_re(t): id="re_extract", marks=[ pytest.mark.notimpl( - ["mssql", "druid", "oracle"], + ["mssql", "druid", "oracle", "exasol"], raises=com.OperationNotDefinedError, ), ], @@ -264,7 +264,7 @@ def uses_java_re(t): id="re_extract_group", marks=[ pytest.mark.notimpl( - ["mssql", "druid", "oracle"], + ["mssql", "druid", "oracle", "exasol"], raises=com.OperationNotDefinedError, ), ], @@ -277,7 +277,7 @@ def uses_java_re(t): id="re_extract_posix", marks=[ pytest.mark.notimpl( - ["mssql", "druid", "oracle"], + ["mssql", "druid", "oracle", "exasol"], raises=com.OperationNotDefinedError, ), ], @@ -288,7 +288,7 @@ def uses_java_re(t): id="re_extract_whole_group", marks=[ pytest.mark.notimpl( - ["mssql", "druid", "oracle"], + ["mssql", "druid", "oracle", "exasol"], raises=com.OperationNotDefinedError, ), ], @@ -301,7 +301,7 @@ def uses_java_re(t): id="re_extract_group_1", marks=[ pytest.mark.notimpl( - ["mssql", "druid", "oracle"], + ["mssql", "druid", "oracle", "exasol"], raises=com.OperationNotDefinedError, ), ], @@ -314,7 +314,7 @@ def uses_java_re(t): id="re_extract_group_2", marks=[ pytest.mark.notimpl( - ["mssql", "druid", "oracle"], + ["mssql", "druid", "oracle", "exasol"], raises=com.OperationNotDefinedError, ), ], @@ -327,7 +327,7 @@ def uses_java_re(t): id="re_extract_group_3", marks=[ pytest.mark.notimpl( - ["mssql", "druid", "oracle"], + ["mssql", "druid", "oracle", "exasol"], raises=com.OperationNotDefinedError, ), ], @@ -338,7 +338,7 @@ def uses_java_re(t): id="re_extract_group_at_beginning", marks=[ pytest.mark.notimpl( - ["mssql", "druid", "oracle"], + ["mssql", "druid", "oracle", "exasol"], raises=com.OperationNotDefinedError, ), ], @@ -349,7 +349,7 @@ def uses_java_re(t): id="re_extract_group_at_end", marks=[ pytest.mark.notimpl( - ["mssql", "druid", "oracle"], + ["mssql", "druid", "oracle", "exasol"], raises=com.OperationNotDefinedError, ), ], @@ -362,7 +362,7 @@ def uses_java_re(t): id="re_replace_posix", marks=[ pytest.mark.notimpl( - ["mysql", "mssql", "druid", "oracle"], + ["mysql", "mssql", "druid", "oracle", "exasol"], raises=com.OperationNotDefinedError, ), ], @@ -373,7 +373,7 @@ def uses_java_re(t): id="re_replace", marks=[ pytest.mark.notimpl( - ["mysql", "mssql", "druid", "oracle"], + ["mysql", "mssql", "druid", "oracle", "exasol"], raises=com.OperationNotDefinedError, ), ], @@ -475,6 +475,7 @@ def uses_java_re(t): "trino", "druid", "oracle", + "exasol", ], raises=com.OperationNotDefinedError, ), @@ -501,6 +502,7 @@ def uses_java_re(t): "trino", "druid", "oracle", + "exasol", ], raises=com.OperationNotDefinedError, ), @@ -831,6 +833,7 @@ def uses_java_re(t): "druid", "oracle", "flink", + "exasol", ], raises=com.OperationNotDefinedError, ), @@ -839,6 +842,10 @@ def uses_java_re(t): lambda t: ibis.literal("-").join(["a", t.string_col, "c"]), lambda t: "a-" + t.string_col + "-c", id="join", + marks=pytest.mark.notimpl( + ["exasol"], + raises=com.OperationNotDefinedError, + ), ), param( lambda t: t.string_col + t.date_string_col, @@ -871,7 +878,7 @@ def test_string(backend, alltypes, df, result_func, expected_func): @pytest.mark.notimpl( - ["mysql", "mssql", "druid", "oracle"], + ["mysql", "mssql", "druid", "oracle", "exasol"], raises=com.OperationNotDefinedError, ) def test_re_replace_global(con): @@ -979,6 +986,7 @@ def test_substr_with_null_values(backend, alltypes, df): [ "bigquery", "duckdb", + "exasol", "mssql", "mysql", "polars", @@ -1008,7 +1016,7 @@ def test_capitalize(con): raises=OperationNotDefinedError, ) @pytest.mark.notyet( - ["impala", "mssql", "mysql", "sqlite"], + ["impala", "mssql", "mysql", "sqlite", "exasol"], reason="no arrays", raises=OperationNotDefinedError, ) @@ -1023,7 +1031,7 @@ def test_array_string_join(con): @pytest.mark.notimpl( - ["mssql", "mysql", "druid", "oracle"], raises=com.OperationNotDefinedError + ["mssql", "mysql", "druid", "oracle", "exasol"], raises=com.OperationNotDefinedError ) def test_subs_with_re_replace(con): expr = ibis.literal("hi").re_replace("i", "a").substitute({"d": "b"}, else_="k") @@ -1052,6 +1060,7 @@ def test_multiple_subs(con): "polars", "sqlite", "flink", + "exasol", ], raises=com.OperationNotDefinedError, ) @@ -1087,7 +1096,7 @@ def test_no_conditional_percent_escape(con, expr): @pytest.mark.notimpl( - ["dask", "pandas", "mssql", "oracle"], raises=com.OperationNotDefinedError + ["dask", "pandas", "mssql", "oracle", "exasol"], raises=com.OperationNotDefinedError ) def test_non_match_regex_search_is_false(con): expr = ibis.literal("foo").re_search("bar") diff --git a/ibis/backends/tests/test_struct.py b/ibis/backends/tests/test_struct.py index ace35261ef2d..d59459f858c2 100644 --- a/ibis/backends/tests/test_struct.py +++ b/ibis/backends/tests/test_struct.py @@ -13,7 +13,7 @@ pytestmark = [ pytest.mark.never(["mysql", "sqlite", "mssql"], reason="No struct support"), pytest.mark.notyet(["impala"]), - pytest.mark.notimpl(["datafusion", "druid", "oracle"]), + pytest.mark.notimpl(["datafusion", "druid", "oracle", "exasol"]), ] diff --git a/ibis/backends/tests/test_temporal.py b/ibis/backends/tests/test_temporal.py index 30d11878fff6..57371718907e 100644 --- a/ibis/backends/tests/test_temporal.py +++ b/ibis/backends/tests/test_temporal.py @@ -65,6 +65,12 @@ Py4JJavaError = None +try: + from pyexasol.exceptions import ExaQueryError +except ImportError: + ExaQueryError = None + + @pytest.mark.parametrize("attr", ["year", "month", "day"]) @pytest.mark.parametrize( "expr_fn", @@ -82,6 +88,7 @@ raises=AttributeError, reason="Can only use .dt accessor with datetimelike values", ) +@pytest.mark.notimpl(["exasol"], raises=sa.exc.DBAPIError) def test_date_extract(backend, alltypes, df, attr, expr_fn): expr = getattr(expr_fn(alltypes.timestamp_col), attr)() expected = getattr(df.timestamp_col.dt, attr).astype("int32") @@ -94,22 +101,38 @@ def test_date_extract(backend, alltypes, df, attr, expr_fn): @pytest.mark.parametrize( "attr", [ - "year", - "month", - "day", + param( + "year", marks=[pytest.mark.notimpl(["exasol"], raises=sa.exc.DBAPIError)] + ), + param( + "month", marks=[pytest.mark.notimpl(["exasol"], raises=sa.exc.DBAPIError)] + ), + param("day", marks=[pytest.mark.notimpl(["exasol"], raises=sa.exc.DBAPIError)]), param( "day_of_year", marks=[ - pytest.mark.notimpl(["impala"], raises=com.OperationNotDefinedError), + pytest.mark.notimpl( + ["exasol", "impala"], raises=com.OperationNotDefinedError + ), pytest.mark.notyet(["oracle"], raises=com.OperationNotDefinedError), ], ), param( - "quarter", marks=pytest.mark.notyet(["oracle"], raises=sa.exc.DatabaseError) + "quarter", + marks=[ + pytest.mark.notyet(["oracle"], raises=sa.exc.DatabaseError), + pytest.mark.notimpl(["exasol"], raises=sa.exc.DBAPIError), + ], + ), + param( + "hour", marks=[pytest.mark.notimpl(["exasol"], raises=sa.exc.DBAPIError)] + ), + param( + "minute", marks=[pytest.mark.notimpl(["exasol"], raises=sa.exc.DBAPIError)] + ), + param( + "second", marks=[pytest.mark.notimpl(["exasol"], raises=sa.exc.DBAPIError)] ), - "hour", - "minute", - "second", ], ) @pytest.mark.notimpl(["druid"], raises=com.OperationNotDefinedError) @@ -141,6 +164,7 @@ def test_timestamp_extract(backend, alltypes, df, attr): raises=sa.exc.CompileError, reason='No literal value renderer is available for literal value "datetime.datetime(2015, 9, 1, 14, 48, 5, 359000)" with datatype DATETIME', ), + pytest.mark.notimpl(["exasol"], raises=ExaQueryError), ], ), param( @@ -153,6 +177,7 @@ def test_timestamp_extract(backend, alltypes, df, attr): raises=sa.exc.CompileError, reason='No literal value renderer is available for literal value "datetime.datetime(2015, 9, 1, 14, 48, 5, 359000)" with datatype DATETIME', ), + pytest.mark.notimpl(["exasol"], raises=ExaQueryError), ], ), param( @@ -165,6 +190,7 @@ def test_timestamp_extract(backend, alltypes, df, attr): raises=sa.exc.CompileError, reason='No literal value renderer is available for literal value "datetime.datetime(2015, 9, 1, 14, 48, 5, 359000)" with datatype DATETIME', ), + pytest.mark.notimpl(["exasol"], raises=ExaQueryError), ], ), param( @@ -177,6 +203,7 @@ def test_timestamp_extract(backend, alltypes, df, attr): raises=sa.exc.CompileError, reason='No literal value renderer is available for literal value "datetime.datetime(2015, 9, 1, 14, 48, 5, 359000)" with datatype DATETIME', ), + pytest.mark.notimpl(["exasol"], raises=ExaQueryError), ], ), param( @@ -189,6 +216,7 @@ def test_timestamp_extract(backend, alltypes, df, attr): raises=sa.exc.CompileError, reason='No literal value renderer is available for literal value "datetime.datetime(2015, 9, 1, 14, 48, 5, 359000)" with datatype DATETIME', ), + pytest.mark.notimpl(["exasol"], raises=ExaQueryError), ], ), param( @@ -201,6 +229,7 @@ def test_timestamp_extract(backend, alltypes, df, attr): raises=sa.exc.CompileError, reason='No literal value renderer is available for literal value "datetime.datetime(2015, 9, 1, 14, 48, 5, 359000)" with datatype DATETIME', ), + pytest.mark.notimpl(["exasol"], raises=ExaQueryError), ], ), param( @@ -209,7 +238,7 @@ def test_timestamp_extract(backend, alltypes, df, attr): id="millisecond", marks=[ pytest.mark.notimpl( - ["druid", "oracle"], raises=com.OperationNotDefinedError + ["druid", "oracle", "exasol"], raises=com.OperationNotDefinedError ), pytest.mark.notimpl( ["pyspark"], @@ -224,7 +253,7 @@ def test_timestamp_extract(backend, alltypes, df, attr): id="day_of_week_index", marks=[ pytest.mark.notimpl( - ["druid", "oracle"], raises=com.OperationNotDefinedError + ["druid", "oracle", "exasol"], raises=com.OperationNotDefinedError ), ], ), @@ -234,7 +263,8 @@ def test_timestamp_extract(backend, alltypes, df, attr): id="day_of_week_full_name", marks=[ pytest.mark.notimpl( - ["mssql", "druid", "oracle"], raises=com.OperationNotDefinedError + ["mssql", "druid", "oracle", "exasol"], + raises=com.OperationNotDefinedError, ), pytest.mark.never( ["flink"], @@ -271,6 +301,7 @@ def test_timestamp_extract_literal(con, func, expected): reason="Impala backend does not support extracting microseconds.", ) @pytest.mark.broken(["sqlite"], raises=AssertionError) +@pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError) def test_timestamp_extract_microseconds(backend, alltypes, df): expr = alltypes.timestamp_col.microsecond().name("microsecond") result = expr.execute() @@ -292,6 +323,7 @@ def test_timestamp_extract_microseconds(backend, alltypes, df): reason="PySpark backend does not support extracting milliseconds.", ) @pytest.mark.broken(["sqlite"], raises=AssertionError) +@pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError) def test_timestamp_extract_milliseconds(backend, alltypes, df): expr = alltypes.timestamp_col.millisecond().name("millisecond") result = expr.execute() @@ -316,6 +348,7 @@ def test_timestamp_extract_milliseconds(backend, alltypes, df): pyspark=["pandas<2.1"], reason="test was adjusted to work with pandas 2.1 output; pyspark doesn't support pandas 2", ) +@pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError) def test_timestamp_extract_epoch_seconds(backend, alltypes, df): expr = alltypes.timestamp_col.epoch_seconds().name("tmp") result = expr.execute() @@ -332,6 +365,7 @@ def test_timestamp_extract_epoch_seconds(backend, alltypes, df): raises=AttributeError, reason="'StringColumn' object has no attribute 'week_of_year'", ) +@pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError) def test_timestamp_extract_week_of_year(backend, alltypes, df): expr = alltypes.timestamp_col.week_of_year().name("tmp") result = expr.execute() @@ -574,6 +608,7 @@ def test_timestamp_extract_week_of_year(backend, alltypes, df): raises=AttributeError, reason="AttributeError: 'StringColumn' object has no attribute 'truncate'", ) +@pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError) def test_timestamp_truncate(backend, alltypes, df, unit): expr = alltypes.timestamp_col.truncate(unit).name("tmp") @@ -665,6 +700,7 @@ def test_timestamp_truncate(backend, alltypes, df, unit): raises=AttributeError, reason="AttributeError: 'StringColumn' object has no attribute 'date'", ) +@pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError) def test_date_truncate(backend, alltypes, df, unit): expr = alltypes.timestamp_col.date().truncate(unit).name("tmp") @@ -879,6 +915,7 @@ def test_date_truncate(backend, alltypes, df, unit): raises=ValidationError, reason="Given argument with datatype interval('h') is not implicitly castable to string", ) +@pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError) def test_integer_to_interval_timestamp( backend, con, alltypes, df, unit, displacement_type ): @@ -959,6 +996,7 @@ def convert_to_offset(offset, displacement_type=displacement_type): "No translation rule for " ), ) +@pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError) def test_integer_to_interval_date(backend, con, alltypes, df, unit): interval = alltypes.int_col.to_interval(unit=unit) array = alltypes.date_string_col.split("/") @@ -1183,6 +1221,7 @@ def convert_to_offset(x): ], ) @pytest.mark.notimpl(["mssql", "oracle"], raises=com.OperationNotDefinedError) +@pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError) def test_temporal_binop(backend, con, alltypes, df, expr_fn, expected_fn): expr = expr_fn(alltypes, backend).name("tmp") expected = expected_fn(df, backend) @@ -1369,6 +1408,7 @@ def test_temporal_binop(backend, con, alltypes, df, expr_fn, expected_fn): ], ) @pytest.mark.notimpl(["sqlite", "mssql", "oracle"], raises=com.OperationNotDefinedError) +@pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError) def test_temporal_binop_pandas_timedelta( backend, con, alltypes, df, timedelta, temporal_fn ): @@ -1411,6 +1451,7 @@ def test_temporal_binop_pandas_timedelta( raises=AttributeError, reason="Can only use .dt accessor with datetimelike values", ) +@pytest.mark.notimpl(["exasol"], raises=sa.exc.DBAPIError) def test_timestamp_comparison_filter(backend, con, alltypes, df, func_name): ts = pd.Timestamp("20100302", tz="UTC").to_pydatetime() @@ -1549,6 +1590,7 @@ def test_timestamp_comparison_filter_numpy(backend, con, alltypes, df, func_name raises=Py4JJavaError, reason="ParseException: Encountered '+ INTERVAL CAST'", ) +@pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError) def test_interval_add_cast_scalar(backend, alltypes): timestamp_date = alltypes.timestamp_col.date() delta = ibis.literal(10).cast("interval('D')") @@ -1570,6 +1612,7 @@ def test_interval_add_cast_scalar(backend, alltypes): raises=AttributeError, reason="'StringColumn' object has no attribute 'date'", ) +@pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError) def test_interval_add_cast_column(backend, alltypes, df): timestamp_date = alltypes.timestamp_col.date() delta = alltypes.bigint_col.cast("interval('D')") @@ -1677,6 +1720,7 @@ def test_interval_add_cast_column(backend, alltypes, df): raises=AttributeError, reason="'StringColumn' object has no attribute 'strftime'", ) +@pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError) def test_strftime(backend, alltypes, df, expr_fn, pandas_pattern): expr = expr_fn(alltypes) expected = df.timestamp_col.dt.strftime(pandas_pattern).rename("formatted") @@ -1767,6 +1811,7 @@ def test_strftime(backend, alltypes, df, expr_fn, pandas_pattern): ["mysql", "postgres", "sqlite", "druid", "oracle"], raises=com.OperationNotDefinedError, ) +@pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError) def test_integer_to_timestamp(backend, con, unit): backend_unit = backend.returned_timestamp_unit factor = unit_factors[unit] @@ -1864,6 +1909,7 @@ def test_integer_to_timestamp(backend, con, unit): ], raises=com.OperationNotDefinedError, ) +@pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError) def test_string_to_timestamp(alltypes, fmt): table = alltypes result = table.mutate(date=table.date_string_col.to_timestamp(fmt)).execute() @@ -1893,6 +1939,7 @@ def test_string_to_timestamp(alltypes, fmt): raises=Py4JJavaError, reason="DayOfWeekName is not supported in Flink", ) +@pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError) def test_day_of_week_scalar(con, date, expected_index, expected_day): expr = ibis.literal(date).cast(dt.date) result_index = con.execute(expr.day_of_week.index().name("tmp")) @@ -1917,6 +1964,7 @@ def test_day_of_week_scalar(con, date, expected_index, expected_day): "Ref: https://nightlies.apache.org/flink/flink-docs-release-1.13/docs/dev/table/functions/systemfunctions/#temporal-functions" ), ) +@pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError) def test_day_of_week_column(backend, alltypes, df): expr = alltypes.timestamp_col.day_of_week @@ -1967,6 +2015,7 @@ def test_day_of_week_column(backend, alltypes, df): raises=AttributeError, reason="'StringColumn' object has no attribute 'day_of_week'", ) +@pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError) def test_day_of_week_column_group_by( backend, alltypes, df, day_of_week_expr, day_of_week_pandas ): @@ -1991,6 +2040,7 @@ def test_day_of_week_column_group_by( @pytest.mark.notimpl( ["datafusion", "druid", "oracle"], raises=com.OperationNotDefinedError ) +@pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError) def test_now(con): expr = ibis.now() result = con.execute(expr.name("tmp")) @@ -2001,6 +2051,7 @@ def test_now(con): @pytest.mark.notimpl( ["datafusion", "druid", "oracle"], raises=com.OperationNotDefinedError ) +@pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError) def test_now_from_projection(alltypes): n = 2 expr = alltypes.select(now=ibis.now()).limit(n) @@ -2044,6 +2095,7 @@ def test_now_from_projection(alltypes): ), ) @pytest.mark.notyet(["impala"], raises=com.OperationNotDefinedError) +@pytest.mark.notimpl(["exasol"], raises=ExaQueryError) def test_date_literal(con, backend): expr = ibis.date(2022, 2, 4) result = con.execute(expr) @@ -2090,6 +2142,7 @@ def test_date_literal(con, backend): ["oracle"], raises=sa.exc.DatabaseError, reason="ORA-00904: MAKE TIMESTAMP invalid" ) @pytest.mark.notyet(["impala"], raises=com.OperationNotDefinedError) +@pytest.mark.notimpl(["exasol"], raises=ExaQueryError) def test_timestamp_literal(con, backend): expr = ibis.timestamp(2022, 2, 4, 16, 20, 0) result = con.execute(expr) @@ -2158,6 +2211,7 @@ def test_timestamp_literal(con, backend): "https://github.com/ibis-project/ibis/pull/6920/files#r1372453059", raises=AssertionError, ) +@pytest.mark.notimpl(["exasol"], raises=ExaQueryError) def test_timestamp_with_timezone_literal(con, timezone, expected): expr = ibis.timestamp(2022, 2, 4, 16, 20, 0).cast(dt.Timestamp(timezone=timezone)) result = con.execute(expr) @@ -2204,6 +2258,7 @@ def test_timestamp_with_timezone_literal(con, timezone, expected): @pytest.mark.broken( ["druid"], raises=sa.exc.ProgrammingError, reason="SQL parse failed" ) +@pytest.mark.notimpl(["exasol"], raises=ExaQueryError) def test_time_literal(con, backend): expr = ibis.time(16, 20, 0) result = con.execute(expr) @@ -2263,6 +2318,7 @@ def test_time_literal(con, backend): ], ids=["second", "subsecond"], ) +@pytest.mark.notimpl(["exasol"], raises=ExaQueryError) def test_extract_time_from_timestamp(con, microsecond): raw_ts = datetime.datetime(2023, 1, 7, 13, 20, 5, microsecond) ts = ibis.timestamp(raw_ts) @@ -2374,6 +2430,7 @@ def test_interval_literal(con, backend): ["oracle"], raises=sa.exc.DatabaseError, reason="ORA-00936: missing expression" ) @pytest.mark.notyet(["impala"], raises=com.OperationNotDefinedError) +@pytest.mark.notimpl(["exasol"], raises=sa.exc.DBAPIError) def test_date_column_from_ymd(backend, con, alltypes, df): c = alltypes.timestamp_col expr = ibis.date(c.year(), c.month(), c.day()) @@ -2402,6 +2459,7 @@ def test_date_column_from_ymd(backend, con, alltypes, df): ["oracle"], raises=sa.exc.DatabaseError, reason="ORA-00904 make timestamp invalid" ) @pytest.mark.notyet(["impala"], raises=com.OperationNotDefinedError) +@pytest.mark.notimpl(["exasol"], raises=sa.exc.DBAPIError) def test_timestamp_column_from_ymdhms(backend, con, alltypes, df): c = alltypes.timestamp_col expr = ibis.timestamp( @@ -2447,6 +2505,7 @@ def test_date_scalar_from_iso(con): raises=sa.exc.DatabaseError, reason="ORA-22849 type CLOB is not supported", ) +@pytest.mark.notimpl(["exasol"], raises=AssertionError, strict=False) def test_date_column_from_iso(backend, con, alltypes, df): expr = ( alltypes.year.cast("string") @@ -2473,6 +2532,7 @@ def test_date_column_from_iso(backend, con, alltypes, df): raises=com.UnsupportedOperationError, reason="PySpark backend does not support extracting milliseconds.", ) +@pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError) def test_timestamp_extract_milliseconds_with_big_value(con): timestamp = ibis.timestamp("2021-01-01 01:30:59.333456") millis = timestamp.millisecond() @@ -2497,6 +2557,7 @@ def test_timestamp_extract_milliseconds_with_big_value(con): raises=sa.exc.ProgrammingError, reason="No match found for function signature to_timestamp()", ) +@pytest.mark.notimpl(["exasol"], raises=sa.exc.DBAPIError) def test_integer_cast_to_timestamp_column(backend, alltypes, df): expr = alltypes.int_col.cast("timestamp") expected = pd.to_datetime(df.int_col, unit="s").rename(expr.get_name()) @@ -2519,6 +2580,7 @@ def test_integer_cast_to_timestamp_column(backend, alltypes, df): reason="No match found for function signature to_timestamp()", ) @pytest.mark.notimpl(["oracle"], raises=sa.exc.DatabaseError) +@pytest.mark.notimpl(["exasol"], raises=sa.exc.DBAPIError) def test_integer_cast_to_timestamp_scalar(alltypes, df): expr = alltypes.int_col.min().cast("timestamp") result = expr.execute() @@ -2645,6 +2707,7 @@ def test_timestamp_date_comparison(backend, alltypes, df, left_fn, right_fn): reason="Casting from timestamp[s] to timestamp[ns] would result in out of bounds timestamp: 81953424000", raises=ArrowInvalid, ) +@pytest.mark.notimpl(["exasol"], raises=ExaQueryError) def test_large_timestamp(con): huge_timestamp = datetime.datetime(year=4567, month=1, day=1) expr = ibis.timestamp("4567-01-01 00:00:00") @@ -2737,6 +2800,7 @@ def test_large_timestamp(con): raises=sa.exc.DatabaseError, reason="ORA-01843: invalid month was specified", ) +@pytest.mark.notimpl(["exasol"], raises=ExaQueryError) def test_timestamp_precision_output(con, ts, scale, unit): dtype = dt.Timestamp(scale=scale) expr = ibis.literal(ts).cast(dtype) @@ -2799,6 +2863,7 @@ def test_timestamp_precision_output(con, ts, scale, unit): ), ], ) +@pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError) def test_delta(con, start, end, unit, expected): expr = end.delta(start, unit) assert con.execute(expr) == expected @@ -2895,6 +2960,7 @@ def test_delta(con, start, end, unit, expected): ), ], ) +@pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError) def test_timestamp_bucket(backend, kws, pd_freq): ts = backend.functional_alltypes.timestamp_col.name("ts").execute() res = backend.functional_alltypes.timestamp_col.bucket(**kws).name("ts").execute() @@ -2929,6 +2995,7 @@ def test_timestamp_bucket(backend, kws, pd_freq): raises=com.UnsupportedOperationError, ) @pytest.mark.parametrize("offset_mins", [2, -2], ids=["pos", "neg"]) +@pytest.mark.notimpl(["exasol"], raises=com.OperationNotDefinedError) def test_timestamp_bucket_offset(backend, offset_mins): ts = backend.functional_alltypes.timestamp_col.name("ts") expr = ts.bucket(minutes=5, offset=ibis.interval(minutes=offset_mins)).name("ts") diff --git a/ibis/backends/tests/test_timecontext.py b/ibis/backends/tests/test_timecontext.py index 754552c5fdd7..a23a024702f8 100644 --- a/ibis/backends/tests/test_timecontext.py +++ b/ibis/backends/tests/test_timecontext.py @@ -20,6 +20,7 @@ "bigquery", "clickhouse", "datafusion", + "exasol", "impala", "mysql", "postgres", diff --git a/ibis/backends/tests/test_udf.py b/ibis/backends/tests/test_udf.py index 9117c6b21459..2c2fac00246c 100644 --- a/ibis/backends/tests/test_udf.py +++ b/ibis/backends/tests/test_udf.py @@ -12,6 +12,7 @@ "clickhouse", "dask", "druid", + "exasol", "flink", "impala", "mssql", diff --git a/ibis/backends/tests/test_uuid.py b/ibis/backends/tests/test_uuid.py index 440b0fd07d02..8a7ad0695bcb 100644 --- a/ibis/backends/tests/test_uuid.py +++ b/ibis/backends/tests/test_uuid.py @@ -19,6 +19,7 @@ UUID_BACKEND_TYPE = { "bigquery": "STRING", "duckdb": "UUID", + "exasol": "UUID", "flink": "CHAR(36) NOT NULL", "sqlite": "text", "snowflake": "VARCHAR", @@ -39,6 +40,7 @@ "dask": TEST_UUID, "oracle": TEST_UUID, "flink": RAW_TEST_UUID, + "exasol": TEST_UUID, } pytestmark = pytest.mark.notimpl( diff --git a/ibis/backends/tests/test_window.py b/ibis/backends/tests/test_window.py index c379f65b0d4b..b10b80527777 100644 --- a/ibis/backends/tests/test_window.py +++ b/ibis/backends/tests/test_window.py @@ -15,7 +15,7 @@ from ibis.legacy.udf.vectorized import analytic, reduction pytestmark = pytest.mark.notimpl( - ["druid"], + ["druid", "exasol"], raises=( sa.exc.ProgrammingError, sa.exc.NoSuchTableError, diff --git a/ibis/util.py b/ibis/util.py index 0f7cb0e45a03..2f121ddc936f 100644 --- a/ibis/util.py +++ b/ibis/util.py @@ -534,9 +534,9 @@ def _absolufy_paths(name): def gen_name(namespace: str) -> str: - """Create a case-insensitive uuid4 unique table name.""" + """Create a unique identifier.""" uid = base64.b32encode(uuid.uuid4().bytes).decode().rstrip("=").lower() - return f"_ibis_{namespace}_{uid}" + return f"ibis_{namespace}_{uid}" def slice_to_limit_offset( diff --git a/poetry-overrides.nix b/poetry-overrides.nix index 664d1134aa21..a16f10dfcaf1 100644 --- a/poetry-overrides.nix +++ b/poetry-overrides.nix @@ -18,4 +18,7 @@ self: super: { }) ]; }); + pyodbc = (super.pyodbc.override { preferWheel = false; }).overridePythonAttrs (attrs: { + nativeBuildInputs = attrs.nativeBuildInputs or [ ] ++ [ self.pkgs.unixODBC ]; + }); } diff --git a/pyproject.toml b/pyproject.toml index bfb3a59ac609..d948a0a0e83b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,6 +96,9 @@ shapely = { version = ">=2,<3", optional = true } snowflake-connector-python = { version = ">=3.0.2,<4,!=3.3.0b1", optional = true } snowflake-sqlalchemy = { version = ">=1.4.1,<2", optional = true } sqlalchemy = { version = ">=1.4,<3", optional = true } +sqlalchemy-exasol = { version = ">=4.6.0", optional = true, extras = [ + "exasol", +] } sqlalchemy-views = { version = ">=0.3.1,<1", optional = true } trino = { version = ">=0.321,<1", optional = true, extras = ["sqlalchemy"] } @@ -179,6 +182,7 @@ all = [ "snowflake-connector-python", "snowflake-sqlalchemy", "sqlalchemy", + "sqlalchemy-exasol", "sqlalchemy-views", "trino", ] @@ -193,6 +197,7 @@ dask = ["dask", "regex"] datafusion = ["datafusion"] druid = ["pydruid", "sqlalchemy"] duckdb = ["duckdb", "duckdb-engine", "sqlalchemy", "sqlalchemy-views"] +exasol = ["sqlalchemy", "sqlalchemy-exasol", "sqlalchemy-views"] flink = [] geospatial = ["geoalchemy2", "geopandas", "shapely"] impala = ["fsspec", "impyla", "requests", "sqlalchemy"] @@ -223,6 +228,7 @@ dask = "ibis.backends.dask" datafusion = "ibis.backends.datafusion" druid = "ibis.backends.druid" duckdb = "ibis.backends.duckdb" +exasol = "ibis.backends.exasol" flink = "ibis.backends.flink" impala = "ibis.backends.impala" mysql = "ibis.backends.mysql" @@ -358,6 +364,7 @@ markers = [ "datafusion: Apache Datafusion tests", "druid: Apache Druid tests", "duckdb: DuckDB tests", + "exasol: ExasolDB tests", "flink: Flink tests", "impala: Apache Impala tests", "mysql: MySQL tests", diff --git a/requirements-dev.txt b/requirements-dev.txt index d17c0e514b9a..1db06feac7b7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -180,11 +180,13 @@ pydantic==2.5.2 ; python_version >= "3.10" and python_version < "3.13" pydata-google-auth==1.8.2 ; python_version >= "3.9" and python_version < "4.0" pydeps==1.12.17 ; python_version >= "3.9" and python_version < "4.0" pydruid[sqlalchemy]==0.6.6 ; python_version >= "3.9" and python_version < "4.0" +pyexasol==0.25.2 ; python_version >= "3.9" and python_version < "4.0" pygments==2.17.2 ; python_version >= "3.9" and python_version < "4.0" pyinstrument==4.6.1 ; python_version >= "3.9" and python_version < "4.0" pyjwt==2.8.0 ; python_version >= "3.9" and python_version < "4.0" pymssql==2.2.11 ; python_version >= "3.9" and python_version < "4.0" pymysql==1.1.0 ; python_version >= "3.9" and python_version < "4.0" +pyodbc==4.0.39 ; python_version >= "3.9" and python_version < "4.0" pyopenssl==23.3.0 ; python_version >= "3.9" and python_version < "4.0" pyparsing==3.1.1 ; python_version >= "3.10" and python_version < "3.13" pyproj==3.6.1 ; python_version >= "3.9" and python_version < "4.0" @@ -237,6 +239,7 @@ snowflake-sqlalchemy==1.5.1 ; python_version >= "3.9" and python_version < "4.0" sortedcontainers==2.4.0 ; python_version >= "3.9" and python_version < "4.0" soupsieve==2.5 ; python_version >= "3.10" and python_version < "3.13" sphobjinv==2.3.1 ; python_version >= "3.10" and python_version < "3.13" +sqlalchemy-exasol[exasol]==4.6.2 ; python_version >= "3.9" and python_version < "4.0" sqlalchemy-views==0.3.2 ; python_version >= "3.9" and python_version < "4.0" sqlalchemy==1.4.50 ; python_version >= "3.9" and python_version < "4.0" sqlglot==20.1.0 ; python_version >= "3.9" and python_version < "4.0" @@ -265,6 +268,7 @@ urllib3==1.26.18 ; python_version >= "3.9" and python_version < "4.0" virtualenv==20.25.0 ; python_version >= "3.9" and python_version < "4.0" watchdog==3.0.0 ; python_version >= "3.10" and python_version < "3.13" wcwidth==0.2.12 ; python_version >= "3.9" and python_version < "4.0" +websocket-client==1.7.0 ; python_version >= "3.9" and python_version < "4.0" werkzeug==3.0.1 ; python_version >= "3.9" and python_version < "4.0" whitebox==2.3.1 ; python_version >= "3.10" and python_version < "3.13" whiteboxgui==2.3.0 ; python_version >= "3.10" and python_version < "3.13"