diff --git a/ibis/backends/clickhouse/__init__.py b/ibis/backends/clickhouse/__init__.py index c80d93be8d97..bce5a8cc5442 100644 --- a/ibis/backends/clickhouse/__init__.py +++ b/ibis/backends/clickhouse/__init__.py @@ -403,7 +403,8 @@ def table(self, name: str, database: str | None = None) -> ir.Table: Table expression """ schema = self.get_schema(name, database=database) - return ClickhouseTable(ops.DatabaseTable(name, schema, self)) + qname = self._fully_qualified_name(name, database) + return ClickhouseTable(ops.DatabaseTable(qname, schema, self)) def raw_sql( self, @@ -446,7 +447,9 @@ def close(self): self.con.close() def _fully_qualified_name(self, name, database): - return sg.table(name, db=database or self.current_database).sql() + return sg.table(name, db=database or self.current_database or None).sql( + dialect="clickhouse" + ) def get_schema( self, @@ -507,13 +510,13 @@ def drop_database(self, name: str, *, force: bool = False) -> None: self.raw_sql(f"DROP DATABASE {'IF EXISTS ' * force}{name}") def truncate_table(self, name: str, database: str | None = None) -> None: - ident = ".".join(filter(None, (database, name))) + ident = self._fully_qualified_name(name, database) self.raw_sql(f"DELETE FROM {ident}") def drop_table( self, name: str, database: str | None = None, force: bool = False ) -> None: - ident = ".".join(filter(None, (database, name))) + ident = self._fully_qualified_name(name, database) self.raw_sql(f"DROP TABLE {'IF EXISTS ' * force}{ident}") def create_table( @@ -526,30 +529,67 @@ def create_table( temp: bool = False, overwrite: bool = False, # backend specific arguments - engine: str | None, + engine: str, order_by: Iterable[str] | None = None, partition_by: Iterable[str] | None = None, sample_by: str | None = None, settings: Mapping[str, Any] | None = None, ) -> ir.Table: + """Create a table in a ClickHouse database. + + Parameters + ---------- + name + Name of the table to create + obj + Optional data to create the table with + schema + Optional names and types of the table + database + Database to create the table in + temp + Create a temporary table. This is not yet supported, and exists for + API compatibility. + overwrite + Whether to overwrite the table + engine + The table engine to use. See [ClickHouse's `CREATE TABLE` documentation](https://clickhouse.com/docs/en/sql-reference/statements/create/table) + for specifics. + order_by + String column names to order by. Required for some table engines like `MergeTree`. + partition_by + String column names to partition by + sample_by + String column names to sample by + settings + Key-value pairs of settings for table creation + + Returns + ------- + Table + The new table + """ + if temp: + raise com.IbisError("ClickHouse temporary tables are not yet supported") + tmp = "TEMPORARY " * temp replace = "OR REPLACE " * overwrite - code = f"CREATE {replace}{tmp}TABLE {name}" + code = ( + f"CREATE {replace}{tmp}TABLE {self._fully_qualified_name(name, database)}" + ) if obj is None and schema is None: raise com.IbisError("The schema or obj parameter is required") if schema is not None: - code += f" ({schema})" + serialized_schema = ", ".join( + f"`{name}` {serialize(typ)}" for name, typ in schema.items() + ) + code += f" ({serialized_schema})" - if isinstance(obj, pd.DataFrame): + if obj is not None and not isinstance(obj, ir.Expr): obj = ibis.memtable(obj, schema=schema) - if obj is not None: - self._register_in_memory_tables(obj) - query = self.compile(obj) - code += f" AS {query}" - code += f" ENGINE = {engine}" if order_by is not None: @@ -565,6 +605,11 @@ def create_table( kvs = ", ".join(f"{name}={value!r}" for name, value in settings.items()) code += f" SETTINGS {kvs}" + if obj is not None: + self._register_in_memory_tables(obj) + query = self.compile(obj) + code += f" AS {query}" + self.raw_sql(code) return self.table(name, database=database) diff --git a/ibis/backends/clickhouse/compiler/relations.py b/ibis/backends/clickhouse/compiler/relations.py index 257d0bb31578..cde7b2a51f7b 100644 --- a/ibis/backends/clickhouse/compiler/relations.py +++ b/ibis/backends/clickhouse/compiler/relations.py @@ -26,7 +26,7 @@ def _dummy(op: ops.DummyTable, **kw): @translate_rel.register(ops.PhysicalTable) def _physical_table(op: ops.PhysicalTable, **_): - return sg.table(op.name) + return sg.parse_one(op.name, into=sg.exp.Table) @translate_rel.register(ops.Selection) diff --git a/ibis/backends/clickhouse/tests/snapshots/test_select/test_complex_array_expr_projection/out.sql b/ibis/backends/clickhouse/tests/snapshots/test_select/test_complex_array_expr_projection/out.sql index e583e68c02e6..5948346555fe 100644 --- a/ibis/backends/clickhouse/tests/snapshots/test_select/test_complex_array_expr_projection/out.sql +++ b/ibis/backends/clickhouse/tests/snapshots/test_select/test_complex_array_expr_projection/out.sql @@ -4,7 +4,7 @@ FROM ( SELECT t0.string_col, COUNT(*) AS count - FROM functional_alltypes AS t0 + FROM ibis_testing.functional_alltypes AS t0 GROUP BY 1 ) AS t1 \ No newline at end of file diff --git a/ibis/backends/clickhouse/tests/snapshots/test_select/test_isin_notin_in_select/out1.sql b/ibis/backends/clickhouse/tests/snapshots/test_select/test_isin_notin_in_select/out1.sql index 6320f23aa1a2..68f805955300 100644 --- a/ibis/backends/clickhouse/tests/snapshots/test_select/test_isin_notin_in_select/out1.sql +++ b/ibis/backends/clickhouse/tests/snapshots/test_select/test_isin_notin_in_select/out1.sql @@ -1,5 +1,5 @@ SELECT * -FROM functional_alltypes AS t0 +FROM ibis_testing.functional_alltypes AS t0 WHERE t0.string_col IN ('foo', 'bar') \ No newline at end of file diff --git a/ibis/backends/clickhouse/tests/snapshots/test_select/test_isin_notin_in_select/out2.sql b/ibis/backends/clickhouse/tests/snapshots/test_select/test_isin_notin_in_select/out2.sql index d89b906f9ea1..6b54ac33071c 100644 --- a/ibis/backends/clickhouse/tests/snapshots/test_select/test_isin_notin_in_select/out2.sql +++ b/ibis/backends/clickhouse/tests/snapshots/test_select/test_isin_notin_in_select/out2.sql @@ -1,5 +1,5 @@ SELECT * -FROM functional_alltypes AS t0 +FROM ibis_testing.functional_alltypes AS t0 WHERE NOT t0.string_col IN ('foo', 'bar') \ No newline at end of file diff --git a/ibis/backends/clickhouse/tests/snapshots/test_select/test_isnull_case_expr_rewrite_failure/out.sql b/ibis/backends/clickhouse/tests/snapshots/test_select/test_isnull_case_expr_rewrite_failure/out.sql index 42b21372b307..4565cd7f3bdb 100644 --- a/ibis/backends/clickhouse/tests/snapshots/test_select/test_isnull_case_expr_rewrite_failure/out.sql +++ b/ibis/backends/clickhouse/tests/snapshots/test_select/test_isnull_case_expr_rewrite_failure/out.sql @@ -1,3 +1,3 @@ SELECT SUM(CASE WHEN isNull(t0.string_col) THEN 1 ELSE 0 END) AS "Sum(Where(IsNull(string_col), 1, 0))" -FROM functional_alltypes AS t0 \ No newline at end of file +FROM ibis_testing.functional_alltypes AS t0 \ No newline at end of file diff --git a/ibis/backends/clickhouse/tests/snapshots/test_select/test_join_self_reference/out.sql b/ibis/backends/clickhouse/tests/snapshots/test_select/test_join_self_reference/out.sql index 4a756ab86ec9..f369044c93e4 100644 --- a/ibis/backends/clickhouse/tests/snapshots/test_select/test_join_self_reference/out.sql +++ b/ibis/backends/clickhouse/tests/snapshots/test_select/test_join_self_reference/out.sql @@ -1,5 +1,5 @@ SELECT t0.* -FROM functional_alltypes AS t0 -INNER JOIN functional_alltypes AS t1 +FROM ibis_testing.functional_alltypes AS t0 +INNER JOIN ibis_testing.functional_alltypes AS t1 ON t0.id = t1.id \ No newline at end of file diff --git a/ibis/backends/clickhouse/tests/snapshots/test_select/test_physical_table_reference_translate/out.sql b/ibis/backends/clickhouse/tests/snapshots/test_select/test_physical_table_reference_translate/out.sql index cdcc673f9d1c..1db32cc34a27 100644 --- a/ibis/backends/clickhouse/tests/snapshots/test_select/test_physical_table_reference_translate/out.sql +++ b/ibis/backends/clickhouse/tests/snapshots/test_select/test_physical_table_reference_translate/out.sql @@ -1,3 +1,3 @@ SELECT * -FROM functional_alltypes \ No newline at end of file +FROM ibis_testing.functional_alltypes \ No newline at end of file diff --git a/ibis/backends/clickhouse/tests/snapshots/test_select/test_self_reference_simple/out.sql b/ibis/backends/clickhouse/tests/snapshots/test_select/test_self_reference_simple/out.sql index 99d5c76e03f3..834569d7bcda 100644 --- a/ibis/backends/clickhouse/tests/snapshots/test_select/test_self_reference_simple/out.sql +++ b/ibis/backends/clickhouse/tests/snapshots/test_select/test_self_reference_simple/out.sql @@ -1,3 +1,3 @@ SELECT * -FROM functional_alltypes AS t0 \ No newline at end of file +FROM ibis_testing.functional_alltypes AS t0 \ No newline at end of file diff --git a/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-awardID-any_inner_join/out.sql b/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-awardID-any_inner_join/out.sql index f879a888124a..0d241ee9d400 100644 --- a/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-awardID-any_inner_join/out.sql +++ b/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-awardID-any_inner_join/out.sql @@ -1,5 +1,5 @@ SELECT t0.* -FROM batting AS t0 -ANY JOIN awards_players AS t1 +FROM ibis_testing.batting AS t0 +ANY JOIN ibis_testing.awards_players AS t1 ON t0.playerID = t1.awardID \ No newline at end of file diff --git a/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-awardID-any_left_join/out.sql b/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-awardID-any_left_join/out.sql index 88c96b29443c..a12ee0ef961d 100644 --- a/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-awardID-any_left_join/out.sql +++ b/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-awardID-any_left_join/out.sql @@ -1,5 +1,5 @@ SELECT t0.* -FROM batting AS t0 -LEFT ANY JOIN awards_players AS t1 +FROM ibis_testing.batting AS t0 +LEFT ANY JOIN ibis_testing.awards_players AS t1 ON t0.playerID = t1.awardID \ No newline at end of file diff --git a/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-awardID-inner_join/out.sql b/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-awardID-inner_join/out.sql index 700f214f0382..8dc68ced27bf 100644 --- a/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-awardID-inner_join/out.sql +++ b/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-awardID-inner_join/out.sql @@ -1,5 +1,5 @@ SELECT t0.* -FROM batting AS t0 -INNER JOIN awards_players AS t1 +FROM ibis_testing.batting AS t0 +INNER JOIN ibis_testing.awards_players AS t1 ON t0.playerID = t1.awardID \ No newline at end of file diff --git a/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-awardID-left_join/out.sql b/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-awardID-left_join/out.sql index 9e158d9dd8a1..35d611b2f340 100644 --- a/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-awardID-left_join/out.sql +++ b/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-awardID-left_join/out.sql @@ -1,5 +1,5 @@ SELECT t0.* -FROM batting AS t0 -LEFT OUTER JOIN awards_players AS t1 +FROM ibis_testing.batting AS t0 +LEFT OUTER JOIN ibis_testing.awards_players AS t1 ON t0.playerID = t1.awardID \ No newline at end of file diff --git a/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-playerID-any_inner_join/out.sql b/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-playerID-any_inner_join/out.sql index 363aaebf890c..41b1278f5f1d 100644 --- a/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-playerID-any_inner_join/out.sql +++ b/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-playerID-any_inner_join/out.sql @@ -1,5 +1,5 @@ SELECT t0.* -FROM batting AS t0 -ANY JOIN awards_players AS t1 +FROM ibis_testing.batting AS t0 +ANY JOIN ibis_testing.awards_players AS t1 ON t0.playerID = t1.playerID \ No newline at end of file diff --git a/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-playerID-any_left_join/out.sql b/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-playerID-any_left_join/out.sql index 3ba9f0d4e06f..701a461d51e1 100644 --- a/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-playerID-any_left_join/out.sql +++ b/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-playerID-any_left_join/out.sql @@ -1,5 +1,5 @@ SELECT t0.* -FROM batting AS t0 -LEFT ANY JOIN awards_players AS t1 +FROM ibis_testing.batting AS t0 +LEFT ANY JOIN ibis_testing.awards_players AS t1 ON t0.playerID = t1.playerID \ No newline at end of file diff --git a/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-playerID-inner_join/out.sql b/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-playerID-inner_join/out.sql index 5d0d8dc31e6e..8c9eed45dfd9 100644 --- a/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-playerID-inner_join/out.sql +++ b/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-playerID-inner_join/out.sql @@ -1,5 +1,5 @@ SELECT t0.* -FROM batting AS t0 -INNER JOIN awards_players AS t1 +FROM ibis_testing.batting AS t0 +INNER JOIN ibis_testing.awards_players AS t1 ON t0.playerID = t1.playerID \ No newline at end of file diff --git a/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-playerID-left_join/out.sql b/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-playerID-left_join/out.sql index cc098eca7bfb..f70a784199a9 100644 --- a/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-playerID-left_join/out.sql +++ b/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_joins/playerID-playerID-left_join/out.sql @@ -1,5 +1,5 @@ SELECT t0.* -FROM batting AS t0 -LEFT OUTER JOIN awards_players AS t1 +FROM ibis_testing.batting AS t0 +LEFT OUTER JOIN ibis_testing.awards_players AS t1 ON t0.playerID = t1.playerID \ No newline at end of file diff --git a/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_scalar_aggregates/out.sql b/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_scalar_aggregates/out.sql index c960b3b70a1a..e9a4aa6b60be 100644 --- a/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_scalar_aggregates/out.sql +++ b/ibis/backends/clickhouse/tests/snapshots/test_select/test_simple_scalar_aggregates/out.sql @@ -1,5 +1,5 @@ SELECT SUM(t0.float_col) AS "Sum(float_col)" -FROM functional_alltypes AS t0 +FROM ibis_testing.functional_alltypes AS t0 WHERE t0.int_col > 0 \ No newline at end of file diff --git a/ibis/backends/clickhouse/tests/snapshots/test_select/test_table_column_unbox/out.sql b/ibis/backends/clickhouse/tests/snapshots/test_select/test_table_column_unbox/out.sql index f3463bec44ea..4c03016deaf5 100644 --- a/ibis/backends/clickhouse/tests/snapshots/test_select/test_table_column_unbox/out.sql +++ b/ibis/backends/clickhouse/tests/snapshots/test_select/test_table_column_unbox/out.sql @@ -4,7 +4,7 @@ FROM ( SELECT t0.string_col, SUM(t0.float_col) AS total - FROM functional_alltypes AS t0 + FROM ibis_testing.functional_alltypes AS t0 WHERE t0.int_col > 0 GROUP BY diff --git a/ibis/backends/clickhouse/tests/snapshots/test_select/test_timestamp_extract_field/out.sql b/ibis/backends/clickhouse/tests/snapshots/test_select/test_timestamp_extract_field/out.sql index 9336d08bda8d..201e06f72f71 100644 --- a/ibis/backends/clickhouse/tests/snapshots/test_select/test_timestamp_extract_field/out.sql +++ b/ibis/backends/clickhouse/tests/snapshots/test_select/test_timestamp_extract_field/out.sql @@ -5,4 +5,4 @@ SELECT toHour(t0.timestamp_col) AS hour, toMinute(t0.timestamp_col) AS minute, toSecond(t0.timestamp_col) AS second -FROM functional_alltypes AS t0 \ No newline at end of file +FROM ibis_testing.functional_alltypes AS t0 \ No newline at end of file diff --git a/ibis/backends/clickhouse/tests/snapshots/test_select/test_where_simple_comparisons/out.sql b/ibis/backends/clickhouse/tests/snapshots/test_select/test_where_simple_comparisons/out.sql index 6806def413a1..86486220be26 100644 --- a/ibis/backends/clickhouse/tests/snapshots/test_select/test_where_simple_comparisons/out.sql +++ b/ibis/backends/clickhouse/tests/snapshots/test_select/test_where_simple_comparisons/out.sql @@ -1,6 +1,6 @@ SELECT * -FROM functional_alltypes AS t0 +FROM ibis_testing.functional_alltypes AS t0 WHERE t0.float_col > 0 AND t0.int_col < ( t0.float_col * 2 diff --git a/ibis/backends/clickhouse/tests/snapshots/test_select/test_where_with_between/out.sql b/ibis/backends/clickhouse/tests/snapshots/test_select/test_where_with_between/out.sql index 7982cdf7c584..27630e4efaa5 100644 --- a/ibis/backends/clickhouse/tests/snapshots/test_select/test_where_with_between/out.sql +++ b/ibis/backends/clickhouse/tests/snapshots/test_select/test_where_with_between/out.sql @@ -1,5 +1,5 @@ SELECT * -FROM functional_alltypes AS t0 +FROM ibis_testing.functional_alltypes AS t0 WHERE t0.int_col > 0 AND t0.float_col BETWEEN 0 AND 1 \ No newline at end of file diff --git a/ibis/backends/clickhouse/tests/test_client.py b/ibis/backends/clickhouse/tests/test_client.py index bb1b9ba55e27..bd3f0e61eae7 100644 --- a/ibis/backends/clickhouse/tests/test_client.py +++ b/ibis/backends/clickhouse/tests/test_client.py @@ -1,6 +1,7 @@ import pandas as pd import pandas.testing as tm import pytest +from pytest import param import ibis import ibis.expr.datatypes as dt @@ -13,6 +14,8 @@ CLICKHOUSE_USER, IBIS_TEST_CLICKHOUSE_DB, ) +from ibis.common.exceptions import IbisError +from ibis.util import gen_name pytest.importorskip("clickhouse_driver") @@ -204,3 +207,56 @@ def test_list_tables_empty_database(con, worker_id): assert not con.list_tables(database=dbname) finally: con.raw_sql(f"DROP DATABASE IF EXISTS {dbname}") + + +@pytest.mark.parametrize( + "temp", + [ + param( + True, + marks=pytest.mark.xfail( + reason="Ibis is likely making incorrect assumptions about object lifetime and cursors", + raises=IbisError, + ), + ), + False, + ], + ids=["temp", "no_temp"], +) +def test_create_table_no_data(con, temp): + name = gen_name("clickhouse_create_table_no_data") + schema = ibis.schema(dict(a="!int", b="string")) + t = con.create_table( + name, schema=schema, temp=temp, engine="Memory", database="tmptables" + ) + try: + assert t.execute().empty + finally: + con.drop_table(name, force=True, database="tmptables") + assert name not in con.list_tables(database="tmptables") + + +@pytest.mark.parametrize( + "data", + [ + {"a": [1, 2, 3], "b": [None, "b", "c"]}, + pd.DataFrame({"a": [1, 2, 3], "b": [None, "b", "c"]}), + ], + ids=["dict", "dataframe"], +) +@pytest.mark.parametrize( + "engine", + ["File(Native)", "File(Parquet)", "Memory"], + ids=["native", "mem", "parquet"], +) +def test_create_table_data(con, data, engine): + name = gen_name("clickhouse_create_table_data") + schema = ibis.schema(dict(a="!int", b="string")) + t = con.create_table( + name, obj=data, schema=schema, engine=engine, database="tmptables" + ) + try: + assert len(t.execute()) == 3 + finally: + con.drop_table(name, force=True, database="tmptables") + assert name not in con.list_tables(database="tmptables") diff --git a/ibis/backends/tests/snapshots/test_string/test_rlike/clickhouse/out.sql b/ibis/backends/tests/snapshots/test_string/test_rlike/clickhouse/out.sql index f0b38274c3c0..4ad1900e8953 100644 --- a/ibis/backends/tests/snapshots/test_string/test_rlike/clickhouse/out.sql +++ b/ibis/backends/tests/snapshots/test_string/test_rlike/clickhouse/out.sql @@ -1,5 +1,5 @@ SELECT * -FROM functional_alltypes AS t0 +FROM ibis_testing.functional_alltypes AS t0 WHERE multiMatchAny(t0.string_col, [ '0']) \ No newline at end of file