Skip to content

Commit

Permalink
test(trino): use tpch.tiny for faster testing of tpch queries
Browse files Browse the repository at this point in the history
  • Loading branch information
cpcloud authored and kszucs committed Oct 22, 2023
1 parent ea5689f commit 04fd088
Show file tree
Hide file tree
Showing 2 changed files with 24 additions and 5 deletions.
10 changes: 9 additions & 1 deletion ibis/backends/tests/tpch/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,13 +59,21 @@ def wrapper(*args, backend, snapshot, **kwargs):
result_expr = test(*args, **kwargs)

result = result_expr.execute()
assert not result.empty

expected = expected_expr.execute()
assert not expected.empty

assert list(map(str.lower, expected.columns)) == result.columns.tolist()
expected.columns = result.columns

assert len(expected) == len(result)
tm.assert_frame_equal(result, expected, check_dtype=False)
tm.assert_frame_equal(
# TODO: we should handle this more precisely
result.replace(float("nan"), None),
expected,
check_dtype=False,
)

# only produce sql if the execution passes
result_expr_sql = ibis.to_sql(result_expr, dialect=backend_name)
Expand Down
19 changes: 15 additions & 4 deletions ibis/backends/trino/tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
import sqlglot as sg

import ibis
import ibis.expr.datatypes as dt
import ibis.selectors as s
from ibis.backends.conftest import TEST_TABLES
from ibis.backends.tests.base import BackendTest, RoundAwayFromZero

Expand Down Expand Up @@ -40,7 +42,7 @@ class TestConf(BackendTest, RoundAwayFromZero):
supports_tpch = True
deps = ("sqlalchemy", "trino.sqlalchemy")

_tpch_data_schema = "tpch.sf1"
_tpch_data_schema = "tpch.tiny"
_tpch_query_schema = "hive.ibis_sf1"

def preload(self):
Expand Down Expand Up @@ -113,14 +115,23 @@ def load_tpch(self) -> None:
con.create_schema(schema, database=database, force=True)

prefixes = {"partsupp": "ps"}

# this is the type duckdb uses for numeric columns in TPC-H data
decimal_type = dt.Decimal(15, 2)

with con.begin() as c:
for table in tables:
prefix = prefixes.get(table, table[0])

t = con.table(table, schema=data_schema)
new_t = t.rename(**{f"{prefix}_{old}": old for old in t.columns})
t = (
con.table(table, schema=data_schema).rename(f"{prefix}_{{}}".format)
# https://github.com/trinodb/trino/issues/19477
.mutate(
s.across(s.of_type(dt.float64), lambda c: c.cast(decimal_type))
)
)

sql = ibis.to_sql(new_t, dialect="trino")
sql = ibis.to_sql(t, dialect="trino")
c.exec_driver_sql(
f"CREATE OR REPLACE VIEW {query_schema}.{table} AS {sql}"
)
Expand Down

0 comments on commit 04fd088

Please sign in to comment.