Skip to content

Commit

Permalink
feat(impala): add tbl_properties to create_table (#9839)
Browse files Browse the repository at this point in the history
`TBLPROPERTIES` can be set on table creation. Exposing this lets us
un-xfail a few tests by setting the required `TBLPROPERTIES` to make
them work.
  • Loading branch information
jcrist authored Aug 14, 2024
1 parent c573392 commit e3d02bd
Show file tree
Hide file tree
Showing 3 changed files with 34 additions and 7 deletions.
5 changes: 5 additions & 0 deletions ibis/backends/impala/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -471,6 +471,7 @@ def create_table(
format="parquet",
location=None,
partition=None,
tbl_properties: Mapping[str, Any] | None = None,
like_parquet=None,
) -> ir.Table:
"""Create a new table using an Ibis table expression or in-memory data.
Expand Down Expand Up @@ -501,6 +502,8 @@ def create_table(
partition
Must pass a schema to use this. Cannot partition from an
expression.
tbl_properties
Table properties to set on table creation.
like_parquet
Can specify instead of a schema
Expand Down Expand Up @@ -534,6 +537,7 @@ def create_table(
format=format,
external=True if location is not None else external,
partition=partition,
tbl_properties=tbl_properties,
path=location,
)
)
Expand All @@ -549,6 +553,7 @@ def create_table(
external=external,
path=location,
partition=partition,
tbl_properties=tbl_properties,
)
)
return self.table(name, database=database or self.current_database)
Expand Down
11 changes: 11 additions & 0 deletions ibis/backends/impala/ddl.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,13 @@ def _create_line(self):
def _location(self):
return f"LOCATION '{self.path}'" if self.path else None

def _tbl_properties(self):
return (
self.format_tblproperties(self.tbl_properties)
if self.tbl_properties
else None
)

def _storage(self):
# By the time we're here, we have a valid format
return f"STORED AS {self.format}"
Expand Down Expand Up @@ -152,6 +159,7 @@ def _pieces(self):
yield self._storage()

yield self._location()
yield self._tbl_properties()


class AlterTable(ImpalaBase, DDL):
Expand Down Expand Up @@ -258,6 +266,7 @@ def __init__(
can_exist=False,
path=None,
partition=None,
tbl_properties=None,
):
super().__init__(
table_name,
Expand All @@ -267,6 +276,7 @@ def __init__(
can_exist=can_exist,
path=path,
partition=partition,
tbl_properties=tbl_properties,
)
self.select = select

Expand All @@ -275,6 +285,7 @@ def _pieces(self):
yield self._partitioned_by()
yield self._storage()
yield self._location()
yield self._tbl_properties()
yield "AS"
yield self.select

Expand Down
25 changes: 18 additions & 7 deletions ibis/backends/impala/tests/test_partition.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
import pandas as pd
import pandas.testing as tm
import pytest
from impala.error import HiveServer2Error

import ibis
from ibis import util
Expand Down Expand Up @@ -142,10 +141,14 @@ def test_create_partitioned_table_from_expr(con, alltypes, tmp_parted):
tm.assert_frame_equal(result, expected)


@pytest.mark.xfail(raises=HiveServer2Error)
def test_add_drop_partition_no_location(con, temp_table):
schema = ibis.schema([("foo", "string"), ("year", "int32"), ("month", "int16")])
con.create_table(temp_table, schema=schema, partition=["year", "month"])
con.create_table(
temp_table,
schema=schema,
partition=["year", "month"],
tbl_properties={"transactional": "false"},
)
table = con.table(temp_table)

part = {"year": 2007, "month": 4}
Expand All @@ -159,10 +162,14 @@ def test_add_drop_partition_no_location(con, temp_table):
assert len(table.partitions()) == 1


@pytest.mark.xfail(raises=HiveServer2Error)
def test_add_drop_partition_owned_by_impala(con, temp_table):
schema = ibis.schema([("foo", "string"), ("year", "int32"), ("month", "int16")])
con.create_table(temp_table, schema=schema, partition=["year", "month"])
con.create_table(
temp_table,
schema=schema,
partition=["year", "month"],
tbl_properties={"transactional": "false"},
)

table = con.table(temp_table)

Expand All @@ -181,10 +188,14 @@ def test_add_drop_partition_owned_by_impala(con, temp_table):
assert len(table.partitions()) == 1


@pytest.mark.xfail(raises=HiveServer2Error)
def test_add_drop_partition_hive_bug(con, temp_table):
schema = ibis.schema([("foo", "string"), ("year", "int32"), ("month", "int16")])
con.create_table(temp_table, schema=schema, partition=["year", "month"])
con.create_table(
temp_table,
schema=schema,
partition=["year", "month"],
tbl_properties={"transactional": "false"},
)

table = con.table(temp_table)

Expand Down

0 comments on commit e3d02bd

Please sign in to comment.