Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(impala): add tbl_properties to create_table #9839

Merged
merged 1 commit into from
Aug 14, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions ibis/backends/impala/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -471,6 +471,7 @@ def create_table(
format="parquet",
location=None,
partition=None,
tbl_properties: Mapping[str, Any] | None = None,
like_parquet=None,
) -> ir.Table:
"""Create a new table using an Ibis table expression or in-memory data.
Expand Down Expand Up @@ -501,6 +502,8 @@ def create_table(
partition
Must pass a schema to use this. Cannot partition from an
expression.
tbl_properties
Table properties to set on table creation.
like_parquet
Can specify instead of a schema

Expand Down Expand Up @@ -534,6 +537,7 @@ def create_table(
format=format,
external=True if location is not None else external,
partition=partition,
tbl_properties=tbl_properties,
path=location,
)
)
Expand All @@ -549,6 +553,7 @@ def create_table(
external=external,
path=location,
partition=partition,
tbl_properties=tbl_properties,
)
)
return self.table(name, database=database or self.current_database)
Expand Down
11 changes: 11 additions & 0 deletions ibis/backends/impala/ddl.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,13 @@
def _location(self):
return f"LOCATION '{self.path}'" if self.path else None

def _tbl_properties(self):
return (

Check warning on line 109 in ibis/backends/impala/ddl.py

View check run for this annotation

Codecov / codecov/patch

ibis/backends/impala/ddl.py#L109

Added line #L109 was not covered by tests
self.format_tblproperties(self.tbl_properties)
if self.tbl_properties
else None
)

def _storage(self):
# By the time we're here, we have a valid format
return f"STORED AS {self.format}"
Expand Down Expand Up @@ -152,6 +159,7 @@
yield self._storage()

yield self._location()
yield self._tbl_properties()

Check warning on line 162 in ibis/backends/impala/ddl.py

View check run for this annotation

Codecov / codecov/patch

ibis/backends/impala/ddl.py#L162

Added line #L162 was not covered by tests


class AlterTable(ImpalaBase, DDL):
Expand Down Expand Up @@ -258,6 +266,7 @@
can_exist=False,
path=None,
partition=None,
tbl_properties=None,
):
super().__init__(
table_name,
Expand All @@ -267,6 +276,7 @@
can_exist=can_exist,
path=path,
partition=partition,
tbl_properties=tbl_properties,
)
self.select = select

Expand All @@ -275,6 +285,7 @@
yield self._partitioned_by()
yield self._storage()
yield self._location()
yield self._tbl_properties()

Check warning on line 288 in ibis/backends/impala/ddl.py

View check run for this annotation

Codecov / codecov/patch

ibis/backends/impala/ddl.py#L288

Added line #L288 was not covered by tests
yield "AS"
yield self.select

Expand Down
25 changes: 18 additions & 7 deletions ibis/backends/impala/tests/test_partition.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
import pandas as pd
import pandas.testing as tm
import pytest
from impala.error import HiveServer2Error

import ibis
from ibis import util
Expand Down Expand Up @@ -142,10 +141,14 @@
tm.assert_frame_equal(result, expected)


@pytest.mark.xfail(raises=HiveServer2Error)
def test_add_drop_partition_no_location(con, temp_table):
schema = ibis.schema([("foo", "string"), ("year", "int32"), ("month", "int16")])
con.create_table(temp_table, schema=schema, partition=["year", "month"])
con.create_table(

Check warning on line 146 in ibis/backends/impala/tests/test_partition.py

View check run for this annotation

Codecov / codecov/patch

ibis/backends/impala/tests/test_partition.py#L146

Added line #L146 was not covered by tests
temp_table,
schema=schema,
partition=["year", "month"],
tbl_properties={"transactional": "false"},
)
table = con.table(temp_table)

part = {"year": 2007, "month": 4}
Expand All @@ -159,10 +162,14 @@
assert len(table.partitions()) == 1


@pytest.mark.xfail(raises=HiveServer2Error)
def test_add_drop_partition_owned_by_impala(con, temp_table):
schema = ibis.schema([("foo", "string"), ("year", "int32"), ("month", "int16")])
con.create_table(temp_table, schema=schema, partition=["year", "month"])
con.create_table(

Check warning on line 167 in ibis/backends/impala/tests/test_partition.py

View check run for this annotation

Codecov / codecov/patch

ibis/backends/impala/tests/test_partition.py#L167

Added line #L167 was not covered by tests
temp_table,
schema=schema,
partition=["year", "month"],
tbl_properties={"transactional": "false"},
)

table = con.table(temp_table)

Expand All @@ -181,10 +188,14 @@
assert len(table.partitions()) == 1


@pytest.mark.xfail(raises=HiveServer2Error)
def test_add_drop_partition_hive_bug(con, temp_table):
schema = ibis.schema([("foo", "string"), ("year", "int32"), ("month", "int16")])
con.create_table(temp_table, schema=schema, partition=["year", "month"])
con.create_table(

Check warning on line 193 in ibis/backends/impala/tests/test_partition.py

View check run for this annotation

Codecov / codecov/patch

ibis/backends/impala/tests/test_partition.py#L193

Added line #L193 was not covered by tests
temp_table,
schema=schema,
partition=["year", "month"],
tbl_properties={"transactional": "false"},
)

table = con.table(temp_table)

Expand Down