Skip to content

Commit

Permalink
feat(bigquery): support in-memory tables
Browse files Browse the repository at this point in the history
  • Loading branch information
krzysztof-kwitt authored and cpcloud committed Jan 29, 2023
1 parent 2819cff commit 37e3279
Show file tree
Hide file tree
Showing 3 changed files with 13 additions and 6 deletions.
9 changes: 7 additions & 2 deletions ibis/backends/base/sql/compiler/query_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,12 @@ def _format_in_memory_table(self, op):
)
for row in op.data.to_frame().itertuples(index=False)
)
rows = ", ".join(f"({raw_row})" for raw_row in raw_rows)
return f"(VALUES {rows})"
if self.context.compiler.support_values_syntax_in_select:
rows = ", ".join(f"({raw_row})" for raw_row in raw_rows)
return f"(VALUES {rows})"
else:
rows = "UNION ALL ".join(f"(SELECT {raw_row})" for raw_row in raw_rows)
return f"({rows})"

def _format_table(self, op):
# TODO: This could probably go in a class and be significantly nicer
Expand Down Expand Up @@ -488,6 +492,7 @@ class Compiler:
difference_class = Difference

cheap_in_memory_tables = False
support_values_syntax_in_select = True

@classmethod
def make_context(cls, params=None):
Expand Down
2 changes: 2 additions & 0 deletions ibis/backends/bigquery/compiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,8 @@ class BigQueryCompiler(sql_compiler.Compiler):
intersect_class = BigQueryIntersection
difference_class = BigQueryDifference

support_values_syntax_in_select = False

@staticmethod
def _generate_setup_queries(expr, context):
"""Generate DDL for temporary resources."""
Expand Down
8 changes: 4 additions & 4 deletions ibis/backends/tests/test_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -692,11 +692,11 @@ def test_deprecated_path_argument(backend_name, tmp_path):
],
)
@pytest.mark.notyet(
["bigquery", "mysql", "sqlite"],
["mysql", "sqlite"],
reason="SQLAlchemy generates incorrect code for `VALUES` projections.",
raises=(sa.exc.ProgrammingError, sa.exc.OperationalError),
)
@pytest.mark.notimpl(["bigquery", "dask", "datafusion", "pandas"])
@pytest.mark.notimpl(["dask", "datafusion", "pandas"])
def test_in_memory_table(backend, con, expr, expected):
result = con.execute(expr)
backend.assert_frame_equal(result, expected)
Expand All @@ -707,7 +707,7 @@ def test_in_memory_table(backend, con, expr, expected):
reason="SQLAlchemy generates incorrect code for `VALUES` projections.",
raises=(sa.exc.ProgrammingError, sa.exc.OperationalError),
)
@pytest.mark.notimpl(["bigquery", "dask", "datafusion", "pandas"])
@pytest.mark.notimpl(["dask", "datafusion", "pandas"])
def test_filter_memory_table(backend, con):
t = ibis.memtable([(1, 2), (3, 4), (5, 6)], columns=["x", "y"])
expr = t.filter(t.x > 1)
Expand All @@ -721,7 +721,7 @@ def test_filter_memory_table(backend, con):
reason="SQLAlchemy generates incorrect code for `VALUES` projections.",
raises=(sa.exc.ProgrammingError, sa.exc.OperationalError),
)
@pytest.mark.notimpl(["bigquery", "dask", "datafusion", "pandas"])
@pytest.mark.notimpl(["dask", "datafusion", "pandas"])
def test_agg_memory_table(con):
t = ibis.memtable([(1, 2), (3, 4), (5, 6)], columns=["x", "y"])
expr = t.x.count()
Expand Down

0 comments on commit 37e3279

Please sign in to comment.