Skip to content

Commit

Permalink
feat(api): add has_operation API to the backend
Browse files Browse the repository at this point in the history
  • Loading branch information
cpcloud authored and kszucs committed Mar 10, 2022
1 parent 802804a commit 4fab014
Show file tree
Hide file tree
Showing 12 changed files with 197 additions and 59 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -97,3 +97,4 @@ ci/udf/.ninja_log
ci/udf/build.ninja
junit.xml
spark-warehouse
docs/backends/support_matrix.csv
3 changes: 0 additions & 3 deletions docs/backends/99_support_matrix.md

This file was deleted.

10 changes: 10 additions & 0 deletions docs/backends/support_matrix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
---
hide:
- toc
---

# Operation Support Matrix

!!! note "Backends are shown in descending order of the number of supported operations"

{{ read_csv("docs/backends/support_matrix.csv") }}
33 changes: 30 additions & 3 deletions docs/stylesheets/extra.css
Original file line number Diff line number Diff line change
Expand Up @@ -5,52 +5,79 @@
.md-typeset details.experimental {
border-color: rgb(43, 155, 70);
}

.md-typeset .experimental > .admonition-title,
.md-typeset .experimental > summary {
background-color: rgba(43, 155, 70, 0.1);
border-color: rgb(43, 155, 70);
}

.md-typeset .experimental > .admonition-title::before,
.md-typeset .experimental > summary::before {
background-color: rgb(43, 155, 70);
-webkit-mask-image: var(--md-admonition-icon--experimental);
mask-image: var(--md-admonition-icon--experimental);
}

.verified {
color: #00c853;
}

.unverified {
color: #ff9100;
}

.bug {
color: #f50057;
}

.cancel {
color: #ff5252;
}

.download-button {
text-align: center;
}

.support-matrix .md-typeset__table {
display: table;
min-width: 100%;
}

.support-matrix .md-typeset table:not([class]) {
display: table;
min-width: 100%;
}

.md-typeset__scrollwrap {
overflow-x: unset;
transform: rotateX(180deg);
overflow-x: auto;
}

.md-typeset__scrollwrap > .md-typeset__table {
transform: rotateX(180deg);
}
body > div.md-container > main > div > div.md-content > article > div > div > div > table > thead > tr > th:nth-child(1) {
min-width: 9.2rem;

body
> div.md-container
> main
> div
> div.md-content
> article
> div.md-typeset__scrollwrap
> div
> table
> thead
> tr
> th:nth-child(1) {
min-width: 8.8rem;
}

.md-typeset table:not([class]) th {
text-align: center;
padding: 0.9375em 0.5em;
}

.md-typeset table:not([class]) td {
text-align: center;
padding: 0.9375em 0.5em;
Expand Down
69 changes: 29 additions & 40 deletions gen_matrix.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
import collections
import importlib
import operator
import io
from pathlib import Path

import mkdocs_gen_files
import tabulate
import pandas as pd
import tomli

import ibis
Expand All @@ -28,43 +25,35 @@ def get_leaf_classes(op):
yield from get_leaf_classes(child_class)


with mkdocs_gen_files.open(
Path("backends") / "99_support_matrix.md",
"w",
) as f:
print("# Operation Support Matrix", file=f)
print('<div class="support-matrix" markdown>', file=f)
ICONS = {
True: ":material-check-decagram:{ .verified }",
False: ":material-cancel:{ .cancel }",
}

support = collections.defaultdict(list)

possible_ops = sorted(
set(get_leaf_classes(ops.ValueOp)), key=operator.attrgetter("__name__")
def main():
dst = Path(__file__).parent.joinpath(
"docs",
"backends",
"support_matrix.csv",
)
possible_ops = set(get_leaf_classes(ops.ValueOp))

for op in possible_ops:
support["operation"].append(f"`{op.__name__}`")
for name, backend in get_backends():
try:
translator = backend.compiler.translator_class
ops = translator._registry.keys() | translator._rewrites.keys()
supported = op in ops
except AttributeError:
if name in ("dask", "pandas"):
execution = importlib.import_module(
f"ibis.backends.{name}.execution"
)
execute_node = execution.execute_node
ops = {op for op, *_ in execute_node.funcs.keys()}
supported = op in ops or any(
issubclass(op, other) for other in ops
)
else:
continue
if supported:
support[name].append(":material-check-decagram:{ .verified }")
else:
support[name].append(":material-cancel:{ .cancel }")
support = {
"operation": [f"`{op.__name__}`" for op in possible_ops],
}
for name, backend in get_backends():
support[name] = list(map(backend.has_operation, possible_ops))

table = tabulate.tabulate(support, headers="keys", tablefmt="pipe")
print(table, file=f)
print('</div>', file=f)
df = pd.DataFrame(support).set_index("operation").sort_index()
counts = df.sum().sort_values(ascending=False)
df = df.loc[:, counts.index].replace(ICONS)
out = io.BytesIO()
df.to_csv(out)
ops_bytes = out.getvalue()

if not dst.exists() or ops_bytes != dst.read_bytes():
dst.write_bytes(ops_bytes)


main()
27 changes: 27 additions & 0 deletions ibis/backends/base/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -566,3 +566,30 @@ def drop_view(
raise NotImplementedError(
f'Backend "{self.name}" does not implement "drop_view"'
)

@classmethod
def has_operation(cls, operation: type[ops.ValueOp]) -> bool:
"""Return whether the backend implements support for `operation`.
Parameters
----------
operation
A class corresponding to an operation.
Returns
-------
bool
Whether the backend implements the operation.
Examples
--------
>>> import ibis
>>> import ibis.expr.operations as ops
>>> ibis.sqlite.has_operation(ops.ArrayIndex)
False
>>> ibis.postgres.has_operation(ops.ArrayIndex)
True
"""
raise NotImplementedError(
f"{cls.name} backend has not implemented `has_operation` API"
)
6 changes: 6 additions & 0 deletions ibis/backends/base/sql/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -252,3 +252,9 @@ def explain(
cur.release()

return '\n'.join(['Query:', util.indent(query, 2), '', *result])

@classmethod
def has_operation(cls, operation: type[ops.ValueOp]) -> bool:
translator = cls.compiler.translator_class
op_classes = translator._registry.keys() | translator._rewrites.keys()
return operation in op_classes
12 changes: 12 additions & 0 deletions ibis/backends/datafusion/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import pyarrow as pa

import ibis.common.exceptions as com
import ibis.expr.operations as ops
import ibis.expr.schema as sch
import ibis.expr.types as ir
from ibis.backends.base import BaseBackend
Expand Down Expand Up @@ -195,3 +196,14 @@ def compile(
**kwargs: Any,
):
return translate(expr)

@classmethod
def has_operation(cls, operation: type[ops.ValueOp]) -> bool:
from .compiler import translate

op_classes = translate.registry
return operation in op_classes or any(
issubclass(operation, op_impl)
for op_impl in op_classes
if issubclass(op_impl, ops.ValueOp)
)
15 changes: 15 additions & 0 deletions ibis/backends/pandas/__init__.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
from __future__ import annotations

import importlib
from typing import Any, MutableMapping

import pandas as pd
from pydantic import Field

import ibis.common.exceptions as com
import ibis.config
import ibis.expr.operations as ops
import ibis.expr.schema as sch
import ibis.expr.types as ir
from ibis.backends.base import BaseBackend
Expand Down Expand Up @@ -158,6 +160,19 @@ def _from_pandas(df: pd.DataFrame) -> pd.DataFrame:
def _convert_object(cls, obj: Any) -> Any:
return cls.backend_table_type(obj)

@classmethod
def has_operation(cls, operation: type[ops.ValueOp]) -> bool:
execution = importlib.import_module(
f"ibis.backends.{cls.name}.execution"
)
execute_node = execution.execute_node
op_classes = {op for op, *_ in execute_node.funcs.keys()}
return operation in op_classes or any(
issubclass(operation, op_impl)
for op_impl in op_classes
if issubclass(op_impl, ops.ValueOp)
)


class Backend(BasePandasBackend):
name = 'pandas'
Expand Down
23 changes: 14 additions & 9 deletions mkdocs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ theme:
- header.autohide
- navigation.indexes
- navigation.instant
- navigation.tracking
- search.highlight
- search.share
- search.suggest
Expand All @@ -25,14 +26,16 @@ theme:
custom_dir: docs/overrides
plugins:
- search
- macros
- autorefs
- exclude:
glob:
- backends/template.md
- backends/support_matrix.csv
- gen-files:
scripts:
- gen_matrix.py
- table-reader
- macros
- mkdocstrings:
enable_inventory: true
handlers:
Expand Down Expand Up @@ -78,13 +81,13 @@ plugins:
show_root_heading: true
show_root_toc_entry: true
show_source: false
# - mkdocs-jupyter:
# execute: true
# ignore:
# - "*.py"
# execute_ignore: "tutorial/*Geospatial*.ipynb"
# include_source: true
# theme: dark
- mkdocs-jupyter:
execute: true
ignore:
- "*.py"
execute_ignore: "tutorial/*Geospatial*.ipynb"
include_source: true
theme: dark
- literate-nav
markdown_extensions:
- admonition
Expand All @@ -110,7 +113,9 @@ markdown_extensions:
- pymdownx.superfences
- pymdownx.tabbed:
alternate_style: true
- toc
- toc:
permalink: true
permalink_title: "Anchor link to this section for reference"
extra:
project_name: "ibis"
team:
Expand Down
Loading

0 comments on commit 4fab014

Please sign in to comment.