diff --git a/.gitignore b/.gitignore index 0daf098a6273..5201c9ad8ee9 100644 --- a/.gitignore +++ b/.gitignore @@ -97,3 +97,4 @@ ci/udf/.ninja_log ci/udf/build.ninja junit.xml spark-warehouse +docs/backends/support_matrix.csv diff --git a/docs/backends/99_support_matrix.md b/docs/backends/99_support_matrix.md deleted file mode 100644 index a8fa2ec3224c..000000000000 --- a/docs/backends/99_support_matrix.md +++ /dev/null @@ -1,3 +0,0 @@ -# Operation Support Matrix - -
diff --git a/docs/backends/support_matrix.md b/docs/backends/support_matrix.md new file mode 100644 index 000000000000..4adc26184859 --- /dev/null +++ b/docs/backends/support_matrix.md @@ -0,0 +1,10 @@ +--- +hide: + - toc +--- + +# Operation Support Matrix + +!!! note "Backends are shown in descending order of the number of supported operations" + +{{ read_csv("docs/backends/support_matrix.csv") }} diff --git a/docs/stylesheets/extra.css b/docs/stylesheets/extra.css index 2191f57efcce..9000e125c331 100644 --- a/docs/stylesheets/extra.css +++ b/docs/stylesheets/extra.css @@ -5,52 +5,79 @@ .md-typeset details.experimental { border-color: rgb(43, 155, 70); } + .md-typeset .experimental > .admonition-title, .md-typeset .experimental > summary { background-color: rgba(43, 155, 70, 0.1); border-color: rgb(43, 155, 70); } + .md-typeset .experimental > .admonition-title::before, .md-typeset .experimental > summary::before { background-color: rgb(43, 155, 70); -webkit-mask-image: var(--md-admonition-icon--experimental); mask-image: var(--md-admonition-icon--experimental); } + .verified { color: #00c853; } + .unverified { color: #ff9100; } + .bug { color: #f50057; } + .cancel { color: #ff5252; } + .download-button { text-align: center; } + .support-matrix .md-typeset__table { display: table; min-width: 100%; } + .support-matrix .md-typeset table:not([class]) { display: table; min-width: 100%; } .md-typeset__scrollwrap { - overflow-x: unset; + transform: rotateX(180deg); + overflow-x: auto; +} + +.md-typeset__scrollwrap > .md-typeset__table { + transform: rotateX(180deg); } -body > div.md-container > main > div > div.md-content > article > div > div > div > table > thead > tr > th:nth-child(1) { - min-width: 9.2rem; + +body + > div.md-container + > main + > div + > div.md-content + > article + > div.md-typeset__scrollwrap + > div + > table + > thead + > tr + > th:nth-child(1) { + min-width: 8.8rem; } .md-typeset table:not([class]) th { text-align: center; padding: 0.9375em 0.5em; } + .md-typeset table:not([class]) td { text-align: center; padding: 0.9375em 0.5em; diff --git a/gen_matrix.py b/gen_matrix.py index 23de1cde4dcf..c39636ab90bc 100644 --- a/gen_matrix.py +++ b/gen_matrix.py @@ -1,10 +1,7 @@ -import collections -import importlib -import operator +import io from pathlib import Path -import mkdocs_gen_files -import tabulate +import pandas as pd import tomli import ibis @@ -28,43 +25,35 @@ def get_leaf_classes(op): yield from get_leaf_classes(child_class) -with mkdocs_gen_files.open( - Path("backends") / "99_support_matrix.md", - "w", -) as f: - print("# Operation Support Matrix", file=f) - print('
', file=f) +ICONS = { + True: ":material-check-decagram:{ .verified }", + False: ":material-cancel:{ .cancel }", +} - support = collections.defaultdict(list) - possible_ops = sorted( - set(get_leaf_classes(ops.ValueOp)), key=operator.attrgetter("__name__") +def main(): + dst = Path(__file__).parent.joinpath( + "docs", + "backends", + "support_matrix.csv", ) + possible_ops = set(get_leaf_classes(ops.ValueOp)) - for op in possible_ops: - support["operation"].append(f"`{op.__name__}`") - for name, backend in get_backends(): - try: - translator = backend.compiler.translator_class - ops = translator._registry.keys() | translator._rewrites.keys() - supported = op in ops - except AttributeError: - if name in ("dask", "pandas"): - execution = importlib.import_module( - f"ibis.backends.{name}.execution" - ) - execute_node = execution.execute_node - ops = {op for op, *_ in execute_node.funcs.keys()} - supported = op in ops or any( - issubclass(op, other) for other in ops - ) - else: - continue - if supported: - support[name].append(":material-check-decagram:{ .verified }") - else: - support[name].append(":material-cancel:{ .cancel }") + support = { + "operation": [f"`{op.__name__}`" for op in possible_ops], + } + for name, backend in get_backends(): + support[name] = list(map(backend.has_operation, possible_ops)) - table = tabulate.tabulate(support, headers="keys", tablefmt="pipe") - print(table, file=f) - print('
', file=f) + df = pd.DataFrame(support).set_index("operation").sort_index() + counts = df.sum().sort_values(ascending=False) + df = df.loc[:, counts.index].replace(ICONS) + out = io.BytesIO() + df.to_csv(out) + ops_bytes = out.getvalue() + + if not dst.exists() or ops_bytes != dst.read_bytes(): + dst.write_bytes(ops_bytes) + + +main() diff --git a/ibis/backends/base/__init__.py b/ibis/backends/base/__init__.py index 17516041c605..c49680aaafd7 100644 --- a/ibis/backends/base/__init__.py +++ b/ibis/backends/base/__init__.py @@ -566,3 +566,30 @@ def drop_view( raise NotImplementedError( f'Backend "{self.name}" does not implement "drop_view"' ) + + @classmethod + def has_operation(cls, operation: type[ops.ValueOp]) -> bool: + """Return whether the backend implements support for `operation`. + + Parameters + ---------- + operation + A class corresponding to an operation. + + Returns + ------- + bool + Whether the backend implements the operation. + + Examples + -------- + >>> import ibis + >>> import ibis.expr.operations as ops + >>> ibis.sqlite.has_operation(ops.ArrayIndex) + False + >>> ibis.postgres.has_operation(ops.ArrayIndex) + True + """ + raise NotImplementedError( + f"{cls.name} backend has not implemented `has_operation` API" + ) diff --git a/ibis/backends/base/sql/__init__.py b/ibis/backends/base/sql/__init__.py index e0be6e7c2851..1174fdfd2de5 100644 --- a/ibis/backends/base/sql/__init__.py +++ b/ibis/backends/base/sql/__init__.py @@ -252,3 +252,9 @@ def explain( cur.release() return '\n'.join(['Query:', util.indent(query, 2), '', *result]) + + @classmethod + def has_operation(cls, operation: type[ops.ValueOp]) -> bool: + translator = cls.compiler.translator_class + op_classes = translator._registry.keys() | translator._rewrites.keys() + return operation in op_classes diff --git a/ibis/backends/datafusion/__init__.py b/ibis/backends/datafusion/__init__.py index bf863597283b..4771fdf609d5 100644 --- a/ibis/backends/datafusion/__init__.py +++ b/ibis/backends/datafusion/__init__.py @@ -8,6 +8,7 @@ import pyarrow as pa import ibis.common.exceptions as com +import ibis.expr.operations as ops import ibis.expr.schema as sch import ibis.expr.types as ir from ibis.backends.base import BaseBackend @@ -195,3 +196,14 @@ def compile( **kwargs: Any, ): return translate(expr) + + @classmethod + def has_operation(cls, operation: type[ops.ValueOp]) -> bool: + from .compiler import translate + + op_classes = translate.registry + return operation in op_classes or any( + issubclass(operation, op_impl) + for op_impl in op_classes + if issubclass(op_impl, ops.ValueOp) + ) diff --git a/ibis/backends/pandas/__init__.py b/ibis/backends/pandas/__init__.py index f0a20875de3c..8a58fec1c247 100644 --- a/ibis/backends/pandas/__init__.py +++ b/ibis/backends/pandas/__init__.py @@ -1,5 +1,6 @@ from __future__ import annotations +import importlib from typing import Any, MutableMapping import pandas as pd @@ -7,6 +8,7 @@ import ibis.common.exceptions as com import ibis.config +import ibis.expr.operations as ops import ibis.expr.schema as sch import ibis.expr.types as ir from ibis.backends.base import BaseBackend @@ -158,6 +160,19 @@ def _from_pandas(df: pd.DataFrame) -> pd.DataFrame: def _convert_object(cls, obj: Any) -> Any: return cls.backend_table_type(obj) + @classmethod + def has_operation(cls, operation: type[ops.ValueOp]) -> bool: + execution = importlib.import_module( + f"ibis.backends.{cls.name}.execution" + ) + execute_node = execution.execute_node + op_classes = {op for op, *_ in execute_node.funcs.keys()} + return operation in op_classes or any( + issubclass(operation, op_impl) + for op_impl in op_classes + if issubclass(op_impl, ops.ValueOp) + ) + class Backend(BasePandasBackend): name = 'pandas' diff --git a/mkdocs.yml b/mkdocs.yml index 8297497fa894..7b55390c2c1b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -11,6 +11,7 @@ theme: - header.autohide - navigation.indexes - navigation.instant + - navigation.tracking - search.highlight - search.share - search.suggest @@ -25,14 +26,16 @@ theme: custom_dir: docs/overrides plugins: - search - - macros - autorefs - exclude: glob: - backends/template.md + - backends/support_matrix.csv - gen-files: scripts: - gen_matrix.py + - table-reader + - macros - mkdocstrings: enable_inventory: true handlers: @@ -78,13 +81,13 @@ plugins: show_root_heading: true show_root_toc_entry: true show_source: false - # - mkdocs-jupyter: - # execute: true - # ignore: - # - "*.py" - # execute_ignore: "tutorial/*Geospatial*.ipynb" - # include_source: true - # theme: dark + - mkdocs-jupyter: + execute: true + ignore: + - "*.py" + execute_ignore: "tutorial/*Geospatial*.ipynb" + include_source: true + theme: dark - literate-nav markdown_extensions: - admonition @@ -110,7 +113,9 @@ markdown_extensions: - pymdownx.superfences - pymdownx.tabbed: alternate_style: true - - toc + - toc: + permalink: true + permalink_title: "Anchor link to this section for reference" extra: project_name: "ibis" team: diff --git a/poetry.lock b/poetry.lock index f67a3e9f28d6..b65d5d1fc6b5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -978,6 +978,17 @@ python-versions = "*" [package.dependencies] mkdocs = "*" +[[package]] +name = "mkdocs-gen-files" +version = "0.3.4" +description = "MkDocs plugin to programmatically generate documentation pages during the build" +category = "dev" +optional = false +python-versions = ">=3.7,<4.0" + +[package.dependencies] +mkdocs = ">=1.0.3,<2.0.0" + [[package]] name = "mkdocs-jupyter" version = "0.20.0" @@ -1045,6 +1056,20 @@ category = "dev" optional = false python-versions = ">=3.6" +[[package]] +name = "mkdocs-table-reader-plugin" +version = "1.0.0" +description = "MkDocs plugin to directly insert tables from files into markdown." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +mkdocs = ">=1.0" +pandas = ">=1.1" +PyYAML = ">=5.4.1" +tabulate = ">=0.8.7" + [[package]] name = "mkdocstrings" version = "0.17.0" @@ -1877,6 +1902,17 @@ pure-eval = "*" [package.extras] tests = ["pytest", "typeguard", "pygments", "littleutils", "cython"] +[[package]] +name = "tabulate" +version = "0.8.9" +description = "Pretty-print tabular data" +category = "dev" +optional = false +python-versions = "*" + +[package.extras] +widechars = ["wcwidth"] + [[package]] name = "termcolor" version = "1.1.0" @@ -2107,7 +2143,7 @@ visualization = ["graphviz"] [metadata] lock-version = "1.1" python-versions = ">=3.8,<3.11" -content-hash = "ebd39cbaa4824940f4ec2acd877af49d2ed2a1af842be884a0a003d9d65005f4" +content-hash = "34e890835e1d400ec9387a4074c914291ef8dd24af0e241553dbe7e327ca4cb2" [metadata.files] appnope = [ @@ -2946,6 +2982,10 @@ mkdocs-autorefs = [ mkdocs-exclude = [ {file = "mkdocs-exclude-1.0.2.tar.gz", hash = "sha256:ba6fab3c80ddbe3fd31d3e579861fd3124513708271180a5f81846da8c7e2a51"}, ] +mkdocs-gen-files = [ + {file = "mkdocs-gen-files-0.3.4.tar.gz", hash = "sha256:c69188486bdc1e74bd2b9b7ebbde9f9eb21052ae7762f1b35420cfbfc6d7122e"}, + {file = "mkdocs_gen_files-0.3.4-py3-none-any.whl", hash = "sha256:07f43245c87a03cfb03884e767655c2a61def24d07e47fb3a8d26b1581524d6a"}, +] mkdocs-jupyter = [ {file = "mkdocs-jupyter-0.20.0.tar.gz", hash = "sha256:4d436d45dedc5e78760e24a54105ceef849aa6414e4ef13ddc99c915e9a8ea73"}, ] @@ -2965,6 +3005,10 @@ mkdocs-material-extensions = [ {file = "mkdocs-material-extensions-1.0.3.tar.gz", hash = "sha256:bfd24dfdef7b41c312ede42648f9eb83476ea168ec163b613f9abd12bbfddba2"}, {file = "mkdocs_material_extensions-1.0.3-py3-none-any.whl", hash = "sha256:a82b70e533ce060b2a5d9eb2bc2e1be201cf61f901f93704b4acf6e3d5983a44"}, ] +mkdocs-table-reader-plugin = [ + {file = "mkdocs-table-reader-plugin-1.0.0.tar.gz", hash = "sha256:ef222ca5e82431a3b98fbdcc935d6d5247e8a8d4ad9a616bdcbc40f20eb18db0"}, + {file = "mkdocs_table_reader_plugin-1.0.0-py3-none-any.whl", hash = "sha256:ea5cc6d642bd80bde733f4418c3ce3a4bb9c9327247810d864502a6912cc2d87"}, +] mkdocstrings = [ {file = "mkdocstrings-0.17.0-py3-none-any.whl", hash = "sha256:103fc1dd58cb23b7e0a6da5292435f01b29dc6fa0ba829132537f3f556f985de"}, {file = "mkdocstrings-0.17.0.tar.gz", hash = "sha256:75b5cfa2039aeaf3a5f5cf0aa438507b0330ce76c8478da149d692daa7213a98"}, @@ -3607,6 +3651,10 @@ stack-data = [ {file = "stack_data-0.2.0-py3-none-any.whl", hash = "sha256:999762f9c3132308789affa03e9271bbbe947bf78311851f4d485d8402ed858e"}, {file = "stack_data-0.2.0.tar.gz", hash = "sha256:45692d41bd633a9503a5195552df22b583caf16f0b27c4e58c98d88c8b648e12"}, ] +tabulate = [ + {file = "tabulate-0.8.9-py3-none-any.whl", hash = "sha256:d7c013fe7abbc5e491394e10fa845f8f32fe54f8dc60c6622c6cf482d25d47e4"}, + {file = "tabulate-0.8.9.tar.gz", hash = "sha256:eb1d13f25760052e8931f2ef80aaf6045a6cceb47514db8beab24cded16f13a7"}, +] termcolor = [ {file = "termcolor-1.1.0.tar.gz", hash = "sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b"}, ] diff --git a/pyproject.toml b/pyproject.toml index 90aa4b34a386..a1ce51d8e764 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,10 +65,12 @@ ipython = ">=7.27.0,<9" isort = ">=5.9.3,<6" mkdocs = ">=1.2.3,<2" mkdocs-exclude = ">=1.0.2,<2" +mkdocs-gen-files = ">=0.3.4,<0.4.0" mkdocs-jupyter = ">=0.20.0,<1" mkdocs-literate-nav = ">=0.4.1,<1" mkdocs-macros-plugin = ">=0.6.3,<1" mkdocs-material = ">=8.2.1,<9" +mkdocs-table-reader-plugin = ">=1.0.0,<2" mkdocstrings = ">=0.17.0,<0.18.0" mypy = ">=0.931,<1" pyarrow = ">=1,<8" @@ -79,16 +81,15 @@ pytest-benchmark = ">=3.4.1,<4" pytest-cov = ">=3.0.0,<4" pytest-mock = ">=3.6.1,<4" pytest-randomly = ">=3.10.1,<4" +pytest-repeat = ">=0.9.1,<0.10" pytest-xdist = ">=2.3.0,<3" pytkdocs = { version = ">=0.15.0,<0.17.0", extras = ["numpy-style"] } pyupgrade = ">=2.26.0,<3" requests = ">=2,<3" setuptools = ">=57,<61" sqlalchemy = ">=1.3,<1.5" +sqlparse = ">=0.4.2,<0.5.0" types-requests = ">=2.27.8,<3" -sqlparse = "^0.4.2" -pytest-repeat = "^0.9.1" -mkdocs-gen-files = "^0.3.4" [tool.poetry.extras] all = [