From 0fc896c71f5ee13899247df57517430fe725e2ba Mon Sep 17 00:00:00 2001 From: Manfred Moser Date: Fri, 2 Jun 2023 17:21:19 -0700 Subject: [PATCH 1/2] Enable deflist markdown extension --- docs/src/main/sphinx/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/src/main/sphinx/conf.py b/docs/src/main/sphinx/conf.py index de296043d9ac..79be66d2717e 100644 --- a/docs/src/main/sphinx/conf.py +++ b/docs/src/main/sphinx/conf.py @@ -115,6 +115,7 @@ def setup(app): myst_enable_extensions = [ "colon_fence", + "deflist" ] # -- Options for HTML output --------------------------------------------------- From 21b9afd3b77903f33e3d6e83cbbbf733f9e76e35 Mon Sep 17 00:00:00 2001 From: Manfred Moser Date: Fri, 2 Jun 2023 17:23:28 -0700 Subject: [PATCH 2/2] Convert more documentation to markdown source * top-level docs * appendix * overview * developer guide --- docs/src/main/sphinx/admin.md | 25 + docs/src/main/sphinx/admin.rst | 27 - docs/src/main/sphinx/appendix.md | 8 + docs/src/main/sphinx/appendix.rst | 10 - docs/src/main/sphinx/appendix/from-hive.md | 188 ++++ docs/src/main/sphinx/appendix/from-hive.rst | 170 ---- .../src/main/sphinx/appendix/legal-notices.md | 57 ++ .../main/sphinx/appendix/legal-notices.rst | 66 -- docs/src/main/sphinx/client.md | 20 + docs/src/main/sphinx/client.rst | 22 - docs/src/main/sphinx/connector.md | 45 + docs/src/main/sphinx/connector.rst | 46 - docs/src/main/sphinx/develop.md | 24 + docs/src/main/sphinx/develop.rst | 25 - .../develop/certificate-authenticator.md | 41 + .../develop/certificate-authenticator.rst | 45 - ...client-protocol.rst => client-protocol.md} | 93 +- docs/src/main/sphinx/develop/connectors.md | 847 +++++++++++++++++ docs/src/main/sphinx/develop/connectors.rst | 854 ------------------ .../src/main/sphinx/develop/event-listener.md | 59 ++ .../main/sphinx/develop/event-listener.rst | 64 -- docs/src/main/sphinx/develop/example-http.md | 102 +++ docs/src/main/sphinx/develop/example-http.rst | 112 --- docs/src/main/sphinx/develop/example-jdbc.md | 64 ++ docs/src/main/sphinx/develop/example-jdbc.rst | 72 -- docs/src/main/sphinx/develop/functions.md | 320 +++++++ docs/src/main/sphinx/develop/functions.rst | 329 ------- .../src/main/sphinx/develop/group-provider.md | 40 + .../main/sphinx/develop/group-provider.rst | 44 - .../sphinx/develop/header-authenticator.md | 42 + .../sphinx/develop/header-authenticator.rst | 46 - docs/src/main/sphinx/develop/insert.md | 28 + docs/src/main/sphinx/develop/insert.rst | 30 - .../sphinx/develop/password-authenticator.md | 41 + .../sphinx/develop/password-authenticator.rst | 45 - docs/src/main/sphinx/develop/spi-overview.md | 113 +++ docs/src/main/sphinx/develop/spi-overview.rst | 120 --- .../main/sphinx/develop/supporting-merge.md | 431 +++++++++ .../main/sphinx/develop/supporting-merge.rst | 420 --------- .../sphinx/develop/system-access-control.md | 49 + .../sphinx/develop/system-access-control.rst | 53 -- .../main/sphinx/develop/table-functions.md | 274 ++++++ .../main/sphinx/develop/table-functions.rst | 289 ------ docs/src/main/sphinx/develop/types.md | 36 + docs/src/main/sphinx/develop/types.rst | 41 - docs/src/main/sphinx/functions.md | 52 ++ docs/src/main/sphinx/functions.rst | 53 -- docs/src/main/sphinx/glossary.md | 213 +++++ docs/src/main/sphinx/glossary.rst | 198 ---- docs/src/main/sphinx/index.md | 26 + docs/src/main/sphinx/index.rst | 26 - .../{installation.rst => installation.md} | 21 +- .../main/sphinx/{language.rst => language.md} | 19 +- docs/src/main/sphinx/optimizer.md | 10 + docs/src/main/sphinx/optimizer.rst | 11 - .../main/sphinx/{overview.rst => overview.md} | 13 +- .../overview/{concepts.rst => concepts.md} | 119 +-- .../overview/{use-cases.rst => use-cases.md} | 12 +- docs/src/main/sphinx/release.md | 345 +++++++ docs/src/main/sphinx/release.rst | 347 ------- docs/src/main/sphinx/security.md | 63 ++ docs/src/main/sphinx/security.rst | 65 -- docs/src/main/sphinx/sql.md | 76 ++ docs/src/main/sphinx/sql.rst | 77 -- 64 files changed, 3761 insertions(+), 3862 deletions(-) create mode 100644 docs/src/main/sphinx/admin.md delete mode 100644 docs/src/main/sphinx/admin.rst create mode 100644 docs/src/main/sphinx/appendix.md delete mode 100644 docs/src/main/sphinx/appendix.rst create mode 100644 docs/src/main/sphinx/appendix/from-hive.md delete mode 100644 docs/src/main/sphinx/appendix/from-hive.rst create mode 100644 docs/src/main/sphinx/appendix/legal-notices.md delete mode 100644 docs/src/main/sphinx/appendix/legal-notices.rst create mode 100644 docs/src/main/sphinx/client.md delete mode 100644 docs/src/main/sphinx/client.rst create mode 100644 docs/src/main/sphinx/connector.md delete mode 100644 docs/src/main/sphinx/connector.rst create mode 100644 docs/src/main/sphinx/develop.md delete mode 100644 docs/src/main/sphinx/develop.rst create mode 100644 docs/src/main/sphinx/develop/certificate-authenticator.md delete mode 100644 docs/src/main/sphinx/develop/certificate-authenticator.rst rename docs/src/main/sphinx/develop/{client-protocol.rst => client-protocol.md} (79%) create mode 100644 docs/src/main/sphinx/develop/connectors.md delete mode 100644 docs/src/main/sphinx/develop/connectors.rst create mode 100644 docs/src/main/sphinx/develop/event-listener.md delete mode 100644 docs/src/main/sphinx/develop/event-listener.rst create mode 100644 docs/src/main/sphinx/develop/example-http.md delete mode 100644 docs/src/main/sphinx/develop/example-http.rst create mode 100644 docs/src/main/sphinx/develop/example-jdbc.md delete mode 100644 docs/src/main/sphinx/develop/example-jdbc.rst create mode 100644 docs/src/main/sphinx/develop/functions.md delete mode 100644 docs/src/main/sphinx/develop/functions.rst create mode 100644 docs/src/main/sphinx/develop/group-provider.md delete mode 100644 docs/src/main/sphinx/develop/group-provider.rst create mode 100644 docs/src/main/sphinx/develop/header-authenticator.md delete mode 100644 docs/src/main/sphinx/develop/header-authenticator.rst create mode 100644 docs/src/main/sphinx/develop/insert.md delete mode 100644 docs/src/main/sphinx/develop/insert.rst create mode 100644 docs/src/main/sphinx/develop/password-authenticator.md delete mode 100644 docs/src/main/sphinx/develop/password-authenticator.rst create mode 100644 docs/src/main/sphinx/develop/spi-overview.md delete mode 100644 docs/src/main/sphinx/develop/spi-overview.rst create mode 100644 docs/src/main/sphinx/develop/supporting-merge.md delete mode 100644 docs/src/main/sphinx/develop/supporting-merge.rst create mode 100644 docs/src/main/sphinx/develop/system-access-control.md delete mode 100644 docs/src/main/sphinx/develop/system-access-control.rst create mode 100644 docs/src/main/sphinx/develop/table-functions.md delete mode 100644 docs/src/main/sphinx/develop/table-functions.rst create mode 100644 docs/src/main/sphinx/develop/types.md delete mode 100644 docs/src/main/sphinx/develop/types.rst create mode 100644 docs/src/main/sphinx/functions.md delete mode 100644 docs/src/main/sphinx/functions.rst create mode 100644 docs/src/main/sphinx/glossary.md delete mode 100644 docs/src/main/sphinx/glossary.rst create mode 100644 docs/src/main/sphinx/index.md delete mode 100644 docs/src/main/sphinx/index.rst rename docs/src/main/sphinx/{installation.rst => installation.md} (57%) rename docs/src/main/sphinx/{language.rst => language.md} (65%) create mode 100644 docs/src/main/sphinx/optimizer.md delete mode 100644 docs/src/main/sphinx/optimizer.rst rename docs/src/main/sphinx/{overview.rst => overview.md} (56%) rename docs/src/main/sphinx/overview/{concepts.rst => concepts.md} (84%) rename docs/src/main/sphinx/overview/{use-cases.rst => use-cases.md} (91%) create mode 100644 docs/src/main/sphinx/release.md delete mode 100644 docs/src/main/sphinx/release.rst create mode 100644 docs/src/main/sphinx/security.md delete mode 100644 docs/src/main/sphinx/security.rst create mode 100644 docs/src/main/sphinx/sql.md delete mode 100644 docs/src/main/sphinx/sql.rst diff --git a/docs/src/main/sphinx/admin.md b/docs/src/main/sphinx/admin.md new file mode 100644 index 000000000000..17d7c1e2b0c9 --- /dev/null +++ b/docs/src/main/sphinx/admin.md @@ -0,0 +1,25 @@ +# Administration + +```{toctree} +:maxdepth: 1 + +admin/web-interface +admin/tuning +admin/jmx +admin/properties +admin/spill +admin/resource-groups +admin/session-property-managers +admin/dist-sort +admin/dynamic-filtering +admin/graceful-shutdown +admin/fault-tolerant-execution +``` + +# Event listeners + +```{toctree} +:titlesonly: true + +admin/event-listeners-http +``` diff --git a/docs/src/main/sphinx/admin.rst b/docs/src/main/sphinx/admin.rst deleted file mode 100644 index a69c92fc1e79..000000000000 --- a/docs/src/main/sphinx/admin.rst +++ /dev/null @@ -1,27 +0,0 @@ -************** -Administration -************** - -.. toctree:: - :maxdepth: 1 - - admin/web-interface - admin/tuning - admin/jmx - admin/properties - admin/spill - admin/resource-groups - admin/session-property-managers - admin/dist-sort - admin/dynamic-filtering - admin/graceful-shutdown - admin/fault-tolerant-execution - -*************** -Event listeners -*************** - -.. toctree:: - :titlesonly: - - admin/event-listeners-http diff --git a/docs/src/main/sphinx/appendix.md b/docs/src/main/sphinx/appendix.md new file mode 100644 index 000000000000..72fe770f3dfa --- /dev/null +++ b/docs/src/main/sphinx/appendix.md @@ -0,0 +1,8 @@ +# Appendix + +```{toctree} +:maxdepth: 1 + +appendix/from-hive +appendix/legal-notices +``` diff --git a/docs/src/main/sphinx/appendix.rst b/docs/src/main/sphinx/appendix.rst deleted file mode 100644 index a8dd5fb476b7..000000000000 --- a/docs/src/main/sphinx/appendix.rst +++ /dev/null @@ -1,10 +0,0 @@ -********* -Appendix -********* - -.. toctree:: - :maxdepth: 1 - - appendix/from-hive - appendix/legal-notices - diff --git a/docs/src/main/sphinx/appendix/from-hive.md b/docs/src/main/sphinx/appendix/from-hive.md new file mode 100644 index 000000000000..03475abf0510 --- /dev/null +++ b/docs/src/main/sphinx/appendix/from-hive.md @@ -0,0 +1,188 @@ +# Migrating from Hive + +Trino uses ANSI SQL syntax and semantics, whereas Hive uses a language similar +to SQL called HiveQL which is loosely modeled after MySQL (which itself has many +differences from ANSI SQL). + +## Use subscript for accessing a dynamic index of an array instead of a udf + +The subscript operator in SQL supports full expressions, unlike Hive (which only supports constants). Therefore you can write queries like: + +``` +SELECT my_array[CARDINALITY(my_array)] as last_element +FROM ... +``` + +## Avoid out of bounds access of arrays + +Accessing out of bounds elements of an array will result in an exception. You can avoid this with an `if` as follows: + +``` +SELECT IF(CARDINALITY(my_array) >= 3, my_array[3], NULL) +FROM ... +``` + +## Use ANSI SQL syntax for arrays + +Arrays are indexed starting from 1, not from 0: + +``` +SELECT my_array[1] AS first_element +FROM ... +``` + +Construct arrays with ANSI syntax: + +``` +SELECT ARRAY[1, 2, 3] AS my_array +``` + +## Use ANSI SQL syntax for identifiers and strings + +Strings are delimited with single quotes and identifiers are quoted with double quotes, not backquotes: + +``` +SELECT name AS "User Name" +FROM "7day_active" +WHERE name = 'foo' +``` + +## Quote identifiers that start with numbers + +Identifiers that start with numbers are not legal in ANSI SQL and must be quoted using double quotes: + +``` +SELECT * +FROM "7day_active" +``` + +## Use the standard string concatenation operator + +Use the ANSI SQL string concatenation operator: + +``` +SELECT a || b || c +FROM ... +``` + +## Use standard types for CAST targets + +The following standard types are supported for `CAST` targets: + +``` +SELECT + CAST(x AS varchar) +, CAST(x AS bigint) +, CAST(x AS double) +, CAST(x AS boolean) +FROM ... +``` + +In particular, use `VARCHAR` instead of `STRING`. + +## Use CAST when dividing integers + +Trino follows the standard behavior of performing integer division when dividing two integers. For example, dividing `7` by `2` will result in `3`, not `3.5`. +To perform floating point division on two integers, cast one of them to a double: + +``` +SELECT CAST(5 AS DOUBLE) / 2 +``` + +## Use WITH for complex expressions or queries + +When you want to re-use a complex output expression as a filter, use either an inline subquery or factor it out using the `WITH` clause: + +``` +WITH a AS ( + SELECT substr(name, 1, 3) x + FROM ... +) +SELECT * +FROM a +WHERE x = 'foo' +``` + +## Use UNNEST to expand arrays and maps + +Trino supports {ref}`unnest` for expanding arrays and maps. +Use `UNNEST` instead of `LATERAL VIEW explode()`. + +Hive query: + +``` +SELECT student, score +FROM tests +LATERAL VIEW explode(scores) t AS score; +``` + +Trino query: + +``` +SELECT student, score +FROM tests +CROSS JOIN UNNEST(scores) AS t (score); +``` + +## Use ANSI SQL syntax for date and time INTERVAL expressions + +Trino supports the ANSI SQL style `INTERVAL` expressions that differs from the implementation used in Hive. + +- The `INTERVAL` keyword is required and is not optional. +- Date and time units must be singular. For example `day` and not `days`. +- Values must be quoted. + +Hive query: + +``` +SELECT cast('2000-08-19' as date) + 14 days; +``` + +Equivalent Trino query: + +``` +SELECT cast('2000-08-19' as date) + INTERVAL '14' day; +``` + +## Caution with datediff + +The Hive `datediff` function returns the difference between the two dates in +days and is declared as: + +```text +datediff(string enddate, string startdate) -> integer +``` + +The equivalent Trino function {ref}`date_diff` +uses a reverse order for the two date parameters and requires a unit. This has +to be taken into account when migrating: + +Hive query: + +``` +datediff(enddate, startdate) +``` + +Trino query: + +``` +date_diff('day', startdate, enddate) +``` + +## Overwriting data on insert + +By default, `INSERT` queries are not allowed to overwrite existing data. You +can use the catalog session property `insert_existing_partitions_behavior` to +allow overwrites. Prepend the name of the catalog using the Hive connector, for +example `hdfs`, and set the property in the session before you run the insert +query: + +``` +SET SESSION hdfs.insert_existing_partitions_behavior = 'OVERWRITE'; +INSERT INTO hdfs.schema.table ... +``` + +The resulting behavior is equivalent to using [INSERT OVERWRITE](https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DML) in Hive. + +Insert overwrite operation is not supported by Trino when the table is stored on +encrypted HDFS, when the table is unpartitioned or table is transactional. diff --git a/docs/src/main/sphinx/appendix/from-hive.rst b/docs/src/main/sphinx/appendix/from-hive.rst deleted file mode 100644 index 2fa7b6f7aa14..000000000000 --- a/docs/src/main/sphinx/appendix/from-hive.rst +++ /dev/null @@ -1,170 +0,0 @@ -=================== -Migrating from Hive -=================== - -Trino uses ANSI SQL syntax and semantics, whereas Hive uses a language similar -to SQL called HiveQL which is loosely modeled after MySQL (which itself has many -differences from ANSI SQL). - -Use subscript for accessing a dynamic index of an array instead of a udf ------------------------------------------------------------------------- - -The subscript operator in SQL supports full expressions, unlike Hive (which only supports constants). Therefore you can write queries like:: - - SELECT my_array[CARDINALITY(my_array)] as last_element - FROM ... - -Avoid out of bounds access of arrays ------------------------------------- - -Accessing out of bounds elements of an array will result in an exception. You can avoid this with an ``if`` as follows:: - - SELECT IF(CARDINALITY(my_array) >= 3, my_array[3], NULL) - FROM ... - -Use ANSI SQL syntax for arrays ------------------------------- - -Arrays are indexed starting from 1, not from 0:: - - SELECT my_array[1] AS first_element - FROM ... - -Construct arrays with ANSI syntax:: - - SELECT ARRAY[1, 2, 3] AS my_array - -Use ANSI SQL syntax for identifiers and strings ------------------------------------------------ - -Strings are delimited with single quotes and identifiers are quoted with double quotes, not backquotes:: - - SELECT name AS "User Name" - FROM "7day_active" - WHERE name = 'foo' - -Quote identifiers that start with numbers ------------------------------------------ - -Identifiers that start with numbers are not legal in ANSI SQL and must be quoted using double quotes:: - - SELECT * - FROM "7day_active" - -Use the standard string concatenation operator ----------------------------------------------- - -Use the ANSI SQL string concatenation operator:: - - SELECT a || b || c - FROM ... - -Use standard types for CAST targets ------------------------------------ - -The following standard types are supported for ``CAST`` targets:: - - SELECT - CAST(x AS varchar) - , CAST(x AS bigint) - , CAST(x AS double) - , CAST(x AS boolean) - FROM ... - -In particular, use ``VARCHAR`` instead of ``STRING``. - -Use CAST when dividing integers -------------------------------- - -Trino follows the standard behavior of performing integer division when dividing two integers. For example, dividing ``7`` by ``2`` will result in ``3``, not ``3.5``. -To perform floating point division on two integers, cast one of them to a double:: - - SELECT CAST(5 AS DOUBLE) / 2 - -Use WITH for complex expressions or queries -------------------------------------------- - -When you want to re-use a complex output expression as a filter, use either an inline subquery or factor it out using the ``WITH`` clause:: - - WITH a AS ( - SELECT substr(name, 1, 3) x - FROM ... - ) - SELECT * - FROM a - WHERE x = 'foo' - -Use UNNEST to expand arrays and maps ------------------------------------- - -Trino supports :ref:`unnest` for expanding arrays and maps. -Use ``UNNEST`` instead of ``LATERAL VIEW explode()``. - -Hive query:: - - SELECT student, score - FROM tests - LATERAL VIEW explode(scores) t AS score; - -Trino query:: - - SELECT student, score - FROM tests - CROSS JOIN UNNEST(scores) AS t (score); - -Use ANSI SQL syntax for date and time INTERVAL expressions ----------------------------------------------------------- - -Trino supports the ANSI SQL style ``INTERVAL`` expressions that differs from the implementation used in Hive. - -* The ``INTERVAL`` keyword is required and is not optional. -* Date and time units must be singular. For example ``day`` and not ``days``. -* Values must be quoted. - -Hive query:: - - SELECT cast('2000-08-19' as date) + 14 days; - -Equivalent Trino query:: - - SELECT cast('2000-08-19' as date) + INTERVAL '14' day; - -Caution with datediff ---------------------- - -The Hive ``datediff`` function returns the difference between the two dates in -days and is declared as: - -.. code-block:: text - - datediff(string enddate, string startdate) -> integer - -The equivalent Trino function :ref:`date_diff` -uses a reverse order for the two date parameters and requires a unit. This has -to be taken into account when migrating: - -Hive query:: - - datediff(enddate, startdate) - -Trino query:: - - date_diff('day', startdate, enddate) - -Overwriting data on insert --------------------------- - -By default, ``INSERT`` queries are not allowed to overwrite existing data. You -can use the catalog session property ``insert_existing_partitions_behavior`` to -allow overwrites. Prepend the name of the catalog using the Hive connector, for -example ``hdfs``, and set the property in the session before you run the insert -query:: - - SET SESSION hdfs.insert_existing_partitions_behavior = 'OVERWRITE'; - INSERT INTO hdfs.schema.table ... - -The resulting behavior is equivalent to using `INSERT OVERWRITE -`_ in Hive. - -Insert overwrite operation is not supported by Trino when the table is stored on -encrypted HDFS, when the table is unpartitioned or table is transactional. diff --git a/docs/src/main/sphinx/appendix/legal-notices.md b/docs/src/main/sphinx/appendix/legal-notices.md new file mode 100644 index 000000000000..2d462dc22603 --- /dev/null +++ b/docs/src/main/sphinx/appendix/legal-notices.md @@ -0,0 +1,57 @@ +# Legal notices + +## License + +Trino is open source software licensed under the +[Apache License 2.0](https://github.com/trinodb/trino/blob/master/LICENSE). + +## Code + +Source code is available at [https://github.com/trinodb](https://github.com/trinodb). + +## Governance + +The project is run by volunteer contributions and supported by the [Trino +Software Foundation](https://trino.io/foundation.html). + +## Trademarks + +Product names, other names, logos and other material used on this site are +registered trademarks of various entities including, but not limited to, the +following trademark owners and names: + +[American National Standards Institute](https://www.ansi.org/) + +- ANSI, and other names + +[Apache Software Foundation](https://apache.org/) + +- Apache Hadoop, Apache Hive, Apache Iceberg, Apache Kafka, and other names + +[Amazon](https://trademarks.amazon.com/) + +- AWS, S3, Glue, EMR, and other names + +[Docker Inc.](https://www.docker.com/) + +- Docker + +[Google](https://www.google.com/permissions/trademark/trademark-list/) + +- GCP, YouTube and other names + +[Linux Mark Institute](http://www.linuxmark.org/) + +- Linux + +[Microsoft](https://www.microsoft.com/en-us/legal/intellectualproperty/Trademarks/EN-US.aspx) + +- Azure, AKS, and others + +[Oracle](https://www.oracle.com/) + +- Java, JVM, OpenJDK, and other names + +[The Linux Foundation](https://www.linuxfoundation.org/trademark-list/) + +- Kubernetes, Presto, and other names diff --git a/docs/src/main/sphinx/appendix/legal-notices.rst b/docs/src/main/sphinx/appendix/legal-notices.rst deleted file mode 100644 index b5960fa502ac..000000000000 --- a/docs/src/main/sphinx/appendix/legal-notices.rst +++ /dev/null @@ -1,66 +0,0 @@ -============= -Legal notices -============= - - -License -------- - -Trino is open source software licensed under the -`Apache License 2.0 `_. - -Code ----- - -Source code is available at `https://github.com/trinodb -`_. - -Governance ----------- - -The project is run by volunteer contributions and supported by the `Trino -Software Foundation `_. - -Trademarks ----------- - -Product names, other names, logos and other material used on this site are -registered trademarks of various entities including, but not limited to, the -following trademark owners and names: - -`American National Standards Institute `_ - -* ANSI, and other names - -`Apache Software Foundation `_ - -* Apache Hadoop, Apache Hive, Apache Iceberg, Apache Kafka, and other names - -`Amazon `_ - -* AWS, S3, Glue, EMR, and other names - -`Docker Inc. `_ - -* Docker - -`Google `_ - -* GCP, YouTube and other names - -`Linux Mark Institute `_ - -* Linux - -`Microsoft `_ - -* Azure, AKS, and others - -`Oracle `_ - -* Java, JVM, OpenJDK, and other names - -`The Linux Foundation `_ - -* Kubernetes, Presto, and other names - diff --git a/docs/src/main/sphinx/client.md b/docs/src/main/sphinx/client.md new file mode 100644 index 000000000000..f458859783ae --- /dev/null +++ b/docs/src/main/sphinx/client.md @@ -0,0 +1,20 @@ +# Clients + +A client is used to send queries to Trino and receive results, or otherwise +interact with Trino and the connected data sources. + +Some clients, such as the {doc}`command line interface `, can +provide a user interface directly. Clients like the {doc}`JDBC driver +`, provide a mechanism for other tools to connect to Trino. + +The following clients are available: + +```{toctree} +:maxdepth: 1 + +client/cli +client/jdbc +``` + +In addition, the community provides [numerous other clients](https://trino.io/resources.html) for platforms such as Python, and these +can in turn be used to connect applications using these platforms. diff --git a/docs/src/main/sphinx/client.rst b/docs/src/main/sphinx/client.rst deleted file mode 100644 index 4515bbc48059..000000000000 --- a/docs/src/main/sphinx/client.rst +++ /dev/null @@ -1,22 +0,0 @@ -******* -Clients -******* - -A client is used to send queries to Trino and receive results, or otherwise -interact with Trino and the connected data sources. - -Some clients, such as the :doc:`command line interface `, can -provide a user interface directly. Clients like the :doc:`JDBC driver -`, provide a mechanism for other tools to connect to Trino. - -The following clients are available: - -.. toctree:: - :maxdepth: 1 - - client/cli - client/jdbc - -In addition, the community provides `numerous other clients -`_ for platforms such as Python, and these -can in turn be used to connect applications using these platforms. diff --git a/docs/src/main/sphinx/connector.md b/docs/src/main/sphinx/connector.md new file mode 100644 index 000000000000..3b86e28f3d56 --- /dev/null +++ b/docs/src/main/sphinx/connector.md @@ -0,0 +1,45 @@ +# Connectors + +This section describes the connectors available in Trino to access data +from different data sources. + +```{toctree} +:maxdepth: 1 + +Accumulo +Atop +BigQuery +Black Hole +Cassandra +ClickHouse +Delta Lake +Druid +Elasticsearch +Google Sheets +Hive +Hudi +Iceberg +Ignite +JMX +Kafka +Kinesis +Kudu +Local File +MariaDB +Memory +MongoDB +MySQL +Oracle +Phoenix +Pinot +PostgreSQL +Prometheus +Redis +Redshift +SingleStore +SQL Server +System +Thrift +TPCDS +TPCH +``` diff --git a/docs/src/main/sphinx/connector.rst b/docs/src/main/sphinx/connector.rst deleted file mode 100644 index 96fb4ce9e091..000000000000 --- a/docs/src/main/sphinx/connector.rst +++ /dev/null @@ -1,46 +0,0 @@ -********** -Connectors -********** - -This section describes the connectors available in Trino to access data -from different data sources. - -.. toctree:: - :maxdepth: 1 - - Accumulo - Atop - BigQuery - Black Hole - Cassandra - ClickHouse - Delta Lake - Druid - Elasticsearch - Google Sheets - Hive - Hudi - Iceberg - Ignite - JMX - Kafka - Kinesis - Kudu - Local File - MariaDB - Memory - MongoDB - MySQL - Oracle - Phoenix - Pinot - PostgreSQL - Prometheus - Redis - Redshift - SingleStore - SQL Server - System - Thrift - TPCDS - TPCH diff --git a/docs/src/main/sphinx/develop.md b/docs/src/main/sphinx/develop.md new file mode 100644 index 000000000000..5f3667c50fa7 --- /dev/null +++ b/docs/src/main/sphinx/develop.md @@ -0,0 +1,24 @@ +# Developer guide + +This guide is intended for Trino contributors and plugin developers. + +```{toctree} +:maxdepth: 1 + +develop/spi-overview +develop/connectors +develop/example-http +develop/example-jdbc +develop/insert +develop/supporting-merge +develop/types +develop/functions +develop/table-functions +develop/system-access-control +develop/password-authenticator +develop/certificate-authenticator +develop/header-authenticator +develop/group-provider +develop/event-listener +develop/client-protocol +``` diff --git a/docs/src/main/sphinx/develop.rst b/docs/src/main/sphinx/develop.rst deleted file mode 100644 index 210e78813dc4..000000000000 --- a/docs/src/main/sphinx/develop.rst +++ /dev/null @@ -1,25 +0,0 @@ -*************** -Developer guide -*************** - -This guide is intended for Trino contributors and plugin developers. - -.. toctree:: - :maxdepth: 1 - - develop/spi-overview - develop/connectors - develop/example-http - develop/example-jdbc - develop/insert - develop/supporting-merge - develop/types - develop/functions - develop/table-functions - develop/system-access-control - develop/password-authenticator - develop/certificate-authenticator - develop/header-authenticator - develop/group-provider - develop/event-listener - develop/client-protocol diff --git a/docs/src/main/sphinx/develop/certificate-authenticator.md b/docs/src/main/sphinx/develop/certificate-authenticator.md new file mode 100644 index 000000000000..722773e86e19 --- /dev/null +++ b/docs/src/main/sphinx/develop/certificate-authenticator.md @@ -0,0 +1,41 @@ +# Certificate authenticator + +Trino supports TLS-based authentication with X509 certificates via a custom +certificate authenticator that extracts the principal from a client certificate. + +## Implementation + +`CertificateAuthenticatorFactory` is responsible for creating a +`CertificateAuthenticator` instance. It also defines the name of this +authenticator which is used by the administrator in a Trino configuration. + +`CertificateAuthenticator` contains a single method, `authenticate()`, +which authenticates the client certificate and returns a `Principal`, which is then +authorized by the {doc}`system-access-control`. + +The implementation of `CertificateAuthenticatorFactory` must be wrapped +as a plugin and installed on the Trino cluster. + +## Configuration + +After a plugin that implements `CertificateAuthenticatorFactory` has been +installed on the coordinator, it is configured using an +`etc/certificate-authenticator.properties` file. All of the +properties other than `certificate-authenticator.name` are specific to the +`CertificateAuthenticatorFactory` implementation. + +The `certificate-authenticator.name` property is used by Trino to find a +registered `CertificateAuthenticatorFactory` based on the name returned by +`CertificateAuthenticatorFactory.getName()`. The remaining properties are +passed as a map to `CertificateAuthenticatorFactory.create()`. + +Example configuration file: + +```text +certificate-authenticator.name=custom +custom-property1=custom-value1 +custom-property2=custom-value2 +``` + +Additionally, the coordinator must be configured to use certificate authentication +and have HTTPS enabled (or HTTPS forwarding enabled). diff --git a/docs/src/main/sphinx/develop/certificate-authenticator.rst b/docs/src/main/sphinx/develop/certificate-authenticator.rst deleted file mode 100644 index e213194e909e..000000000000 --- a/docs/src/main/sphinx/develop/certificate-authenticator.rst +++ /dev/null @@ -1,45 +0,0 @@ -========================= -Certificate authenticator -========================= - -Trino supports TLS-based authentication with X509 certificates via a custom -certificate authenticator that extracts the principal from a client certificate. - -Implementation --------------- - -``CertificateAuthenticatorFactory`` is responsible for creating a -``CertificateAuthenticator`` instance. It also defines the name of this -authenticator which is used by the administrator in a Trino configuration. - -``CertificateAuthenticator`` contains a single method, ``authenticate()``, -which authenticates the client certificate and returns a ``Principal``, which is then -authorized by the :doc:`system-access-control`. - -The implementation of ``CertificateAuthenticatorFactory`` must be wrapped -as a plugin and installed on the Trino cluster. - -Configuration -------------- - -After a plugin that implements ``CertificateAuthenticatorFactory`` has been -installed on the coordinator, it is configured using an -``etc/certificate-authenticator.properties`` file. All of the -properties other than ``certificate-authenticator.name`` are specific to the -``CertificateAuthenticatorFactory`` implementation. - -The ``certificate-authenticator.name`` property is used by Trino to find a -registered ``CertificateAuthenticatorFactory`` based on the name returned by -``CertificateAuthenticatorFactory.getName()``. The remaining properties are -passed as a map to ``CertificateAuthenticatorFactory.create()``. - -Example configuration file: - -.. code-block:: text - - certificate-authenticator.name=custom - custom-property1=custom-value1 - custom-property2=custom-value2 - -Additionally, the coordinator must be configured to use certificate authentication -and have HTTPS enabled (or HTTPS forwarding enabled). diff --git a/docs/src/main/sphinx/develop/client-protocol.rst b/docs/src/main/sphinx/develop/client-protocol.md similarity index 79% rename from docs/src/main/sphinx/develop/client-protocol.rst rename to docs/src/main/sphinx/develop/client-protocol.md index fc0f6304d1cf..d113d70dd613 100644 --- a/docs/src/main/sphinx/develop/client-protocol.rst +++ b/docs/src/main/sphinx/develop/client-protocol.md @@ -1,6 +1,4 @@ -====================== -Trino client REST API -====================== +# Trino client REST API The REST API allows clients to submit SQL queries to Trino and receive the results. Clients include the CLI, the JDBC driver, and others provided by @@ -8,23 +6,21 @@ the community. The preferred method to interact with Trino is using these existing clients. This document provides details about the API for reference. It can also be used to implement your own client, if necessary. -HTTP methods ------------- +## HTTP methods -* A ``POST`` to ``/v1/statement`` runs the query string in the ``POST`` body, +- A `POST` to `/v1/statement` runs the query string in the `POST` body, and returns a JSON document containing the query results. If there are more - results, the JSON document contains a ``nextUri`` URL attribute. -* A ``GET`` to the ``nextUri`` attribute returns the next batch of query results. -* A ``DELETE`` to ``nextUri`` terminates a running query. + results, the JSON document contains a `nextUri` URL attribute. +- A `GET` to the `nextUri` attribute returns the next batch of query results. +- A `DELETE` to `nextUri` terminates a running query. -Overview of query processing ----------------------------- +## Overview of query processing -A Trino client request is initiated by an HTTP ``POST`` to the endpoint -``/v1/statement``, with a ``POST`` body consisting of the SQL query string. -The caller may set various :ref:`client-request-headers`. The headers are -only required on the initial ``POST`` request, and not when following the -``nextUri`` links. +A Trino client request is initiated by an HTTP `POST` to the endpoint +`/v1/statement`, with a `POST` body consisting of the SQL query string. +The caller may set various {ref}`client-request-headers`. The headers are +only required on the initial `POST` request, and not when following the +`nextUri` links. If the client request returns an HTTP 502, 503 or 504, that means there was intermittent problem processing request and the client should try again @@ -33,39 +29,39 @@ but those can be generated by gateways/load balancers in front of Trino. Any HTTP status other than 502, 503, 504 or 200 means that query processing has failed. -The ``/v1/statement`` ``POST`` request returns a JSON document of type -``QueryResults``, as well as a collection of response headers. The -``QueryResults`` document contains an ``error`` field of type -``QueryError`` if the query has failed, and if that object is not present, -the query succeeded. Important members of ``QueryResults`` are documented +The `/v1/statement` `POST` request returns a JSON document of type +`QueryResults`, as well as a collection of response headers. The +`QueryResults` document contains an `error` field of type +`QueryError` if the query has failed, and if that object is not present, +the query succeeded. Important members of `QueryResults` are documented in the following sections. -If the ``data`` field of the JSON document is set, it contains a list of the -rows of data. The ``columns`` field is set to a list of the +If the `data` field of the JSON document is set, it contains a list of the +rows of data. The `columns` field is set to a list of the names and types of the columns returned by the query. Most of the response headers are treated like browser cookies by the client, and echoed back as request headers in subsequent client requests, as documented below. -If the JSON document returned by the ``POST`` to ``/v1/statement`` does not -contain a ``nextUri`` link, the query has completed, either successfully or +If the JSON document returned by the `POST` to `/v1/statement` does not +contain a `nextUri` link, the query has completed, either successfully or unsuccessfully, and no additional requests need to be made. If the -``nextUri`` link is present in the document, there are more query results -to be fetched. The client should loop executing a ``GET`` request -to the ``nextUri`` returned in the ``QueryResults`` response object until -``nextUri`` is absent from the response. +`nextUri` link is present in the document, there are more query results +to be fetched. The client should loop executing a `GET` request +to the `nextUri` returned in the `QueryResults` response object until +`nextUri` is absent from the response. -The ``status`` field of the JSON document is for human consumption only, and +The `status` field of the JSON document is for human consumption only, and provides a hint about the query state. It can not be used to tell if the query is finished. -Important ``QueryResults`` attributes -------------------------------------- +## Important `QueryResults` attributes -The most important attributes of the ``QueryResults`` JSON document returned by +The most important attributes of the `QueryResults` JSON document returned by the REST API endpoints are listed in this table. For more details, refer to the -class ``io.trino.client.QueryResults`` in module ``trino-client`` in the -``client`` directory of the Trino source code. +class `io.trino.client.QueryResults` in module `trino-client` in the +`client` directory of the Trino source code. +```{eval-rst} .. list-table:: ``QueryResults attributes`` :widths: 25, 55 :header-rows: 1 @@ -95,12 +91,13 @@ class ``io.trino.client.QueryResults`` in module ``trino-client`` in the about the error. See the ``io.trino.client.QueryError`` class in module ``trino-client`` in the ``client`` directory for more details. +``` -``QueryResults`` diagnostic attributes --------------------------------------- +## `QueryResults` diagnostic attributes -These ``QueryResults`` data members may be useful in tracking down problems: +These `QueryResults` data members may be useful in tracking down problems: +```{eval-rst} .. list-table:: ``QueryResults diagnostic attributes`` :widths: 20, 20, 40 :header-rows: 1 @@ -125,16 +122,17 @@ These ``QueryResults`` data members may be useful in tracking down problems: particular interest is ``StatementStats.rootStage``, of type ``StageStats``, providing statistics on the execution of each of the stages of query processing. +``` -.. _client-request-headers: +(client-request-headers)= -Client request headers ----------------------- +## Client request headers This table lists all supported client request headers. Many of the headers can be updated in the client as response headers, and supplied in subsequent requests, just like browser cookies. +```{eval-rst} .. list-table:: Client request headers :widths: 30, 50 :header-rows: 1 @@ -210,14 +208,15 @@ in subsequent requests, just like browser cookies. - Provides extra credentials to the connector. The header is a name=value string that is saved in the session ``Identity`` object. The name and value are only meaningful to the connector. +``` -Client response headers ------------------------ +## Client response headers This table lists the supported client response headers. After receiving a response, a client must update the request headers used in subsequent requests to be consistent with the response headers received. +```{eval-rst} .. list-table:: Client response headers :widths: 30, 50 :header-rows: 1 @@ -261,10 +260,10 @@ subsequent requests to be consistent with the response headers received. * - ``X-Trino-Clear-Transaction-Id`` - Instructs the client to clear the ``X-Trino-Transaction-Id`` request header in subsequent requests. +``` -``ProtocolHeaders`` -------------------- +## `ProtocolHeaders` -Class ``io.trino.client.ProtocolHeaders`` in module ``trino-client`` in the -``client`` directory of Trino source enumerates all the HTTP request and +Class `io.trino.client.ProtocolHeaders` in module `trino-client` in the +`client` directory of Trino source enumerates all the HTTP request and response headers allowed by the Trino client REST API. diff --git a/docs/src/main/sphinx/develop/connectors.md b/docs/src/main/sphinx/develop/connectors.md new file mode 100644 index 000000000000..b3612eb35a92 --- /dev/null +++ b/docs/src/main/sphinx/develop/connectors.md @@ -0,0 +1,847 @@ +# Connectors + +Connectors are the source of all data for queries in Trino. Even if your data +source doesn't have underlying tables backing it, as long as you adapt your data +source to the API expected by Trino, you can write queries against this data. + +## ConnectorFactory + +Instances of your connector are created by a `ConnectorFactory` instance which +is created when Trino calls `getConnectorFactory()` on the plugin. The +connector factory is a simple interface responsible for providing the connector +name and creating an instance of a `Connector` object. A basic connector +implementation that only supports reading, but not writing data, should return +instances of the following services: + +- {ref}`connector-metadata` +- {ref}`connector-split-manager` +- {ref}`connector-record-set-provider` or {ref}`connector-page-source-provider` + +### Configuration + +The `create()` method of the connector factory receives a `config` map, +containing all properties from the catalog properties file. It can be used +to configure the connector, but because all the values are strings, they +might require additional processing if they represent other data types. +It also doesn't validate if all the provided properties are known. This +can lead to the connector behaving differently than expected when a +connector ignores a property due to the user making a mistake in +typing the name of the property. + +To make the configuration more robust, define a Configuration class. This +class describes all the available properties, their types, and additional +validation rules. + +```java +import io.airlift.configuration.Config; +import io.airlift.configuration.ConfigDescription; +import io.airlift.configuration.ConfigSecuritySensitive; +import io.airlift.units.Duration; +import io.airlift.units.MaxDuration; +import io.airlift.units.MinDuration; + +import javax.validation.constraints.NotNull; + +public class ExampleConfig +{ + private String secret; + private Duration timeout = Duration.succinctDuration(10, TimeUnit.SECONDS); + + public String getSecret() + { + return secret; + } + + @Config("secret") + @ConfigDescription("Secret required to access the data source") + @ConfigSecuritySensitive + public ExampleConfig setSecret(String secret) + { + this.secret = secret; + return this; + } + + @NotNull + @MaxDuration("10m") + @MinDuration("1ms") + public Duration getTimeout() + { + return timeout; + } + + @Config("timeout") + public ExampleConfig setTimeout(Duration timeout) + { + this.timeout = timeout; + return this; + } +} +``` + +The preceding example defines two configuration properties and makes +the connector more robust by: + +- defining all supported properties, which allows detecting spelling mistakes + in the configuration on server startup +- defining a default timeout value, to prevent connections getting stuck + indefinitely +- preventing invalid timeout values, like 0 ms, that would make + all requests fail +- parsing timeout values in different units, detecting invalid values +- preventing logging the secret value in plain text + +The configuration class needs to be bound in a Guice module: + +```java +import com.google.inject.Binder; +import com.google.inject.Module; + +import static io.airlift.configuration.ConfigBinder.configBinder; + +public class ExampleModule + implements Module +{ + public ExampleModule() + { + } + + @Override + public void configure(Binder binder) + { + configBinder(binder).bindConfig(ExampleConfig.class); + } +} +``` + +And then the module needs to be initialized in the connector factory, when +creating a new instance of the connector: + +```java +@Override +public Connector create(String connectorName, Map config, ConnectorContext context) +{ + requireNonNull(config, "config is null"); + Bootstrap app = new Bootstrap(new ExampleModule()); + Injector injector = app + .doNotInitializeLogging() + .setRequiredConfigurationProperties(config) + .initialize(); + + return injector.getInstance(ExampleConnector.class); +} +``` + +:::{note} +Environment variables in the catalog properties file +(ex. `secret=${ENV:SECRET}`) are resolved only when using +the `io.airlift.bootstrap.Bootstrap` class to initialize the module. +See {doc}`/security/secrets` for more information. +::: + +If you end up needing to define multiple catalogs using the same connector +just to change one property, consider adding support for schema and/or +table properties. That would allow a more fine-grained configuration. +If a connector doesn't support managing the schema, query predicates for +selected columns could be used as a way of passing the required configuration +at run time. + +For example, when building a connector to read commits from a Git repository, +the repository URL could be a configuration property. But this would result +in a catalog being able to return data only from a single repository. +Alternatively, it can be a column, where every select query would require +a predicate for it: + +```sql +SELECT * +FROM git.default.commits +WHERE url = 'https://github.com/trinodb/trino.git' +``` + +(connector-metadata)= + +## ConnectorMetadata + +The connector metadata interface allows Trino to get a lists of schemas, +tables, columns, and other metadata about a particular data source. + +A basic read-only connector should implement the following methods: + +- `listSchemaNames` +- `listTables` +- `streamTableColumns` +- `getTableHandle` +- `getTableMetadata` +- `getColumnHandles` +- `getColumnMetadata` + +If you are interested in seeing strategies for implementing more methods, +look at the {doc}`example-http` and the Cassandra connector. If your underlying +data source supports schemas, tables, and columns, this interface should be +straightforward to implement. If you are attempting to adapt something that +isn't a relational database, as the Example HTTP connector does, you may +need to get creative about how you map your data source to Trino's schema, +table, and column concepts. + +The connector metadata interface allows to also implement other connector +features, like: + +- Schema management, which is creating, altering and dropping schemas, tables, + table columns, views, and materialized views. + +- Support for table and column comments, and properties. + +- Schema, table and view authorization. + +- Executing {doc}`table-functions`. + +- Providing table statistics used by the Cost Based Optimizer (CBO) + and collecting statistics during writes and when analyzing selected tables. + +- Data modification, which is: + + - inserting, updating, and deleting rows in tables, + - refreshing materialized views, + - truncating whole tables, + - and creating tables from query results. + +- Role and grant management. + +- Pushing down: + + - {ref}`Limit and Top N - limit with sort items ` + - {ref}`Predicates ` + - Projections + - Sampling + - Aggregations + - Joins + - Table function invocation + +Note that data modification also requires implementing +a {ref}`connector-page-sink-provider`. + +When Trino receives a `SELECT` query, it parses it into an Intermediate +Representation (IR). Then, during optimization, it checks if connectors +can handle operations related to SQL clauses by calling one of the following +methods of the `ConnectorMetadata` service: + +- `applyLimit` +- `applyTopN` +- `applyFilter` +- `applyProjection` +- `applySample` +- `applyAggregation` +- `applyJoin` +- `applyTableFunction` +- `applyTableScanRedirect` + +Connectors can indicate that they don't support a particular pushdown or that +the action had no effect by returning `Optional.empty()`. Connectors should +expect these methods to be called multiple times during the optimization of +a given query. + +:::{warning} +It's critical for connectors to return `Optional.empty()` if calling +this method has no effect for that invocation, even if the connector generally +supports a particular pushdown. Doing otherwise can cause the optimizer +to loop indefinitely. +::: + +Otherwise, these methods return a result object containing a new table handle. +The new table handle represents the virtual table derived from applying the +operation (filter, project, limit, etc.) to the table produced by the table +scan node. Once the query actually runs, `ConnectorRecordSetProvider` or +`ConnectorPageSourceProvider` can use whatever optimizations were pushed down to +`ConnectorTableHandle`. + +The returned table handle is later passed to other services that the connector +implements, like the `ConnectorRecordSetProvider` or +`ConnectorPageSourceProvider`. + +(connector-limit-pushdown)= + +### Limit and top-N pushdown + +When executing a `SELECT` query with `LIMIT` or `ORDER BY` clauses, +the query plan may contain a `Sort` or `Limit` operations. + +When the plan contains a `Sort` and `Limit` operations, the engine +tries to push down the limit into the connector by calling the `applyTopN` +method of the connector metadata service. If there's no `Sort` operation, but +only a `Limit`, the `applyLimit` method is called, and the connector can +return results in an arbitrary order. + +If the connector could benefit from the information passed to these methods but +can't guarantee that it's be able to produce fewer rows than the provided +limit, it should return a non-empty result containing a new handle for the +derived table and the `limitGuaranteed` (in `LimitApplicationResult`) or +`topNGuaranteed` (in `TopNApplicationResult`) flag set to false. + +If the connector can guarantee to produce fewer rows than the provided +limit, it should return a non-empty result with the "limit guaranteed" or +"topN guaranteed" flag set to true. + +:::{note} +The `applyTopN` is the only method that receives sort items from the +`Sort` operation. +::: + +In a query, the `ORDER BY` section can include any column with any order. +But the data source for the connector might only support limited combinations. +Plugin authors have to decide if the connector should ignore the pushdown, +return all the data and let the engine sort it, or throw an exception +to inform the user that particular order isn't supported, if fetching all +the data would be too expensive or time consuming. When throwing +an exception, use the `TrinoException` class with the `INVALID_ORDER_BY` +error code and an actionable message, to let users know how to write a valid +query. + +(dev-predicate-pushdown)= + +### Predicate pushdown + +When executing a query with a `WHERE` clause, the query plan can +contain a `ScanFilterProject` plan node/node with a predicate constraint. + +A predicate constraint is a description of the constraint imposed on the +results of the stage/fragment as expressed in the `WHERE` clause. For example, +`WHERE x > 5 AND y = 3` translates into a constraint where the +`summary` field means the `x` column's domain must be greater than +`5` and the `y` column domain equals `3`. + +When the query plan contains a `ScanFilterProject` operation, Trino +tries to optimize the query by pushing down the predicate constraint +into the connector by calling the `applyFilter` method of the +connector metadata service. This method receives a table handle with +all optimizations applied thus far, and returns either +`Optional.empty()` or a response with a new table handle derived from +the old one. + +The query optimizer may call `applyFilter` for a single query multiple times, +as it searches for an optimal query plan. Connectors must +return `Optional.empty()` from `applyFilter` if they cannot apply the +constraint for this invocation, even if they support `ScanFilterProject` +pushdown in general. Connectors must also return `Optional.empty()` if the +constraint has already been applied. + +A constraint contains the following elements: + +- A `TupleDomain` defining the mapping between columns and their domains. + A `Domain` is either a list of possible values, or a list of ranges, and + also contains information about nullability. +- Expression for pushing down function calls. +- Map of assignments from variables in the expression to columns. +- (optional) Predicate which tests a map of columns and their values; + it cannot be held on to after the `applyFilter` call returns. +- (optional) Set of columns the predicate depends on; must be present + if predicate is present. + +If both a predicate and a summary are available, the predicate is guaranteed to +be more strict in filtering of values, and can provide a significant boost to +query performance if used. + +However it is not possible to store a predicate in the table handle and use +it later, as the predicate cannot be held on to after the `applyFilter` +call returns. It is used for filtering of entire partitions, and is not pushed +down. The summary can be pushed down instead by storing it in the table handle. + +This overlap between the predicate and summary is due to historical reasons, +as simple comparison pushdown was implemented first via summary, and more +complex filters such as `LIKE` which required more expressive predicates +were added later. + +If a constraint can only be partially pushed down, for example when a connector +for a database that does not support range matching is used in a query with +`WHERE x = 2 AND y > 5`, the `y` column constraint must be +returned in the `ConstraintApplicationResult` from `applyFilter`. +In this case the `y > 5` condition is applied in Trino, +and not pushed down. + +The following is a simple example which only looks at `TupleDomain`: + +```java +@Override +public Optional> applyFilter( + ConnectorSession session, + ConnectorTableHandle tableHandle, + Constraint constraint) +{ + ExampleTableHandle handle = (ExampleTableHandle) tableHandle; + + TupleDomain oldDomain = handle.getConstraint(); + TupleDomain newDomain = oldDomain.intersect(constraint.getSummary()); + if (oldDomain.equals(newDomain)) { + // Nothing has changed, return empty Option + return Optional.empty(); + } + + handle = new ExampleTableHandle(newDomain); + return Optional.of(new ConstraintApplicationResult<>(handle, TupleDomain.all(), false)); +} +``` + +The `TupleDomain` from the constraint is intersected with the `TupleDomain` +already applied to the `TableHandle` to form `newDomain`. +If filtering has not changed, an `Optional.empty()` result is returned to +notify the planner that this optimization path has reached its end. + +In this example, the connector pushes down the `TupleDomain` +with all Trino data types supported with same semantics in the +data source. As a result, no filters are needed in Trino, +and the `ConstraintApplicationResult` sets `remainingFilter` to +`TupleDomain.all()`. + +This pushdown implementation is quite similar to many Trino connectors, +including `MongoMetadata`, `BigQueryMetadata`, `KafkaMetadata`. + +The following, more complex example shows data types from Trino that are +not available directly in the underlying data source, and must be mapped: + +```java +@Override +public Optional> applyFilter( + ConnectorSession session, + ConnectorTableHandle table, + Constraint constraint) +{ + JdbcTableHandle handle = (JdbcTableHandle) table; + + TupleDomain oldDomain = handle.getConstraint(); + TupleDomain newDomain = oldDomain.intersect(constraint.getSummary()); + TupleDomain remainingFilter; + if (newDomain.isNone()) { + newConstraintExpressions = ImmutableList.of(); + remainingFilter = TupleDomain.all(); + remainingExpression = Optional.of(Constant.TRUE); + } + else { + // We need to decide which columns to push down. + // Since this is a base class for many JDBC-based connectors, each + // having different Trino type mappings and comparison semantics + // it needs to be flexible. + + Map domains = newDomain.getDomains().orElseThrow(); + List columnHandles = domains.keySet().stream() + .map(JdbcColumnHandle.class::cast) + .collect(toImmutableList()); + + // Get information about how to push down every column based on its + // JDBC data type + List columnMappings = jdbcClient.toColumnMappings( + session, + columnHandles.stream() + .map(JdbcColumnHandle::getJdbcTypeHandle) + .collect(toImmutableList())); + + // Calculate the domains which can be safely pushed down (supported) + // and those which need to be filtered in Trino (unsupported) + Map supported = new HashMap<>(); + Map unsupported = new HashMap<>(); + for (int i = 0; i < columnHandles.size(); i++) { + JdbcColumnHandle column = columnHandles.get(i); + DomainPushdownResult pushdownResult = + columnMappings.get(i).getPredicatePushdownController().apply( + session, + domains.get(column)); + supported.put(column, pushdownResult.getPushedDown()); + unsupported.put(column, pushdownResult.getRemainingFilter()); + } + + newDomain = TupleDomain.withColumnDomains(supported); + remainingFilter = TupleDomain.withColumnDomains(unsupported); + } + + // Return empty Optional if nothing changed in filtering + if (oldDomain.equals(newDomain)) { + return Optional.empty(); + } + + handle = new JdbcTableHandle( + handle.getRelationHandle(), + newDomain, + ...); + + return Optional.of( + new ConstraintApplicationResult<>( + handle, + remainingFilter)); +} +``` + +This example illustrates implementing a base class for many JDBC connectors +while handling the specific requirements of multiple JDBC-compliant data sources. +It ensures that if a constraint gets pushed down, it works exactly the same in +the underlying data source, and produces the same results as it would in Trino. +For example, in databases where string comparisons are case-insensitive, +pushdown does not work, as string comparison operations in Trino are +case-sensitive. + +The `PredicatePushdownController` interface determines if a column domain can +be pushed down in JDBC-compliant data sources. In the preceding example, it is +called from a `JdbcClient` implementation specific to that database. +In non-JDBC-compliant data sources, type-based push downs are implemented +directly, without going through the `PredicatePushdownController` interface. + +The following example adds expression pushdown enabled by a session flag: + +```java +@Override +public Optional> applyFilter( + ConnectorSession session, + ConnectorTableHandle table, + Constraint constraint) +{ + JdbcTableHandle handle = (JdbcTableHandle) table; + + TupleDomain oldDomain = handle.getConstraint(); + TupleDomain newDomain = oldDomain.intersect(constraint.getSummary()); + List newConstraintExpressions; + TupleDomain remainingFilter; + Optional remainingExpression; + if (newDomain.isNone()) { + newConstraintExpressions = ImmutableList.of(); + remainingFilter = TupleDomain.all(); + remainingExpression = Optional.of(Constant.TRUE); + } + else { + // We need to decide which columns to push down. + // Since this is a base class for many JDBC-based connectors, each + // having different Trino type mappings and comparison semantics + // it needs to be flexible. + + Map domains = newDomain.getDomains().orElseThrow(); + List columnHandles = domains.keySet().stream() + .map(JdbcColumnHandle.class::cast) + .collect(toImmutableList()); + + // Get information about how to push down every column based on its + // JDBC data type + List columnMappings = jdbcClient.toColumnMappings( + session, + columnHandles.stream() + .map(JdbcColumnHandle::getJdbcTypeHandle) + .collect(toImmutableList())); + + // Calculate the domains which can be safely pushed down (supported) + // and those which need to be filtered in Trino (unsupported) + Map supported = new HashMap<>(); + Map unsupported = new HashMap<>(); + for (int i = 0; i < columnHandles.size(); i++) { + JdbcColumnHandle column = columnHandles.get(i); + DomainPushdownResult pushdownResult = + columnMappings.get(i).getPredicatePushdownController().apply( + session, + domains.get(column)); + supported.put(column, pushdownResult.getPushedDown()); + unsupported.put(column, pushdownResult.getRemainingFilter()); + } + + newDomain = TupleDomain.withColumnDomains(supported); + remainingFilter = TupleDomain.withColumnDomains(unsupported); + + // Do we want to handle expression pushdown? + if (isComplexExpressionPushdown(session)) { + List newExpressions = new ArrayList<>(); + List remainingExpressions = new ArrayList<>(); + // Each expression can be broken down into a list of conjuncts + // joined with AND. We handle each conjunct separately. + for (ConnectorExpression expression : extractConjuncts(constraint.getExpression())) { + // Try to convert the conjunct into something which is + // understood by the underlying JDBC data source + Optional converted = jdbcClient.convertPredicate( + session, + expression, + constraint.getAssignments()); + if (converted.isPresent()) { + newExpressions.add(converted.get()); + } + else { + remainingExpressions.add(expression); + } + } + // Calculate which parts of the expression can be pushed down + // and which need to be calculated in Trino engine + newConstraintExpressions = ImmutableSet.builder() + .addAll(handle.getConstraintExpressions()) + .addAll(newExpressions) + .build().asList(); + remainingExpression = Optional.of(and(remainingExpressions)); + } + else { + newConstraintExpressions = ImmutableList.of(); + remainingExpression = Optional.empty(); + } + } + + // Return empty Optional if nothing changed in filtering + if (oldDomain.equals(newDomain) && + handle.getConstraintExpressions().equals(newConstraintExpressions)) { + return Optional.empty(); + } + + handle = new JdbcTableHandle( + handle.getRelationHandle(), + newDomain, + newConstraintExpressions, + ...); + + return Optional.of( + remainingExpression.isPresent() + ? new ConstraintApplicationResult<>( + handle, + remainingFilter, + remainingExpression.get()) + : new ConstraintApplicationResult<>( + handle, + remainingFilter)); +} +``` + +`ConnectorExpression` is split similarly to `TupleDomain`. +Each expression can be broken down into independent *conjuncts*. Conjuncts are +smaller expressions which, if joined together using an `AND` operator, are +equivalent to the original expression. Every conjunct can be handled +individually. Each one is converted using connector-specific rules, as defined +by the `JdbcClient` implementation, to be more flexible. Unconverted +conjuncts are returned as `remainingExpression` and are evaluated by +the Trino engine. + +(connector-split-manager)= + +## ConnectorSplitManager + +The split manager partitions the data for a table into the individual chunks +that Trino distributes to workers for processing. For example, the Hive +connector lists the files for each Hive partition and creates one or more +splits per file. For data sources that don't have partitioned data, a good +strategy here is to simply return a single split for the entire table. This is +the strategy employed by the Example HTTP connector. + +(connector-record-set-provider)= + +## ConnectorRecordSetProvider + +Given a split, a table handle, and a list of columns, the record set provider +is responsible for delivering data to the Trino execution engine. + +The table and column handles represent a virtual table. They're created by the +connector's metadata service, called by Trino during query planning and +optimization. Such a virtual table doesn't have to map directly to a single +collection in the connector's data source. If the connector supports pushdowns, +there can be multiple virtual tables derived from others, presenting a different +view of the underlying data. + +The provider creates a `RecordSet`, which in turn creates a `RecordCursor` +that's used by Trino to read the column values for each row. + +The provided record set must only include requested columns in the order +matching the list of column handles passed to the +`ConnectorRecordSetProvider.getRecordSet()` method. The record set must return +all the rows contained in the "virtual table" represented by the TableHandle +associated with the TableScan operation. + +For simple connectors, where performance isn't critical, the record set +provider can return an instance of `InMemoryRecordSet`. The in-memory record +set can be built using lists of values for every row, which can be simpler than +implementing a `RecordCursor`. + +A `RecordCursor` implementation needs to keep track of the current record. +It return values for columns by a numerical position, in the data type matching +the column definition in the table. When the engine is done reading the current +record it calls `advanceNextPosition` on the cursor. + +### Type mapping + +The built-in SQL data types use different Java types as carrier types. + +```{eval-rst} +.. list-table:: SQL type to carrier type mapping + :widths: 45, 55 + :header-rows: 1 + + * - SQL type + - Java type + * - ``BOOLEAN`` + - ``boolean`` + * - ``TINYINT`` + - ``long`` + * - ``SMALLINT`` + - ``long`` + * - ``INTEGER`` + - ``long`` + * - ``BIGINT`` + - ``long`` + * - ``REAL`` + - ``double`` + * - ``DOUBLE`` + - ``double`` + * - ``DECIMAL`` + - ``long`` for precision up to 19, inclusive; + ``Int128`` for precision greater than 19 + * - ``VARCHAR`` + - ``Slice`` + * - ``CHAR`` + - ``Slice`` + * - ``VARBINARY`` + - ``Slice`` + * - ``JSON`` + - ``Slice`` + * - ``DATE`` + - ``long`` + * - ``TIME(P)`` + - ``long`` + * - ``TIME WITH TIME ZONE`` + - ``long`` for precision up to 9; + ``LongTimeWithTimeZone`` for precision greater than 9 + * - ``TIMESTAMP(P)`` + - ``long`` for precision up to 6; + ``LongTimestamp`` for precision greater than 6 + * - ``TIMESTAMP(P) WITH TIME ZONE`` + - ``long`` for precision up to 3; + ``LongTimestampWithTimeZone`` for precision greater than 3 + * - ``INTERVAL YEAR TO MONTH`` + - ``long`` + * - ``INTERVAL DAY TO SECOND`` + - ``long`` + * - ``ARRAY`` + - ``Block`` + * - ``MAP`` + - ``Block`` + * - ``ROW`` + - ``Block`` + * - ``IPADDRESS`` + - ``Slice`` + * - ``UUID`` + - ``Slice`` + * - ``HyperLogLog`` + - ``Slice`` + * - ``P4HyperLogLog`` + - ``Slice`` + * - ``SetDigest`` + - ``Slice`` + * - ``QDigest`` + - ``Slice`` + * - ``TDigest`` + - ``TDigest`` +``` + +The `RecordCursor.getType(int field)` method returns the SQL type for a field +and the field value is returned by one of the following methods, matching +the carrier type: + +- `getBoolean(int field)` +- `getLong(int field)` +- `getDouble(int field)` +- `getSlice(int field)` +- `getObject(int field)` + +Values for the `timestamp(p) with time zone` and `time(p) with time zone` +types of regular precision can be converted into `long` using static methods +from the `io.trino.spi.type.DateTimeEncoding` class, like `pack()` or +`packDateTimeWithZone()`. + +UTF-8 encoded strings can be converted to Slices using +the `Slices.utf8Slice()` static method. + +:::{note} +The `Slice` class is provided by the `io.airlift:slice` package. +::: + +`Int128` objects can be created using the `Int128.valueOf()` method. + +The following example creates a block for an `array(varchar)` column: + +```java +private Block encodeArray(List names) +{ + BlockBuilder builder = VARCHAR.createBlockBuilder(null, names.size()); + for (String name : names) { + if (name == null) { + builder.appendNull(); + } + else { + VARCHAR.writeString(builder, name); + } + } + return builder.build(); +} +``` + +The following example creates a block for a `map(varchar, varchar)` column: + +```java +private Block encodeMap(Map map) +{ + MapType mapType = typeManager.getType(TypeSignature.mapType( + VARCHAR.getTypeSignature(), + VARCHAR.getTypeSignature())); + BlockBuilder values = mapType.createBlockBuilder(null, map != null ? map.size() : 0); + if (map == null) { + values.appendNull(); + return values.build().getObject(0, Block.class); + } + BlockBuilder builder = values.beginBlockEntry(); + for (Map.Entry entry : map.entrySet()) { + VARCHAR.writeString(builder, entry.getKey()); + Object value = entry.getValue(); + if (value == null) { + builder.appendNull(); + } + else { + VARCHAR.writeString(builder, value.toString()); + } + } + values.closeEntry(); + return values.build().getObject(0, Block.class); +} +``` + +(connector-page-source-provider)= + +## ConnectorPageSourceProvider + +Given a split, a table handle, and a list of columns, the page source provider +is responsible for delivering data to the Trino execution engine. It creates +a `ConnectorPageSource`, which in turn creates `Page` objects that are used +by Trino to read the column values. + +If not implemented, a default `RecordPageSourceProvider` is used. +Given a record set provider, it returns an instance of `RecordPageSource` +that builds `Page` objects from records in a record set. + +A connector should implement a page source provider instead of a record set +provider when it's possible to create pages directly. The conversion of +individual records from a record set provider into pages adds overheads during +query execution. + +(connector-page-sink-provider)= + +## ConnectorPageSinkProvider + +Given an insert table handle, the page sink provider is responsible for +consuming data from the Trino execution engine. +It creates a `ConnectorPageSink`, which in turn accepts `Page` objects +that contains the column values. + +Example that shows how to iterate over the page to access single values: + +```java +@Override +public CompletableFuture appendPage(Page page) +{ + for (int channel = 0; channel < page.getChannelCount(); channel++) { + Block block = page.getBlock(channel); + for (int position = 0; position < page.getPositionCount(); position++) { + if (block.isNull(position)) { + // or handle this differently + continue; + } + + // channel should match the column number in the table + // use it to determine the expected column type + String value = VARCHAR.getSlice(block, position).toStringUtf8(); + // TODO do something with the value + } + } + return NOT_BLOCKED; +} +``` diff --git a/docs/src/main/sphinx/develop/connectors.rst b/docs/src/main/sphinx/develop/connectors.rst deleted file mode 100644 index 5ccd361c6ba0..000000000000 --- a/docs/src/main/sphinx/develop/connectors.rst +++ /dev/null @@ -1,854 +0,0 @@ -========== -Connectors -========== - -Connectors are the source of all data for queries in Trino. Even if your data -source doesn't have underlying tables backing it, as long as you adapt your data -source to the API expected by Trino, you can write queries against this data. - -ConnectorFactory ----------------- - -Instances of your connector are created by a ``ConnectorFactory`` instance which -is created when Trino calls ``getConnectorFactory()`` on the plugin. The -connector factory is a simple interface responsible for providing the connector -name and creating an instance of a ``Connector`` object. A basic connector -implementation that only supports reading, but not writing data, should return -instances of the following services: - -* :ref:`connector-metadata` -* :ref:`connector-split-manager` -* :ref:`connector-record-set-provider` or :ref:`connector-page-source-provider` - -Configuration -^^^^^^^^^^^^^ - -The ``create()`` method of the connector factory receives a ``config`` map, -containing all properties from the catalog properties file. It can be used -to configure the connector, but because all the values are strings, they -might require additional processing if they represent other data types. -It also doesn't validate if all the provided properties are known. This -can lead to the connector behaving differently than expected when a -connector ignores a property due to the user making a mistake in -typing the name of the property. - -To make the configuration more robust, define a Configuration class. This -class describes all the available properties, their types, and additional -validation rules. - - -.. code-block:: java - - import io.airlift.configuration.Config; - import io.airlift.configuration.ConfigDescription; - import io.airlift.configuration.ConfigSecuritySensitive; - import io.airlift.units.Duration; - import io.airlift.units.MaxDuration; - import io.airlift.units.MinDuration; - - import javax.validation.constraints.NotNull; - - public class ExampleConfig - { - private String secret; - private Duration timeout = Duration.succinctDuration(10, TimeUnit.SECONDS); - - public String getSecret() - { - return secret; - } - - @Config("secret") - @ConfigDescription("Secret required to access the data source") - @ConfigSecuritySensitive - public ExampleConfig setSecret(String secret) - { - this.secret = secret; - return this; - } - - @NotNull - @MaxDuration("10m") - @MinDuration("1ms") - public Duration getTimeout() - { - return timeout; - } - - @Config("timeout") - public ExampleConfig setTimeout(Duration timeout) - { - this.timeout = timeout; - return this; - } - } - -The preceding example defines two configuration properties and makes -the connector more robust by: - -* defining all supported properties, which allows detecting spelling mistakes - in the configuration on server startup -* defining a default timeout value, to prevent connections getting stuck - indefinitely -* preventing invalid timeout values, like 0 ms, that would make - all requests fail -* parsing timeout values in different units, detecting invalid values -* preventing logging the secret value in plain text - -The configuration class needs to be bound in a Guice module: - -.. code-block:: java - - import com.google.inject.Binder; - import com.google.inject.Module; - - import static io.airlift.configuration.ConfigBinder.configBinder; - - public class ExampleModule - implements Module - { - public ExampleModule() - { - } - - @Override - public void configure(Binder binder) - { - configBinder(binder).bindConfig(ExampleConfig.class); - } - } - - -And then the module needs to be initialized in the connector factory, when -creating a new instance of the connector: - -.. code-block:: java - - @Override - public Connector create(String connectorName, Map config, ConnectorContext context) - { - requireNonNull(config, "config is null"); - Bootstrap app = new Bootstrap(new ExampleModule()); - Injector injector = app - .doNotInitializeLogging() - .setRequiredConfigurationProperties(config) - .initialize(); - - return injector.getInstance(ExampleConnector.class); - } - -.. note:: - - Environment variables in the catalog properties file - (ex. ``secret=${ENV:SECRET}``) are resolved only when using - the ``io.airlift.bootstrap.Bootstrap`` class to initialize the module. - See :doc:`/security/secrets` for more information. - -If you end up needing to define multiple catalogs using the same connector -just to change one property, consider adding support for schema and/or -table properties. That would allow a more fine-grained configuration. -If a connector doesn't support managing the schema, query predicates for -selected columns could be used as a way of passing the required configuration -at run time. - -For example, when building a connector to read commits from a Git repository, -the repository URL could be a configuration property. But this would result -in a catalog being able to return data only from a single repository. -Alternatively, it can be a column, where every select query would require -a predicate for it: - -.. code-block:: sql - - SELECT * - FROM git.default.commits - WHERE url = 'https://github.com/trinodb/trino.git' - - -.. _connector-metadata: - -ConnectorMetadata ------------------ - -The connector metadata interface allows Trino to get a lists of schemas, -tables, columns, and other metadata about a particular data source. - -A basic read-only connector should implement the following methods: - -* ``listSchemaNames`` -* ``listTables`` -* ``streamTableColumns`` -* ``getTableHandle`` -* ``getTableMetadata`` -* ``getColumnHandles`` -* ``getColumnMetadata`` - -If you are interested in seeing strategies for implementing more methods, -look at the :doc:`example-http` and the Cassandra connector. If your underlying -data source supports schemas, tables, and columns, this interface should be -straightforward to implement. If you are attempting to adapt something that -isn't a relational database, as the Example HTTP connector does, you may -need to get creative about how you map your data source to Trino's schema, -table, and column concepts. - -The connector metadata interface allows to also implement other connector -features, like: - -* Schema management, which is creating, altering and dropping schemas, tables, - table columns, views, and materialized views. -* Support for table and column comments, and properties. -* Schema, table and view authorization. -* Executing :doc:`table-functions`. -* Providing table statistics used by the Cost Based Optimizer (CBO) - and collecting statistics during writes and when analyzing selected tables. -* Data modification, which is: - - * inserting, updating, and deleting rows in tables, - * refreshing materialized views, - * truncating whole tables, - * and creating tables from query results. - -* Role and grant management. -* Pushing down: - - * :ref:`Limit and Top N - limit with sort items ` - * :ref:`Predicates ` - * Projections - * Sampling - * Aggregations - * Joins - * Table function invocation - -Note that data modification also requires implementing -a :ref:`connector-page-sink-provider`. - -When Trino receives a ``SELECT`` query, it parses it into an Intermediate -Representation (IR). Then, during optimization, it checks if connectors -can handle operations related to SQL clauses by calling one of the following -methods of the ``ConnectorMetadata`` service: - -* ``applyLimit`` -* ``applyTopN`` -* ``applyFilter`` -* ``applyProjection`` -* ``applySample`` -* ``applyAggregation`` -* ``applyJoin`` -* ``applyTableFunction`` -* ``applyTableScanRedirect`` - -Connectors can indicate that they don't support a particular pushdown or that -the action had no effect by returning ``Optional.empty()``. Connectors should -expect these methods to be called multiple times during the optimization of -a given query. - -.. warning:: - - It's critical for connectors to return ``Optional.empty()`` if calling - this method has no effect for that invocation, even if the connector generally - supports a particular pushdown. Doing otherwise can cause the optimizer - to loop indefinitely. - -Otherwise, these methods return a result object containing a new table handle. -The new table handle represents the virtual table derived from applying the -operation (filter, project, limit, etc.) to the table produced by the table -scan node. Once the query actually runs, ``ConnectorRecordSetProvider`` or -``ConnectorPageSourceProvider`` can use whatever optimizations were pushed down to -``ConnectorTableHandle``. - -The returned table handle is later passed to other services that the connector -implements, like the ``ConnectorRecordSetProvider`` or -``ConnectorPageSourceProvider``. - -.. _connector-limit-pushdown: - -Limit and top-N pushdown -^^^^^^^^^^^^^^^^^^^^^^^^ - -When executing a ``SELECT`` query with ``LIMIT`` or ``ORDER BY`` clauses, -the query plan may contain a ``Sort`` or ``Limit`` operations. - -When the plan contains a ``Sort`` and ``Limit`` operations, the engine -tries to push down the limit into the connector by calling the ``applyTopN`` -method of the connector metadata service. If there's no ``Sort`` operation, but -only a ``Limit``, the ``applyLimit`` method is called, and the connector can -return results in an arbitrary order. - -If the connector could benefit from the information passed to these methods but -can't guarantee that it's be able to produce fewer rows than the provided -limit, it should return a non-empty result containing a new handle for the -derived table and the ``limitGuaranteed`` (in ``LimitApplicationResult``) or -``topNGuaranteed`` (in ``TopNApplicationResult``) flag set to false. - -If the connector can guarantee to produce fewer rows than the provided -limit, it should return a non-empty result with the "limit guaranteed" or -"topN guaranteed" flag set to true. - -.. note:: - - The ``applyTopN`` is the only method that receives sort items from the - ``Sort`` operation. - -In a query, the ``ORDER BY`` section can include any column with any order. -But the data source for the connector might only support limited combinations. -Plugin authors have to decide if the connector should ignore the pushdown, -return all the data and let the engine sort it, or throw an exception -to inform the user that particular order isn't supported, if fetching all -the data would be too expensive or time consuming. When throwing -an exception, use the ``TrinoException`` class with the ``INVALID_ORDER_BY`` -error code and an actionable message, to let users know how to write a valid -query. - -.. _dev-predicate-pushdown: - -Predicate pushdown -^^^^^^^^^^^^^^^^^^ - -When executing a query with a ``WHERE`` clause, the query plan can -contain a ``ScanFilterProject`` plan node/node with a predicate constraint. - -A predicate constraint is a description of the constraint imposed on the -results of the stage/fragment as expressed in the ``WHERE`` clause. For example, -``WHERE x > 5 AND y = 3`` translates into a constraint where the -``summary`` field means the ``x`` column's domain must be greater than -``5`` and the ``y`` column domain equals ``3``. - -When the query plan contains a ``ScanFilterProject`` operation, Trino -tries to optimize the query by pushing down the predicate constraint -into the connector by calling the ``applyFilter`` method of the -connector metadata service. This method receives a table handle with -all optimizations applied thus far, and returns either -``Optional.empty()`` or a response with a new table handle derived from -the old one. - -The query optimizer may call ``applyFilter`` for a single query multiple times, -as it searches for an optimal query plan. Connectors must -return ``Optional.empty()`` from ``applyFilter`` if they cannot apply the -constraint for this invocation, even if they support ``ScanFilterProject`` -pushdown in general. Connectors must also return ``Optional.empty()`` if the -constraint has already been applied. - -A constraint contains the following elements: - -* A ``TupleDomain`` defining the mapping between columns and their domains. - A ``Domain`` is either a list of possible values, or a list of ranges, and - also contains information about nullability. -* Expression for pushing down function calls. -* Map of assignments from variables in the expression to columns. -* (optional) Predicate which tests a map of columns and their values; - it cannot be held on to after the ``applyFilter`` call returns. -* (optional) Set of columns the predicate depends on; must be present - if predicate is present. - -If both a predicate and a summary are available, the predicate is guaranteed to -be more strict in filtering of values, and can provide a significant boost to -query performance if used. - -However it is not possible to store a predicate in the table handle and use -it later, as the predicate cannot be held on to after the ``applyFilter`` -call returns. It is used for filtering of entire partitions, and is not pushed -down. The summary can be pushed down instead by storing it in the table handle. - -This overlap between the predicate and summary is due to historical reasons, -as simple comparison pushdown was implemented first via summary, and more -complex filters such as ``LIKE`` which required more expressive predicates -were added later. - -If a constraint can only be partially pushed down, for example when a connector -for a database that does not support range matching is used in a query with -``WHERE x = 2 AND y > 5``, the ``y`` column constraint must be -returned in the ``ConstraintApplicationResult`` from ``applyFilter``. -In this case the ``y > 5`` condition is applied in Trino, -and not pushed down. - -The following is a simple example which only looks at ``TupleDomain``: - -.. code-block:: java - - @Override - public Optional> applyFilter( - ConnectorSession session, - ConnectorTableHandle tableHandle, - Constraint constraint) - { - ExampleTableHandle handle = (ExampleTableHandle) tableHandle; - - TupleDomain oldDomain = handle.getConstraint(); - TupleDomain newDomain = oldDomain.intersect(constraint.getSummary()); - if (oldDomain.equals(newDomain)) { - // Nothing has changed, return empty Option - return Optional.empty(); - } - - handle = new ExampleTableHandle(newDomain); - return Optional.of(new ConstraintApplicationResult<>(handle, TupleDomain.all(), false)); - } - -The ``TupleDomain`` from the constraint is intersected with the ``TupleDomain`` -already applied to the ``TableHandle`` to form ``newDomain``. -If filtering has not changed, an ``Optional.empty()`` result is returned to -notify the planner that this optimization path has reached its end. - -In this example, the connector pushes down the ``TupleDomain`` -with all Trino data types supported with same semantics in the -data source. As a result, no filters are needed in Trino, -and the ``ConstraintApplicationResult`` sets ``remainingFilter`` to -``TupleDomain.all()``. - -This pushdown implementation is quite similar to many Trino connectors, -including ``MongoMetadata``, ``BigQueryMetadata``, ``KafkaMetadata``. - -The following, more complex example shows data types from Trino that are -not available directly in the underlying data source, and must be mapped: - -.. code-block:: java - - @Override - public Optional> applyFilter( - ConnectorSession session, - ConnectorTableHandle table, - Constraint constraint) - { - JdbcTableHandle handle = (JdbcTableHandle) table; - - TupleDomain oldDomain = handle.getConstraint(); - TupleDomain newDomain = oldDomain.intersect(constraint.getSummary()); - TupleDomain remainingFilter; - if (newDomain.isNone()) { - newConstraintExpressions = ImmutableList.of(); - remainingFilter = TupleDomain.all(); - remainingExpression = Optional.of(Constant.TRUE); - } - else { - // We need to decide which columns to push down. - // Since this is a base class for many JDBC-based connectors, each - // having different Trino type mappings and comparison semantics - // it needs to be flexible. - - Map domains = newDomain.getDomains().orElseThrow(); - List columnHandles = domains.keySet().stream() - .map(JdbcColumnHandle.class::cast) - .collect(toImmutableList()); - - // Get information about how to push down every column based on its - // JDBC data type - List columnMappings = jdbcClient.toColumnMappings( - session, - columnHandles.stream() - .map(JdbcColumnHandle::getJdbcTypeHandle) - .collect(toImmutableList())); - - // Calculate the domains which can be safely pushed down (supported) - // and those which need to be filtered in Trino (unsupported) - Map supported = new HashMap<>(); - Map unsupported = new HashMap<>(); - for (int i = 0; i < columnHandles.size(); i++) { - JdbcColumnHandle column = columnHandles.get(i); - DomainPushdownResult pushdownResult = - columnMappings.get(i).getPredicatePushdownController().apply( - session, - domains.get(column)); - supported.put(column, pushdownResult.getPushedDown()); - unsupported.put(column, pushdownResult.getRemainingFilter()); - } - - newDomain = TupleDomain.withColumnDomains(supported); - remainingFilter = TupleDomain.withColumnDomains(unsupported); - } - - // Return empty Optional if nothing changed in filtering - if (oldDomain.equals(newDomain)) { - return Optional.empty(); - } - - handle = new JdbcTableHandle( - handle.getRelationHandle(), - newDomain, - ...); - - return Optional.of( - new ConstraintApplicationResult<>( - handle, - remainingFilter)); - } - -This example illustrates implementing a base class for many JDBC connectors -while handling the specific requirements of multiple JDBC-compliant data sources. -It ensures that if a constraint gets pushed down, it works exactly the same in -the underlying data source, and produces the same results as it would in Trino. -For example, in databases where string comparisons are case-insensitive, -pushdown does not work, as string comparison operations in Trino are -case-sensitive. - -The ``PredicatePushdownController`` interface determines if a column domain can -be pushed down in JDBC-compliant data sources. In the preceding example, it is -called from a ``JdbcClient`` implementation specific to that database. -In non-JDBC-compliant data sources, type-based push downs are implemented -directly, without going through the ``PredicatePushdownController`` interface. - -The following example adds expression pushdown enabled by a session flag: - -.. code-block:: java - - @Override - public Optional> applyFilter( - ConnectorSession session, - ConnectorTableHandle table, - Constraint constraint) - { - JdbcTableHandle handle = (JdbcTableHandle) table; - - TupleDomain oldDomain = handle.getConstraint(); - TupleDomain newDomain = oldDomain.intersect(constraint.getSummary()); - List newConstraintExpressions; - TupleDomain remainingFilter; - Optional remainingExpression; - if (newDomain.isNone()) { - newConstraintExpressions = ImmutableList.of(); - remainingFilter = TupleDomain.all(); - remainingExpression = Optional.of(Constant.TRUE); - } - else { - // We need to decide which columns to push down. - // Since this is a base class for many JDBC-based connectors, each - // having different Trino type mappings and comparison semantics - // it needs to be flexible. - - Map domains = newDomain.getDomains().orElseThrow(); - List columnHandles = domains.keySet().stream() - .map(JdbcColumnHandle.class::cast) - .collect(toImmutableList()); - - // Get information about how to push down every column based on its - // JDBC data type - List columnMappings = jdbcClient.toColumnMappings( - session, - columnHandles.stream() - .map(JdbcColumnHandle::getJdbcTypeHandle) - .collect(toImmutableList())); - - // Calculate the domains which can be safely pushed down (supported) - // and those which need to be filtered in Trino (unsupported) - Map supported = new HashMap<>(); - Map unsupported = new HashMap<>(); - for (int i = 0; i < columnHandles.size(); i++) { - JdbcColumnHandle column = columnHandles.get(i); - DomainPushdownResult pushdownResult = - columnMappings.get(i).getPredicatePushdownController().apply( - session, - domains.get(column)); - supported.put(column, pushdownResult.getPushedDown()); - unsupported.put(column, pushdownResult.getRemainingFilter()); - } - - newDomain = TupleDomain.withColumnDomains(supported); - remainingFilter = TupleDomain.withColumnDomains(unsupported); - - // Do we want to handle expression pushdown? - if (isComplexExpressionPushdown(session)) { - List newExpressions = new ArrayList<>(); - List remainingExpressions = new ArrayList<>(); - // Each expression can be broken down into a list of conjuncts - // joined with AND. We handle each conjunct separately. - for (ConnectorExpression expression : extractConjuncts(constraint.getExpression())) { - // Try to convert the conjunct into something which is - // understood by the underlying JDBC data source - Optional converted = jdbcClient.convertPredicate( - session, - expression, - constraint.getAssignments()); - if (converted.isPresent()) { - newExpressions.add(converted.get()); - } - else { - remainingExpressions.add(expression); - } - } - // Calculate which parts of the expression can be pushed down - // and which need to be calculated in Trino engine - newConstraintExpressions = ImmutableSet.builder() - .addAll(handle.getConstraintExpressions()) - .addAll(newExpressions) - .build().asList(); - remainingExpression = Optional.of(and(remainingExpressions)); - } - else { - newConstraintExpressions = ImmutableList.of(); - remainingExpression = Optional.empty(); - } - } - - // Return empty Optional if nothing changed in filtering - if (oldDomain.equals(newDomain) && - handle.getConstraintExpressions().equals(newConstraintExpressions)) { - return Optional.empty(); - } - - handle = new JdbcTableHandle( - handle.getRelationHandle(), - newDomain, - newConstraintExpressions, - ...); - - return Optional.of( - remainingExpression.isPresent() - ? new ConstraintApplicationResult<>( - handle, - remainingFilter, - remainingExpression.get()) - : new ConstraintApplicationResult<>( - handle, - remainingFilter)); - } - -``ConnectorExpression`` is split similarly to ``TupleDomain``. -Each expression can be broken down into independent *conjuncts*. Conjuncts are -smaller expressions which, if joined together using an ``AND`` operator, are -equivalent to the original expression. Every conjunct can be handled -individually. Each one is converted using connector-specific rules, as defined -by the ``JdbcClient`` implementation, to be more flexible. Unconverted -conjuncts are returned as ``remainingExpression`` and are evaluated by -the Trino engine. - -.. _connector-split-manager: - -ConnectorSplitManager ---------------------- - -The split manager partitions the data for a table into the individual chunks -that Trino distributes to workers for processing. For example, the Hive -connector lists the files for each Hive partition and creates one or more -splits per file. For data sources that don't have partitioned data, a good -strategy here is to simply return a single split for the entire table. This is -the strategy employed by the Example HTTP connector. - -.. _connector-record-set-provider: - -ConnectorRecordSetProvider --------------------------- - -Given a split, a table handle, and a list of columns, the record set provider -is responsible for delivering data to the Trino execution engine. - -The table and column handles represent a virtual table. They're created by the -connector's metadata service, called by Trino during query planning and -optimization. Such a virtual table doesn't have to map directly to a single -collection in the connector's data source. If the connector supports pushdowns, -there can be multiple virtual tables derived from others, presenting a different -view of the underlying data. - -The provider creates a ``RecordSet``, which in turn creates a ``RecordCursor`` -that's used by Trino to read the column values for each row. - -The provided record set must only include requested columns in the order -matching the list of column handles passed to the -``ConnectorRecordSetProvider.getRecordSet()`` method. The record set must return -all the rows contained in the "virtual table" represented by the TableHandle -associated with the TableScan operation. - -For simple connectors, where performance isn't critical, the record set -provider can return an instance of ``InMemoryRecordSet``. The in-memory record -set can be built using lists of values for every row, which can be simpler than -implementing a ``RecordCursor``. - -A ``RecordCursor`` implementation needs to keep track of the current record. -It return values for columns by a numerical position, in the data type matching -the column definition in the table. When the engine is done reading the current -record it calls ``advanceNextPosition`` on the cursor. - -Type mapping -^^^^^^^^^^^^ - -The built-in SQL data types use different Java types as carrier types. - -.. list-table:: SQL type to carrier type mapping - :widths: 45, 55 - :header-rows: 1 - - * - SQL type - - Java type - * - ``BOOLEAN`` - - ``boolean`` - * - ``TINYINT`` - - ``long`` - * - ``SMALLINT`` - - ``long`` - * - ``INTEGER`` - - ``long`` - * - ``BIGINT`` - - ``long`` - * - ``REAL`` - - ``double`` - * - ``DOUBLE`` - - ``double`` - * - ``DECIMAL`` - - ``long`` for precision up to 19, inclusive; - ``Int128`` for precision greater than 19 - * - ``VARCHAR`` - - ``Slice`` - * - ``CHAR`` - - ``Slice`` - * - ``VARBINARY`` - - ``Slice`` - * - ``JSON`` - - ``Slice`` - * - ``DATE`` - - ``long`` - * - ``TIME(P)`` - - ``long`` - * - ``TIME WITH TIME ZONE`` - - ``long`` for precision up to 9; - ``LongTimeWithTimeZone`` for precision greater than 9 - * - ``TIMESTAMP(P)`` - - ``long`` for precision up to 6; - ``LongTimestamp`` for precision greater than 6 - * - ``TIMESTAMP(P) WITH TIME ZONE`` - - ``long`` for precision up to 3; - ``LongTimestampWithTimeZone`` for precision greater than 3 - * - ``INTERVAL YEAR TO MONTH`` - - ``long`` - * - ``INTERVAL DAY TO SECOND`` - - ``long`` - * - ``ARRAY`` - - ``Block`` - * - ``MAP`` - - ``Block`` - * - ``ROW`` - - ``Block`` - * - ``IPADDRESS`` - - ``Slice`` - * - ``UUID`` - - ``Slice`` - * - ``HyperLogLog`` - - ``Slice`` - * - ``P4HyperLogLog`` - - ``Slice`` - * - ``SetDigest`` - - ``Slice`` - * - ``QDigest`` - - ``Slice`` - * - ``TDigest`` - - ``TDigest`` - -The ``RecordCursor.getType(int field)`` method returns the SQL type for a field -and the field value is returned by one of the following methods, matching -the carrier type: - -* ``getBoolean(int field)`` -* ``getLong(int field)`` -* ``getDouble(int field)`` -* ``getSlice(int field)`` -* ``getObject(int field)`` - -Values for the ``timestamp(p) with time zone`` and ``time(p) with time zone`` -types of regular precision can be converted into ``long`` using static methods -from the ``io.trino.spi.type.DateTimeEncoding`` class, like ``pack()`` or -``packDateTimeWithZone()``. - -UTF-8 encoded strings can be converted to Slices using -the ``Slices.utf8Slice()`` static method. - -.. note:: - - The ``Slice`` class is provided by the ``io.airlift:slice`` package. - -``Int128`` objects can be created using the ``Int128.valueOf()`` method. - -The following example creates a block for an ``array(varchar)`` column: - -.. code-block:: java - - private Block encodeArray(List names) - { - BlockBuilder builder = VARCHAR.createBlockBuilder(null, names.size()); - for (String name : names) { - if (name == null) { - builder.appendNull(); - } - else { - VARCHAR.writeString(builder, name); - } - } - return builder.build(); - } - -The following example creates a block for a ``map(varchar, varchar)`` column: - -.. code-block:: java - - private Block encodeMap(Map map) - { - MapType mapType = typeManager.getType(TypeSignature.mapType( - VARCHAR.getTypeSignature(), - VARCHAR.getTypeSignature())); - BlockBuilder values = mapType.createBlockBuilder(null, map != null ? map.size() : 0); - if (map == null) { - values.appendNull(); - return values.build().getObject(0, Block.class); - } - BlockBuilder builder = values.beginBlockEntry(); - for (Map.Entry entry : map.entrySet()) { - VARCHAR.writeString(builder, entry.getKey()); - Object value = entry.getValue(); - if (value == null) { - builder.appendNull(); - } - else { - VARCHAR.writeString(builder, value.toString()); - } - } - values.closeEntry(); - return values.build().getObject(0, Block.class); - } - -.. _connector-page-source-provider: - -ConnectorPageSourceProvider ---------------------------- - -Given a split, a table handle, and a list of columns, the page source provider -is responsible for delivering data to the Trino execution engine. It creates -a ``ConnectorPageSource``, which in turn creates ``Page`` objects that are used -by Trino to read the column values. - -If not implemented, a default ``RecordPageSourceProvider`` is used. -Given a record set provider, it returns an instance of ``RecordPageSource`` -that builds ``Page`` objects from records in a record set. - -A connector should implement a page source provider instead of a record set -provider when it's possible to create pages directly. The conversion of -individual records from a record set provider into pages adds overheads during -query execution. - -.. _connector-page-sink-provider: - -ConnectorPageSinkProvider -------------------------- - -Given an insert table handle, the page sink provider is responsible for -consuming data from the Trino execution engine. -It creates a ``ConnectorPageSink``, which in turn accepts ``Page`` objects -that contains the column values. - -Example that shows how to iterate over the page to access single values: - -.. code-block:: java - - @Override - public CompletableFuture appendPage(Page page) - { - for (int channel = 0; channel < page.getChannelCount(); channel++) { - Block block = page.getBlock(channel); - for (int position = 0; position < page.getPositionCount(); position++) { - if (block.isNull(position)) { - // or handle this differently - continue; - } - - // channel should match the column number in the table - // use it to determine the expected column type - String value = VARCHAR.getSlice(block, position).toStringUtf8(); - // TODO do something with the value - } - } - return NOT_BLOCKED; - } diff --git a/docs/src/main/sphinx/develop/event-listener.md b/docs/src/main/sphinx/develop/event-listener.md new file mode 100644 index 000000000000..72f22d644165 --- /dev/null +++ b/docs/src/main/sphinx/develop/event-listener.md @@ -0,0 +1,59 @@ +# Event listener + +Trino supports custom event listeners that are invoked for the following +events: + +- Query creation +- Query completion (success or failure) +- Split completion (success or failure) + +Event details include session, query execution, resource utilization, timeline, +and more. + +This functionality enables development of custom logging, debugging and +performance analysis plugins. + +## Implementation + +`EventListenerFactory` is responsible for creating an +`EventListener` instance. It also defines an `EventListener` +name which is used by the administrator in a Trino configuration. +Implementations of `EventListener` implement methods for the event types +they are interested in handling. + +The implementation of `EventListener` and `EventListenerFactory` +must be wrapped as a plugin and installed on the Trino cluster. + +## Configuration + +After a plugin that implements `EventListener` and +`EventListenerFactory` has been installed on the coordinator, it is +configured using an `etc/event-listener.properties` file. All of the +properties other than `event-listener.name` are specific to the +`EventListener` implementation. + +The `event-listener.name` property is used by Trino to find a registered +`EventListenerFactory` based on the name returned by +`EventListenerFactory.getName()`. The remaining properties are passed +as a map to `EventListenerFactory.create()`. + +Example configuration file: + +```text +event-listener.name=custom-event-listener +custom-property1=custom-value1 +custom-property2=custom-value2 +``` + +(multiple-listeners)= + +## Multiple event listeners + +Trino supports multiple instances of the same or different event listeners. +Install and configure multiple instances by setting +`event-listener.config-files` in {ref}`config-properties` to a comma-separated +list of the event listener configuration files: + +```text +event-listener.config-files=etc/event-listener.properties,etc/event-listener-second.properties +``` diff --git a/docs/src/main/sphinx/develop/event-listener.rst b/docs/src/main/sphinx/develop/event-listener.rst deleted file mode 100644 index 847eb13fadb7..000000000000 --- a/docs/src/main/sphinx/develop/event-listener.rst +++ /dev/null @@ -1,64 +0,0 @@ -============== -Event listener -============== - -Trino supports custom event listeners that are invoked for the following -events: - -* Query creation -* Query completion (success or failure) -* Split completion (success or failure) - -Event details include session, query execution, resource utilization, timeline, -and more. - -This functionality enables development of custom logging, debugging and -performance analysis plugins. - -Implementation --------------- - -``EventListenerFactory`` is responsible for creating an -``EventListener`` instance. It also defines an ``EventListener`` -name which is used by the administrator in a Trino configuration. -Implementations of ``EventListener`` implement methods for the event types -they are interested in handling. - -The implementation of ``EventListener`` and ``EventListenerFactory`` -must be wrapped as a plugin and installed on the Trino cluster. - -Configuration -------------- - -After a plugin that implements ``EventListener`` and -``EventListenerFactory`` has been installed on the coordinator, it is -configured using an ``etc/event-listener.properties`` file. All of the -properties other than ``event-listener.name`` are specific to the -``EventListener`` implementation. - -The ``event-listener.name`` property is used by Trino to find a registered -``EventListenerFactory`` based on the name returned by -``EventListenerFactory.getName()``. The remaining properties are passed -as a map to ``EventListenerFactory.create()``. - -Example configuration file: - -.. code-block:: text - - event-listener.name=custom-event-listener - custom-property1=custom-value1 - custom-property2=custom-value2 - -.. _multiple-listeners: - -Multiple event listeners ------------------------- - -Trino supports multiple instances of the same or different event listeners. -Install and configure multiple instances by setting -``event-listener.config-files`` in :ref:`config-properties` to a comma-separated -list of the event listener configuration files: - -.. code-block:: text - - event-listener.config-files=etc/event-listener.properties,etc/event-listener-second.properties diff --git a/docs/src/main/sphinx/develop/example-http.md b/docs/src/main/sphinx/develop/example-http.md new file mode 100644 index 000000000000..d68d5bb00a8d --- /dev/null +++ b/docs/src/main/sphinx/develop/example-http.md @@ -0,0 +1,102 @@ +# Example HTTP connector + +The Example HTTP connector has a simple goal: it reads comma-separated +data over HTTP. For example, if you have a large amount of data in a +CSV format, you can point the example HTTP connector at this data and +write a query to process it. + +## Code + +The Example HTTP connector can be found in the [trino-example-http](https://github.com/trinodb/trino/tree/master/plugin/trino-example-http) +directory within the Trino source tree. + +## Plugin implementation + +The plugin implementation in the Example HTTP connector looks very +similar to other plugin implementations. Most of the implementation is +devoted to handling optional configuration and the only function of +interest is the following: + +```java +@Override +public Iterable getConnectorFactories() +{ + return ImmutableList.of(new ExampleConnectorFactory()); +} +``` + +Note that the `ImmutableList` class is a utility class from Guava. + +As with all connectors, this plugin overrides the `getConnectorFactories()` method +and returns an `ExampleConnectorFactory`. + +## ConnectorFactory implementation + +In Trino, the primary object that handles the connection between +Trino and a particular type of data source is the `Connector` object, +which are created using `ConnectorFactory`. + +This implementation is available in the class `ExampleConnectorFactory`. +The first thing the connector factory implementation does is specify the +name of this connector. This is the same string used to reference this +connector in Trino configuration. + +```java +@Override +public String getName() +{ + return "example_http"; +} +``` + +The real work in a connector factory happens in the `create()` +method. In the `ExampleConnectorFactory` class, the `create()` method +configures the connector and then asks Guice to create the object. +This is the meat of the `create()` method without parameter validation +and exception handling: + +```java +// A plugin is not required to use Guice; it is just very convenient +Bootstrap app = new Bootstrap( + new JsonModule(), + new ExampleModule(catalogName)); + +Injector injector = app + .doNotInitializeLogging() + .setRequiredConfigurationProperties(requiredConfig) + .initialize(); + +return injector.getInstance(ExampleConnector.class); +``` + +### Connector: ExampleConnector + +This class allows Trino to obtain references to the various services +provided by the connector. + +### Metadata: ExampleMetadata + +This class is responsible for reporting table names, table metadata, +column names, column metadata and other information about the schemas +that are provided by this connector. `ConnectorMetadata` is also called +by Trino to ensure that a particular connector can understand and +handle a given table name. + +The `ExampleMetadata` implementation delegates many of these calls to +`ExampleClient`, a class that implements much of the core functionality +of the connector. + +### Split manager: ExampleSplitManager + +The split manager partitions the data for a table into the individual +chunks that Trino will distribute to workers for processing. +In the case of the Example HTTP connector, each table contains one or +more URIs pointing at the actual data. One split is created per URI. + +### Record set provider: ExampleRecordSetProvider + +The record set provider creates a record set which in turn creates a +record cursor that returns the actual data to Trino. +`ExampleRecordCursor` reads data from a URI via HTTP. Each line +corresponds to a single row. Lines are split on comma into individual +field values which are then returned to Trino. diff --git a/docs/src/main/sphinx/develop/example-http.rst b/docs/src/main/sphinx/develop/example-http.rst deleted file mode 100644 index 07cfa66eedf0..000000000000 --- a/docs/src/main/sphinx/develop/example-http.rst +++ /dev/null @@ -1,112 +0,0 @@ -====================== -Example HTTP connector -====================== - -The Example HTTP connector has a simple goal: it reads comma-separated -data over HTTP. For example, if you have a large amount of data in a -CSV format, you can point the example HTTP connector at this data and -write a query to process it. - -Code ----- - -The Example HTTP connector can be found in the `trino-example-http -`_ -directory within the Trino source tree. - -Plugin implementation ---------------------- - -The plugin implementation in the Example HTTP connector looks very -similar to other plugin implementations. Most of the implementation is -devoted to handling optional configuration and the only function of -interest is the following: - -.. code-block:: java - - @Override - public Iterable getConnectorFactories() - { - return ImmutableList.of(new ExampleConnectorFactory()); - } - -Note that the ``ImmutableList`` class is a utility class from Guava. - -As with all connectors, this plugin overrides the ``getConnectorFactories()`` method -and returns an ``ExampleConnectorFactory``. - -ConnectorFactory implementation -------------------------------- - -In Trino, the primary object that handles the connection between -Trino and a particular type of data source is the ``Connector`` object, -which are created using ``ConnectorFactory``. - -This implementation is available in the class ``ExampleConnectorFactory``. -The first thing the connector factory implementation does is specify the -name of this connector. This is the same string used to reference this -connector in Trino configuration. - -.. code-block:: java - - @Override - public String getName() - { - return "example_http"; - } - -The real work in a connector factory happens in the ``create()`` -method. In the ``ExampleConnectorFactory`` class, the ``create()`` method -configures the connector and then asks Guice to create the object. -This is the meat of the ``create()`` method without parameter validation -and exception handling: - -.. code-block:: java - - // A plugin is not required to use Guice; it is just very convenient - Bootstrap app = new Bootstrap( - new JsonModule(), - new ExampleModule(catalogName)); - - Injector injector = app - .doNotInitializeLogging() - .setRequiredConfigurationProperties(requiredConfig) - .initialize(); - - return injector.getInstance(ExampleConnector.class); - -Connector: ExampleConnector -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This class allows Trino to obtain references to the various services -provided by the connector. - -Metadata: ExampleMetadata -^^^^^^^^^^^^^^^^^^^^^^^^^ - -This class is responsible for reporting table names, table metadata, -column names, column metadata and other information about the schemas -that are provided by this connector. ``ConnectorMetadata`` is also called -by Trino to ensure that a particular connector can understand and -handle a given table name. - -The ``ExampleMetadata`` implementation delegates many of these calls to -``ExampleClient``, a class that implements much of the core functionality -of the connector. - -Split manager: ExampleSplitManager -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The split manager partitions the data for a table into the individual -chunks that Trino will distribute to workers for processing. -In the case of the Example HTTP connector, each table contains one or -more URIs pointing at the actual data. One split is created per URI. - -Record set provider: ExampleRecordSetProvider -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The record set provider creates a record set which in turn creates a -record cursor that returns the actual data to Trino. -``ExampleRecordCursor`` reads data from a URI via HTTP. Each line -corresponds to a single row. Lines are split on comma into individual -field values which are then returned to Trino. diff --git a/docs/src/main/sphinx/develop/example-jdbc.md b/docs/src/main/sphinx/develop/example-jdbc.md new file mode 100644 index 000000000000..a3b2314589c7 --- /dev/null +++ b/docs/src/main/sphinx/develop/example-jdbc.md @@ -0,0 +1,64 @@ +# Example JDBC connector + +The Example JDBC connector shows how to extend the base `JdbcPlugin` +to read data from a source using a JDBC driver, without having +to implement different Trino SPI services, like `ConnectorMetadata` +or `ConnectorRecordSetProvider`. + +:::{note} +This connector is just an example. It supports a very limited set of data +types and does not support any advanced functions, like predicacte or other +kind of pushdowns. +::: + +## Code + +The Example JDBC connector can be found in the [trino-example-jdbc](https://github.com/trinodb/trino/tree/master/plugin/trino-example-jdbc) +directory within the Trino source tree. + +## Plugin implementation + +The plugin implementation in the Example JDBC connector extends +the `JdbcPlugin` class and uses the `ExampleClientModule`. + +The module: + +- binds the `ExampleClient` class so it can be used by the base JDBC + connector; +- provides a connection factory that will create new connections using a JDBC + driver based on the JDBC URL specified in configuration properties. + +## JdbcClient implementation + +The base JDBC plugin maps the Trino SPI calls to the JDBC API. Operations like +reading table and columns names are well defined in JDBC so the base JDBC plugin +can implement it in a way that works for most JDBC drivers. + +One behavior that is not implemented by default is mapping of the data types +when reading and writing data. The Example JDBC connector implements +the `JdbcClient` interface in the `ExampleClient` class that extends +the `BaseJdbcClient` and implements two methods. + +### toColumnMapping + +`toColumnMapping` is used when reading data from the connector. +Given a `ConnectorSession`, `Connection` and a `JdbcTypeHandle`, +it returns a `ColumnMapping`, if there is a matching data type. + +The column mapping includes: + +- a Trino type, +- a write function, used to set query parameter values when preparing a + JDBC statement to execute in the data source, +- and a read function, used to read a value from the JDBC statement result set, + and return it using an internal Trino representation (for example, a Slice). + +### toWriteMapping + +`toWriteMapping` is used when writing data to the connector. Given a +`ConnectorSession` and a Trino type, it returns a `WriteMapping`. + +The mapping includes: + +- a data type name +- a write function diff --git a/docs/src/main/sphinx/develop/example-jdbc.rst b/docs/src/main/sphinx/develop/example-jdbc.rst deleted file mode 100644 index 777fea49074f..000000000000 --- a/docs/src/main/sphinx/develop/example-jdbc.rst +++ /dev/null @@ -1,72 +0,0 @@ -====================== -Example JDBC connector -====================== - -The Example JDBC connector shows how to extend the base ``JdbcPlugin`` -to read data from a source using a JDBC driver, without having -to implement different Trino SPI services, like ``ConnectorMetadata`` -or ``ConnectorRecordSetProvider``. - -.. note:: - - This connector is just an example. It supports a very limited set of data - types and does not support any advanced functions, like predicacte or other - kind of pushdowns. - -Code ----- - -The Example JDBC connector can be found in the `trino-example-jdbc -`_ -directory within the Trino source tree. - -Plugin implementation ---------------------- - -The plugin implementation in the Example JDBC connector extends -the ``JdbcPlugin`` class and uses the ``ExampleClientModule``. - -The module: - -* binds the ``ExampleClient`` class so it can be used by the base JDBC - connector; -* provides a connection factory that will create new connections using a JDBC - driver based on the JDBC URL specified in configuration properties. - -JdbcClient implementation -------------------------- - -The base JDBC plugin maps the Trino SPI calls to the JDBC API. Operations like -reading table and columns names are well defined in JDBC so the base JDBC plugin -can implement it in a way that works for most JDBC drivers. - -One behavior that is not implemented by default is mapping of the data types -when reading and writing data. The Example JDBC connector implements -the ``JdbcClient`` interface in the ``ExampleClient`` class that extends -the ``BaseJdbcClient`` and implements two methods. - -toColumnMapping -^^^^^^^^^^^^^^^ - -``toColumnMapping`` is used when reading data from the connector. -Given a ``ConnectorSession``, ``Connection`` and a ``JdbcTypeHandle``, -it returns a ``ColumnMapping``, if there is a matching data type. - -The column mapping includes: - -* a Trino type, -* a write function, used to set query parameter values when preparing a - JDBC statement to execute in the data source, -* and a read function, used to read a value from the JDBC statement result set, - and return it using an internal Trino representation (for example, a Slice). - -toWriteMapping -^^^^^^^^^^^^^^ - -``toWriteMapping`` is used when writing data to the connector. Given a -``ConnectorSession`` and a Trino type, it returns a ``WriteMapping``. - -The mapping includes: - -* a data type name -* a write function diff --git a/docs/src/main/sphinx/develop/functions.md b/docs/src/main/sphinx/develop/functions.md new file mode 100644 index 000000000000..26555000b6ed --- /dev/null +++ b/docs/src/main/sphinx/develop/functions.md @@ -0,0 +1,320 @@ +# Functions + +## Plugin implementation + +The function framework is used to implement SQL functions. Trino includes a +number of built-in functions. In order to implement new functions, you can +write a plugin that returns one or more functions from `getFunctions()`: + +```java +public class ExampleFunctionsPlugin + implements Plugin +{ + @Override + public Set> getFunctions() + { + return ImmutableSet.>builder() + .add(ExampleNullFunction.class) + .add(IsNullFunction.class) + .add(IsEqualOrNullFunction.class) + .add(ExampleStringFunction.class) + .add(ExampleAverageFunction.class) + .build(); + } +} +``` + +Note that the `ImmutableSet` class is a utility class from Guava. +The `getFunctions()` method contains all of the classes for the functions +that we will implement below in this tutorial. + +For a full example in the codebase, see either the `trino-ml` module for +machine learning functions or the `trino-teradata-functions` module for +Teradata-compatible functions, both in the `plugin` directory of the Trino +source. + +## Scalar function implementation + +The function framework uses annotations to indicate relevant information +about functions, including name, description, return type and parameter +types. Below is a sample function which implements `is_null`: + +```java +public class ExampleNullFunction +{ + @ScalarFunction("is_null", deterministic = true) + @Description("Returns TRUE if the argument is NULL") + @SqlType(StandardTypes.BOOLEAN) + public static boolean isNull( + @SqlNullable @SqlType(StandardTypes.VARCHAR) Slice string) + { + return (string == null); + } +} +``` + +The function `is_null` takes a single `VARCHAR` argument and returns a +`BOOLEAN` indicating if the argument was `NULL`. Note that the argument to +the function is of type `Slice`. `VARCHAR` uses `Slice`, which is essentially +a wrapper around `byte[]`, rather than `String` for its native container type. + +The `deterministic` argument indicates that a function has no side effects and, +for subsequent calls with the same argument(s), the function returns the exact +same value(s). + +In Trino, deterministic functions don't rely on any changing state +and don't modify any state. The `deterministic` flag is optional and defaults +to `true`. + +For example, the function {func}`shuffle` is non-deterministic, since it uses random +values. On the other hand, {func}`now` is deterministic, because subsequent calls in a +single query return the same timestamp. + +Any function with non-deterministic behavior is required to set `deterministic = false` +to avoid unexpected results. + +- `@SqlType`: + + The `@SqlType` annotation is used to declare the return type and the argument + types. Note that the return type and arguments of the Java code must match + the native container types of the corresponding annotations. + +- `@SqlNullable`: + + The `@SqlNullable` annotation indicates that the argument may be `NULL`. Without + this annotation the framework assumes that all functions return `NULL` if + any of their arguments are `NULL`. When working with a `Type` that has a + primitive native container type, such as `BigintType`, use the object wrapper for the + native container type when using `@SqlNullable`. The method must be annotated with + `@SqlNullable` if it can return `NULL` when the arguments are non-null. + +## Parametric scalar functions + +Scalar functions that have type parameters have some additional complexity. +To make our previous example work with any type we need the following: + +```java +@ScalarFunction(name = "is_null") +@Description("Returns TRUE if the argument is NULL") +public final class IsNullFunction +{ + @TypeParameter("T") + @SqlType(StandardTypes.BOOLEAN) + public static boolean isNullSlice(@SqlNullable @SqlType("T") Slice value) + { + return (value == null); + } + + @TypeParameter("T") + @SqlType(StandardTypes.BOOLEAN) + public static boolean isNullLong(@SqlNullable @SqlType("T") Long value) + { + return (value == null); + } + + @TypeParameter("T") + @SqlType(StandardTypes.BOOLEAN) + public static boolean isNullDouble(@SqlNullable @SqlType("T") Double value) + { + return (value == null); + } + + // ...and so on for each native container type +} +``` + +- `@TypeParameter`: + + The `@TypeParameter` annotation is used to declare a type parameter which can + be used in the argument types `@SqlType` annotation, or return type of the function. + It can also be used to annotate a parameter of type `Type`. At runtime, the engine + will bind the concrete type to this parameter. `@OperatorDependency` may be used + to declare that an additional function for operating on the given type parameter is needed. + For example, the following function will only bind to types which have an equals function + defined: + +```java +@ScalarFunction(name = "is_equal_or_null") +@Description("Returns TRUE if arguments are equal or both NULL") +public final class IsEqualOrNullFunction +{ + @TypeParameter("T") + @SqlType(StandardTypes.BOOLEAN) + public static boolean isEqualOrNullSlice( + @OperatorDependency( + operator = OperatorType.EQUAL, + returnType = StandardTypes.BOOLEAN, + argumentTypes = {"T", "T"}) MethodHandle equals, + @SqlNullable @SqlType("T") Slice value1, + @SqlNullable @SqlType("T") Slice value2) + { + if (value1 == null && value2 == null) { + return true; + } + if (value1 == null || value2 == null) { + return false; + } + return (boolean) equals.invokeExact(value1, value2); + } + + // ...and so on for each native container type +} +``` + +## Another scalar function example + +The `lowercaser` function takes a single `VARCHAR` argument and returns a +`VARCHAR`, which is the argument converted to lower case: + +```java +public class ExampleStringFunction +{ + @ScalarFunction("lowercaser") + @Description("Converts the string to alternating case") + @SqlType(StandardTypes.VARCHAR) + public static Slice lowercaser(@SqlType(StandardTypes.VARCHAR) Slice slice) + { + String argument = slice.toStringUtf8(); + return Slices.utf8Slice(argument.toLowerCase()); + } +} +``` + +Note that for most common string functions, including converting a string to +lower case, the Slice library also provides implementations that work directly +on the underlying `byte[]`, which have much better performance. This function +has no `@SqlNullable` annotations, meaning that if the argument is `NULL`, +the result will automatically be `NULL` (the function will not be called). + +## Aggregation function implementation + +Aggregation functions use a similar framework to scalar functions, but are +a bit more complex. + +- `AccumulatorState`: + + All aggregation functions accumulate input rows into a state object; this + object must implement `AccumulatorState`. For simple aggregations, just + extend `AccumulatorState` into a new interface with the getters and setters + you want, and the framework will generate all the implementations and + serializers for you. If you need a more complex state object, you will need + to implement `AccumulatorStateFactory` and `AccumulatorStateSerializer` + and provide these via the `AccumulatorStateMetadata` annotation. + +The following code implements the aggregation function `avg_double` which computes the +average of a `DOUBLE` column: + +```java +@AggregationFunction("avg_double") +public class AverageAggregation +{ + @InputFunction + public static void input( + LongAndDoubleState state, + @SqlType(StandardTypes.DOUBLE) double value) + { + state.setLong(state.getLong() + 1); + state.setDouble(state.getDouble() + value); + } + + @CombineFunction + public static void combine( + LongAndDoubleState state, + LongAndDoubleState otherState) + { + state.setLong(state.getLong() + otherState.getLong()); + state.setDouble(state.getDouble() + otherState.getDouble()); + } + + @OutputFunction(StandardTypes.DOUBLE) + public static void output(LongAndDoubleState state, BlockBuilder out) + { + long count = state.getLong(); + if (count == 0) { + out.appendNull(); + } + else { + double value = state.getDouble(); + DOUBLE.writeDouble(out, value / count); + } + } +} +``` + +The average has two parts: the sum of the `DOUBLE` in each row of the column +and the `LONG` count of the number of rows seen. `LongAndDoubleState` is an interface +which extends `AccumulatorState`: + +```java +public interface LongAndDoubleState + extends AccumulatorState +{ + long getLong(); + + void setLong(long value); + + double getDouble(); + + void setDouble(double value); +} +``` + +As stated above, for simple `AccumulatorState` objects, it is sufficient to +just to define the interface with the getters and setters, and the framework +will generate the implementation for you. + +An in-depth look at the various annotations relevant to writing an aggregation +function follows: + +- `@InputFunction`: + + The `@InputFunction` annotation declares the function which accepts input + rows and stores them in the `AccumulatorState`. Similar to scalar functions + you must annotate the arguments with `@SqlType`. Note that, unlike in the above + scalar example where `Slice` is used to hold `VARCHAR`, the primitive + `double` type is used for the argument to input. In this example, the input + function simply keeps track of the running count of rows (via `setLong()`) + and the running sum (via `setDouble()`). + +- `@CombineFunction`: + + The `@CombineFunction` annotation declares the function used to combine two + state objects. This function is used to merge all the partial aggregation states. + It takes two state objects, and merges the results into the first one (in the + above example, just by adding them together). + +- `@OutputFunction`: + + The `@OutputFunction` is the last function called when computing an + aggregation. It takes the final state object (the result of merging all + partial states) and writes the result to a `BlockBuilder`. + +- Where does serialization happen, and what is `GroupedAccumulatorState`? + + The `@InputFunction` is usually run on a different worker from the + `@CombineFunction`, so the state objects are serialized and transported + between these workers by the aggregation framework. `GroupedAccumulatorState` + is used when performing a `GROUP BY` aggregation, and an implementation + will be automatically generated for you, if you don't specify a + `AccumulatorStateFactory` + +## Deprecated function + +The `@Deprecated` annotation has to be used on any function that should no longer be +used. The annotation causes Trino to generate a warning whenever SQL statements +use a deprecated function. When a function is deprecated, the `@Description` +needs to be replaced with a note about the deprecation and the replacement function: + +```java +public class ExampleDeprecatedFunction +{ + @Deprecated + @ScalarFunction("bad_function") + @Description("(DEPRECATED) Use good_function() instead") + @SqlType(StandardTypes.BOOLEAN) + public static boolean bad_function() + { + return false; + } +} +``` diff --git a/docs/src/main/sphinx/develop/functions.rst b/docs/src/main/sphinx/develop/functions.rst deleted file mode 100644 index a814166ab23b..000000000000 --- a/docs/src/main/sphinx/develop/functions.rst +++ /dev/null @@ -1,329 +0,0 @@ -========= -Functions -========= - -Plugin implementation ---------------------- - -The function framework is used to implement SQL functions. Trino includes a -number of built-in functions. In order to implement new functions, you can -write a plugin that returns one or more functions from ``getFunctions()``: - -.. code-block:: java - - public class ExampleFunctionsPlugin - implements Plugin - { - @Override - public Set> getFunctions() - { - return ImmutableSet.>builder() - .add(ExampleNullFunction.class) - .add(IsNullFunction.class) - .add(IsEqualOrNullFunction.class) - .add(ExampleStringFunction.class) - .add(ExampleAverageFunction.class) - .build(); - } - } - -Note that the ``ImmutableSet`` class is a utility class from Guava. -The ``getFunctions()`` method contains all of the classes for the functions -that we will implement below in this tutorial. - -For a full example in the codebase, see either the ``trino-ml`` module for -machine learning functions or the ``trino-teradata-functions`` module for -Teradata-compatible functions, both in the ``plugin`` directory of the Trino -source. - -Scalar function implementation ------------------------------- - -The function framework uses annotations to indicate relevant information -about functions, including name, description, return type and parameter -types. Below is a sample function which implements ``is_null``: - -.. code-block:: java - - public class ExampleNullFunction - { - @ScalarFunction("is_null", deterministic = true) - @Description("Returns TRUE if the argument is NULL") - @SqlType(StandardTypes.BOOLEAN) - public static boolean isNull( - @SqlNullable @SqlType(StandardTypes.VARCHAR) Slice string) - { - return (string == null); - } - } - -The function ``is_null`` takes a single ``VARCHAR`` argument and returns a -``BOOLEAN`` indicating if the argument was ``NULL``. Note that the argument to -the function is of type ``Slice``. ``VARCHAR`` uses ``Slice``, which is essentially -a wrapper around ``byte[]``, rather than ``String`` for its native container type. - -The ``deterministic`` argument indicates that a function has no side effects and, -for subsequent calls with the same argument(s), the function returns the exact -same value(s). - -In Trino, deterministic functions don't rely on any changing state -and don't modify any state. The ``deterministic`` flag is optional and defaults -to ``true``. - -For example, the function :func:`shuffle` is non-deterministic, since it uses random -values. On the other hand, :func:`now` is deterministic, because subsequent calls in a -single query return the same timestamp. - -Any function with non-deterministic behavior is required to set ``deterministic = false`` -to avoid unexpected results. - -* ``@SqlType``: - - The ``@SqlType`` annotation is used to declare the return type and the argument - types. Note that the return type and arguments of the Java code must match - the native container types of the corresponding annotations. - -* ``@SqlNullable``: - - The ``@SqlNullable`` annotation indicates that the argument may be ``NULL``. Without - this annotation the framework assumes that all functions return ``NULL`` if - any of their arguments are ``NULL``. When working with a ``Type`` that has a - primitive native container type, such as ``BigintType``, use the object wrapper for the - native container type when using ``@SqlNullable``. The method must be annotated with - ``@SqlNullable`` if it can return ``NULL`` when the arguments are non-null. - -Parametric scalar functions ---------------------------- - -Scalar functions that have type parameters have some additional complexity. -To make our previous example work with any type we need the following: - -.. code-block:: java - - @ScalarFunction(name = "is_null") - @Description("Returns TRUE if the argument is NULL") - public final class IsNullFunction - { - @TypeParameter("T") - @SqlType(StandardTypes.BOOLEAN) - public static boolean isNullSlice(@SqlNullable @SqlType("T") Slice value) - { - return (value == null); - } - - @TypeParameter("T") - @SqlType(StandardTypes.BOOLEAN) - public static boolean isNullLong(@SqlNullable @SqlType("T") Long value) - { - return (value == null); - } - - @TypeParameter("T") - @SqlType(StandardTypes.BOOLEAN) - public static boolean isNullDouble(@SqlNullable @SqlType("T") Double value) - { - return (value == null); - } - - // ...and so on for each native container type - } - -* ``@TypeParameter``: - - The ``@TypeParameter`` annotation is used to declare a type parameter which can - be used in the argument types ``@SqlType`` annotation, or return type of the function. - It can also be used to annotate a parameter of type ``Type``. At runtime, the engine - will bind the concrete type to this parameter. ``@OperatorDependency`` may be used - to declare that an additional function for operating on the given type parameter is needed. - For example, the following function will only bind to types which have an equals function - defined: - -.. code-block:: java - - @ScalarFunction(name = "is_equal_or_null") - @Description("Returns TRUE if arguments are equal or both NULL") - public final class IsEqualOrNullFunction - { - @TypeParameter("T") - @SqlType(StandardTypes.BOOLEAN) - public static boolean isEqualOrNullSlice( - @OperatorDependency( - operator = OperatorType.EQUAL, - returnType = StandardTypes.BOOLEAN, - argumentTypes = {"T", "T"}) MethodHandle equals, - @SqlNullable @SqlType("T") Slice value1, - @SqlNullable @SqlType("T") Slice value2) - { - if (value1 == null && value2 == null) { - return true; - } - if (value1 == null || value2 == null) { - return false; - } - return (boolean) equals.invokeExact(value1, value2); - } - - // ...and so on for each native container type - } - -Another scalar function example -------------------------------- - -The ``lowercaser`` function takes a single ``VARCHAR`` argument and returns a -``VARCHAR``, which is the argument converted to lower case: - -.. code-block:: java - - public class ExampleStringFunction - { - @ScalarFunction("lowercaser") - @Description("Converts the string to alternating case") - @SqlType(StandardTypes.VARCHAR) - public static Slice lowercaser(@SqlType(StandardTypes.VARCHAR) Slice slice) - { - String argument = slice.toStringUtf8(); - return Slices.utf8Slice(argument.toLowerCase()); - } - } - -Note that for most common string functions, including converting a string to -lower case, the Slice library also provides implementations that work directly -on the underlying ``byte[]``, which have much better performance. This function -has no ``@SqlNullable`` annotations, meaning that if the argument is ``NULL``, -the result will automatically be ``NULL`` (the function will not be called). - -Aggregation function implementation ------------------------------------ - -Aggregation functions use a similar framework to scalar functions, but are -a bit more complex. - -* ``AccumulatorState``: - - All aggregation functions accumulate input rows into a state object; this - object must implement ``AccumulatorState``. For simple aggregations, just - extend ``AccumulatorState`` into a new interface with the getters and setters - you want, and the framework will generate all the implementations and - serializers for you. If you need a more complex state object, you will need - to implement ``AccumulatorStateFactory`` and ``AccumulatorStateSerializer`` - and provide these via the ``AccumulatorStateMetadata`` annotation. - -The following code implements the aggregation function ``avg_double`` which computes the -average of a ``DOUBLE`` column: - -.. code-block:: java - - @AggregationFunction("avg_double") - public class AverageAggregation - { - @InputFunction - public static void input( - LongAndDoubleState state, - @SqlType(StandardTypes.DOUBLE) double value) - { - state.setLong(state.getLong() + 1); - state.setDouble(state.getDouble() + value); - } - - @CombineFunction - public static void combine( - LongAndDoubleState state, - LongAndDoubleState otherState) - { - state.setLong(state.getLong() + otherState.getLong()); - state.setDouble(state.getDouble() + otherState.getDouble()); - } - - @OutputFunction(StandardTypes.DOUBLE) - public static void output(LongAndDoubleState state, BlockBuilder out) - { - long count = state.getLong(); - if (count == 0) { - out.appendNull(); - } - else { - double value = state.getDouble(); - DOUBLE.writeDouble(out, value / count); - } - } - } - - -The average has two parts: the sum of the ``DOUBLE`` in each row of the column -and the ``LONG`` count of the number of rows seen. ``LongAndDoubleState`` is an interface -which extends ``AccumulatorState``: - -.. code-block:: java - - public interface LongAndDoubleState - extends AccumulatorState - { - long getLong(); - - void setLong(long value); - - double getDouble(); - - void setDouble(double value); - } - -As stated above, for simple ``AccumulatorState`` objects, it is sufficient to -just to define the interface with the getters and setters, and the framework -will generate the implementation for you. - -An in-depth look at the various annotations relevant to writing an aggregation -function follows: - -* ``@InputFunction``: - - The ``@InputFunction`` annotation declares the function which accepts input - rows and stores them in the ``AccumulatorState``. Similar to scalar functions - you must annotate the arguments with ``@SqlType``. Note that, unlike in the above - scalar example where ``Slice`` is used to hold ``VARCHAR``, the primitive - ``double`` type is used for the argument to input. In this example, the input - function simply keeps track of the running count of rows (via ``setLong()``) - and the running sum (via ``setDouble()``). - -* ``@CombineFunction``: - - The ``@CombineFunction`` annotation declares the function used to combine two - state objects. This function is used to merge all the partial aggregation states. - It takes two state objects, and merges the results into the first one (in the - above example, just by adding them together). - -* ``@OutputFunction``: - - The ``@OutputFunction`` is the last function called when computing an - aggregation. It takes the final state object (the result of merging all - partial states) and writes the result to a ``BlockBuilder``. - -* Where does serialization happen, and what is ``GroupedAccumulatorState``? - - The ``@InputFunction`` is usually run on a different worker from the - ``@CombineFunction``, so the state objects are serialized and transported - between these workers by the aggregation framework. ``GroupedAccumulatorState`` - is used when performing a ``GROUP BY`` aggregation, and an implementation - will be automatically generated for you, if you don't specify a - ``AccumulatorStateFactory`` - -Deprecated function -------------------- - -The ``@Deprecated`` annotation has to be used on any function that should no longer be -used. The annotation causes Trino to generate a warning whenever SQL statements -use a deprecated function. When a function is deprecated, the ``@Description`` -needs to be replaced with a note about the deprecation and the replacement function: - -.. code-block:: java - - public class ExampleDeprecatedFunction - { - @Deprecated - @ScalarFunction("bad_function") - @Description("(DEPRECATED) Use good_function() instead") - @SqlType(StandardTypes.BOOLEAN) - public static boolean bad_function() - { - return false; - } - } diff --git a/docs/src/main/sphinx/develop/group-provider.md b/docs/src/main/sphinx/develop/group-provider.md new file mode 100644 index 000000000000..63915ce803fa --- /dev/null +++ b/docs/src/main/sphinx/develop/group-provider.md @@ -0,0 +1,40 @@ +# Group provider + +Trino can map user names onto groups for easier access control management. +This mapping is performed by a `GroupProvider` implementation. + +## Implementation + +`GroupProviderFactory` is responsible for creating a `GroupProvider` instance. +It also defines the name of the group provider as used in the configuration file. + +`GroupProvider` contains a one method, `getGroups(String user)` +which returns a `Set` of group names. +This set of group names becomes part of the `Identity` and `ConnectorIdentity` +objects representing the user, and can then be used by {doc}`system-access-control`. + +The implementation of `GroupProvider` and its corresponding `GroupProviderFactory` +must be wrapped as a Trino plugin and installed on the cluster. + +## Configuration + +After a plugin that implements `GroupProviderFactory` has been installed on the coordinator, +it is configured using an `etc/group-provider.properties` file. +All of the properties other than `group-provider.name` are specific to +the `GroupProviderFactory` implementation. + +The `group-provider.name` property is used by Trino to find a registered +`GroupProviderFactory` based on the name returned by `GroupProviderFactory.getName()`. +The remaining properties are passed as a map to +`GroupProviderFactory.create(Map)`. + +Example configuration file: + +```text +group-provider.name=custom-group-provider +custom-property1=custom-value1 +custom-property2=custom-value2 +``` + +With that file in place, Trino will attempt user group name resolution, +and will be able to use the group names while evaluating access control rules. diff --git a/docs/src/main/sphinx/develop/group-provider.rst b/docs/src/main/sphinx/develop/group-provider.rst deleted file mode 100644 index a183f996ec27..000000000000 --- a/docs/src/main/sphinx/develop/group-provider.rst +++ /dev/null @@ -1,44 +0,0 @@ -============== -Group provider -============== - -Trino can map user names onto groups for easier access control management. -This mapping is performed by a ``GroupProvider`` implementation. - -Implementation --------------- - -``GroupProviderFactory`` is responsible for creating a ``GroupProvider`` instance. -It also defines the name of the group provider as used in the configuration file. - -``GroupProvider`` contains a one method, ``getGroups(String user)`` -which returns a ``Set`` of group names. -This set of group names becomes part of the ``Identity`` and ``ConnectorIdentity`` -objects representing the user, and can then be used by :doc:`system-access-control`. - -The implementation of ``GroupProvider`` and its corresponding ``GroupProviderFactory`` -must be wrapped as a Trino plugin and installed on the cluster. - -Configuration -------------- - -After a plugin that implements ``GroupProviderFactory`` has been installed on the coordinator, -it is configured using an ``etc/group-provider.properties`` file. -All of the properties other than ``group-provider.name`` are specific to -the ``GroupProviderFactory`` implementation. - -The ``group-provider.name`` property is used by Trino to find a registered -``GroupProviderFactory`` based on the name returned by ``GroupProviderFactory.getName()``. -The remaining properties are passed as a map to -``GroupProviderFactory.create(Map)``. - -Example configuration file: - -.. code-block:: text - - group-provider.name=custom-group-provider - custom-property1=custom-value1 - custom-property2=custom-value2 - -With that file in place, Trino will attempt user group name resolution, -and will be able to use the group names while evaluating access control rules. diff --git a/docs/src/main/sphinx/develop/header-authenticator.md b/docs/src/main/sphinx/develop/header-authenticator.md new file mode 100644 index 000000000000..ec8777222dd2 --- /dev/null +++ b/docs/src/main/sphinx/develop/header-authenticator.md @@ -0,0 +1,42 @@ +# Header authenticator + +Trino supports header authentication over TLS via a custom header authenticator +that extracts the principal from a predefined header(s), performs any validation it needs and creates +an authenticated principal. + +## Implementation + +`HeaderAuthenticatorFactory` is responsible for creating a +`HeaderAuthenticator` instance. It also defines the name of this +authenticator which is used by the administrator in a Trino configuration. + +`HeaderAuthenticator` contains a single method, `createAuthenticatedPrincipal()`, +which validates the request headers wrapped by the Headers interface; has the method getHeader(String name) +and returns a `Principal`, which is then authorized by the {doc}`system-access-control`. + +The implementation of `HeaderAuthenticatorFactory` must be wrapped +as a plugin and installed on the Trino cluster. + +## Configuration + +After a plugin that implements `HeaderAuthenticatorFactory` has been +installed on the coordinator, it is configured using an +`etc/header-authenticator.properties` file. All of the +properties other than `header-authenticator.name` are specific to the +`HeaderAuthenticatorFactory` implementation. + +The `header-authenticator.name` property is used by Trino to find a +registered `HeaderAuthenticatorFactory` based on the name returned by +`HeaderAuthenticatorFactory.getName()`. The remaining properties are +passed as a map to `HeaderAuthenticatorFactory.create()`. + +Example configuration file: + +```none +header-authenticator.name=custom +custom-property1=custom-value1 +custom-property2=custom-value2 +``` + +Additionally, the coordinator must be configured to use header authentication +and have HTTPS enabled (or HTTPS forwarding enabled). diff --git a/docs/src/main/sphinx/develop/header-authenticator.rst b/docs/src/main/sphinx/develop/header-authenticator.rst deleted file mode 100644 index ab915d108aed..000000000000 --- a/docs/src/main/sphinx/develop/header-authenticator.rst +++ /dev/null @@ -1,46 +0,0 @@ -==================== -Header authenticator -==================== - -Trino supports header authentication over TLS via a custom header authenticator -that extracts the principal from a predefined header(s), performs any validation it needs and creates -an authenticated principal. - -Implementation --------------- - -``HeaderAuthenticatorFactory`` is responsible for creating a -``HeaderAuthenticator`` instance. It also defines the name of this -authenticator which is used by the administrator in a Trino configuration. - -``HeaderAuthenticator`` contains a single method, ``createAuthenticatedPrincipal()``, -which validates the request headers wrapped by the Headers interface; has the method getHeader(String name) -and returns a ``Principal``, which is then authorized by the :doc:`system-access-control`. - -The implementation of ``HeaderAuthenticatorFactory`` must be wrapped -as a plugin and installed on the Trino cluster. - -Configuration -------------- - -After a plugin that implements ``HeaderAuthenticatorFactory`` has been -installed on the coordinator, it is configured using an -``etc/header-authenticator.properties`` file. All of the -properties other than ``header-authenticator.name`` are specific to the -``HeaderAuthenticatorFactory`` implementation. - -The ``header-authenticator.name`` property is used by Trino to find a -registered ``HeaderAuthenticatorFactory`` based on the name returned by -``HeaderAuthenticatorFactory.getName()``. The remaining properties are -passed as a map to ``HeaderAuthenticatorFactory.create()``. - -Example configuration file: - -.. code-block:: none - - header-authenticator.name=custom - custom-property1=custom-value1 - custom-property2=custom-value2 - -Additionally, the coordinator must be configured to use header authentication -and have HTTPS enabled (or HTTPS forwarding enabled). diff --git a/docs/src/main/sphinx/develop/insert.md b/docs/src/main/sphinx/develop/insert.md new file mode 100644 index 000000000000..cab389ee48a3 --- /dev/null +++ b/docs/src/main/sphinx/develop/insert.md @@ -0,0 +1,28 @@ +# Supporting `INSERT` and `CREATE TABLE AS` + +To support `INSERT`, a connector must implement: + +- `beginInsert()` and `finishInsert()` from the `ConnectorMetadata` + interface; +- a `ConnectorPageSinkProvider` that receives a table handle and returns + a `ConnectorPageSink`. + +When executing an `INSERT` statement, the engine calls the `beginInsert()` +method in the connector, which receives a table handle and a list of columns. +It should return a `ConnectorInsertTableHandle`, that can carry any +connector specific information, and it's passed to the page sink provider. +The `PageSinkProvider` creates a page sink, that accepts `Page` objects. + +When all the pages for a specific split have been processed, Trino calls +`ConnectorPageSink.finish()`, which returns a `Collection` +of fragments representing connector-specific information about the processed +rows. + +When all pages for all splits have been processed, Trino calls +`ConnectorMetadata.finishInsert()`, passing a collection containing all +the fragments from all the splits. The connector does what is required +to finalize the operation, for example, committing the transaction. + +To support `CREATE TABLE AS`, the `ConnectorPageSinkProvider` must also +return a page sink when receiving a `ConnectorOutputTableHandle`. This handle +is returned from `ConnectorMetadata.beginCreateTable()`. diff --git a/docs/src/main/sphinx/develop/insert.rst b/docs/src/main/sphinx/develop/insert.rst deleted file mode 100644 index 87403ae769d7..000000000000 --- a/docs/src/main/sphinx/develop/insert.rst +++ /dev/null @@ -1,30 +0,0 @@ -============================================= -Supporting ``INSERT`` and ``CREATE TABLE AS`` -============================================= - -To support ``INSERT``, a connector must implement: - -* ``beginInsert()`` and ``finishInsert()`` from the ``ConnectorMetadata`` - interface; -* a ``ConnectorPageSinkProvider`` that receives a table handle and returns - a ``ConnectorPageSink``. - -When executing an ``INSERT`` statement, the engine calls the ``beginInsert()`` -method in the connector, which receives a table handle and a list of columns. -It should return a ``ConnectorInsertTableHandle``, that can carry any -connector specific information, and it's passed to the page sink provider. -The ``PageSinkProvider`` creates a page sink, that accepts ``Page`` objects. - -When all the pages for a specific split have been processed, Trino calls -``ConnectorPageSink.finish()``, which returns a ``Collection`` -of fragments representing connector-specific information about the processed -rows. - -When all pages for all splits have been processed, Trino calls -``ConnectorMetadata.finishInsert()``, passing a collection containing all -the fragments from all the splits. The connector does what is required -to finalize the operation, for example, committing the transaction. - -To support ``CREATE TABLE AS``, the ``ConnectorPageSinkProvider`` must also -return a page sink when receiving a ``ConnectorOutputTableHandle``. This handle -is returned from ``ConnectorMetadata.beginCreateTable()``. diff --git a/docs/src/main/sphinx/develop/password-authenticator.md b/docs/src/main/sphinx/develop/password-authenticator.md new file mode 100644 index 000000000000..8095e39abbff --- /dev/null +++ b/docs/src/main/sphinx/develop/password-authenticator.md @@ -0,0 +1,41 @@ +# Password authenticator + +Trino supports authentication with a username and password via a custom +password authenticator that validates the credentials and creates a principal. + +## Implementation + +`PasswordAuthenticatorFactory` is responsible for creating a +`PasswordAuthenticator` instance. It also defines the name of this +authenticator which is used by the administrator in a Trino configuration. + +`PasswordAuthenticator` contains a single method, `createAuthenticatedPrincipal()`, +that validates the credential and returns a `Principal`, which is then +authorized by the {doc}`system-access-control`. + +The implementation of `PasswordAuthenticatorFactory` must be wrapped +as a plugin and installed on the Trino cluster. + +## Configuration + +After a plugin that implements `PasswordAuthenticatorFactory` has been +installed on the coordinator, it is configured using an +`etc/password-authenticator.properties` file. All of the +properties other than `password-authenticator.name` are specific to the +`PasswordAuthenticatorFactory` implementation. + +The `password-authenticator.name` property is used by Trino to find a +registered `PasswordAuthenticatorFactory` based on the name returned by +`PasswordAuthenticatorFactory.getName()`. The remaining properties are +passed as a map to `PasswordAuthenticatorFactory.create()`. + +Example configuration file: + +```text +password-authenticator.name=custom-access-control +custom-property1=custom-value1 +custom-property2=custom-value2 +``` + +Additionally, the coordinator must be configured to use password authentication +and have HTTPS enabled (or HTTPS forwarding enabled). diff --git a/docs/src/main/sphinx/develop/password-authenticator.rst b/docs/src/main/sphinx/develop/password-authenticator.rst deleted file mode 100644 index b7b21255e06e..000000000000 --- a/docs/src/main/sphinx/develop/password-authenticator.rst +++ /dev/null @@ -1,45 +0,0 @@ -====================== -Password authenticator -====================== - -Trino supports authentication with a username and password via a custom -password authenticator that validates the credentials and creates a principal. - -Implementation --------------- - -``PasswordAuthenticatorFactory`` is responsible for creating a -``PasswordAuthenticator`` instance. It also defines the name of this -authenticator which is used by the administrator in a Trino configuration. - -``PasswordAuthenticator`` contains a single method, ``createAuthenticatedPrincipal()``, -that validates the credential and returns a ``Principal``, which is then -authorized by the :doc:`system-access-control`. - -The implementation of ``PasswordAuthenticatorFactory`` must be wrapped -as a plugin and installed on the Trino cluster. - -Configuration -------------- - -After a plugin that implements ``PasswordAuthenticatorFactory`` has been -installed on the coordinator, it is configured using an -``etc/password-authenticator.properties`` file. All of the -properties other than ``password-authenticator.name`` are specific to the -``PasswordAuthenticatorFactory`` implementation. - -The ``password-authenticator.name`` property is used by Trino to find a -registered ``PasswordAuthenticatorFactory`` based on the name returned by -``PasswordAuthenticatorFactory.getName()``. The remaining properties are -passed as a map to ``PasswordAuthenticatorFactory.create()``. - -Example configuration file: - -.. code-block:: text - - password-authenticator.name=custom-access-control - custom-property1=custom-value1 - custom-property2=custom-value2 - -Additionally, the coordinator must be configured to use password authentication -and have HTTPS enabled (or HTTPS forwarding enabled). diff --git a/docs/src/main/sphinx/develop/spi-overview.md b/docs/src/main/sphinx/develop/spi-overview.md new file mode 100644 index 000000000000..872619f2dda1 --- /dev/null +++ b/docs/src/main/sphinx/develop/spi-overview.md @@ -0,0 +1,113 @@ +# SPI overview + +When you implement a new Trino plugin, you implement interfaces and +override methods defined by the Service Provider Interface (SPI). + +Plugins can provide additional: + +- {doc}`connectors`, +- block encodings, +- {doc}`types`, +- {doc}`functions`, +- {doc}`system-access-control`, +- {doc}`group-provider`, +- {doc}`password-authenticator`, +- {doc}`header-authenticator`, +- {doc}`certificate-authenticator`, +- {doc}`event-listener`, +- resource group configuration managers, +- session property configuration managers, +- and exchange managers. + +In particular, connectors are the source of all data for queries in +Trino: they back each catalog available to Trino. + +## Code + +The SPI source can be found in the `core/trino-spi` directory in the Trino +source tree. + +## Plugin metadata + +Each plugin identifies an entry point: an implementation of the +`Plugin` interface. This class name is provided to Trino via +the standard Java `ServiceLoader` interface: the classpath contains +a resource file named `io.trino.spi.Plugin` in the +`META-INF/services` directory. The content of this file is a +single line listing the name of the plugin class: + +```text +com.example.plugin.ExamplePlugin +``` + +For a built-in plugin that is included in the Trino source code, +this resource file is created whenever the `pom.xml` file of a plugin +contains the following line: + +```xml +trino-plugin +``` + +## Plugin + +The `Plugin` interface is a good starting place for developers looking +to understand the Trino SPI. It contains access methods to retrieve +various classes that a Plugin can provide. For example, the `getConnectorFactories()` +method is a top-level function that Trino calls to retrieve a `ConnectorFactory` when Trino +is ready to create an instance of a connector to back a catalog. There are similar +methods for `Type`, `ParametricType`, `Function`, `SystemAccessControl`, and +`EventListenerFactory` objects. + +## Building plugins via Maven + +Plugins depend on the SPI from Trino: + +```xml + + io.trino + trino-spi + provided + +``` + +The plugin uses the Maven `provided` scope because Trino provides +the classes from the SPI at runtime and thus the plugin should not +include them in the plugin assembly. + +There are a few other dependencies that are provided by Trino, +including Slice and Jackson annotations. In particular, Jackson is +used for serializing connector handles and thus plugins must use the +annotations version provided by Trino. + +All other dependencies are based on what the plugin needs for its +own implementation. Plugins are loaded in a separate class loader +to provide isolation and to allow plugins to use a different version +of a library that Trino uses internally. + +For an example `pom.xml` file, see the example HTTP connector in the +`plugin/trino-example-http` directory in the Trino source tree. + +## Deploying a custom plugin + +Because Trino plugins use the `trino-plugin` packaging type, building +a plugin will create a ZIP file in the `target` directory. This file +contains the plugin JAR and all its dependencies JAR files. + +In order to add a custom plugin to a Trino installation, extract the plugin +ZIP file and move the extracted directory into the Trino plugin directory. +For example, for a plugin called `my-functions`, with a version of 1.0, +you would extract `my-functions-1.0.zip` and then move `my-functions-1.0` +to `my-functions` in the Trino plugin directory. + +:::{note} +Every Trino plugin should be in a separate directory. Do not put JAR files +directly into the `plugin` directory. Plugins should only contain JAR files, +so any subdirectories will not be traversed and will be ignored. +::: + +By default, the plugin directory is the `plugin` directory relative to the +directory in which Trino is installed, but it is configurable using the +configuration variable `plugin.dir`. In order for Trino to pick up +the new plugin, you must restart Trino. + +Plugins must be installed on all nodes in the Trino cluster (coordinator and workers). diff --git a/docs/src/main/sphinx/develop/spi-overview.rst b/docs/src/main/sphinx/develop/spi-overview.rst deleted file mode 100644 index d2646e260be1..000000000000 --- a/docs/src/main/sphinx/develop/spi-overview.rst +++ /dev/null @@ -1,120 +0,0 @@ -============ -SPI overview -============ - -When you implement a new Trino plugin, you implement interfaces and -override methods defined by the Service Provider Interface (SPI). - -Plugins can provide additional: - -* :doc:`connectors`, -* block encodings, -* :doc:`types`, -* :doc:`functions`, -* :doc:`system-access-control`, -* :doc:`group-provider`, -* :doc:`password-authenticator`, -* :doc:`header-authenticator`, -* :doc:`certificate-authenticator`, -* :doc:`event-listener`, -* resource group configuration managers, -* session property configuration managers, -* and exchange managers. - -In particular, connectors are the source of all data for queries in -Trino: they back each catalog available to Trino. - -Code ----- - -The SPI source can be found in the ``core/trino-spi`` directory in the Trino -source tree. - -Plugin metadata ---------------- - -Each plugin identifies an entry point: an implementation of the -``Plugin`` interface. This class name is provided to Trino via -the standard Java ``ServiceLoader`` interface: the classpath contains -a resource file named ``io.trino.spi.Plugin`` in the -``META-INF/services`` directory. The content of this file is a -single line listing the name of the plugin class: - -.. code-block:: text - - com.example.plugin.ExamplePlugin - -For a built-in plugin that is included in the Trino source code, -this resource file is created whenever the ``pom.xml`` file of a plugin -contains the following line: - -.. code-block:: xml - - trino-plugin - -Plugin ------- - -The ``Plugin`` interface is a good starting place for developers looking -to understand the Trino SPI. It contains access methods to retrieve -various classes that a Plugin can provide. For example, the ``getConnectorFactories()`` -method is a top-level function that Trino calls to retrieve a ``ConnectorFactory`` when Trino -is ready to create an instance of a connector to back a catalog. There are similar -methods for ``Type``, ``ParametricType``, ``Function``, ``SystemAccessControl``, and -``EventListenerFactory`` objects. - -Building plugins via Maven --------------------------- - -Plugins depend on the SPI from Trino: - -.. code-block:: xml - - - io.trino - trino-spi - provided - - -The plugin uses the Maven ``provided`` scope because Trino provides -the classes from the SPI at runtime and thus the plugin should not -include them in the plugin assembly. - -There are a few other dependencies that are provided by Trino, -including Slice and Jackson annotations. In particular, Jackson is -used for serializing connector handles and thus plugins must use the -annotations version provided by Trino. - -All other dependencies are based on what the plugin needs for its -own implementation. Plugins are loaded in a separate class loader -to provide isolation and to allow plugins to use a different version -of a library that Trino uses internally. - -For an example ``pom.xml`` file, see the example HTTP connector in the -``plugin/trino-example-http`` directory in the Trino source tree. - -Deploying a custom plugin -------------------------- - -Because Trino plugins use the ``trino-plugin`` packaging type, building -a plugin will create a ZIP file in the ``target`` directory. This file -contains the plugin JAR and all its dependencies JAR files. - -In order to add a custom plugin to a Trino installation, extract the plugin -ZIP file and move the extracted directory into the Trino plugin directory. -For example, for a plugin called ``my-functions``, with a version of 1.0, -you would extract ``my-functions-1.0.zip`` and then move ``my-functions-1.0`` -to ``my-functions`` in the Trino plugin directory. - -.. note:: - - Every Trino plugin should be in a separate directory. Do not put JAR files - directly into the ``plugin`` directory. Plugins should only contain JAR files, - so any subdirectories will not be traversed and will be ignored. - -By default, the plugin directory is the ``plugin`` directory relative to the -directory in which Trino is installed, but it is configurable using the -configuration variable ``plugin.dir``. In order for Trino to pick up -the new plugin, you must restart Trino. - -Plugins must be installed on all nodes in the Trino cluster (coordinator and workers). diff --git a/docs/src/main/sphinx/develop/supporting-merge.md b/docs/src/main/sphinx/develop/supporting-merge.md new file mode 100644 index 000000000000..b33d2b5e1f4c --- /dev/null +++ b/docs/src/main/sphinx/develop/supporting-merge.md @@ -0,0 +1,431 @@ +# Supporting `MERGE` + +The Trino engine provides APIs to support row-level SQL `MERGE`. +To implement `MERGE`, a connector must provide the following: + +- An implementation of `ConnectorMergeSink`, which is typically + layered on top of a `ConnectorPageSink`. +- Methods in `ConnectorMetadata` to get a "rowId" column handle, get the + row change paradigm, and to start and complete the `MERGE` operation. + +The Trino engine machinery used to implement SQL `MERGE` is also used to +support SQL `DELETE` and `UPDATE`. This means that all a connector needs to +do is implement support for SQL `MERGE`, and the connector gets all the Data +Modification Language (DML) operations. + +## Standard SQL `MERGE` + +Different query engines support varying definitions of SQL `MERGE`. +Trino supports the strict SQL specification `ISO/IEC 9075`, published +in 2016. As a simple example, given tables `target_table` and +`source_table` defined as: + +``` +CREATE TABLE accounts ( + customer VARCHAR, + purchases DECIMAL, + address VARCHAR); +INSERT INTO accounts (customer, purchases, address) VALUES ...; +CREATE TABLE monthly_accounts_update ( + customer VARCHAR, + purchases DECIMAL, + address VARCHAR); +INSERT INTO monthly_accounts_update (customer, purchases, address) VALUES ...; +``` + +Here is a possible `MERGE` operation, from `monthly_accounts_update` to +`accounts`: + +``` +MERGE INTO accounts t USING monthly_accounts_update s + ON (t.customer = s.customer) + WHEN MATCHED AND s.address = 'Berkeley' THEN + DELETE + WHEN MATCHED AND s.customer = 'Joe Shmoe' THEN + UPDATE SET purchases = purchases + 100.0 + WHEN MATCHED THEN + UPDATE + SET purchases = s.purchases + t.purchases, address = s.address + WHEN NOT MATCHED THEN + INSERT (customer, purchases, address) + VALUES (s.customer, s.purchases, s.address); +``` + +SQL `MERGE` tries to match each `WHEN` clause in source order. When +a match is found, the corresponding `DELETE`, `INSERT` or `UPDATE` +is executed and subsequent `WHEN` clauses are ignored. + +SQL `MERGE` supports two operations on the target table and source +when a row from the source table or query matches a row in the target table: + +- `UPDATE`, in which the columns in the target row are updated. +- `DELETE`, in which the target row is deleted. + +In the `NOT MATCHED` case, SQL `MERGE` supports only `INSERT` +operations. The values inserted are arbitrary but usually come from +the unmatched row of the source table or query. + +## `RowChangeParadigm` + +Different connectors have different ways of representing row updates, +imposed by the underlying storage systems. The Trino engine classifies +these different paradigms as elements of the `RowChangeParadigm` +enumeration, returned by enumeration, returned by method +`ConnectorMetadata.getRowChangeParadigm(...)`. + +The `RowChangeParadigm` enumeration values are: + +- `CHANGE_ONLY_UPDATED_COLUMNS`, intended for connectors that can update + individual columns of rows identified by a `rowId`. The corresponding + merge processor class is `ChangeOnlyUpdatedColumnsMergeProcessor`. +- `DELETE_ROW_AND_INSERT_ROW`, intended for connectors that represent a + row change as a row deletion paired with a row insertion. The corresponding + merge processor class is `DeleteAndInsertMergeProcessor`. + +## Overview of `MERGE` processing + +A `MERGE` statement is processed by creating a `RIGHT JOIN` between the +target table and the source, on the `MERGE` criteria. The source may be +a table or an arbitrary query. For each row in the source table or query, +`MERGE` produces a `ROW` object containing: + +- the data column values from the `UPDATE` or `INSERT` cases. For the + `DELETE` cases, only the partition columns, which determine + partitioning and bucketing, are non-null. +- a boolean column containing `true` for source rows that matched some + target row, and `false` otherwise. +- an integer that identifies whether the merge case operation is `UPDATE`, + `DELETE` or `INSERT`, or a source row for which no case matched. If a + source row doesn't match any merge case, all data column values except + those that determine distribution are null, and the operation number + is -1. + +A `SearchedCaseExpression` is constructed from `RIGHT JOIN` result +to represent the `WHEN` clauses of the `MERGE`. In the example preceding +the `MERGE` is executed as if the `SearchedCaseExpression` were written as: + +``` +SELECT + CASE + WHEN present AND s.address = 'Berkeley' THEN + -- Null values for delete; present=true; operation DELETE=2, case_number=0 + row(null, null, null, true, 2, 0) + WHEN present AND s.customer = 'Joe Shmoe' THEN + -- Update column values; present=true; operation UPDATE=3, case_number=1 + row(t.customer, t.purchases + 100.0, t.address, true, 3, 1) + WHEN present THEN + -- Update column values; present=true; operation UPDATE=3, case_number=2 + row(t.customer, s.purchases + t.purchases, s.address, true, 3, 2) + WHEN (present IS NULL) THEN + -- Insert column values; present=false; operation INSERT=1, case_number=3 + row(s.customer, s.purchases, s.address, false, 1, 3) + ELSE + -- Null values for no case matched; present=false; operation=-1, + -- case_number=-1 + row(null, null, null, false, -1, -1) + END + FROM (SELECT *, true AS present FROM target_table) t + RIGHT JOIN source_table s ON s.customer = t.customer; +``` + +The Trino engine executes the `RIGHT JOIN` and `CASE` expression, +and ensures that no target table row matches more than one source expression +row, and ultimately creates a sequence of pages to be routed to the node that +runs the `ConnectorMergeSink.storeMergedRows(...)` method. + +Like `DELETE` and `UPDATE`, `MERGE` target table rows are identified by +a connector-specific `rowId` column handle. For `MERGE`, the `rowId` +handle is returned by `ConnectorMetadata.getMergeRowIdColumnHandle(...)`. + +## `MERGE` redistribution + +The Trino `MERGE` implementation allows `UPDATE` to change +the values of columns that determine partitioning and/or bucketing, and so +it must "redistribute" rows from the `MERGE` operation to the worker +nodes responsible for writing rows with the merged partitioning and/or +bucketing columns. + +Since the `MERGE` process in general requires redistribution of +merged rows among Trino nodes, the order of rows in pages to be stored +are indeterminate. Connectors like Hive that depend on an ascending +rowId order for deleted rows must sort the deleted rows before storing +them. + +To ensure that all inserted rows for a given partition end up on a +single node, the redistribution hash on the partition key/bucket columns +is applied to the page partition keys. As a result of the hash, all +rows for a specific partition/bucket hash together, whether they +were `MATCHED` rows or `NOT MATCHED` rows. + +For connectors whose `RowChangeParadigm` is `DELETE_ROW_AND_INSERT_ROW`, +inserted rows are distributed using the layout supplied by +`ConnectorMetadata.getInsertLayout()`. For some connectors, the same +layout is used for updated rows. Other connectors require a special +layout for updated rows, supplied by `ConnectorMetadata.getUpdateLayout()`. + +### Connector support for `MERGE` + +To start `MERGE` processing, the Trino engine calls: + +- `ConnectorMetadata.getMergeRowIdColumnHandle(...)` to get the + `rowId` column handle. +- `ConnectorMetadata.getRowChangeParadigm(...)` to get the paradigm + supported by the connector for changing existing table rows. +- `ConnectorMetadata.beginMerge(...)` to get the a + `ConnectorMergeTableHandle` for the merge operation. That + `ConnectorMergeTableHandle` object contains whatever information the + connector needs to specify the `MERGE` operation. +- `ConnectorMetadata.getInsertLayout(...)`, from which it extracts the + the list of partition or table columns that impact write redistribution. +- `ConnectorMetadata.getUpdateLayout(...)`. If that layout is non-empty, + it is used to distribute updated rows resulting from the `MERGE` + operation. + +On nodes that are targets of the hash, the Trino engine calls +`ConnectorPageSinkProvider.createMergeSink(...)` to create a +`ConnectorMergeSink`. + +To write out each page of merged rows, the Trino engine calls +`ConnectorMergeSink.storeMergedRows(Page)`. The `storeMergedRows(Page)` +method iterates over the rows in the page, performing updates and deletes +in the `MATCHED` cases, and inserts in the `NOT MATCHED` cases. + +When using `RowChangeParadigm.DELETE_ROW_AND_INSERT_ROW`, the engine +translates `UPDATE` operations into a pair of `DELETE` and `INSERT` +operations before `storeMergedRows(Page)` is called. + +To complete the `MERGE` operation, the Trino engine calls +`ConnectorMetadata.finishMerge(...)`, passing the table handle +and a collection of JSON objects encoded as `Slice` instances. These +objects contain connector-specific information specifying what was changed +by the `MERGE` operation. Typically this JSON object contains the files +written and table and partition statistics generated by the `MERGE` +operation. The connector takes appropriate actions, if any. + +## `RowChangeProcessor` implementation for `MERGE` + +In the `MERGE` implementation, each `RowChangeParadigm` +corresponds to an internal Trino engine class that implements interface +`RowChangeProcessor`. `RowChangeProcessor` has one interesting method: +`Page transformPage(Page)`. The format of the output page depends +on the `RowChangeParadigm`. + +The connector has no access to the `RowChangeProcessor` instance -- it +is used inside the Trino engine to transform the merge page rows into rows +to be stored, based on the connector's choice of `RowChangeParadigm`. + +The page supplied to `transformPage()` consists of: + +- The write redistribution columns if any +- For partitioned or bucketed tables, a long hash value column. +- The `rowId` column for the row from the target table if matched, or + null if not matched +- The merge case `RowBlock` +- The integer case number block +- The byte `is_distinct` block, with value 0 if not distinct. + +The merge case `RowBlock` has the following layout: + +- Blocks for each column in the table, including partition columns, in + table column order. +- A block containing the boolean "present" value which is true if the + source row matched a target row, and false otherwise. +- A block containing the `MERGE` case operation number, encoded as + `INSERT` = 1, `DELETE` = 2, `UPDATE` = 3 and if no `MERGE` + case matched, -1. +- A block containing the number, starting with 0, for the + `WHEN` clause that matched for the row, or -1 if no clause + matched. + +The page returned from `transformPage` consists of: + +- All table columns, in table column order. +- The merge case operation block. +- The rowId block. +- A byte block containing 1 if the row is an insert derived from an + update operation, and 0 otherwise. This block is used to correctly + calculate the count of rows changed for connectors that represent + updates and deletes plus inserts. + +`transformPage` +must ensure that there are no rows whose operation number is -1 in +the page it returns. + +## Detecting duplicate matching target rows + +The SQL `MERGE` specification requires that in each `MERGE` case, +a single target table row must match at most one source row, after +applying the `MERGE` case condition expression. The first step +toward finding these error is done by labeling each row in the target +table with a unique id, using an `AssignUniqueId` node above the +target table scan. The projected results from the `RIGHT JOIN` +have these unique ids for matched target table rows as well as +the `WHEN` clause number. A `MarkDistinct` node adds an +`is_distinct` column which is true if no other row has the same +unique id and `WHEN` clause number, and false otherwise. If +any row has `is_distinct` equal to false, a +`MERGE_TARGET_ROW_MULTIPLE_MATCHES` exception is raised and +the `MERGE` operation fails. + +## `ConnectorMergeTableHandle` API + +Interface `ConnectorMergeTableHandle` defines one method, +`getTableHandle()` to retrieve the `ConnectorTableHandle` +originally passed to `ConnectorMetadata.beginMerge()`. + +## `ConnectorPageSinkProvider` API + +To support SQL `MERGE`, `ConnectorPageSinkProvider` must implement +the method that creates the `ConnectorMergeSink`: + +- `createMergeSink`: + + ``` + ConnectorMergeSink createMergeSink( + ConnectorTransactionHandle transactionHandle, + ConnectorSession session, + ConnectorMergeTableHandle mergeHandle) + ``` + +## `ConnectorMergeSink` API + +To support `MERGE`, the connector must define an +implementation of `ConnectorMergeSink`, usually layered over the +connector's `ConnectorPageSink`. + +The `ConnectorMergeSink` is created by a call to +`ConnectorPageSinkProvider.createMergeSink()`. + +The only interesting methods are: + +- `storeMergedRows`: + + ``` + void storeMergedRows(Page page) + ``` + + The Trino engine calls the `storeMergedRows(Page)` method of the + `ConnectorMergeSink` instance returned by + `ConnectorPageSinkProvider.createMergeSink()`, passing the page + generated by the `RowChangeProcessor.transformPage()` method. + That page consists of all table columns, in table column order, + followed by the `TINYINT` operation column, followed by the rowId column. + + The job of `storeMergedRows()` is iterate over the rows in the page, + and process them based on the value of the operation column, `INSERT`, + `DELETE`, `UPDATE`, or ignore the row. By choosing appropriate + paradigm, the connector can request that the UPDATE operation be + transformed into `DELETE` and `INSERT` operations. + +- `finish`: + + ``` + CompletableFuture> finish() + ``` + + The Trino engine calls `finish()` when all the data has been processed by + a specific `ConnectorMergeSink` instance. The connector returns a future + containing a collection of `Slice`, representing connector-specific + information about the rows processed. Usually this includes the row count, + and might include information like the files or partitions created or + changed. + +## `ConnectorMetadata` `MERGE` API + +A connector implementing `MERGE` must implement these `ConnectorMetadata` +methods. + +- `getRowChangeParadigm()`: + + ``` + RowChangeParadigm getRowChangeParadigm( + ConnectorSession session, + ConnectorTableHandle tableHandle) + ``` + + This method is called as the engine starts processing a `MERGE` statement. + The connector must return a `RowChangeParadigm` enumeration instance. If + the connector doesn't support `MERGE`, then it should throw a + `NOT_SUPPORTED` exception to indicate that SQL `MERGE` isn't supported by + the connector. Note that the default implementation already throws this + exception when the method isn't implemented. + +- `getMergeRowIdColumnHandle()`: + + ``` + ColumnHandle getMergeRowIdColumnHandle( + ConnectorSession session, + ConnectorTableHandle tableHandle) + ``` + + This method is called in the early stages of query planning for `MERGE` + statements. The ColumnHandle returned provides the `rowId` used by the + connector to identify rows to be merged, as well as any other fields of + the row that the connector needs to complete the `MERGE` operation. + +- `getInsertLayout()`: + + ``` + Optional getInsertLayout( + ConnectorSession session, + ConnectorTableHandle tableHandle) + ``` + + This method is called during query planning to get the table layout to be + used for rows inserted by the `MERGE` operation. For some connectors, + this layout is used for rows deleted as well. + +- `getUpdateLayout()`: + + ``` + Optional getUpdateLayout( + ConnectorSession session, + ConnectorTableHandle tableHandle) + ``` + + This method is called during query planning to get the table layout to be + used for rows deleted by the `MERGE` operation. If the optional return + value is present, the Trino engine uses the layout for updated rows. + Otherwise, it uses the result of `ConnectorMetadata.getInsertLayout` to + distribute updated rows. + +- `beginMerge()`: + + ``` + ConnectorMergeTableHandle beginMerge( + ConnectorSession session, + ConnectorTableHandle tableHandle) + ``` + + As the last step in creating the `MERGE` execution plan, the connector's + `beginMerge()` method is called, passing the `session`, and the + `tableHandle`. + + `beginMerge()` performs any orchestration needed in the connector to + start processing the `MERGE`. This orchestration varies from connector + to connector. In the case of Hive connector operating on transactional tables, + for example, `beginMerge()` checks that the table is transactional and + starts a Hive Metastore transaction. + + `beginMerge()` returns a `ConnectorMergeTableHandle` with any added + information the connector needs when the handle is passed back to + `finishMerge()` and the split generation machinery. For most + connectors, the returned table handle contains at least a flag identifying + the table handle as a table handle for a `MERGE` operation. + +- `finishMerge()`: + + ``` + void finishMerge( + ConnectorSession session, + ConnectorMergeTableHandle tableHandle, + Collection fragments) + ``` + + During `MERGE` processing, the Trino engine accumulates the `Slice` + collections returned by `ConnectorMergeSink.finish()`. The engine calls + `finishMerge()`, passing the table handle and that collection of + `Slice` fragments. In response, the connector takes appropriate actions + to complete the `MERGE` operation. Those actions might include + committing an underlying transaction, if any, or freeing any other + resources. diff --git a/docs/src/main/sphinx/develop/supporting-merge.rst b/docs/src/main/sphinx/develop/supporting-merge.rst deleted file mode 100644 index 7d838f6a55fa..000000000000 --- a/docs/src/main/sphinx/develop/supporting-merge.rst +++ /dev/null @@ -1,420 +0,0 @@ -==================== -Supporting ``MERGE`` -==================== - -The Trino engine provides APIs to support row-level SQL ``MERGE``. -To implement ``MERGE``, a connector must provide the following: - -* An implementation of ``ConnectorMergeSink``, which is typically - layered on top of a ``ConnectorPageSink``. -* Methods in ``ConnectorMetadata`` to get a "rowId" column handle, get the - row change paradigm, and to start and complete the ``MERGE`` operation. - -The Trino engine machinery used to implement SQL ``MERGE`` is also used to -support SQL ``DELETE`` and ``UPDATE``. This means that all a connector needs to -do is implement support for SQL ``MERGE``, and the connector gets all the Data -Modification Language (DML) operations. - -Standard SQL ``MERGE`` ----------------------- - -Different query engines support varying definitions of SQL ``MERGE``. -Trino supports the strict SQL specification ``ISO/IEC 9075``, published -in 2016. As a simple example, given tables ``target_table`` and -``source_table`` defined as:: - - CREATE TABLE accounts ( - customer VARCHAR, - purchases DECIMAL, - address VARCHAR); - INSERT INTO accounts (customer, purchases, address) VALUES ...; - CREATE TABLE monthly_accounts_update ( - customer VARCHAR, - purchases DECIMAL, - address VARCHAR); - INSERT INTO monthly_accounts_update (customer, purchases, address) VALUES ...; - -Here is a possible ``MERGE`` operation, from ``monthly_accounts_update`` to -``accounts``:: - - MERGE INTO accounts t USING monthly_accounts_update s - ON (t.customer = s.customer) - WHEN MATCHED AND s.address = 'Berkeley' THEN - DELETE - WHEN MATCHED AND s.customer = 'Joe Shmoe' THEN - UPDATE SET purchases = purchases + 100.0 - WHEN MATCHED THEN - UPDATE - SET purchases = s.purchases + t.purchases, address = s.address - WHEN NOT MATCHED THEN - INSERT (customer, purchases, address) - VALUES (s.customer, s.purchases, s.address); - -SQL ``MERGE`` tries to match each ``WHEN`` clause in source order. When -a match is found, the corresponding ``DELETE``, ``INSERT`` or ``UPDATE`` -is executed and subsequent ``WHEN`` clauses are ignored. - -SQL ``MERGE`` supports two operations on the target table and source -when a row from the source table or query matches a row in the target table: - -* ``UPDATE``, in which the columns in the target row are updated. -* ``DELETE``, in which the target row is deleted. - -In the ``NOT MATCHED`` case, SQL ``MERGE`` supports only ``INSERT`` -operations. The values inserted are arbitrary but usually come from -the unmatched row of the source table or query. - -``RowChangeParadigm`` ---------------------- - -Different connectors have different ways of representing row updates, -imposed by the underlying storage systems. The Trino engine classifies -these different paradigms as elements of the ``RowChangeParadigm`` -enumeration, returned by enumeration, returned by method -``ConnectorMetadata.getRowChangeParadigm(...)``. - -The ``RowChangeParadigm`` enumeration values are: - -* ``CHANGE_ONLY_UPDATED_COLUMNS``, intended for connectors that can update - individual columns of rows identified by a ``rowId``. The corresponding - merge processor class is ``ChangeOnlyUpdatedColumnsMergeProcessor``. -* ``DELETE_ROW_AND_INSERT_ROW``, intended for connectors that represent a - row change as a row deletion paired with a row insertion. The corresponding - merge processor class is ``DeleteAndInsertMergeProcessor``. - -Overview of ``MERGE`` processing --------------------------------- - -A ``MERGE`` statement is processed by creating a ``RIGHT JOIN`` between the -target table and the source, on the ``MERGE`` criteria. The source may be -a table or an arbitrary query. For each row in the source table or query, -``MERGE`` produces a ``ROW`` object containing: - -* the data column values from the ``UPDATE`` or ``INSERT`` cases. For the - ``DELETE`` cases, only the partition columns, which determine - partitioning and bucketing, are non-null. -* a boolean column containing ``true`` for source rows that matched some - target row, and ``false`` otherwise. -* an integer that identifies whether the merge case operation is ``UPDATE``, - ``DELETE`` or ``INSERT``, or a source row for which no case matched. If a - source row doesn't match any merge case, all data column values except - those that determine distribution are null, and the operation number - is -1. - -A ``SearchedCaseExpression`` is constructed from ``RIGHT JOIN`` result -to represent the ``WHEN`` clauses of the ``MERGE``. In the example preceding -the ``MERGE`` is executed as if the ``SearchedCaseExpression`` were written as:: - - SELECT - CASE - WHEN present AND s.address = 'Berkeley' THEN - -- Null values for delete; present=true; operation DELETE=2, case_number=0 - row(null, null, null, true, 2, 0) - WHEN present AND s.customer = 'Joe Shmoe' THEN - -- Update column values; present=true; operation UPDATE=3, case_number=1 - row(t.customer, t.purchases + 100.0, t.address, true, 3, 1) - WHEN present THEN - -- Update column values; present=true; operation UPDATE=3, case_number=2 - row(t.customer, s.purchases + t.purchases, s.address, true, 3, 2) - WHEN (present IS NULL) THEN - -- Insert column values; present=false; operation INSERT=1, case_number=3 - row(s.customer, s.purchases, s.address, false, 1, 3) - ELSE - -- Null values for no case matched; present=false; operation=-1, - -- case_number=-1 - row(null, null, null, false, -1, -1) - END - FROM (SELECT *, true AS present FROM target_table) t - RIGHT JOIN source_table s ON s.customer = t.customer; - -The Trino engine executes the ``RIGHT JOIN`` and ``CASE`` expression, -and ensures that no target table row matches more than one source expression -row, and ultimately creates a sequence of pages to be routed to the node that -runs the ``ConnectorMergeSink.storeMergedRows(...)`` method. - -Like ``DELETE`` and ``UPDATE``, ``MERGE`` target table rows are identified by -a connector-specific ``rowId`` column handle. For ``MERGE``, the ``rowId`` -handle is returned by ``ConnectorMetadata.getMergeRowIdColumnHandle(...)``. - -``MERGE`` redistribution ------------------------- - -The Trino ``MERGE`` implementation allows ``UPDATE`` to change -the values of columns that determine partitioning and/or bucketing, and so -it must "redistribute" rows from the ``MERGE`` operation to the worker -nodes responsible for writing rows with the merged partitioning and/or -bucketing columns. - -Since the ``MERGE`` process in general requires redistribution of -merged rows among Trino nodes, the order of rows in pages to be stored -are indeterminate. Connectors like Hive that depend on an ascending -rowId order for deleted rows must sort the deleted rows before storing -them. - -To ensure that all inserted rows for a given partition end up on a -single node, the redistribution hash on the partition key/bucket columns -is applied to the page partition keys. As a result of the hash, all -rows for a specific partition/bucket hash together, whether they -were ``MATCHED`` rows or ``NOT MATCHED`` rows. - -For connectors whose ``RowChangeParadigm`` is ``DELETE_ROW_AND_INSERT_ROW``, -inserted rows are distributed using the layout supplied by -``ConnectorMetadata.getInsertLayout()``. For some connectors, the same -layout is used for updated rows. Other connectors require a special -layout for updated rows, supplied by ``ConnectorMetadata.getUpdateLayout()``. - -Connector support for ``MERGE`` -=============================== - -To start ``MERGE`` processing, the Trino engine calls: - -* ``ConnectorMetadata.getMergeRowIdColumnHandle(...)`` to get the - ``rowId`` column handle. -* ``ConnectorMetadata.getRowChangeParadigm(...)`` to get the paradigm - supported by the connector for changing existing table rows. -* ``ConnectorMetadata.beginMerge(...)`` to get the a - ``ConnectorMergeTableHandle`` for the merge operation. That - ``ConnectorMergeTableHandle`` object contains whatever information the - connector needs to specify the ``MERGE`` operation. -* ``ConnectorMetadata.getInsertLayout(...)``, from which it extracts the - the list of partition or table columns that impact write redistribution. -* ``ConnectorMetadata.getUpdateLayout(...)``. If that layout is non-empty, - it is used to distribute updated rows resulting from the ``MERGE`` - operation. - -On nodes that are targets of the hash, the Trino engine calls -``ConnectorPageSinkProvider.createMergeSink(...)`` to create a -``ConnectorMergeSink``. - -To write out each page of merged rows, the Trino engine calls -``ConnectorMergeSink.storeMergedRows(Page)``. The ``storeMergedRows(Page)`` -method iterates over the rows in the page, performing updates and deletes -in the ``MATCHED`` cases, and inserts in the ``NOT MATCHED`` cases. - -When using ``RowChangeParadigm.DELETE_ROW_AND_INSERT_ROW``, the engine -translates ``UPDATE`` operations into a pair of ``DELETE`` and ``INSERT`` -operations before ``storeMergedRows(Page)`` is called. - -To complete the ``MERGE`` operation, the Trino engine calls -``ConnectorMetadata.finishMerge(...)``, passing the table handle -and a collection of JSON objects encoded as ``Slice`` instances. These -objects contain connector-specific information specifying what was changed -by the ``MERGE`` operation. Typically this JSON object contains the files -written and table and partition statistics generated by the ``MERGE`` -operation. The connector takes appropriate actions, if any. - -``RowChangeProcessor`` implementation for ``MERGE`` ---------------------------------------------------- - -In the ``MERGE`` implementation, each ``RowChangeParadigm`` -corresponds to an internal Trino engine class that implements interface -``RowChangeProcessor``. ``RowChangeProcessor`` has one interesting method: -``Page transformPage(Page)``. The format of the output page depends -on the ``RowChangeParadigm``. - -The connector has no access to the ``RowChangeProcessor`` instance -- it -is used inside the Trino engine to transform the merge page rows into rows -to be stored, based on the connector's choice of ``RowChangeParadigm``. - -The page supplied to ``transformPage()`` consists of: - -* The write redistribution columns if any -* For partitioned or bucketed tables, a long hash value column. -* The ``rowId`` column for the row from the target table if matched, or - null if not matched -* The merge case ``RowBlock`` -* The integer case number block -* The byte ``is_distinct`` block, with value 0 if not distinct. - -The merge case ``RowBlock`` has the following layout: - -* Blocks for each column in the table, including partition columns, in - table column order. -* A block containing the boolean "present" value which is true if the - source row matched a target row, and false otherwise. -* A block containing the ``MERGE`` case operation number, encoded as - ``INSERT`` = 1, ``DELETE`` = 2, ``UPDATE`` = 3 and if no ``MERGE`` - case matched, -1. -* A block containing the number, starting with 0, for the - ``WHEN`` clause that matched for the row, or -1 if no clause - matched. - -The page returned from ``transformPage`` consists of: - -* All table columns, in table column order. -* The merge case operation block. -* The rowId block. -* A byte block containing 1 if the row is an insert derived from an - update operation, and 0 otherwise. This block is used to correctly - calculate the count of rows changed for connectors that represent - updates and deletes plus inserts. - -``transformPage`` -must ensure that there are no rows whose operation number is -1 in -the page it returns. - -Detecting duplicate matching target rows ----------------------------------------- - -The SQL ``MERGE`` specification requires that in each ``MERGE`` case, -a single target table row must match at most one source row, after -applying the ``MERGE`` case condition expression. The first step -toward finding these error is done by labeling each row in the target -table with a unique id, using an ``AssignUniqueId`` node above the -target table scan. The projected results from the ``RIGHT JOIN`` -have these unique ids for matched target table rows as well as -the ``WHEN`` clause number. A ``MarkDistinct`` node adds an -``is_distinct`` column which is true if no other row has the same -unique id and ``WHEN`` clause number, and false otherwise. If -any row has ``is_distinct`` equal to false, a -``MERGE_TARGET_ROW_MULTIPLE_MATCHES`` exception is raised and -the ``MERGE`` operation fails. - -``ConnectorMergeTableHandle`` API ---------------------------------- - -Interface ``ConnectorMergeTableHandle`` defines one method, -``getTableHandle()`` to retrieve the ``ConnectorTableHandle`` -originally passed to ``ConnectorMetadata.beginMerge()``. - -``ConnectorPageSinkProvider`` API ---------------------------------- - -To support SQL ``MERGE``, ``ConnectorPageSinkProvider`` must implement -the method that creates the ``ConnectorMergeSink``: - -* ``createMergeSink``:: - - ConnectorMergeSink createMergeSink( - ConnectorTransactionHandle transactionHandle, - ConnectorSession session, - ConnectorMergeTableHandle mergeHandle) - -``ConnectorMergeSink`` API --------------------------- - -To support ``MERGE``, the connector must define an -implementation of ``ConnectorMergeSink``, usually layered over the -connector's ``ConnectorPageSink``. - -The ``ConnectorMergeSink`` is created by a call to -``ConnectorPageSinkProvider.createMergeSink()``. - -The only interesting methods are: - -* ``storeMergedRows``:: - - void storeMergedRows(Page page) - - The Trino engine calls the ``storeMergedRows(Page)`` method of the - ``ConnectorMergeSink`` instance returned by - ``ConnectorPageSinkProvider.createMergeSink()``, passing the page - generated by the ``RowChangeProcessor.transformPage()`` method. - That page consists of all table columns, in table column order, - followed by the ``TINYINT`` operation column, followed by the rowId column. - - The job of ``storeMergedRows()`` is iterate over the rows in the page, - and process them based on the value of the operation column, ``INSERT``, - ``DELETE``, ``UPDATE``, or ignore the row. By choosing appropriate - paradigm, the connector can request that the UPDATE operation be - transformed into ``DELETE`` and ``INSERT`` operations. - -* ``finish``:: - - CompletableFuture> finish() - - The Trino engine calls ``finish()`` when all the data has been processed by - a specific ``ConnectorMergeSink`` instance. The connector returns a future - containing a collection of ``Slice``, representing connector-specific - information about the rows processed. Usually this includes the row count, - and might include information like the files or partitions created or - changed. - -``ConnectorMetadata`` ``MERGE`` API ------------------------------------ - -A connector implementing ``MERGE`` must implement these ``ConnectorMetadata`` -methods. - -* ``getRowChangeParadigm()``:: - - RowChangeParadigm getRowChangeParadigm( - ConnectorSession session, - ConnectorTableHandle tableHandle) - - This method is called as the engine starts processing a ``MERGE`` statement. - The connector must return a ``RowChangeParadigm`` enumeration instance. If - the connector doesn't support ``MERGE``, then it should throw a - ``NOT_SUPPORTED`` exception to indicate that SQL ``MERGE`` isn't supported by - the connector. Note that the default implementation already throws this - exception when the method isn't implemented. - -* ``getMergeRowIdColumnHandle()``:: - - ColumnHandle getMergeRowIdColumnHandle( - ConnectorSession session, - ConnectorTableHandle tableHandle) - - This method is called in the early stages of query planning for ``MERGE`` - statements. The ColumnHandle returned provides the ``rowId`` used by the - connector to identify rows to be merged, as well as any other fields of - the row that the connector needs to complete the ``MERGE`` operation. - -* ``getInsertLayout()``:: - - Optional getInsertLayout( - ConnectorSession session, - ConnectorTableHandle tableHandle) - - This method is called during query planning to get the table layout to be - used for rows inserted by the ``MERGE`` operation. For some connectors, - this layout is used for rows deleted as well. - -* ``getUpdateLayout()``:: - - Optional getUpdateLayout( - ConnectorSession session, - ConnectorTableHandle tableHandle) - - This method is called during query planning to get the table layout to be - used for rows deleted by the ``MERGE`` operation. If the optional return - value is present, the Trino engine uses the layout for updated rows. - Otherwise, it uses the result of ``ConnectorMetadata.getInsertLayout`` to - distribute updated rows. - -* ``beginMerge()``:: - - ConnectorMergeTableHandle beginMerge( - ConnectorSession session, - ConnectorTableHandle tableHandle) - - As the last step in creating the ``MERGE`` execution plan, the connector's - ``beginMerge()`` method is called, passing the ``session``, and the - ``tableHandle``. - - ``beginMerge()`` performs any orchestration needed in the connector to - start processing the ``MERGE``. This orchestration varies from connector - to connector. In the case of Hive connector operating on transactional tables, - for example, ``beginMerge()`` checks that the table is transactional and - starts a Hive Metastore transaction. - - ``beginMerge()`` returns a ``ConnectorMergeTableHandle`` with any added - information the connector needs when the handle is passed back to - ``finishMerge()`` and the split generation machinery. For most - connectors, the returned table handle contains at least a flag identifying - the table handle as a table handle for a ``MERGE`` operation. - -* ``finishMerge()``:: - - void finishMerge( - ConnectorSession session, - ConnectorMergeTableHandle tableHandle, - Collection fragments) - - During ``MERGE`` processing, the Trino engine accumulates the ``Slice`` - collections returned by ``ConnectorMergeSink.finish()``. The engine calls - ``finishMerge()``, passing the table handle and that collection of - ``Slice`` fragments. In response, the connector takes appropriate actions - to complete the ``MERGE`` operation. Those actions might include - committing an underlying transaction, if any, or freeing any other - resources. diff --git a/docs/src/main/sphinx/develop/system-access-control.md b/docs/src/main/sphinx/develop/system-access-control.md new file mode 100644 index 000000000000..54050491d894 --- /dev/null +++ b/docs/src/main/sphinx/develop/system-access-control.md @@ -0,0 +1,49 @@ +# System access control + +Trino separates the concept of the principal who authenticates to the +coordinator from the username that is responsible for running queries. When +running the Trino CLI, for example, the Trino username can be specified using +the `--user` option. + +By default, the Trino coordinator allows any principal to run queries as any +Trino user. In a secure environment, this is probably not desirable behavior +and likely requires customization. + +## Implementation + +`SystemAccessControlFactory` is responsible for creating a +`SystemAccessControl` instance. It also defines a `SystemAccessControl` +name which is used by the administrator in a Trino configuration. + +`SystemAccessControl` implementations have several responsibilities: + +- Verifying whether or not a given principal is authorized to execute queries as a specific user. +- Determining whether or not a given user can alter values for a given system property. +- Performing access checks across all catalogs. These access checks happen before + any connector specific checks and thus can deny permissions that would otherwise + be allowed by `ConnectorAccessControl`. + +The implementation of `SystemAccessControl` and `SystemAccessControlFactory` +must be wrapped as a plugin and installed on the Trino cluster. + +## Configuration + +After a plugin that implements `SystemAccessControl` and +`SystemAccessControlFactory` has been installed on the coordinator, it is +configured using the file(s) specified by the `access-control.config-files` +property (the default is a single `etc/access-control.properties` file). +All of the properties other than `access-control.name` are specific to +the `SystemAccessControl` implementation. + +The `access-control.name` property is used by Trino to find a registered +`SystemAccessControlFactory` based on the name returned by +`SystemAccessControlFactory.getName()`. The remaining properties are passed +as a map to `SystemAccessControlFactory.create()`. + +Example configuration file: + +```text +access-control.name=custom-access-control +custom-property1=custom-value1 +custom-property2=custom-value2 +``` diff --git a/docs/src/main/sphinx/develop/system-access-control.rst b/docs/src/main/sphinx/develop/system-access-control.rst deleted file mode 100644 index fbe1013bef97..000000000000 --- a/docs/src/main/sphinx/develop/system-access-control.rst +++ /dev/null @@ -1,53 +0,0 @@ -===================== -System access control -===================== - -Trino separates the concept of the principal who authenticates to the -coordinator from the username that is responsible for running queries. When -running the Trino CLI, for example, the Trino username can be specified using -the ``--user`` option. - -By default, the Trino coordinator allows any principal to run queries as any -Trino user. In a secure environment, this is probably not desirable behavior -and likely requires customization. - -Implementation --------------- - -``SystemAccessControlFactory`` is responsible for creating a -``SystemAccessControl`` instance. It also defines a ``SystemAccessControl`` -name which is used by the administrator in a Trino configuration. - -``SystemAccessControl`` implementations have several responsibilities: - -* Verifying whether or not a given principal is authorized to execute queries as a specific user. -* Determining whether or not a given user can alter values for a given system property. -* Performing access checks across all catalogs. These access checks happen before - any connector specific checks and thus can deny permissions that would otherwise - be allowed by ``ConnectorAccessControl``. - -The implementation of ``SystemAccessControl`` and ``SystemAccessControlFactory`` -must be wrapped as a plugin and installed on the Trino cluster. - -Configuration -------------- - -After a plugin that implements ``SystemAccessControl`` and -``SystemAccessControlFactory`` has been installed on the coordinator, it is -configured using the file(s) specified by the ``access-control.config-files`` -property (the default is a single ``etc/access-control.properties`` file). -All of the properties other than ``access-control.name`` are specific to -the ``SystemAccessControl`` implementation. - -The ``access-control.name`` property is used by Trino to find a registered -``SystemAccessControlFactory`` based on the name returned by -``SystemAccessControlFactory.getName()``. The remaining properties are passed -as a map to ``SystemAccessControlFactory.create()``. - -Example configuration file: - -.. code-block:: text - - access-control.name=custom-access-control - custom-property1=custom-value1 - custom-property2=custom-value2 diff --git a/docs/src/main/sphinx/develop/table-functions.md b/docs/src/main/sphinx/develop/table-functions.md new file mode 100644 index 000000000000..7b5048fa2767 --- /dev/null +++ b/docs/src/main/sphinx/develop/table-functions.md @@ -0,0 +1,274 @@ +# Table functions + +Table functions return tables. They allow users to dynamically invoke custom +logic from within the SQL query. They are invoked in the `FROM` clause of a +query, and the calling convention is similar to a scalar function call. For +description of table functions usage, see +{doc}`table functions`. + +Trino supports adding custom table functions. They are declared by connectors +through implementing dedicated interfaces. + +## Table function declaration + +To declare a table function, you need to implement `ConnectorTableFunction`. +Subclassing `AbstractConnectorTableFunction` is a convenient way to do it. +The connector's `getTableFunctions()` method must return a set of your +implementations. + +### The constructor + +```java +public class MyFunction + extends AbstractConnectorTableFunction +{ + public MyFunction() + { + super( + "system", + "my_function", + List.of( + ScalarArgumentSpecification.builder() + .name("COLUMN_COUNT") + .type(INTEGER) + .defaultValue(2) + .build(), + ScalarArgumentSpecification.builder() + .name("ROW_COUNT") + .type(INTEGER) + .build()), + GENERIC_TABLE); + } +} +``` + +The constructor takes the following arguments: + +- **schema name** + +The schema name helps you organize functions, and it is used for function +resolution. When a table function is invoked, the right implementation is +identified by the catalog name, the schema name, and the function name. + +The function can use the schema name, for example to use data from the +indicated schema, or ignore it. + +- **function name** +- **list of expected arguments** + +Three different types of arguments are supported: scalar arguments, descriptor +arguments, and table arguments. See {ref}`tf-argument-types` for details. You can +specify default values for scalar and descriptor arguments. The arguments with +specified default can be skipped during table function invocation. + +- **returned row type** + +It describes the row type produced by the table function. + +If a table function takes table arguments, it can additionally pass the columns +of the input tables to output using the *pass-through mechanism*. The returned +row type is supposed to describe only the columns produced by the function, as +opposed to the pass-through columns. + +In the example, the returned row type is `GENERIC_TABLE`, which means that +the row type is not known statically, and it is determined dynamically based on +the passed arguments. + +When the returned row type is known statically, you can declare it using: + +```java +new DescribedTable(descriptor) +``` + +If a table function does not produce any columns, and it only outputs the +pass-through columns, use `ONLY_PASS_THROUGH` as the returned row type. + +:::{note} +A table function must return at least one column. It can either be a proper +column, i.e. produced by the function, or a pass-through column. +::: + +(tf-argument-types)= + +### Argument types + +Table functions take three types of arguments: +{ref}`scalar arguments`, +{ref}`descriptor arguments`, and +{ref}`table arguments`. + +(tf-scalar-arguments)= + +#### Scalar arguments + +They can be of any supported data type. You can specify a default value. + +```java +ScalarArgumentSpecification.builder() + .name("COLUMN_COUNT") + .type(INTEGER) + .defaultValue(2) + .build() +``` + +```java +ScalarArgumentSpecification.builder() + .name("ROW_COUNT") + .type(INTEGER) + .build() +``` + +(tf-descriptor-arguments)= + +#### Descriptor arguments + +Descriptors consist of fields with names and optional data types. They are a +convenient way to pass the required result row type to the function, or for +example inform the function which input columns it should use. You can specify +default values for descriptor arguments. Descriptor argument can be `null`. + +```java +DescriptorArgumentSpecification.builder() + .name("SCHEMA") + .defaultValue(null) + .build() +``` + +(tf-table-arguments)= + +#### Table arguments + +A table function can take any number of input relations. It allows you to +process multiple data sources simultaneously. + +When declaring a table argument, you must specify characteristics to determine +how the input table is processed. Also note that you cannot specify a default +value for a table argument. + +```java +TableArgumentSpecification.builder() + .name("INPUT") + .rowSemantics() + .pruneWhenEmpty() + .passThroughColumns() + .build() +``` + +(tf-set-or-row-semantics)= + +##### Set or row semantics + +Set semantics is the default for table arguments. A table argument with set +semantics is processed on a partition-by-partition basis. During function +invocation, the user can specify partitioning and ordering for the argument. If +no partitioning is specified, the argument is processed as a single partition. + +A table argument with row semantics is processed on a row-by-row basis. +Partitioning or ordering is not applicable. + +##### Prune or keep when empty + +The *prune when empty* property indicates that if the given table argument is +empty, the function returns empty result. This property is used to optimize +queries involving table functions. The *keep when empty* property indicates +that the function should be executed even if the table argument is empty. The +user can override this property when invoking the function. Using the *keep +when empty* property can negatively affect performance when the table argument +is not empty. + +##### Pass-through columns + +If a table argument has *pass-through columns*, all of its columns are passed +on output. For a table argument without this property, only the partitioning +columns are passed on output. + +### The `analyze()` method + +In order to provide all the necessary information to the Trino engine, the +class must implement the `analyze()` method. This method is called by the +engine during the analysis phase of query processing. The `analyze()` method +is also the place to perform custom checks on the arguments: + +```java +@Override +public TableFunctionAnalysis analyze(ConnectorSession session, ConnectorTransactionHandle transaction, Map arguments) +{ + long columnCount = (long) ((ScalarArgument) arguments.get("COLUMN_COUNT")).getValue(); + long rowCount = (long) ((ScalarArgument) arguments.get("ROW_COUNT")).getValue(); + + // custom validation of arguments + if (columnCount < 1 || columnCount > 3) { + throw new TrinoException(INVALID_FUNCTION_ARGUMENT, "column_count must be in range [1, 3]"); + } + + if (rowCount < 1) { + throw new TrinoException(INVALID_FUNCTION_ARGUMENT, "row_count must be positive"); + } + + // determine the returned row type + List fields = List.of("col_a", "col_b", "col_c").subList(0, (int) columnCount).stream() + .map(name -> new Descriptor.Field(name, Optional.of(BIGINT))) + .collect(toList()); + + Descriptor returnedType = new Descriptor(fields); + + return TableFunctionAnalysis.builder() + .returnedType(returnedType) + .handle(new MyHandle(columnCount, rowCount)) + .build(); +} +``` + +The `analyze()` method returns a `TableFunctionAnalysis` object, which +comprises all the information required by the engine to analyze, plan, and +execute the table function invocation: + +- The returned row type, specified as an optional `Descriptor`. It should be + passed if and only if the table function is declared with the + `GENERIC_TABLE` returned type. +- Required columns from the table arguments, specified as a map of table + argument names to lists of column indexes. +- Any information gathered during analysis that is useful during planning or + execution, in the form of a `ConnectorTableFunctionHandle`. + `ConnectorTableFunctionHandle` is a marker interface intended to carry + information throughout subsequent phases of query processing in a manner that + is opaque to the engine. + +## Table function execution + +There are two paths of execution available for table functions. + +1. Pushdown to the connector + +The connector that provides the table function implements the +`applyTableFunction()` method. This method is called during the optimization +phase of query processing. It returns a `ConnectorTableHandle` and a list of +`ColumnHandle` s representing the table function result. The table function +invocation is then replaced with a `TableScanNode`. + +This execution path is convenient for table functions whose results are easy to +represent as a `ConnectorTableHandle`, for example query pass-through. It +only supports scalar and descriptor arguments. + +2. Execution by operator + +Trino has a dedicated operator for table functions. It can handle table +functions with any number of table arguments as well as scalar and descriptor +arguments. To use this execution path, you provide an implementation of a +processor. + +If your table function has one or more table arguments, you must implement +`TableFunctionDataProcessor`. It processes pages of input data. + +If your table function is a source operator (it does not have table arguments), +you must implement `TableFunctionSplitProcessor`. It processes splits. The +connector that provides the function must provide a `ConnectorSplitSource` +for the function. With splits, the task can be divided so that each split +represents a subtask. + +## Access control + +The access control for table functions can be provided both on system and +connector level. It is based on the fully qualified table function name, +which consists of the catalog name, the schema name, and the function name, +in the syntax of `catalog.schema.function`. diff --git a/docs/src/main/sphinx/develop/table-functions.rst b/docs/src/main/sphinx/develop/table-functions.rst deleted file mode 100644 index 70f2547ea3f4..000000000000 --- a/docs/src/main/sphinx/develop/table-functions.rst +++ /dev/null @@ -1,289 +0,0 @@ - -=============== -Table functions -=============== - -Table functions return tables. They allow users to dynamically invoke custom -logic from within the SQL query. They are invoked in the ``FROM`` clause of a -query, and the calling convention is similar to a scalar function call. For -description of table functions usage, see -:doc:`table functions`. - -Trino supports adding custom table functions. They are declared by connectors -through implementing dedicated interfaces. - -Table function declaration --------------------------- - -To declare a table function, you need to implement ``ConnectorTableFunction``. -Subclassing ``AbstractConnectorTableFunction`` is a convenient way to do it. -The connector's ``getTableFunctions()`` method must return a set of your -implementations. - -The constructor -^^^^^^^^^^^^^^^ - -.. code-block:: java - - public class MyFunction - extends AbstractConnectorTableFunction - { - public MyFunction() - { - super( - "system", - "my_function", - List.of( - ScalarArgumentSpecification.builder() - .name("COLUMN_COUNT") - .type(INTEGER) - .defaultValue(2) - .build(), - ScalarArgumentSpecification.builder() - .name("ROW_COUNT") - .type(INTEGER) - .build()), - GENERIC_TABLE); - } - } - -The constructor takes the following arguments: - -- **schema name** - -The schema name helps you organize functions, and it is used for function -resolution. When a table function is invoked, the right implementation is -identified by the catalog name, the schema name, and the function name. - -The function can use the schema name, for example to use data from the -indicated schema, or ignore it. - -- **function name** -- **list of expected arguments** - -Three different types of arguments are supported: scalar arguments, descriptor -arguments, and table arguments. See :ref:`tf-argument-types` for details. You can -specify default values for scalar and descriptor arguments. The arguments with -specified default can be skipped during table function invocation. - -- **returned row type** - -It describes the row type produced by the table function. - -If a table function takes table arguments, it can additionally pass the columns -of the input tables to output using the *pass-through mechanism*. The returned -row type is supposed to describe only the columns produced by the function, as -opposed to the pass-through columns. - -In the example, the returned row type is ``GENERIC_TABLE``, which means that -the row type is not known statically, and it is determined dynamically based on -the passed arguments. - -When the returned row type is known statically, you can declare it using: - -.. code-block:: java - - new DescribedTable(descriptor) - -If a table function does not produce any columns, and it only outputs the -pass-through columns, use ``ONLY_PASS_THROUGH`` as the returned row type. - -.. note:: - - A table function must return at least one column. It can either be a proper - column, i.e. produced by the function, or a pass-through column. - -.. _tf-argument-types: - -Argument types -^^^^^^^^^^^^^^ - -Table functions take three types of arguments: -:ref:`scalar arguments`, -:ref:`descriptor arguments`, and -:ref:`table arguments`. - -.. _tf-scalar-arguments: - -Scalar arguments -++++++++++++++++ - -They can be of any supported data type. You can specify a default value. - -.. code-block:: java - - ScalarArgumentSpecification.builder() - .name("COLUMN_COUNT") - .type(INTEGER) - .defaultValue(2) - .build() - -.. code-block:: java - - ScalarArgumentSpecification.builder() - .name("ROW_COUNT") - .type(INTEGER) - .build() - -.. _tf-descriptor-arguments: - -Descriptor arguments -++++++++++++++++++++ - -Descriptors consist of fields with names and optional data types. They are a -convenient way to pass the required result row type to the function, or for -example inform the function which input columns it should use. You can specify -default values for descriptor arguments. Descriptor argument can be ``null``. - -.. code-block:: java - - DescriptorArgumentSpecification.builder() - .name("SCHEMA") - .defaultValue(null) - .build() - -.. _tf-table-arguments: - -Table arguments -+++++++++++++++ - -A table function can take any number of input relations. It allows you to -process multiple data sources simultaneously. - -When declaring a table argument, you must specify characteristics to determine -how the input table is processed. Also note that you cannot specify a default -value for a table argument. - -.. code-block:: java - - TableArgumentSpecification.builder() - .name("INPUT") - .rowSemantics() - .pruneWhenEmpty() - .passThroughColumns() - .build() - -.. _tf-set-or-row-semantics: - -Set or row semantics -==================== - -Set semantics is the default for table arguments. A table argument with set -semantics is processed on a partition-by-partition basis. During function -invocation, the user can specify partitioning and ordering for the argument. If -no partitioning is specified, the argument is processed as a single partition. - -A table argument with row semantics is processed on a row-by-row basis. -Partitioning or ordering is not applicable. - -Prune or keep when empty -======================== - -The *prune when empty* property indicates that if the given table argument is -empty, the function returns empty result. This property is used to optimize -queries involving table functions. The *keep when empty* property indicates -that the function should be executed even if the table argument is empty. The -user can override this property when invoking the function. Using the *keep -when empty* property can negatively affect performance when the table argument -is not empty. - -Pass-through columns -==================== - -If a table argument has *pass-through columns*, all of its columns are passed -on output. For a table argument without this property, only the partitioning -columns are passed on output. - -The ``analyze()`` method -^^^^^^^^^^^^^^^^^^^^^^^^ - -In order to provide all the necessary information to the Trino engine, the -class must implement the ``analyze()`` method. This method is called by the -engine during the analysis phase of query processing. The ``analyze()`` method -is also the place to perform custom checks on the arguments: - -.. code-block:: java - - @Override - public TableFunctionAnalysis analyze(ConnectorSession session, ConnectorTransactionHandle transaction, Map arguments) - { - long columnCount = (long) ((ScalarArgument) arguments.get("COLUMN_COUNT")).getValue(); - long rowCount = (long) ((ScalarArgument) arguments.get("ROW_COUNT")).getValue(); - - // custom validation of arguments - if (columnCount < 1 || columnCount > 3) { - throw new TrinoException(INVALID_FUNCTION_ARGUMENT, "column_count must be in range [1, 3]"); - } - - if (rowCount < 1) { - throw new TrinoException(INVALID_FUNCTION_ARGUMENT, "row_count must be positive"); - } - - // determine the returned row type - List fields = List.of("col_a", "col_b", "col_c").subList(0, (int) columnCount).stream() - .map(name -> new Descriptor.Field(name, Optional.of(BIGINT))) - .collect(toList()); - - Descriptor returnedType = new Descriptor(fields); - - return TableFunctionAnalysis.builder() - .returnedType(returnedType) - .handle(new MyHandle(columnCount, rowCount)) - .build(); - } - -The ``analyze()`` method returns a ``TableFunctionAnalysis`` object, which -comprises all the information required by the engine to analyze, plan, and -execute the table function invocation: - -- The returned row type, specified as an optional ``Descriptor``. It should be - passed if and only if the table function is declared with the - ``GENERIC_TABLE`` returned type. -- Required columns from the table arguments, specified as a map of table - argument names to lists of column indexes. -- Any information gathered during analysis that is useful during planning or - execution, in the form of a ``ConnectorTableFunctionHandle``. - ``ConnectorTableFunctionHandle`` is a marker interface intended to carry - information throughout subsequent phases of query processing in a manner that - is opaque to the engine. - -Table function execution ------------------------- - -There are two paths of execution available for table functions. - -1. Pushdown to the connector - -The connector that provides the table function implements the -``applyTableFunction()`` method. This method is called during the optimization -phase of query processing. It returns a ``ConnectorTableHandle`` and a list of -``ColumnHandle`` s representing the table function result. The table function -invocation is then replaced with a ``TableScanNode``. - -This execution path is convenient for table functions whose results are easy to -represent as a ``ConnectorTableHandle``, for example query pass-through. It -only supports scalar and descriptor arguments. - -2. Execution by operator - -Trino has a dedicated operator for table functions. It can handle table -functions with any number of table arguments as well as scalar and descriptor -arguments. To use this execution path, you provide an implementation of a -processor. - -If your table function has one or more table arguments, you must implement -``TableFunctionDataProcessor``. It processes pages of input data. - -If your table function is a source operator (it does not have table arguments), -you must implement ``TableFunctionSplitProcessor``. It processes splits. The -connector that provides the function must provide a ``ConnectorSplitSource`` -for the function. With splits, the task can be divided so that each split -represents a subtask. - -Access control --------------- - -The access control for table functions can be provided both on system and -connector level. It is based on the fully qualified table function name, -which consists of the catalog name, the schema name, and the function name, -in the syntax of ``catalog.schema.function``. diff --git a/docs/src/main/sphinx/develop/types.md b/docs/src/main/sphinx/develop/types.md new file mode 100644 index 000000000000..1ae357d27de1 --- /dev/null +++ b/docs/src/main/sphinx/develop/types.md @@ -0,0 +1,36 @@ +# Types + +The `Type` interface in Trino is used to implement a type in the SQL language. +Trino ships with a number of built-in types, like `VarcharType` and `BigintType`. +The `ParametricType` interface is used to provide type parameters for types, to +allow types like `VARCHAR(10)` or `DECIMAL(22, 5)`. A `Plugin` can provide +new `Type` objects by returning them from `getTypes()` and new `ParametricType` +objects by returning them from `getParametricTypes()`. + +Below is a high level overview of the `Type` interface. For more details, see the +JavaDocs for `Type`. + +## Native container type + +All types define the `getJavaType()` method, frequently referred to as the +"native container type". This is the Java type used to hold values during execution +and to store them in a `Block`. For example, this is the type used in +the Java code that implements functions that produce or consume this `Type`. + +## Native encoding + +The interpretation of a value in its native container type form is defined by its +`Type`. For some types, such as `BigintType`, it matches the Java +interpretation of the native container type (64bit 2's complement). However, for other +types such as `TimestampWithTimeZoneType`, which also uses `long` for its +native container type, the value stored in the `long` is a 8byte binary value +combining the timezone and the milliseconds since the unix epoch. In particular, +this means that you cannot compare two native values and expect a meaningful +result, without knowing the native encoding. + +## Type signature + +The signature of a type defines its identity, and also encodes some general +information about the type, such as its type parameters (if it's parametric), +and its literal parameters. The literal parameters are used in types like +`VARCHAR(10)`. diff --git a/docs/src/main/sphinx/develop/types.rst b/docs/src/main/sphinx/develop/types.rst deleted file mode 100644 index 77add5472cf0..000000000000 --- a/docs/src/main/sphinx/develop/types.rst +++ /dev/null @@ -1,41 +0,0 @@ -===== -Types -===== - -The ``Type`` interface in Trino is used to implement a type in the SQL language. -Trino ships with a number of built-in types, like ``VarcharType`` and ``BigintType``. -The ``ParametricType`` interface is used to provide type parameters for types, to -allow types like ``VARCHAR(10)`` or ``DECIMAL(22, 5)``. A ``Plugin`` can provide -new ``Type`` objects by returning them from ``getTypes()`` and new ``ParametricType`` -objects by returning them from ``getParametricTypes()``. - -Below is a high level overview of the ``Type`` interface. For more details, see the -JavaDocs for ``Type``. - -Native container type ----------------------- - -All types define the ``getJavaType()`` method, frequently referred to as the -"native container type". This is the Java type used to hold values during execution -and to store them in a ``Block``. For example, this is the type used in -the Java code that implements functions that produce or consume this ``Type``. - -Native encoding ---------------- - -The interpretation of a value in its native container type form is defined by its -``Type``. For some types, such as ``BigintType``, it matches the Java -interpretation of the native container type (64bit 2's complement). However, for other -types such as ``TimestampWithTimeZoneType``, which also uses ``long`` for its -native container type, the value stored in the ``long`` is a 8byte binary value -combining the timezone and the milliseconds since the unix epoch. In particular, -this means that you cannot compare two native values and expect a meaningful -result, without knowing the native encoding. - -Type signature --------------- - -The signature of a type defines its identity, and also encodes some general -information about the type, such as its type parameters (if it's parametric), -and its literal parameters. The literal parameters are used in types like -``VARCHAR(10)``. diff --git a/docs/src/main/sphinx/functions.md b/docs/src/main/sphinx/functions.md new file mode 100644 index 000000000000..91885e47236f --- /dev/null +++ b/docs/src/main/sphinx/functions.md @@ -0,0 +1,52 @@ +# Functions and operators + +This section describes the built-in SQL functions and operators supported by +Trino. They allow you to implement complex capabilities and behavior of the +queries executed by Trino operating on the underlying data sources. + +If you are looking for a specific function or operator, see the {doc}`full +alphabetical list` or the {doc}`full list by +topic`. Using {doc}`SHOW FUNCTIONS +` returns a list of all available functions, including +custom functions, with all supported arguments and a short description. + +Also see the {doc}`SQL data types` +and the {doc}`SQL statement and syntax reference`. + +```{toctree} +:maxdepth: 1 + +Aggregate +Array +Binary +Bitwise +Color +Comparison +Conditional +Conversion +Date and time +Decimal +Geospatial +HyperLogLog +IP Address +JSON +Lambda +Logical +Machine learning +Map +Math +Quantile digest +Regular expression +Session +Set Digest +String +System +Table +Teradata +T-Digest +URL +UUID +Window +functions/list +functions/list-by-topic +``` diff --git a/docs/src/main/sphinx/functions.rst b/docs/src/main/sphinx/functions.rst deleted file mode 100644 index e4875630dda1..000000000000 --- a/docs/src/main/sphinx/functions.rst +++ /dev/null @@ -1,53 +0,0 @@ -*********************** -Functions and operators -*********************** - -This section describes the built-in SQL functions and operators supported by -Trino. They allow you to implement complex capabilities and behavior of the -queries executed by Trino operating on the underlying data sources. - -If you are looking for a specific function or operator, see the :doc:`full -alphabetical list` or the :doc:`full list by -topic`. Using :doc:`SHOW FUNCTIONS -` returns a list of all available functions, including -custom functions, with all supported arguments and a short description. - -Also see the :doc:`SQL data types` -and the :doc:`SQL statement and syntax reference`. - -.. toctree:: - :maxdepth: 1 - - Aggregate - Array - Binary - Bitwise - Color - Comparison - Conditional - Conversion - Date and time - Decimal - Geospatial - HyperLogLog - IP Address - JSON - Lambda - Logical - Machine learning - Map - Math - Quantile digest - Regular expression - Session - Set Digest - String - System - Table - Teradata - T-Digest - URL - UUID - Window - functions/list - functions/list-by-topic diff --git a/docs/src/main/sphinx/glossary.md b/docs/src/main/sphinx/glossary.md new file mode 100644 index 000000000000..a47d746740b1 --- /dev/null +++ b/docs/src/main/sphinx/glossary.md @@ -0,0 +1,213 @@ +# Glossary + +The glossary contains a list of key Trino terms and definitions. + +(glosscatalog)= + +Catalog + +: Catalogs define and name a configuration for connecting to a data source, + allowing users to query the connected data. Each catalog's configuration + specifies a {ref}`connector ` to define which data source + the catalog connects to. For more information about catalogs, see + {ref}`trino-concept-catalog`. + +(glosscert)= + +Certificate + +: A public key [certificate](https://wikipedia.org/wiki/Public_key_certificate) issued by a {ref}`CA + `, sometimes abbreviated as cert, that verifies the ownership of a + server's private keys. Certificate format is specified in the [X.509](https://wikipedia.org/wiki/X.509) standard. + +(glossca)= + +Certificate Authority (CA) + +: A trusted organization that signs and issues certificates. Its signatures + can be used to verify the validity of {ref}`certificates `. + +Cluster + +: A Trino cluster provides the resources to run queries against numerous data + sources. Clusters define the number of nodes, the configuration for the JVM + runtime, configured data sources, and others aspects. For more information, + see {ref}`trino-concept-cluster`. + +(glossconnector)= + +Connector + +: Translates data from a data source into Trino schemas, tables, columns, + rows, and data types. A {doc}`connector ` is specific to a data + source, and is used in {ref}`catalog ` configurations to + define what data source the catalog connects to. A connector is one of many + types of {ref}`plugins ` + +Container + +: A lightweight virtual package of software that contains libraries, binaries, + code, configuration files, and other dependencies needed to deploy an + application. A running container does not include an operating system, + instead using the operating system of the host machine. To learn more, read + read about [containers](https://kubernetes.io/docs/concepts/containers/) + in the Kubernetes documentation. + +(glossdatasource)= + +Data source + +: A system from which data is retrieved - for example, PostgreSQL or Iceberg + on S3 data. In Trino, users query data sources with {ref}`catalogs + ` that connect to each source. See + {ref}`trino-concept-data-sources` for more information. + +(glossdatavirtualization)= + +Data virtualization + +: [Data virtualization](https://wikipedia.org/wiki/Data_virtualization) is a + method of abstracting an interaction with multiple {ref}`heterogeneous data + sources `, without needing to know the distributed nature + of the data, its format, or any other technical details involved in + presenting the data. + +(glossgzip)= + +gzip + +: [gzip](https://wikipedia.org/wiki/Gzip) is a compression format and + software that compresses and decompresses files. This format is used several + ways in Trino, including deployment and compressing files in {ref}`object + storage `. The most common extension for gzip-compressed + files is `.gz`. + +(glosshdfs)= + +HDFS + +: [Hadoop Distributed Filesystem (HDFS)](https://wikipedia.org/wiki/Apache_Hadoop#HDFS) is a scalable {ref}`open + source ` filesystem that was one of the earliest + distributed big data systems created to store large amounts of data for the + [Hadoop ecosystem](https://wikipedia.org/wiki/Apache_Hadoop). + +(glossjks)= + +Java KeyStore (JKS) + +: The system of public key cryptography supported as one part of the Java + security APIs. The legacy JKS system recognizes keys and {ref}`certificates + ` stored in *keystore* files, typically with the `.jks` + extension, and by default relies on a system-level list of {ref}`CAs + ` in *truststore* files installed as part of the current Java + installation. + +Key + +: A cryptographic key specified as a pair of public and private strings + generally used in the context of {ref}`TLS ` to secure public + network traffic. + +(glosslb)= + +Load Balancer (LB) + +: Software or a hardware device that sits on a network edge and accepts + network connections on behalf of servers behind that wall, distributing + traffic across network and server infrastructure to balance the load on + networked services. + +(glossobjectstorage)= + +Object storage + +: [Object storage](https://en.wikipedia.org/wiki/Object_storage) is a file + storage mechanism. Examples of compatible object stores include the + following: + + - [Amazon S3](https://aws.amazon.com/s3) + - [Google Cloud Storage](https://cloud.google.com/storage) + - [Azure Blob Storage](https://azure.microsoft.com/en-us/products/storage/blobs) + - [MinIO](https://min.io/) and other S3-compatible stores + - {ref}`HDFS ` + +(glossopensource)= + +Open-source + +: Typically refers to [open-source software](https://wikipedia.org/wiki/Open-source_software). which is software that + has the source code made available for others to see, use, and contribute + to. Allowed usage varies depending on the license that the software is + licensed under. Trino is licensed under the [Apache license](https://wikipedia.org/wiki/Apache_License), and is therefore maintained + by a community of contributors from all across the globe. + +(glosspem)= + +PEM file format + +: A format for storing and sending cryptographic keys and certificates. PEM + format can contain both a key and its certificate, plus the chain of + certificates from authorities back to the root {ref}`CA `, or back + to a CA vendor's intermediate CA. + +(glosspkcs12)= + +PKCS #12 + +: A binary archive used to store keys and certificates or certificate chains + that validate a key. [PKCS #12](https://wikipedia.org/wiki/PKCS_12) files + have `.p12` or `.pfx` extensions. This format is a less popular + alternative to {ref}`PEM `. + +(glossplugin)= + +Plugin + +: A bundle of code implementing the Trino {doc}`Service Provider Interface + (SPI) ` that is used to add new {ref}`connectors + `, {doc}`data types `, {doc}`functions`, + {doc}`access control implementations `, and + other features of Trino. + +Presto and PrestoSQL + +: The old name for Trino. To learn more about the name change to Trino, read + [the history](). + +Query federation + +: A type of {ref}`data virtualization ` that provides a + common access point and data model across two or more heterogeneous data + sources. A popular data model used by many query federation engines is + translating different data sources to {ref}`SQL ` tables. + +(glossssl)= + +Secure Sockets Layer (SSL) + +: Now superseded by {ref}`TLS `, but still recognized as the term + for what TLS does. + +(glosssql)= + +Structured Query Language (SQL) + +: The standard language used with relational databases. For more information, + see {doc}`SQL `. + +Tarball + +: A common abbreviation for [TAR file](), which is a common software + distribution mechanism. This file format is a collection of multiple files + distributed as a single file, commonly compressed using {ref}`gzip + ` compression. + +(glosstls)= + +Transport Layer Security (TLS) + +: [TLS](https://wikipedia.org/wiki/Transport_Layer_Security) is a security + protocol designed to provide secure communications over a network. It is the + successor to {ref}`SSL `, and used in many applications like + HTTPS, email, and Trino. These security topics use the term TLS to refer to + both TLS and SSL. diff --git a/docs/src/main/sphinx/glossary.rst b/docs/src/main/sphinx/glossary.rst deleted file mode 100644 index 7a5f7ebf6ee8..000000000000 --- a/docs/src/main/sphinx/glossary.rst +++ /dev/null @@ -1,198 +0,0 @@ -======== -Glossary -======== - -The glossary contains a list of key Trino terms and definitions. - -.. _glossCatalog: - -Catalog - Catalogs define and name a configuration for connecting to a data source, - allowing users to query the connected data. Each catalog's configuration - specifies a :ref:`connector ` to define which data source - the catalog connects to. For more information about catalogs, see - :ref:`trino-concept-catalog`. - -.. _glossCert: - -Certificate - A public key `certificate - `_ issued by a :ref:`CA - `, sometimes abbreviated as cert, that verifies the ownership of a - server's private keys. Certificate format is specified in the `X.509 - `_ standard. - -.. _glossCA: - -Certificate Authority (CA) - A trusted organization that signs and issues certificates. Its signatures - can be used to verify the validity of :ref:`certificates `. - -Cluster - A Trino cluster provides the resources to run queries against numerous data - sources. Clusters define the number of nodes, the configuration for the JVM - runtime, configured data sources, and others aspects. For more information, - see :ref:`trino-concept-cluster`. - -.. _glossConnector: - -Connector - Translates data from a data source into Trino schemas, tables, columns, - rows, and data types. A :doc:`connector ` is specific to a data - source, and is used in :ref:`catalog ` configurations to - define what data source the catalog connects to. A connector is one of many - types of :ref:`plugins ` - -Container - A lightweight virtual package of software that contains libraries, binaries, - code, configuration files, and other dependencies needed to deploy an - application. A running container does not include an operating system, - instead using the operating system of the host machine. To learn more, read - read about `containers `_ - in the Kubernetes documentation. - -.. _glossDataSource: - -Data source - A system from which data is retrieved - for example, PostgreSQL or Iceberg - on S3 data. In Trino, users query data sources with :ref:`catalogs - ` that connect to each source. See - :ref:`trino-concept-data-sources` for more information. - -.. _glossDataVirtualization: - -Data virtualization - `Data virtualization `_ is a - method of abstracting an interaction with multiple :ref:`heterogeneous data - sources `, without needing to know the distributed nature - of the data, its format, or any other technical details involved in - presenting the data. - -.. _glossGzip: - -gzip - `gzip `_ is a compression format and - software that compresses and decompresses files. This format is used several - ways in Trino, including deployment and compressing files in :ref:`object - storage `. The most common extension for gzip-compressed - files is ``.gz``. - -.. _glossHDFS: - -HDFS - `Hadoop Distributed Filesystem (HDFS) - `_ is a scalable :ref:`open - source ` filesystem that was one of the earliest - distributed big data systems created to store large amounts of data for the - `Hadoop ecosystem `_. - -.. _glossJKS: - -Java KeyStore (JKS) - The system of public key cryptography supported as one part of the Java - security APIs. The legacy JKS system recognizes keys and :ref:`certificates - ` stored in *keystore* files, typically with the ``.jks`` - extension, and by default relies on a system-level list of :ref:`CAs - ` in *truststore* files installed as part of the current Java - installation. - -Key - A cryptographic key specified as a pair of public and private strings - generally used in the context of :ref:`TLS ` to secure public - network traffic. - -.. _glossLB: - -Load Balancer (LB) - Software or a hardware device that sits on a network edge and accepts - network connections on behalf of servers behind that wall, distributing - traffic across network and server infrastructure to balance the load on - networked services. - -.. _glossObjectStorage: - -Object storage - `Object storage `_ is a file - storage mechanism. Examples of compatible object stores include the - following: - - * `Amazon S3 `_ - * `Google Cloud Storage `_ - * `Azure Blob Storage `_ - * `MinIO `_ and other S3-compatible stores - * :ref:`HDFS ` - -.. _glossOpenSource: - -Open-source - Typically refers to `open-source software - `_. which is software that - has the source code made available for others to see, use, and contribute - to. Allowed usage varies depending on the license that the software is - licensed under. Trino is licensed under the `Apache license - `_, and is therefore maintained - by a community of contributors from all across the globe. - -.. _glossPEM: - -PEM file format - A format for storing and sending cryptographic keys and certificates. PEM - format can contain both a key and its certificate, plus the chain of - certificates from authorities back to the root :ref:`CA `, or back - to a CA vendor's intermediate CA. - -.. _glossPKCS12: - -PKCS #12 - A binary archive used to store keys and certificates or certificate chains - that validate a key. `PKCS #12 `_ files - have ``.p12`` or ``.pfx`` extensions. This format is a less popular - alternative to :ref:`PEM `. - -.. _glossPlugin: - -Plugin - A bundle of code implementing the Trino :doc:`Service Provider Interface - (SPI) ` that is used to add new :ref:`connectors - `, :doc:`data types `, :doc:`functions`, - :doc:`access control implementations `, and - other features of Trino. - -Presto and PrestoSQL - The old name for Trino. To learn more about the name change to Trino, read - `the history - `_. - -Query federation - A type of :ref:`data virtualization ` that provides a - common access point and data model across two or more heterogeneous data - sources. A popular data model used by many query federation engines is - translating different data sources to :ref:`SQL ` tables. - -.. _glossSSL: - -Secure Sockets Layer (SSL) - Now superseded by :ref:`TLS `, but still recognized as the term - for what TLS does. - -.. _glossSQL: - -Structured Query Language (SQL) - The standard language used with relational databases. For more information, - see :doc:`SQL `. - -Tarball - A common abbreviation for `TAR file - `_, which is a common software - distribution mechanism. This file format is a collection of multiple files - distributed as a single file, commonly compressed using :ref:`gzip - ` compression. - -.. _glossTLS: - -Transport Layer Security (TLS) - `TLS `_ is a security - protocol designed to provide secure communications over a network. It is the - successor to :ref:`SSL `, and used in many applications like - HTTPS, email, and Trino. These security topics use the term TLS to refer to - both TLS and SSL. diff --git a/docs/src/main/sphinx/index.md b/docs/src/main/sphinx/index.md new file mode 100644 index 000000000000..8c485a678dbd --- /dev/null +++ b/docs/src/main/sphinx/index.md @@ -0,0 +1,26 @@ +# Trino documentation + +```{toctree} +:titlesonly: true + +overview +installation +client +security +admin +optimizer +connector +functions +language +sql +develop +glossary +appendix +``` + +```{toctree} +:maxdepth: 1 +:titlesonly: true + +release +``` diff --git a/docs/src/main/sphinx/index.rst b/docs/src/main/sphinx/index.rst deleted file mode 100644 index 3abf7a78c0a2..000000000000 --- a/docs/src/main/sphinx/index.rst +++ /dev/null @@ -1,26 +0,0 @@ -#################### -Trino documentation -#################### - -.. toctree:: - :titlesonly: - - overview - installation - client - security - admin - optimizer - connector - functions - language - sql - develop - glossary - appendix - -.. toctree:: - :titlesonly: - :maxdepth: 1 - - release diff --git a/docs/src/main/sphinx/installation.rst b/docs/src/main/sphinx/installation.md similarity index 57% rename from docs/src/main/sphinx/installation.rst rename to docs/src/main/sphinx/installation.md index c8b6200ccf6f..01c76e095819 100644 --- a/docs/src/main/sphinx/installation.rst +++ b/docs/src/main/sphinx/installation.md @@ -1,21 +1,20 @@ -************ -Installation -************ +# Installation A Trino server can be installed and deployed on a number of different platforms. Typically you run a cluster of machines with one coordinator and many workers. You can find instructions for deploying such a cluster, and related information, in the following sections: -.. toctree:: - :maxdepth: 1 +```{toctree} +:maxdepth: 1 - installation/deployment - installation/containers - installation/kubernetes - installation/rpm - installation/query-resiliency +installation/deployment +installation/containers +installation/kubernetes +installation/rpm +installation/query-resiliency +``` Once you have a completed the deployment, or if you have access to a running -cluster already, you can proceed to configure your :doc:`client application +cluster already, you can proceed to configure your {doc}`client application `. diff --git a/docs/src/main/sphinx/language.rst b/docs/src/main/sphinx/language.md similarity index 65% rename from docs/src/main/sphinx/language.rst rename to docs/src/main/sphinx/language.md index dd1dbee576d6..15286eb208c1 100644 --- a/docs/src/main/sphinx/language.rst +++ b/docs/src/main/sphinx/language.md @@ -1,6 +1,4 @@ -************ -SQL language -************ +# SQL language Trino is an ANSI SQL compliant query engine. This standard compliance allows Trino users to integrate their favorite data tools, including BI and ETL tools @@ -12,14 +10,15 @@ operations on the connected data source. This section provides a reference to the supported SQL data types and other general characteristics of the SQL support of Trino. -A :doc:`full SQL statement and syntax reference` is +A {doc}`full SQL statement and syntax reference` is available in a separate section. -Trino also provides :doc:`numerous SQL functions and operators`. +Trino also provides {doc}`numerous SQL functions and operators`. -.. toctree:: - :maxdepth: 2 +```{toctree} +:maxdepth: 2 - language/sql-support - language/types - language/reserved +language/sql-support +language/types +language/reserved +``` diff --git a/docs/src/main/sphinx/optimizer.md b/docs/src/main/sphinx/optimizer.md new file mode 100644 index 000000000000..346d0fd82eed --- /dev/null +++ b/docs/src/main/sphinx/optimizer.md @@ -0,0 +1,10 @@ +# Query optimizer + +```{toctree} +:maxdepth: 1 + +optimizer/statistics +optimizer/cost-in-explain +optimizer/cost-based-optimizations +optimizer/pushdown +``` diff --git a/docs/src/main/sphinx/optimizer.rst b/docs/src/main/sphinx/optimizer.rst deleted file mode 100644 index e3b3bd64b319..000000000000 --- a/docs/src/main/sphinx/optimizer.rst +++ /dev/null @@ -1,11 +0,0 @@ -*************** -Query optimizer -*************** - -.. toctree:: - :maxdepth: 1 - - optimizer/statistics - optimizer/cost-in-explain - optimizer/cost-based-optimizations - optimizer/pushdown diff --git a/docs/src/main/sphinx/overview.rst b/docs/src/main/sphinx/overview.md similarity index 56% rename from docs/src/main/sphinx/overview.rst rename to docs/src/main/sphinx/overview.md index d7eb7d4ee082..4367ab6e6941 100644 --- a/docs/src/main/sphinx/overview.rst +++ b/docs/src/main/sphinx/overview.md @@ -1,12 +1,11 @@ -******** -Overview -******** +# Overview Trino is a distributed SQL query engine designed to query large data sets distributed over one or more heterogeneous data sources. -.. toctree:: - :maxdepth: 1 +```{toctree} +:maxdepth: 1 - overview/use-cases - overview/concepts +overview/use-cases +overview/concepts +``` diff --git a/docs/src/main/sphinx/overview/concepts.rst b/docs/src/main/sphinx/overview/concepts.md similarity index 84% rename from docs/src/main/sphinx/overview/concepts.rst rename to docs/src/main/sphinx/overview/concepts.md index f82fc8c565a2..b81439e07c00 100644 --- a/docs/src/main/sphinx/overview/concepts.rst +++ b/docs/src/main/sphinx/overview/concepts.md @@ -1,9 +1,6 @@ -============== -Trino concepts -============== +# Trino concepts -Overview --------- +## Overview To understand Trino, you must first understand the terms and concepts used throughout the Trino documentation. @@ -19,46 +16,41 @@ This section provides a solid definition for the core concepts referenced throughout Trino, and these sections are sorted from most general to most specific. -.. note:: +:::{note} +The book [Trino: The Definitive Guide](https://trino.io/trino-the-definitive-guide.html) and the research +paper [Presto: SQL on Everything](https://trino.io/paper.html) can +provide further information about Trino and the concepts in use. +::: - The book `Trino: The Definitive Guide - `_ and the research - paper `Presto: SQL on Everything `_ can - provide further information about Trino and the concepts in use. +(trino-concept-architecture)= - -.. _trino-concept-architecture: - -Architecture ------------- +## Architecture Trino is a distributed query engine that processes data in parallel across multiple servers. There are two types of Trino servers, -:ref:`coordinators ` and -:ref:`workers `. The following sections describe these +{ref}`coordinators ` and +{ref}`workers `. The following sections describe these servers and other components of Trino's architecture. -.. _trino-concept-cluster: +(trino-concept-cluster)= -Cluster -^^^^^^^ +### Cluster -A Trino cluster consists of a :ref:`coordinator ` and -many :ref:`workers `. Users connect to the coordinator -with their :ref:`SQL ` query tool. The coordinator collaborates with the +A Trino cluster consists of a {ref}`coordinator ` and +many {ref}`workers `. Users connect to the coordinator +with their {ref}`SQL ` query tool. The coordinator collaborates with the workers. The coordinator and the workers access the connected -:ref:`data sources `. This access is configured in -:ref:`catalogs `. +{ref}`data sources `. This access is configured in +{ref}`catalogs `. Processing each query is a stateful operation. The workload is orchestrated by the coordinator and spread parallel across all workers in the cluster. Each node runs Trino in one JVM instance, and processing is parallelized further using threads. -.. _trino-concept-coordinator: +(trino-concept-coordinator)= -Coordinator -^^^^^^^^^^^ +### Coordinator The Trino coordinator is the server that is responsible for parsing statements, planning queries, and managing Trino worker nodes. It is @@ -76,10 +68,9 @@ Trino workers. Coordinators communicate with workers and clients using a REST API. -.. _trino-concept-worker: +(trino-concept-worker)= -Worker -^^^^^^ +### Worker A Trino worker is a server in a Trino installation, which is responsible for executing tasks and processing data. Worker nodes fetch data from @@ -94,36 +85,34 @@ for task execution. Workers communicate with other workers and Trino coordinators using a REST API. -.. _trino-concept-data-sources: +(trino-concept-data-sources)= -Data sources ------------- +## Data sources Throughout this documentation, you'll read terms such as connector, catalog, schema, and table. These fundamental concepts cover Trino's model of a particular data source and are described in the following section. -Connector -^^^^^^^^^ +### Connector A connector adapts Trino to a data source such as Hive or a relational database. You can think of a connector the same way you think of a driver for a database. It is an implementation of Trino's -:doc:`SPI `, which allows Trino to interact +{doc}`SPI `, which allows Trino to interact with a resource using a standard API. Trino contains several built-in connectors: a connector for -:doc:`JMX `, a :doc:`System ` +{doc}`JMX `, a {doc}`System ` connector which provides access to built-in system tables, -a :doc:`Hive ` connector, and a -:doc:`TPCH ` connector designed to serve TPC-H benchmark +a {doc}`Hive ` connector, and a +{doc}`TPCH ` connector designed to serve TPC-H benchmark data. Many third-party developers have contributed connectors so that Trino can access data in a variety of data sources. Every catalog is associated with a specific connector. If you examine a catalog configuration file, you see that each contains a -mandatory property ``connector.name``, which is used by the catalog +mandatory property `connector.name`, which is used by the catalog manager to create a connector for a given catalog. It is possible to have more than one catalog use the same connector to access two different instances of a similar database. For example, if you have @@ -131,10 +120,9 @@ two Hive clusters, you can configure two catalogs in a single Trino cluster that both use the Hive connector, allowing you to query data from both Hive clusters, even within the same SQL query. -.. _trino-concept-catalog: +(trino-concept-catalog)= -Catalog -^^^^^^^ +### Catalog A Trino catalog contains schemas and references a data source via a connector. For example, you can configure a JMX catalog to provide @@ -145,14 +133,13 @@ Hive data source. When addressing a table in Trino, the fully-qualified table name is always rooted in a catalog. For example, a fully-qualified table name -of ``hive.test_data.test`` refers to the ``test`` table in the -``test_data`` schema in the ``hive`` catalog. +of `hive.test_data.test` refers to the `test` table in the +`test_data` schema in the `hive` catalog. Catalogs are defined in properties files stored in the Trino configuration directory. -Schema -^^^^^^ +### Schema Schemas are a way to organize tables. Together, a catalog and schema define a set of tables that can be queried. When accessing Hive or a @@ -161,21 +148,18 @@ the same concept in the target database. Other types of connectors may choose to organize tables into schemas in a way that makes sense for the underlying data source. -Table -^^^^^ +### Table A table is a set of unordered rows, which are organized into named columns with types. This is the same as in any relational database. The mapping from source data to tables is defined by the connector. -Query execution model ---------------------- +## Query execution model Trino executes SQL statements and turns these statements into queries, that are executed across a distributed cluster of coordinator and workers. -Statement -^^^^^^^^^ +### Statement Trino executes ANSI-compatible SQL statements. When the Trino documentation refers to a statement, it is referring to statements as @@ -189,8 +173,7 @@ in SQL. When a statement is executed, Trino creates a query along with a query plan that is then distributed across a series of Trino workers. -Query -^^^^^ +### Query When Trino parses a statement, it converts it into a query and creates a distributed query plan, which is then realized as a series of @@ -206,10 +189,9 @@ that statement. A query encompasses stages, tasks, splits, connectors, and other components and data sources working in concert to produce a result. -.. _trino-concept-stage: +(trino-concept-stage)= -Stage -^^^^^ +### Stage When Trino executes a query, it does so by breaking up the execution into a hierarchy of stages. For example, if Trino needs to aggregate @@ -224,10 +206,9 @@ the output from other stages. Stages are what the coordinator uses to model a distributed query plan, but stages themselves don't run on Trino workers. -.. _trino-concept-task: +(trino-concept-task)= -Task -^^^^ +### Task As mentioned in the previous section, stages model a particular section of a distributed query plan, but stages themselves don't @@ -242,10 +223,9 @@ task has inputs and outputs, and just as a stage can be executed in parallel by a series of tasks, a task is executing in parallel with a series of drivers. -.. _trino-concept-splits: +(trino-concept-splits)= -Split -^^^^^ +### Split Tasks operate on splits, which are sections of a larger data set. Stages at the lowest level of a distributed query plan retrieve @@ -257,8 +237,7 @@ connector for a list of all splits that are available for a table. The coordinator keeps track of which machines are running which tasks, and what splits are being processed by which tasks. -Driver -^^^^^^ +### Driver Tasks contain one or more parallel drivers. Drivers act upon data and combine operators to produce output that is then aggregated by a task @@ -268,16 +247,14 @@ physical set of operators in memory. It is the lowest level of parallelism in the Trino architecture. A driver has one input and one output. -Operator -^^^^^^^^ +### Operator An operator consumes, transforms and produces data. For example, a table scan fetches data from a connector and produces data that can be consumed by other operators, and a filter operator consumes data and produces a subset by applying a predicate over the input data. -Exchange -^^^^^^^^ +### Exchange Exchanges transfer data between Trino nodes for different stages of a query. Tasks produce data into an output buffer and consume data diff --git a/docs/src/main/sphinx/overview/use-cases.rst b/docs/src/main/sphinx/overview/use-cases.md similarity index 91% rename from docs/src/main/sphinx/overview/use-cases.rst rename to docs/src/main/sphinx/overview/use-cases.md index 47cb4bf4818e..9ab8422a5824 100644 --- a/docs/src/main/sphinx/overview/use-cases.rst +++ b/docs/src/main/sphinx/overview/use-cases.md @@ -1,13 +1,9 @@ -========= -Use cases -========= +# Use cases This section puts Trino into perspective, so that prospective administrators and end users know what to expect from Trino. ------------------ -What Trino is not ------------------ +## What Trino is not Since Trino is being called a *database* by many members of the community, it makes sense to begin with a definition of what Trino is not. @@ -19,9 +15,7 @@ PostgreSQL or Oracle. Trino was not designed to handle Online Transaction Processing (OLTP). This is also true for many other databases designed and optimized for data warehousing or analytics. -------------- -What Trino is -------------- +## What Trino is Trino is a tool designed to efficiently query vast amounts of data using distributed queries. If you work with terabytes or petabytes of diff --git a/docs/src/main/sphinx/release.md b/docs/src/main/sphinx/release.md new file mode 100644 index 000000000000..77721302c739 --- /dev/null +++ b/docs/src/main/sphinx/release.md @@ -0,0 +1,345 @@ +# Release notes + +(releases-2023)= + +## 2023 + +```{toctree} +:maxdepth: 1 + +release/release-418 +release/release-417 +release/release-416 +release/release-415 +release/release-414 +release/release-413 +release/release-412 +release/release-411 +release/release-410 +release/release-409 +release/release-408 +release/release-407 +release/release-406 +``` + +(releases-2022)= + +## 2022 + +```{toctree} +:maxdepth: 1 + +release/release-405 +release/release-404 +release/release-403 +release/release-402 +release/release-401 +release/release-400 +release/release-399 +release/release-398 +release/release-397 +release/release-396 +release/release-395 +release/release-394 +release/release-393 +release/release-392 +release/release-391 +release/release-390 +release/release-389 +release/release-388 +release/release-387 +release/release-386 +release/release-385 +release/release-384 +release/release-383 +release/release-382 +release/release-381 +release/release-380 +release/release-379 +release/release-378 +release/release-377 +release/release-376 +release/release-375 +release/release-374 +release/release-373 +release/release-372 +release/release-371 +release/release-370 +release/release-369 +release/release-368 +``` + +(releases-2021)= + +## 2021 + +```{toctree} +:maxdepth: 1 + +release/release-367 +release/release-366 +release/release-365 +release/release-364 +release/release-363 +release/release-362 +release/release-361 +release/release-360 +release/release-359 +release/release-358 +release/release-357 +release/release-356 +release/release-355 +release/release-354 +release/release-353 +release/release-352 +release/release-351 +``` + +(releases-2020)= + +## 2020 + +```{toctree} +:maxdepth: 1 + +release/release-350 +release/release-349 +release/release-348 +release/release-347 +release/release-346 +release/release-345 +release/release-344 +release/release-343 +release/release-342 +release/release-341 +release/release-340 +release/release-339 +release/release-338 +release/release-337 +release/release-336 +release/release-335 +release/release-334 +release/release-333 +release/release-332 +release/release-331 +release/release-330 +release/release-329 +release/release-328 +``` + +(releases-2019)= + +## 2019 + +```{toctree} +:maxdepth: 1 + +release/release-327 +release/release-326 +release/release-325 +release/release-324 +release/release-323 +release/release-322 +release/release-321 +release/release-320 +release/release-319 +release/release-318 +release/release-317 +release/release-316 +release/release-315 +release/release-314 +release/release-313 +release/release-312 +release/release-311 +release/release-310 +release/release-309 +release/release-308 +release/release-307 +release/release-306 +release/release-305 +release/release-304 +release/release-303 +release/release-302 +release/release-301 +release/release-300 +``` + +## Before 2019 + +```{toctree} +:maxdepth: 1 + +release/release-0.215 +release/release-0.214 +release/release-0.213 +release/release-0.212 +release/release-0.211 +release/release-0.210 +release/release-0.209 +release/release-0.208 +release/release-0.207 +release/release-0.206 +release/release-0.205 +release/release-0.204 +release/release-0.203 +release/release-0.202 +release/release-0.201 +release/release-0.200 +release/release-0.199 +release/release-0.198 +release/release-0.197 +release/release-0.196 +release/release-0.195 +release/release-0.194 +release/release-0.193 +release/release-0.192 +release/release-0.191 +release/release-0.190 +release/release-0.189 +release/release-0.188 +release/release-0.187 +release/release-0.186 +release/release-0.185 +release/release-0.184 +release/release-0.183 +release/release-0.182 +release/release-0.181 +release/release-0.180 +release/release-0.179 +release/release-0.178 +release/release-0.177 +release/release-0.176 +release/release-0.175 +release/release-0.174 +release/release-0.173 +release/release-0.172 +release/release-0.171 +release/release-0.170 +release/release-0.169 +release/release-0.168 +release/release-0.167 +release/release-0.166 +release/release-0.165 +release/release-0.164 +release/release-0.163 +release/release-0.162 +release/release-0.161 +release/release-0.160 +release/release-0.159 +release/release-0.158 +release/release-0.157.1 +release/release-0.157 +release/release-0.156 +release/release-0.155 +release/release-0.154 +release/release-0.153 +release/release-0.152.3 +release/release-0.152.2 +release/release-0.152.1 +release/release-0.152 +release/release-0.151 +release/release-0.150 +release/release-0.149 +release/release-0.148 +release/release-0.147 +release/release-0.146 +release/release-0.145 +release/release-0.144.7 +release/release-0.144.6 +release/release-0.144.5 +release/release-0.144.4 +release/release-0.144.3 +release/release-0.144.2 +release/release-0.144.1 +release/release-0.144 +release/release-0.143 +release/release-0.142 +release/release-0.141 +release/release-0.140 +release/release-0.139 +release/release-0.138 +release/release-0.137 +release/release-0.136 +release/release-0.135 +release/release-0.134 +release/release-0.133 +release/release-0.132 +release/release-0.131 +release/release-0.130 +release/release-0.129 +release/release-0.128 +release/release-0.127 +release/release-0.126 +release/release-0.125 +release/release-0.124 +release/release-0.123 +release/release-0.122 +release/release-0.121 +release/release-0.120 +release/release-0.119 +release/release-0.118 +release/release-0.117 +release/release-0.116 +release/release-0.115 +release/release-0.114 +release/release-0.113 +release/release-0.112 +release/release-0.111 +release/release-0.110 +release/release-0.109 +release/release-0.108 +release/release-0.107 +release/release-0.106 +release/release-0.105 +release/release-0.104 +release/release-0.103 +release/release-0.102 +release/release-0.101 +release/release-0.100 +release/release-0.99 +release/release-0.98 +release/release-0.97 +release/release-0.96 +release/release-0.95 +release/release-0.94 +release/release-0.93 +release/release-0.92 +release/release-0.91 +release/release-0.90 +release/release-0.89 +release/release-0.88 +release/release-0.87 +release/release-0.86 +release/release-0.85 +release/release-0.84 +release/release-0.83 +release/release-0.82 +release/release-0.81 +release/release-0.80 +release/release-0.79 +release/release-0.78 +release/release-0.77 +release/release-0.76 +release/release-0.75 +release/release-0.74 +release/release-0.73 +release/release-0.72 +release/release-0.71 +release/release-0.70 +release/release-0.69 +release/release-0.68 +release/release-0.67 +release/release-0.66 +release/release-0.65 +release/release-0.64 +release/release-0.63 +release/release-0.62 +release/release-0.61 +release/release-0.60 +release/release-0.59 +release/release-0.58 +release/release-0.57 +release/release-0.56 +release/release-0.55 +release/release-0.54 +``` diff --git a/docs/src/main/sphinx/release.rst b/docs/src/main/sphinx/release.rst deleted file mode 100644 index 8c829715f258..000000000000 --- a/docs/src/main/sphinx/release.rst +++ /dev/null @@ -1,347 +0,0 @@ -************* -Release notes -************* - -.. _releases-2023: - -2023 -==== - -.. toctree:: - :maxdepth: 1 - - release/release-418 - release/release-417 - release/release-416 - release/release-415 - release/release-414 - release/release-413 - release/release-412 - release/release-411 - release/release-410 - release/release-409 - release/release-408 - release/release-407 - release/release-406 - -.. _releases-2022: - -2022 -==== - -.. toctree:: - :maxdepth: 1 - - release/release-405 - release/release-404 - release/release-403 - release/release-402 - release/release-401 - release/release-400 - release/release-399 - release/release-398 - release/release-397 - release/release-396 - release/release-395 - release/release-394 - release/release-393 - release/release-392 - release/release-391 - release/release-390 - release/release-389 - release/release-388 - release/release-387 - release/release-386 - release/release-385 - release/release-384 - release/release-383 - release/release-382 - release/release-381 - release/release-380 - release/release-379 - release/release-378 - release/release-377 - release/release-376 - release/release-375 - release/release-374 - release/release-373 - release/release-372 - release/release-371 - release/release-370 - release/release-369 - release/release-368 - -.. _releases-2021: - -2021 -==== - -.. toctree:: - :maxdepth: 1 - - release/release-367 - release/release-366 - release/release-365 - release/release-364 - release/release-363 - release/release-362 - release/release-361 - release/release-360 - release/release-359 - release/release-358 - release/release-357 - release/release-356 - release/release-355 - release/release-354 - release/release-353 - release/release-352 - release/release-351 - -.. _releases-2020: - -2020 -==== - -.. toctree:: - :maxdepth: 1 - - release/release-350 - release/release-349 - release/release-348 - release/release-347 - release/release-346 - release/release-345 - release/release-344 - release/release-343 - release/release-342 - release/release-341 - release/release-340 - release/release-339 - release/release-338 - release/release-337 - release/release-336 - release/release-335 - release/release-334 - release/release-333 - release/release-332 - release/release-331 - release/release-330 - release/release-329 - release/release-328 - -.. _releases-2019: - -2019 -==== - -.. toctree:: - :maxdepth: 1 - - release/release-327 - release/release-326 - release/release-325 - release/release-324 - release/release-323 - release/release-322 - release/release-321 - release/release-320 - release/release-319 - release/release-318 - release/release-317 - release/release-316 - release/release-315 - release/release-314 - release/release-313 - release/release-312 - release/release-311 - release/release-310 - release/release-309 - release/release-308 - release/release-307 - release/release-306 - release/release-305 - release/release-304 - release/release-303 - release/release-302 - release/release-301 - release/release-300 - -Before 2019 -=========== - -.. toctree:: - :maxdepth: 1 - - release/release-0.215 - release/release-0.214 - release/release-0.213 - release/release-0.212 - release/release-0.211 - release/release-0.210 - release/release-0.209 - release/release-0.208 - release/release-0.207 - release/release-0.206 - release/release-0.205 - release/release-0.204 - release/release-0.203 - release/release-0.202 - release/release-0.201 - release/release-0.200 - release/release-0.199 - release/release-0.198 - release/release-0.197 - release/release-0.196 - release/release-0.195 - release/release-0.194 - release/release-0.193 - release/release-0.192 - release/release-0.191 - release/release-0.190 - release/release-0.189 - release/release-0.188 - release/release-0.187 - release/release-0.186 - release/release-0.185 - release/release-0.184 - release/release-0.183 - release/release-0.182 - release/release-0.181 - release/release-0.180 - release/release-0.179 - release/release-0.178 - release/release-0.177 - release/release-0.176 - release/release-0.175 - release/release-0.174 - release/release-0.173 - release/release-0.172 - release/release-0.171 - release/release-0.170 - release/release-0.169 - release/release-0.168 - release/release-0.167 - release/release-0.166 - release/release-0.165 - release/release-0.164 - release/release-0.163 - release/release-0.162 - release/release-0.161 - release/release-0.160 - release/release-0.159 - release/release-0.158 - release/release-0.157.1 - release/release-0.157 - release/release-0.156 - release/release-0.155 - release/release-0.154 - release/release-0.153 - release/release-0.152.3 - release/release-0.152.2 - release/release-0.152.1 - release/release-0.152 - release/release-0.151 - release/release-0.150 - release/release-0.149 - release/release-0.148 - release/release-0.147 - release/release-0.146 - release/release-0.145 - release/release-0.144.7 - release/release-0.144.6 - release/release-0.144.5 - release/release-0.144.4 - release/release-0.144.3 - release/release-0.144.2 - release/release-0.144.1 - release/release-0.144 - release/release-0.143 - release/release-0.142 - release/release-0.141 - release/release-0.140 - release/release-0.139 - release/release-0.138 - release/release-0.137 - release/release-0.136 - release/release-0.135 - release/release-0.134 - release/release-0.133 - release/release-0.132 - release/release-0.131 - release/release-0.130 - release/release-0.129 - release/release-0.128 - release/release-0.127 - release/release-0.126 - release/release-0.125 - release/release-0.124 - release/release-0.123 - release/release-0.122 - release/release-0.121 - release/release-0.120 - release/release-0.119 - release/release-0.118 - release/release-0.117 - release/release-0.116 - release/release-0.115 - release/release-0.114 - release/release-0.113 - release/release-0.112 - release/release-0.111 - release/release-0.110 - release/release-0.109 - release/release-0.108 - release/release-0.107 - release/release-0.106 - release/release-0.105 - release/release-0.104 - release/release-0.103 - release/release-0.102 - release/release-0.101 - release/release-0.100 - release/release-0.99 - release/release-0.98 - release/release-0.97 - release/release-0.96 - release/release-0.95 - release/release-0.94 - release/release-0.93 - release/release-0.92 - release/release-0.91 - release/release-0.90 - release/release-0.89 - release/release-0.88 - release/release-0.87 - release/release-0.86 - release/release-0.85 - release/release-0.84 - release/release-0.83 - release/release-0.82 - release/release-0.81 - release/release-0.80 - release/release-0.79 - release/release-0.78 - release/release-0.77 - release/release-0.76 - release/release-0.75 - release/release-0.74 - release/release-0.73 - release/release-0.72 - release/release-0.71 - release/release-0.70 - release/release-0.69 - release/release-0.68 - release/release-0.67 - release/release-0.66 - release/release-0.65 - release/release-0.64 - release/release-0.63 - release/release-0.62 - release/release-0.61 - release/release-0.60 - release/release-0.59 - release/release-0.58 - release/release-0.57 - release/release-0.56 - release/release-0.55 - release/release-0.54 diff --git a/docs/src/main/sphinx/security.md b/docs/src/main/sphinx/security.md new file mode 100644 index 000000000000..5b1ecf62fb23 --- /dev/null +++ b/docs/src/main/sphinx/security.md @@ -0,0 +1,63 @@ +# Security + +## Introduction + +```{toctree} +:maxdepth: 1 + +security/overview +``` + +## Cluster access security + +```{toctree} +:maxdepth: 1 + +security/tls +security/inspect-pem +security/inspect-jks +``` + +(security-authentication)= + +## Authentication + +```{toctree} +:maxdepth: 1 + +security/authentication-types +security/password-file +security/ldap +security/salesforce +security/oauth2 +security/kerberos +security/certificate +security/jwt +``` + +## User name management + +```{toctree} +:maxdepth: 1 + +security/user-mapping +security/group-file +``` + +## Access control + +```{toctree} +:maxdepth: 1 + +security/built-in-system-access-control +security/file-system-access-control +``` + +## Security inside the cluster + +```{toctree} +:maxdepth: 1 + +security/internal-communication +security/secrets +``` diff --git a/docs/src/main/sphinx/security.rst b/docs/src/main/sphinx/security.rst deleted file mode 100644 index bbf6285e988a..000000000000 --- a/docs/src/main/sphinx/security.rst +++ /dev/null @@ -1,65 +0,0 @@ -******** -Security -******** - -Introduction -============ - -.. toctree:: - :maxdepth: 1 - - security/overview - -Cluster access security -======================= - -.. toctree:: - :maxdepth: 1 - - security/tls - security/inspect-pem - security/inspect-jks - -.. _security-authentication: - -Authentication -============== - -.. toctree:: - :maxdepth: 1 - - security/authentication-types - security/password-file - security/ldap - security/salesforce - security/oauth2 - security/kerberos - security/certificate - security/jwt - -User name management -==================== - -.. toctree:: - :maxdepth: 1 - - security/user-mapping - security/group-file - -Access control -============== - -.. toctree:: - :maxdepth: 1 - - security/built-in-system-access-control - security/file-system-access-control - -Security inside the cluster -=========================== - -.. toctree:: - :maxdepth: 1 - - security/internal-communication - security/secrets diff --git a/docs/src/main/sphinx/sql.md b/docs/src/main/sphinx/sql.md new file mode 100644 index 000000000000..a5f5ac675cfe --- /dev/null +++ b/docs/src/main/sphinx/sql.md @@ -0,0 +1,76 @@ +# SQL statement syntax + +This section describes the SQL syntax used in Trino. + +A {doc}`reference to the supported SQL data types` is available. + +Trino also provides {doc}`numerous SQL functions and operators`. + +```{toctree} +:maxdepth: 1 + +sql/alter-materialized-view +sql/alter-schema +sql/alter-table +sql/alter-view +sql/analyze +sql/call +sql/comment +sql/commit +sql/create-materialized-view +sql/create-role +sql/create-schema +sql/create-table +sql/create-table-as +sql/create-view +sql/deallocate-prepare +sql/delete +sql/deny +sql/describe +sql/describe-input +sql/describe-output +sql/drop-materialized-view +sql/drop-role +sql/drop-schema +sql/drop-table +sql/drop-view +sql/execute +sql/execute-immediate +sql/explain +sql/explain-analyze +sql/grant +sql/grant-roles +sql/insert +sql/match-recognize +sql/merge +sql/pattern-recognition-in-window +sql/prepare +sql/refresh-materialized-view +sql/reset-session +sql/revoke +sql/revoke-roles +sql/rollback +sql/select +sql/set-role +sql/set-session +sql/set-time-zone +sql/show-catalogs +sql/show-columns +sql/show-create-materialized-view +sql/show-create-schema +sql/show-create-table +sql/show-create-view +sql/show-functions +sql/show-grants +sql/show-role-grants +sql/show-roles +sql/show-schemas +sql/show-session +sql/show-stats +sql/show-tables +sql/start-transaction +sql/truncate +sql/update +sql/use +sql/values +``` diff --git a/docs/src/main/sphinx/sql.rst b/docs/src/main/sphinx/sql.rst deleted file mode 100644 index ef3ab367fe4e..000000000000 --- a/docs/src/main/sphinx/sql.rst +++ /dev/null @@ -1,77 +0,0 @@ -******************** -SQL statement syntax -******************** - -This section describes the SQL syntax used in Trino. - -A :doc:`reference to the supported SQL data types` is available. - -Trino also provides :doc:`numerous SQL functions and operators`. - -.. toctree:: - :maxdepth: 1 - - sql/alter-materialized-view - sql/alter-schema - sql/alter-table - sql/alter-view - sql/analyze - sql/call - sql/comment - sql/commit - sql/create-materialized-view - sql/create-role - sql/create-schema - sql/create-table - sql/create-table-as - sql/create-view - sql/deallocate-prepare - sql/delete - sql/deny - sql/describe - sql/describe-input - sql/describe-output - sql/drop-materialized-view - sql/drop-role - sql/drop-schema - sql/drop-table - sql/drop-view - sql/execute - sql/execute-immediate - sql/explain - sql/explain-analyze - sql/grant - sql/grant-roles - sql/insert - sql/match-recognize - sql/merge - sql/pattern-recognition-in-window - sql/prepare - sql/refresh-materialized-view - sql/reset-session - sql/revoke - sql/revoke-roles - sql/rollback - sql/select - sql/set-role - sql/set-session - sql/set-time-zone - sql/show-catalogs - sql/show-columns - sql/show-create-materialized-view - sql/show-create-schema - sql/show-create-table - sql/show-create-view - sql/show-functions - sql/show-grants - sql/show-role-grants - sql/show-roles - sql/show-schemas - sql/show-session - sql/show-stats - sql/show-tables - sql/start-transaction - sql/truncate - sql/update - sql/use - sql/values