From 11680a0861293b710fdc7c4771a5a4f4675e423a Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Sat, 27 Jul 2024 04:39:32 -0600 Subject: [PATCH] Upgrade protobuf definitions to flightsql 17.0 (#6133) * Update FlightSql.proto to version 17.0 Adds new message CommandStatementIngest and removes `experimental` from other messages. * Regenerate flight sql protocol This upgrades the file to version 17.0 of the protobuf definition. --- .../src/sql/arrow.flight.protocol.sql.rs | 177 +++++++++++++++++- format/FlightSql.proto | 112 +++++++---- 2 files changed, 252 insertions(+), 37 deletions(-) diff --git a/arrow-flight/src/sql/arrow.flight.protocol.sql.rs b/arrow-flight/src/sql/arrow.flight.protocol.sql.rs index 5e6f198df75c..3eeed6ff4b12 100644 --- a/arrow-flight/src/sql/arrow.flight.protocol.sql.rs +++ b/arrow-flight/src/sql/arrow.flight.protocol.sql.rs @@ -798,9 +798,157 @@ pub struct CommandPreparedStatementUpdate { pub prepared_statement_handle: ::prost::bytes::Bytes, } /// -/// Returned from the RPC call DoPut when a CommandStatementUpdate -/// CommandPreparedStatementUpdate was in the request, containing -/// results from the update. +/// Represents a bulk ingestion request. Used in the command member of FlightDescriptor +/// for the the RPC call DoPut to cause the server load the contents of the stream's +/// FlightData into the target destination. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CommandStatementIngest { + /// The behavior for handling the table definition. + #[prost(message, optional, tag = "1")] + pub table_definition_options: ::core::option::Option< + command_statement_ingest::TableDefinitionOptions, + >, + /// The table to load data into. + #[prost(string, tag = "2")] + pub table: ::prost::alloc::string::String, + /// The db_schema of the destination table to load data into. If unset, a backend-specific default may be used. + #[prost(string, optional, tag = "3")] + pub schema: ::core::option::Option<::prost::alloc::string::String>, + /// The catalog of the destination table to load data into. If unset, a backend-specific default may be used. + #[prost(string, optional, tag = "4")] + pub catalog: ::core::option::Option<::prost::alloc::string::String>, + /// + /// Store ingested data in a temporary table. + /// The effect of setting temporary is to place the table in a backend-defined namespace, and to drop the table at the end of the session. + /// The namespacing may make use of a backend-specific schema and/or catalog. + /// The server should return an error if an explicit choice of schema or catalog is incompatible with the server's namespacing decision. + #[prost(bool, tag = "5")] + pub temporary: bool, + /// Perform the ingestion as part of this transaction. If specified, results should not be committed in the event of an error/cancellation. + #[prost(bytes = "bytes", optional, tag = "6")] + pub transaction_id: ::core::option::Option<::prost::bytes::Bytes>, + /// Backend-specific options. + #[prost(map = "string, string", tag = "1000")] + pub options: ::std::collections::HashMap< + ::prost::alloc::string::String, + ::prost::alloc::string::String, + >, +} +/// Nested message and enum types in `CommandStatementIngest`. +pub mod command_statement_ingest { + /// Options for table definition behavior + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, Copy, PartialEq, ::prost::Message)] + pub struct TableDefinitionOptions { + #[prost( + enumeration = "table_definition_options::TableNotExistOption", + tag = "1" + )] + pub if_not_exist: i32, + #[prost(enumeration = "table_definition_options::TableExistsOption", tag = "2")] + pub if_exists: i32, + } + /// Nested message and enum types in `TableDefinitionOptions`. + pub mod table_definition_options { + /// The action to take if the target table does not exist + #[derive( + Clone, + Copy, + Debug, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + ::prost::Enumeration + )] + #[repr(i32)] + pub enum TableNotExistOption { + /// Do not use. Servers should error if this is specified by a client. + Unspecified = 0, + /// Create the table if it does not exist + Create = 1, + /// Fail if the table does not exist + Fail = 2, + } + impl TableNotExistOption { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + TableNotExistOption::Unspecified => { + "TABLE_NOT_EXIST_OPTION_UNSPECIFIED" + } + TableNotExistOption::Create => "TABLE_NOT_EXIST_OPTION_CREATE", + TableNotExistOption::Fail => "TABLE_NOT_EXIST_OPTION_FAIL", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "TABLE_NOT_EXIST_OPTION_UNSPECIFIED" => Some(Self::Unspecified), + "TABLE_NOT_EXIST_OPTION_CREATE" => Some(Self::Create), + "TABLE_NOT_EXIST_OPTION_FAIL" => Some(Self::Fail), + _ => None, + } + } + } + /// The action to take if the target table already exists + #[derive( + Clone, + Copy, + Debug, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + ::prost::Enumeration + )] + #[repr(i32)] + pub enum TableExistsOption { + /// Do not use. Servers should error if this is specified by a client. + Unspecified = 0, + /// Fail if the table already exists + Fail = 1, + /// Append to the table if it already exists + Append = 2, + /// Drop and recreate the table if it already exists + Replace = 3, + } + impl TableExistsOption { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + TableExistsOption::Unspecified => "TABLE_EXISTS_OPTION_UNSPECIFIED", + TableExistsOption::Fail => "TABLE_EXISTS_OPTION_FAIL", + TableExistsOption::Append => "TABLE_EXISTS_OPTION_APPEND", + TableExistsOption::Replace => "TABLE_EXISTS_OPTION_REPLACE", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "TABLE_EXISTS_OPTION_UNSPECIFIED" => Some(Self::Unspecified), + "TABLE_EXISTS_OPTION_FAIL" => Some(Self::Fail), + "TABLE_EXISTS_OPTION_APPEND" => Some(Self::Append), + "TABLE_EXISTS_OPTION_REPLACE" => Some(Self::Replace), + _ => None, + } + } + } + } +} +/// +/// Returned from the RPC call DoPut when a CommandStatementUpdate, +/// CommandPreparedStatementUpdate, or CommandStatementIngest was +/// in the request, containing results from the update. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct DoPutUpdateResult { @@ -972,6 +1120,19 @@ pub enum SqlInfo { /// query cancellation (the CancelQuery action). FlightSqlServerCancel = 9, /// + /// Retrieves a boolean value indicating whether the Flight SQL Server supports executing + /// bulk ingestion. + FlightSqlServerBulkIngestion = 10, + /// + /// Retrieves a boolean value indicating whether transactions are supported for bulk ingestion. If not, invoking + /// the method commit in the context of a bulk ingestion is a noop, and the isolation level is + /// `arrow.flight.protocol.sql.SqlTransactionIsolationLevel.TRANSACTION_NONE`. + /// + /// Returns: + /// - false: if bulk ingestion transactions are unsupported; + /// - true: if bulk ingestion transactions are supported. + FlightSqlServerIngestTransactionsSupported = 11, + /// /// Retrieves an int32 indicating the timeout (in milliseconds) for prepared statement handles. /// /// If 0, there is no timeout. Servers should reset the timeout when the handle is used in a command. @@ -1542,6 +1703,10 @@ impl SqlInfo { } SqlInfo::FlightSqlServerTransaction => "FLIGHT_SQL_SERVER_TRANSACTION", SqlInfo::FlightSqlServerCancel => "FLIGHT_SQL_SERVER_CANCEL", + SqlInfo::FlightSqlServerBulkIngestion => "FLIGHT_SQL_SERVER_BULK_INGESTION", + SqlInfo::FlightSqlServerIngestTransactionsSupported => { + "FLIGHT_SQL_SERVER_INGEST_TRANSACTIONS_SUPPORTED" + } SqlInfo::FlightSqlServerStatementTimeout => { "FLIGHT_SQL_SERVER_STATEMENT_TIMEOUT" } @@ -1674,6 +1839,12 @@ impl SqlInfo { } "FLIGHT_SQL_SERVER_TRANSACTION" => Some(Self::FlightSqlServerTransaction), "FLIGHT_SQL_SERVER_CANCEL" => Some(Self::FlightSqlServerCancel), + "FLIGHT_SQL_SERVER_BULK_INGESTION" => { + Some(Self::FlightSqlServerBulkIngestion) + } + "FLIGHT_SQL_SERVER_INGEST_TRANSACTIONS_SUPPORTED" => { + Some(Self::FlightSqlServerIngestTransactionsSupported) + } "FLIGHT_SQL_SERVER_STATEMENT_TIMEOUT" => { Some(Self::FlightSqlServerStatementTimeout) } diff --git a/format/FlightSql.proto b/format/FlightSql.proto index 4fc68f2a5db0..8f9e1c8d829b 100644 --- a/format/FlightSql.proto +++ b/format/FlightSql.proto @@ -43,7 +43,6 @@ * where there is one row per requested piece of metadata information. */ message CommandGetSqlInfo { - option (experimental) = true; /* * Values are modelled after ODBC's SQLGetInfo() function. This information is intended to provide @@ -135,6 +134,23 @@ */ FLIGHT_SQL_SERVER_CANCEL = 9; + /* + * Retrieves a boolean value indicating whether the Flight SQL Server supports executing + * bulk ingestion. + */ + FLIGHT_SQL_SERVER_BULK_INGESTION = 10; + + /* + * Retrieves a boolean value indicating whether transactions are supported for bulk ingestion. If not, invoking + * the method commit in the context of a bulk ingestion is a noop, and the isolation level is + * `arrow.flight.protocol.sql.SqlTransactionIsolationLevel.TRANSACTION_NONE`. + * + * Returns: + * - false: if bulk ingestion transactions are unsupported; + * - true: if bulk ingestion transactions are supported. + */ + FLIGHT_SQL_SERVER_INGEST_TRANSACTIONS_SUPPORTED = 11; + /* * Retrieves an int32 indicating the timeout (in milliseconds) for prepared statement handles. * @@ -1114,7 +1130,6 @@ * The returned data should be ordered by data_type and then by type_name. */ message CommandGetXdbcTypeInfo { - option (experimental) = true; /* * Specifies the data type to search for the info. @@ -1136,7 +1151,6 @@ * The returned data should be ordered by catalog_name. */ message CommandGetCatalogs { - option (experimental) = true; } /* @@ -1154,7 +1168,6 @@ * The returned data should be ordered by catalog_name, then db_schema_name. */ message CommandGetDbSchemas { - option (experimental) = true; /* * Specifies the Catalog to search for the tables. @@ -1202,7 +1215,6 @@ * The returned data should be ordered by catalog_name, db_schema_name, table_name, then table_type, followed by table_schema if requested. */ message CommandGetTables { - option (experimental) = true; /* * Specifies the Catalog to search for the tables. @@ -1255,7 +1267,6 @@ * The returned data should be ordered by table_type. */ message CommandGetTableTypes { - option (experimental) = true; } /* @@ -1276,7 +1287,6 @@ * The returned data should be ordered by catalog_name, db_schema_name, table_name, key_name, then key_sequence. */ message CommandGetPrimaryKeys { - option (experimental) = true; /* * Specifies the catalog to search for the table. @@ -1331,7 +1341,6 @@ * update_rule and delete_rule returns a byte that is equivalent to actions declared on UpdateDeleteRules enum. */ message CommandGetExportedKeys { - option (experimental) = true; /* * Specifies the catalog to search for the foreign key table. @@ -1382,7 +1391,6 @@ * - 4 = SET DEFAULT */ message CommandGetImportedKeys { - option (experimental) = true; /* * Specifies the catalog to search for the primary key table. @@ -1435,7 +1443,6 @@ * - 4 = SET DEFAULT */ message CommandGetCrossReference { - option (experimental) = true; /** * The catalog name where the parent table is. @@ -1482,7 +1489,6 @@ * Request message for the "CreatePreparedStatement" action on a Flight SQL enabled backend. */ message ActionCreatePreparedStatementRequest { - option (experimental) = true; // The valid SQL string to create a prepared statement for. string query = 1; @@ -1495,7 +1501,6 @@ * An embedded message describing a Substrait plan to execute. */ message SubstraitPlan { - option (experimental) = true; // The serialized substrait.Plan to create a prepared statement for. // XXX(ARROW-16902): this is bytes instead of an embedded message @@ -1512,7 +1517,6 @@ * Request message for the "CreatePreparedSubstraitPlan" action on a Flight SQL enabled backend. */ message ActionCreatePreparedSubstraitPlanRequest { - option (experimental) = true; // The serialized substrait.Plan to create a prepared statement for. SubstraitPlan plan = 1; @@ -1531,7 +1535,6 @@ * The result should be wrapped in a google.protobuf.Any message. */ message ActionCreatePreparedStatementResult { - option (experimental) = true; // Opaque handle for the prepared statement on the server. bytes prepared_statement_handle = 1; @@ -1553,7 +1556,6 @@ * Closes server resources associated with the prepared statement handle. */ message ActionClosePreparedStatementRequest { - option (experimental) = true; // Opaque handle for the prepared statement on the server. bytes prepared_statement_handle = 1; @@ -1564,7 +1566,6 @@ * Begins a transaction. */ message ActionBeginTransactionRequest { - option (experimental) = true; } /* @@ -1575,7 +1576,6 @@ * FLIGHT_SQL_TRANSACTION_SUPPORT_SAVEPOINT. */ message ActionBeginSavepointRequest { - option (experimental) = true; // The transaction to which a savepoint belongs. bytes transaction_id = 1; @@ -1593,7 +1593,6 @@ * The result should be wrapped in a google.protobuf.Any message. */ message ActionBeginTransactionResult { - option (experimental) = true; // Opaque handle for the transaction on the server. bytes transaction_id = 1; @@ -1609,7 +1608,6 @@ * The result should be wrapped in a google.protobuf.Any message. */ message ActionBeginSavepointResult { - option (experimental) = true; // Opaque handle for the savepoint on the server. bytes savepoint_id = 1; @@ -1624,7 +1622,6 @@ * invalidated, as are all associated savepoints. */ message ActionEndTransactionRequest { - option (experimental) = true; enum EndTransaction { END_TRANSACTION_UNSPECIFIED = 0; @@ -1650,7 +1647,6 @@ * savepoints created after the current savepoint. */ message ActionEndSavepointRequest { - option (experimental) = true; enum EndSavepoint { END_SAVEPOINT_UNSPECIFIED = 0; @@ -1685,7 +1681,6 @@ * - GetFlightInfo: execute the query. */ message CommandStatementQuery { - option (experimental) = true; // The SQL syntax. string query = 1; @@ -1712,7 +1707,6 @@ * - DoPut: execute the query. */ message CommandStatementSubstraitPlan { - option (experimental) = true; // A serialized substrait.Plan SubstraitPlan plan = 1; @@ -1725,7 +1719,6 @@ * This should be used only once and treated as an opaque value, that is, clients should not attempt to parse this. */ message TicketStatementQuery { - option (experimental) = true; // Unique identifier for the instance of the statement to execute. bytes statement_handle = 1; @@ -1753,7 +1746,6 @@ * - GetFlightInfo: execute the prepared statement instance. */ message CommandPreparedStatementQuery { - option (experimental) = true; // Opaque handle for the prepared statement on the server. bytes prepared_statement_handle = 1; @@ -1764,7 +1756,6 @@ * for the RPC call DoPut to cause the server to execute the included SQL update. */ message CommandStatementUpdate { - option (experimental) = true; // The SQL syntax. string query = 1; @@ -1778,19 +1769,75 @@ * prepared statement handle as an update. */ message CommandPreparedStatementUpdate { - option (experimental) = true; // Opaque handle for the prepared statement on the server. bytes prepared_statement_handle = 1; } /* - * Returned from the RPC call DoPut when a CommandStatementUpdate - * CommandPreparedStatementUpdate was in the request, containing - * results from the update. + * Represents a bulk ingestion request. Used in the command member of FlightDescriptor + * for the the RPC call DoPut to cause the server load the contents of the stream's + * FlightData into the target destination. + */ +message CommandStatementIngest { + + // Options for table definition behavior + message TableDefinitionOptions { + // The action to take if the target table does not exist + enum TableNotExistOption { + // Do not use. Servers should error if this is specified by a client. + TABLE_NOT_EXIST_OPTION_UNSPECIFIED = 0; + // Create the table if it does not exist + TABLE_NOT_EXIST_OPTION_CREATE = 1; + // Fail if the table does not exist + TABLE_NOT_EXIST_OPTION_FAIL = 2; + } + // The action to take if the target table already exists + enum TableExistsOption { + // Do not use. Servers should error if this is specified by a client. + TABLE_EXISTS_OPTION_UNSPECIFIED = 0; + // Fail if the table already exists + TABLE_EXISTS_OPTION_FAIL = 1; + // Append to the table if it already exists + TABLE_EXISTS_OPTION_APPEND = 2; + // Drop and recreate the table if it already exists + TABLE_EXISTS_OPTION_REPLACE = 3; + } + + TableNotExistOption if_not_exist = 1; + TableExistsOption if_exists = 2; + } + + // The behavior for handling the table definition. + TableDefinitionOptions table_definition_options = 1; + // The table to load data into. + string table = 2; + // The db_schema of the destination table to load data into. If unset, a backend-specific default may be used. + optional string schema = 3; + // The catalog of the destination table to load data into. If unset, a backend-specific default may be used. + optional string catalog = 4; + /* + * Store ingested data in a temporary table. + * The effect of setting temporary is to place the table in a backend-defined namespace, and to drop the table at the end of the session. + * The namespacing may make use of a backend-specific schema and/or catalog. + * The server should return an error if an explicit choice of schema or catalog is incompatible with the server's namespacing decision. + */ + bool temporary = 5; + // Perform the ingestion as part of this transaction. If specified, results should not be committed in the event of an error/cancellation. + optional bytes transaction_id = 6; + + // Future extensions to the parameters of CommandStatementIngest should be added here, at a lower index than the generic 'options' parameter. + + // Backend-specific options. + map options = 1000; +} + +/* + * Returned from the RPC call DoPut when a CommandStatementUpdate, + * CommandPreparedStatementUpdate, or CommandStatementIngest was + * in the request, containing results from the update. */ message DoPutUpdateResult { - option (experimental) = true; // The number of records updated. A return value of -1 represents // an unknown updated record count. @@ -1804,7 +1851,6 @@ * can continue as though the fields in this message were not provided or set to sensible default values. */ message DoPutPreparedStatementResult { - option (experimental) = true; // Represents a (potentially updated) opaque handle for the prepared statement on the server. // Because the handle could potentially be updated, any previous handles for this prepared @@ -1836,7 +1882,6 @@ */ message ActionCancelQueryRequest { option deprecated = true; - option (experimental) = true; // The result of the GetFlightInfo RPC that initiated the query. // XXX(ARROW-16902): this must be a serialized FlightInfo, but is @@ -1855,7 +1900,6 @@ */ message ActionCancelQueryResult { option deprecated = true; - option (experimental) = true; enum CancelResult { // The cancellation status is unknown. Servers should avoid using