From 20be27f926d28d8f9918d758cb04f326e1ca66f6 Mon Sep 17 00:00:00 2001 From: Rafael Telles Date: Thu, 9 Dec 2021 17:19:40 -0300 Subject: [PATCH] Add missing metadata on Arrow schemas returned by Flight SQL's GetTables and query execution methods. (#226) This add an auxiliary class FlightSqlColumnMetadata meant to read and write known metadata for Arrow schema fields, such as CATALOG_NAME, SCHEMA_NAME, TABLE_NAME, PRECISION, SCALE, IS_AUTO_INCREMENT, IS_CASE_SENSITIVE, IS_READ_ONLY and IS_SEARCHABLE. --- format/FlightSql.proto | 54 +++++++------- ...owFlightJdbcVectorSchemaRootResultSet.java | 70 ++++++++++++++++++- .../flight/sql/FlightSqlColumnMetadata.java | 27 ++++--- .../arrow/flight/sql/FlightSqlUtils.java | 1 + 4 files changed, 110 insertions(+), 42 deletions(-) diff --git a/format/FlightSql.proto b/format/FlightSql.proto index 193a2b607169c..1bfc64788fd30 100644 --- a/format/FlightSql.proto +++ b/format/FlightSql.proto @@ -1115,15 +1115,15 @@ message CommandGetDbSchemas { * it is serialized as an IPC message.) * > * Fields on table_schema may contain the following metadata: - * - ARROW:FLIGHT:SQL:CATALOG_NAME - Table's catalog name - * - ARROW:FLIGHT:SQL:DB_SCHEMA_NAME - Database schema name - * - ARROW:FLIGHT:SQL:TABLE_NAME - Table name - * - ARROW:FLIGHT:SQL:PRECISION - Column precision/size - * - ARROW:FLIGHT:SQL:SCALE - Column scale/decimal digits if applicable - * - ARROW:FLIGHT:SQL:IS_AUTO_INCREMENT - "1" indicates if the column is auto incremented, "0" otherwise. - * - ARROW:FLIGHT:SQL:IS_CASE_SENSITIVE - "1" indicates if the column is case sensitive, "0" otherwise. - * - ARROW:FLIGHT:SQL:IS_READ_ONLY - "1" indicates if the column is read only, "0" otherwise. - * - ARROW:FLIGHT:SQL:IS_SEARCHABLE - "1" indicates if the column is searchable via WHERE clause, "0" otherwise. + * - CATALOG_NAME - Table's catalog name + * - SCHEMA_NAME - Table's schema name + * - TABLE_NAME - Table name + * - PRECISION - Column precision/size + * - SCALE - Column scale/decimal digits + * - IS_AUTO_INCREMENT - "1" if column is auto incremented, "0" otherwise. + * - IS_CASE_SENSITIVE - "1" if column is case sensitive, "0" otherwise. + * - IS_READ_ONLY - "1" if column is read only, "0" otherwise. + * - IS_SEARCHABLE - "1" if column is searchable, "0" otherwise. * The returned data should be ordered by catalog_name, db_schema_name, table_name, then table_type, followed by table_schema if requested. */ message CommandGetTables { @@ -1454,15 +1454,15 @@ message ActionClosePreparedStatementRequest { * for the following RPC calls: * - GetSchema: return the Arrow schema of the query. * Fields on this schema may contain the following metadata: - * - ARROW:FLIGHT:SQL:CATALOG_NAME - Table's catalog name - * - ARROW:FLIGHT:SQL:DB_SCHEMA_NAME - Database schema name - * - ARROW:FLIGHT:SQL:TABLE_NAME - Table name - * - ARROW:FLIGHT:SQL:PRECISION - Column precision/size - * - ARROW:FLIGHT:SQL:SCALE - Column scale/decimal digits if applicable - * - ARROW:FLIGHT:SQL:IS_AUTO_INCREMENT - "1" indicates if the column is auto incremented, "0" otherwise. - * - ARROW:FLIGHT:SQL:IS_CASE_SENSITIVE - "1" indicates if the column is case sensitive, "0" otherwise. - * - ARROW:FLIGHT:SQL:IS_READ_ONLY - "1" indicates if the column is read only, "0" otherwise. - * - ARROW:FLIGHT:SQL:IS_SEARCHABLE - "1" indicates if the column is searchable via WHERE clause, "0" otherwise. + * - CATALOG_NAME - Table's catalog name + * - SCHEMA_NAME - Table's schema name + * - TABLE_NAME - Table name + * - PRECISION - Column precision/size + * - SCALE - Column scale/decimal digits + * - IS_AUTO_INCREMENT - "1" if column is auto incremented, "0" otherwise. + * - IS_CASE_SENSITIVE - "1" if column is case sensitive, "0" otherwise. + * - IS_READ_ONLY - "1" if column is read only, "0" otherwise. + * - IS_SEARCHABLE - "1" if column is searchable, "0" otherwise. * - GetFlightInfo: execute the query. */ message CommandStatementQuery { @@ -1488,15 +1488,15 @@ message TicketStatementQuery { * the following RPC calls: * - GetSchema: return the Arrow schema of the query. * Fields on this schema may contain the following metadata: - * - ARROW:FLIGHT:SQL:CATALOG_NAME - Table's catalog name - * - ARROW:FLIGHT:SQL:DB_SCHEMA_NAME - Database schema name - * - ARROW:FLIGHT:SQL:TABLE_NAME - Table name - * - ARROW:FLIGHT:SQL:PRECISION - Column precision/size - * - ARROW:FLIGHT:SQL:SCALE - Column scale/decimal digits if applicable - * - ARROW:FLIGHT:SQL:IS_AUTO_INCREMENT - "1" indicates if the column is auto incremented, "0" otherwise. - * - ARROW:FLIGHT:SQL:IS_CASE_SENSITIVE - "1" indicates if the column is case sensitive, "0" otherwise. - * - ARROW:FLIGHT:SQL:IS_READ_ONLY - "1" indicates if the column is read only, "0" otherwise. - * - ARROW:FLIGHT:SQL:IS_SEARCHABLE - "1" indicates if the column is searchable via WHERE clause, "0" otherwise. + * - CATALOG_NAME - Table's catalog name + * - SCHEMA_NAME - Table's schema name + * - TABLE_NAME - Table name + * - PRECISION - Column precision/size + * - SCALE - Column scale/decimal digits + * - IS_AUTO_INCREMENT - "1" if column is auto incremented, "0" otherwise. + * - IS_CASE_SENSITIVE - "1" if column is case sensitive, "0" otherwise. + * - IS_READ_ONLY - "1" if column is read only, "0" otherwise. + * - IS_SEARCHABLE - "1" if column is searchable, "0" otherwise. * - DoPut: bind parameter values. All of the bound parameter sets will be executed as a single atomic execution. * - GetFlightInfo: execute the prepared statement instance. */ diff --git a/java/flight/flight-jdbc-driver/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java b/java/flight/flight-jdbc-driver/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java index 9e377e51decc9..83bc774bf8ff5 100644 --- a/java/flight/flight-jdbc-driver/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java +++ b/java/flight/flight-jdbc-driver/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java @@ -24,10 +24,12 @@ import java.sql.SQLException; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.TimeZone; -import org.apache.arrow.driver.jdbc.utils.ConvertUtils; +import org.apache.arrow.driver.jdbc.utils.SqlTypes; +import org.apache.arrow.flight.sql.FlightSqlColumnMetadata; import org.apache.arrow.util.AutoCloseables; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.types.pojo.Field; @@ -87,6 +89,72 @@ public static ArrowFlightJdbcVectorSchemaRootResultSet fromVectorSchemaRoot( return resultSet; } + private static List convertArrowFieldsToColumnMetaDataList( + final List fields) { + return Stream.iterate(0, Math::incrementExact).limit(fields.size()) + .map(index -> { + final Field field = fields.get(index); + final ArrowType.ArrowTypeID fieldTypeId = field.getType().getTypeID(); + + final Common.ColumnMetaData.Builder builder = Common.ColumnMetaData.newBuilder(); + builder.setOrdinal(index); + builder.setColumnName(field.getName()); + builder.setLabel(field.getName()); + + setOnColumnMetaDataBuilder(builder, field.getMetadata()); + + builder.setType(Common.AvaticaType.newBuilder() + .setId(SqlTypes.getSqlTypeIdFromArrowType(field.getType())) + .setName(fieldTypeId.name()) + .build()); + + return ColumnMetaData.fromProto(builder.build()); + }).collect(Collectors.toList()); + } + + private static void setOnColumnMetaDataBuilder(Common.ColumnMetaData.Builder builder, + Map metadataMap) { + FlightSqlColumnMetadata columnMetadata = new FlightSqlColumnMetadata(metadataMap); + String catalogName = columnMetadata.getCatalogName(); + if (catalogName != null) { + builder.setCatalogName(catalogName); + } + String schemaName = columnMetadata.getSchemaName(); + if (schemaName != null) { + builder.setSchemaName(schemaName); + } + String tableName = columnMetadata.getTableName(); + if (tableName != null) { + builder.setTableName(tableName); + } + + Integer precision = columnMetadata.getPrecision(); + if (precision != null) { + builder.setPrecision(precision); + } + Integer scale = columnMetadata.getScale(); + if (scale != null) { + builder.setScale(scale); + } + + Boolean isAutoIncrement = columnMetadata.isAutoIncrement(); + if (isAutoIncrement != null) { + builder.setAutoIncrement(isAutoIncrement); + } + Boolean caseSensitive = columnMetadata.isCaseSensitive(); + if (caseSensitive != null) { + builder.setCaseSensitive(caseSensitive); + } + Boolean readOnly = columnMetadata.isReadOnly(); + if (readOnly != null) { + builder.setReadOnly(readOnly); + } + Boolean searchable = columnMetadata.isSearchable(); + if (searchable != null) { + builder.setSearchable(searchable); + } + } + @Override protected AvaticaResultSet execute() throws SQLException { throw new RuntimeException("Can only execute with execute(VectorSchemaRoot)"); diff --git a/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlColumnMetadata.java b/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlColumnMetadata.java index f085e7db4ed8e..d5db2fb43491c 100644 --- a/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlColumnMetadata.java +++ b/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlColumnMetadata.java @@ -17,7 +17,6 @@ package org.apache.arrow.flight.sql; -import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -42,15 +41,15 @@ */ public class FlightSqlColumnMetadata { - private static final String CATALOG_NAME = "ARROW:FLIGHT:SQL:CATALOG_NAME"; - private static final String SCHEMA_NAME = "ARROW:FLIGHT:SQL:SCHEMA_NAME"; - private static final String TABLE_NAME = "ARROW:FLIGHT:SQL:TABLE_NAME"; - private static final String PRECISION = "ARROW:FLIGHT:SQL:PRECISION"; - private static final String SCALE = "ARROW:FLIGHT:SQL:SCALE"; - private static final String IS_AUTO_INCREMENT = "ARROW:FLIGHT:SQL:IS_AUTO_INCREMENT"; - private static final String IS_CASE_SENSITIVE = "ARROW:FLIGHT:SQL:IS_CASE_SENSITIVE"; - private static final String IS_READ_ONLY = "ARROW:FLIGHT:SQL:IS_READ_ONLY"; - private static final String IS_SEARCHABLE = "ARROW:FLIGHT:SQL:IS_SEARCHABLE"; + private static final String CATALOG_NAME = "CATALOG_NAME"; + private static final String SCHEMA_NAME = "SCHEMA_NAME"; + private static final String TABLE_NAME = "TABLE_NAME"; + private static final String PRECISION = "PRECISION"; + private static final String SCALE = "SCALE"; + private static final String IS_AUTO_INCREMENT = "IS_AUTO_INCREMENT"; + private static final String IS_CASE_SENSITIVE = "IS_CASE_SENSITIVE"; + private static final String IS_READ_ONLY = "IS_READ_ONLY"; + private static final String IS_SEARCHABLE = "IS_SEARCHABLE"; private static final String BOOLEAN_TRUE_STR = "1"; private static final String BOOLEAN_FALSE_STR = "0"; @@ -61,7 +60,7 @@ public class FlightSqlColumnMetadata { * Creates a new instance of FlightSqlColumnMetadata. */ public FlightSqlColumnMetadata(Map metadataMap) { - this.metadataMap = new HashMap<>(metadataMap); + this.metadataMap = metadataMap; } /** @@ -69,7 +68,7 @@ public FlightSqlColumnMetadata(Map metadataMap) { * @return The metadata map. */ public Map getMetadataMap() { - return Collections.unmodifiableMap(metadataMap); + return metadataMap; } /** @@ -286,8 +285,8 @@ public FlightSqlColumnMetadata build() { } } - private static String booleanToString(boolean boolValue) { - return boolValue ? BOOLEAN_TRUE_STR : BOOLEAN_FALSE_STR; + private static String booleanToString(boolean isSearchable) { + return isSearchable ? BOOLEAN_TRUE_STR : BOOLEAN_FALSE_STR; } private static boolean stringToBoolean(String value) { diff --git a/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlUtils.java b/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlUtils.java index 25affa8f08aaa..fbffb6aeedb0e 100644 --- a/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlUtils.java +++ b/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlUtils.java @@ -31,6 +31,7 @@ * Utilities to work with Flight SQL semantics. */ public final class FlightSqlUtils { + public static final ActionType FLIGHT_SQL_CREATE_PREPARED_STATEMENT = new ActionType("CreatePreparedStatement", "Creates a reusable prepared statement resource on the server. \n" + "Request Message: ActionCreatePreparedStatementRequest\n" +