Skip to content

Commit

Permalink
Add missing metadata on Arrow schemas returned by Flight SQL's GetTab…
Browse files Browse the repository at this point in the history 
…les and query execution methods. (apache#226)

This add an auxiliary class FlightSqlColumnMetadata meant to read and write known metadata for Arrow schema fields, such as CATALOG_NAME, SCHEMA_NAME, TABLE_NAME, PRECISION, SCALE, IS_AUTO_INCREMENT, IS_CASE_SENSITIVE, IS_READ_ONLY and IS_SEARCHABLE.
  • Loading branch information
@rafael-telles @vfraga
rafael-telles authored and vfraga committed Mar 29, 2022
1 parent 3de26e5 commit 20be27f
Show file tree
Hide file tree
Showing 4 changed files with 110 additions and 42 deletions.
54 changes: 27 additions & 27 deletions format/FlightSql.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1115,15 +1115,15 @@ message CommandGetDbSchemas {
* it is serialized as an IPC message.)
* >
* Fields on table_schema may contain the following metadata:
* - ARROW:FLIGHT:SQL:CATALOG_NAME - Table's catalog name
* - ARROW:FLIGHT:SQL:DB_SCHEMA_NAME - Database schema name
* - ARROW:FLIGHT:SQL:TABLE_NAME - Table name
* - ARROW:FLIGHT:SQL:PRECISION - Column precision/size
* - ARROW:FLIGHT:SQL:SCALE - Column scale/decimal digits if applicable
* - ARROW:FLIGHT:SQL:IS_AUTO_INCREMENT - "1" indicates if the column is auto incremented, "0" otherwise.
* - ARROW:FLIGHT:SQL:IS_CASE_SENSITIVE - "1" indicates if the column is case sensitive, "0" otherwise.
* - ARROW:FLIGHT:SQL:IS_READ_ONLY - "1" indicates if the column is read only, "0" otherwise.
* - ARROW:FLIGHT:SQL:IS_SEARCHABLE - "1" indicates if the column is searchable via WHERE clause, "0" otherwise.
* - CATALOG_NAME - Table's catalog name
* - SCHEMA_NAME - Table's schema name
* - TABLE_NAME - Table name
* - PRECISION - Column precision/size
* - SCALE - Column scale/decimal digits
* - IS_AUTO_INCREMENT - "1" if column is auto incremented, "0" otherwise.
* - IS_CASE_SENSITIVE - "1" if column is case sensitive, "0" otherwise.
* - IS_READ_ONLY - "1" if column is read only, "0" otherwise.
* - IS_SEARCHABLE - "1" if column is searchable, "0" otherwise.
* The returned data should be ordered by catalog_name, db_schema_name, table_name, then table_type, followed by table_schema if requested.
*/
message CommandGetTables {
Expand Down Expand Up @@ -1454,15 +1454,15 @@ message ActionClosePreparedStatementRequest {
* for the following RPC calls:
* - GetSchema: return the Arrow schema of the query.
* Fields on this schema may contain the following metadata:
* - ARROW:FLIGHT:SQL:CATALOG_NAME - Table's catalog name
* - ARROW:FLIGHT:SQL:DB_SCHEMA_NAME - Database schema name
* - ARROW:FLIGHT:SQL:TABLE_NAME - Table name
* - ARROW:FLIGHT:SQL:PRECISION - Column precision/size
* - ARROW:FLIGHT:SQL:SCALE - Column scale/decimal digits if applicable
* - ARROW:FLIGHT:SQL:IS_AUTO_INCREMENT - "1" indicates if the column is auto incremented, "0" otherwise.
* - ARROW:FLIGHT:SQL:IS_CASE_SENSITIVE - "1" indicates if the column is case sensitive, "0" otherwise.
* - ARROW:FLIGHT:SQL:IS_READ_ONLY - "1" indicates if the column is read only, "0" otherwise.
* - ARROW:FLIGHT:SQL:IS_SEARCHABLE - "1" indicates if the column is searchable via WHERE clause, "0" otherwise.
* - CATALOG_NAME - Table's catalog name
* - SCHEMA_NAME - Table's schema name
* - TABLE_NAME - Table name
* - PRECISION - Column precision/size
* - SCALE - Column scale/decimal digits
* - IS_AUTO_INCREMENT - "1" if column is auto incremented, "0" otherwise.
* - IS_CASE_SENSITIVE - "1" if column is case sensitive, "0" otherwise.
* - IS_READ_ONLY - "1" if column is read only, "0" otherwise.
* - IS_SEARCHABLE - "1" if column is searchable, "0" otherwise.
* - GetFlightInfo: execute the query.
*/
message CommandStatementQuery {
Expand All @@ -1488,15 +1488,15 @@ message TicketStatementQuery {
* the following RPC calls:
* - GetSchema: return the Arrow schema of the query.
* Fields on this schema may contain the following metadata:
* - ARROW:FLIGHT:SQL:CATALOG_NAME - Table's catalog name
* - ARROW:FLIGHT:SQL:DB_SCHEMA_NAME - Database schema name
* - ARROW:FLIGHT:SQL:TABLE_NAME - Table name
* - ARROW:FLIGHT:SQL:PRECISION - Column precision/size
* - ARROW:FLIGHT:SQL:SCALE - Column scale/decimal digits if applicable
* - ARROW:FLIGHT:SQL:IS_AUTO_INCREMENT - "1" indicates if the column is auto incremented, "0" otherwise.
* - ARROW:FLIGHT:SQL:IS_CASE_SENSITIVE - "1" indicates if the column is case sensitive, "0" otherwise.
* - ARROW:FLIGHT:SQL:IS_READ_ONLY - "1" indicates if the column is read only, "0" otherwise.
* - ARROW:FLIGHT:SQL:IS_SEARCHABLE - "1" indicates if the column is searchable via WHERE clause, "0" otherwise.
* - CATALOG_NAME - Table's catalog name
* - SCHEMA_NAME - Table's schema name
* - TABLE_NAME - Table name
* - PRECISION - Column precision/size
* - SCALE - Column scale/decimal digits
* - IS_AUTO_INCREMENT - "1" if column is auto incremented, "0" otherwise.
* - IS_CASE_SENSITIVE - "1" if column is case sensitive, "0" otherwise.
* - IS_READ_ONLY - "1" if column is read only, "0" otherwise.
* - IS_SEARCHABLE - "1" if column is searchable, "0" otherwise.
* - DoPut: bind parameter values. All of the bound parameter sets will be executed as a single atomic execution.
* - GetFlightInfo: execute the prepared statement instance.
*/
Expand Down
70 changes: 69 additions & 1 deletion .../src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,12 @@
import java.sql.SQLException;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TimeZone;

import org.apache.arrow.driver.jdbc.utils.ConvertUtils;
import org.apache.arrow.driver.jdbc.utils.SqlTypes;
import org.apache.arrow.flight.sql.FlightSqlColumnMetadata;
import org.apache.arrow.util.AutoCloseables;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.types.pojo.Field;
Expand Down Expand Up @@ -87,6 +89,72 @@ public static ArrowFlightJdbcVectorSchemaRootResultSet fromVectorSchemaRoot(
return resultSet;
}

private static List<ColumnMetaData> convertArrowFieldsToColumnMetaDataList(
final List<Field> fields) {
return Stream.iterate(0, Math::incrementExact).limit(fields.size())
.map(index -> {
final Field field = fields.get(index);
final ArrowType.ArrowTypeID fieldTypeId = field.getType().getTypeID();

final Common.ColumnMetaData.Builder builder = Common.ColumnMetaData.newBuilder();
builder.setOrdinal(index);
builder.setColumnName(field.getName());
builder.setLabel(field.getName());

setOnColumnMetaDataBuilder(builder, field.getMetadata());

builder.setType(Common.AvaticaType.newBuilder()
.setId(SqlTypes.getSqlTypeIdFromArrowType(field.getType()))
.setName(fieldTypeId.name())
.build());

return ColumnMetaData.fromProto(builder.build());
}).collect(Collectors.toList());
}

private static void setOnColumnMetaDataBuilder(Common.ColumnMetaData.Builder builder,
Map<String, String> metadataMap) {
FlightSqlColumnMetadata columnMetadata = new FlightSqlColumnMetadata(metadataMap);
String catalogName = columnMetadata.getCatalogName();
if (catalogName != null) {
builder.setCatalogName(catalogName);
}
String schemaName = columnMetadata.getSchemaName();
if (schemaName != null) {
builder.setSchemaName(schemaName);
}
String tableName = columnMetadata.getTableName();
if (tableName != null) {
builder.setTableName(tableName);
}

Integer precision = columnMetadata.getPrecision();
if (precision != null) {
builder.setPrecision(precision);
}
Integer scale = columnMetadata.getScale();
if (scale != null) {
builder.setScale(scale);
}

Boolean isAutoIncrement = columnMetadata.isAutoIncrement();
if (isAutoIncrement != null) {
builder.setAutoIncrement(isAutoIncrement);
}
Boolean caseSensitive = columnMetadata.isCaseSensitive();
if (caseSensitive != null) {
builder.setCaseSensitive(caseSensitive);
}
Boolean readOnly = columnMetadata.isReadOnly();
if (readOnly != null) {
builder.setReadOnly(readOnly);
}
Boolean searchable = columnMetadata.isSearchable();
if (searchable != null) {
builder.setSearchable(searchable);
}
}

@Override
protected AvaticaResultSet execute() throws SQLException {
throw new RuntimeException("Can only execute with execute(VectorSchemaRoot)");
Expand Down
27 changes: 13 additions & 14 deletions .../flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlColumnMetadata.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@

package org.apache.arrow.flight.sql;

import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

Expand All @@ -42,15 +41,15 @@
*/
public class FlightSqlColumnMetadata {

private static final String CATALOG_NAME = "ARROW:FLIGHT:SQL:CATALOG_NAME";
private static final String SCHEMA_NAME = "ARROW:FLIGHT:SQL:SCHEMA_NAME";
private static final String TABLE_NAME = "ARROW:FLIGHT:SQL:TABLE_NAME";
private static final String PRECISION = "ARROW:FLIGHT:SQL:PRECISION";
private static final String SCALE = "ARROW:FLIGHT:SQL:SCALE";
private static final String IS_AUTO_INCREMENT = "ARROW:FLIGHT:SQL:IS_AUTO_INCREMENT";
private static final String IS_CASE_SENSITIVE = "ARROW:FLIGHT:SQL:IS_CASE_SENSITIVE";
private static final String IS_READ_ONLY = "ARROW:FLIGHT:SQL:IS_READ_ONLY";
private static final String IS_SEARCHABLE = "ARROW:FLIGHT:SQL:IS_SEARCHABLE";
private static final String CATALOG_NAME = "CATALOG_NAME";
private static final String SCHEMA_NAME = "SCHEMA_NAME";
private static final String TABLE_NAME = "TABLE_NAME";
private static final String PRECISION = "PRECISION";
private static final String SCALE = "SCALE";
private static final String IS_AUTO_INCREMENT = "IS_AUTO_INCREMENT";
private static final String IS_CASE_SENSITIVE = "IS_CASE_SENSITIVE";
private static final String IS_READ_ONLY = "IS_READ_ONLY";
private static final String IS_SEARCHABLE = "IS_SEARCHABLE";

private static final String BOOLEAN_TRUE_STR = "1";
private static final String BOOLEAN_FALSE_STR = "0";
Expand All @@ -61,15 +60,15 @@ public class FlightSqlColumnMetadata {
* Creates a new instance of FlightSqlColumnMetadata.
*/
public FlightSqlColumnMetadata(Map<String, String> metadataMap) {
this.metadataMap = new HashMap<>(metadataMap);
this.metadataMap = metadataMap;
}

/**
* Returns the metadata map.
* @return The metadata map.
*/
public Map<String, String> getMetadataMap() {
return Collections.unmodifiableMap(metadataMap);
return metadataMap;
}

/**
Expand Down Expand Up @@ -286,8 +285,8 @@ public FlightSqlColumnMetadata build() {
}
}

private static String booleanToString(boolean boolValue) {
return boolValue ? BOOLEAN_TRUE_STR : BOOLEAN_FALSE_STR;
private static String booleanToString(boolean isSearchable) {
return isSearchable ? BOOLEAN_TRUE_STR : BOOLEAN_FALSE_STR;
}

private static boolean stringToBoolean(String value) {
Expand Down
1 change: 1 addition & 0 deletions java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
* Utilities to work with Flight SQL semantics.
*/
public final class FlightSqlUtils {

public static final ActionType FLIGHT_SQL_CREATE_PREPARED_STATEMENT = new ActionType("CreatePreparedStatement",
"Creates a reusable prepared statement resource on the server. \n" +
"Request Message: ActionCreatePreparedStatementRequest\n" +
Expand Down

0 comments on commit 20be27f

Please sign in to comment.