diff --git a/.github/workflows/github_actions.yml b/.github/workflows/github_actions.yml index 2c9f063..82e83f9 100644 --- a/.github/workflows/github_actions.yml +++ b/.github/workflows/github_actions.yml @@ -14,7 +14,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ ubuntu-18.04, macos-10.15 ] + os: [ ubuntu-latest, macos-10.15 ] steps: - uses: actions/checkout@v2 diff --git a/README.md b/README.md index 22e5198..f460ffb 100644 --- a/README.md +++ b/README.md @@ -138,46 +138,6 @@ See [docs/Limitations.md](docs/Limitations.md). When a multi-dimensional array is queried in Presto, the dimensions are converted to table columns for the result set. TileDB array attributes attributes are also returned as columns. -### Dense Arrays - -Consider the following example 2D `4x2` dense array with `dim1` and `dim2` -as the dimensions and a single attribute `a`: - -``` -+-------+-------+ -| | | -| a:1 | a:2 | -| | | -+---------------+ -| | | -| a:3 | a:4 | -| | | -+---------------+ -| | | -| a:5 | a:6 | -| | | -+---------------+ -| | | -| a:7 | a:8 | -| | | -+-------+-------+ -```` - -When queried via Presto the results are mapped to the following table: - -``` - dim1 | dim2 | a -------+------+--- - 1 | 1 | 1 - 1 | 2 | 2 - 2 | 1 | 3 - 2 | 2 | 4 - 3 | 1 | 5 - 3 | 2 | 6 - 4 | 1 | 7 - 4 | 2 | 8 -``` - ### Sparse Arrays A sparse array is materialized similarly to dense arrays. The following example diff --git a/pom.xml b/pom.xml index 2821b56..58c967a 100644 --- a/pom.xml +++ b/pom.xml @@ -100,7 +100,7 @@ io.tiledb tiledb-java - 0.5.1 + 0.9.0 diff --git a/src/main/java/com/facebook/presto/plugin/tiledb/TileDBMetadata.java b/src/main/java/com/facebook/presto/plugin/tiledb/TileDBMetadata.java index 3149dfa..3b68ceb 100644 --- a/src/main/java/com/facebook/presto/plugin/tiledb/TileDBMetadata.java +++ b/src/main/java/com/facebook/presto/plugin/tiledb/TileDBMetadata.java @@ -49,7 +49,6 @@ import io.airlift.slice.Slice; import io.tiledb.java.api.Array; import io.tiledb.java.api.ArraySchema; -import io.tiledb.java.api.ArrayType; import io.tiledb.java.api.Attribute; import io.tiledb.java.api.Context; import io.tiledb.java.api.Datatype; @@ -91,7 +90,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static io.airlift.slice.Slices.utf8Slice; -import static io.tiledb.java.api.ArrayType.TILEDB_DENSE; import static io.tiledb.java.api.ArrayType.TILEDB_SPARSE; import static io.tiledb.java.api.Constants.TILEDB_VAR_NUM; import static io.tiledb.java.api.QueryType.TILEDB_READ; @@ -537,29 +535,11 @@ public TileDBOutputTableHandle beginCreateArray(ConnectorSession session, Connec encryptionKey = (String) properties.get(TileDBTableProperties.EncryptionKey); } - ArrayType arrayType; - // Get array type from table properties - String arrayTypeStr = ((String) properties.get(TileDBTableProperties.ArrayType)).toUpperCase(); - - // Set array type based on string value - if (arrayTypeStr.equals("DENSE")) { - arrayType = TILEDB_DENSE; - } - else if (arrayTypeStr.equals("SPARSE")) { - arrayType = TILEDB_SPARSE; - } - else { - throw new TileDBError("Invalid array type set, must be one of [DENSE, SPARSE]"); - } - // Create array schema - ArraySchema arraySchema = new ArraySchema(localCtx, arrayType); + ArraySchema arraySchema = new ArraySchema(localCtx, TILEDB_SPARSE); io.tiledb.java.api.Domain domain = new io.tiledb.java.api.Domain(localCtx); - // If we have a sparse array we need to set capacity - if (arrayType == TILEDB_SPARSE) { - arraySchema.setCapacity((long) properties.get(TileDBTableProperties.Capacity)); - } + arraySchema.setCapacity((long) properties.get(TileDBTableProperties.Capacity)); if (properties.containsKey(TileDBTableProperties.OffsetsFilterList)) { String filters = TileDBTableProperties.getOffsetsFilterList(properties); diff --git a/src/main/java/com/facebook/presto/plugin/tiledb/TileDBRecordCursor.java b/src/main/java/com/facebook/presto/plugin/tiledb/TileDBRecordCursor.java index a3c1838..3739a03 100644 --- a/src/main/java/com/facebook/presto/plugin/tiledb/TileDBRecordCursor.java +++ b/src/main/java/com/facebook/presto/plugin/tiledb/TileDBRecordCursor.java @@ -275,7 +275,7 @@ private void initializeQuery(TileDBSplit split) throws TileDBError for (int i = 0; i < columnHandles.size(); i++) { columnIndexLookup.put(columnHandles.get(i).getColumnName(), i); } - HashMap> estimations = new HashMap<>(); + HashMap> estimations = new HashMap<>(); String name; // Build attribute array to avoid making calls to ArraySchema.getAttribute(string) @@ -337,7 +337,7 @@ private void initializeQuery(TileDBSplit split) throws TileDBError totalNumRecordsUB = estimations.values().iterator().next().getSecond(); // Build buffers for each column (attribute) in the query. - for (Map.Entry> maxSize : estimations.entrySet()) { + for (Map.Entry> maxSize : estimations.entrySet()) { String columnName = maxSize.getKey(); // Check to see if column is in request list, if not don't set a buffer @@ -356,7 +356,7 @@ private void initializeQuery(TileDBSplit split) throws TileDBError /** * Allocates a NativeArray buffer for the given attribute and adds it to the query object. */ - private void initQueryBufferForField(String field, Pair maxBufferElements) throws TileDBError + private void initQueryBufferForField(String field, Pair maxBufferElements) throws TileDBError { Pair timer = startTimer(); boolean isAttribute = arraySchema.getAttributes().containsKey(field); diff --git a/src/test/java/com/facebook/presto/plugin/tiledb/TestTileDBQueries.java b/src/test/java/com/facebook/presto/plugin/tiledb/TestTileDBQueries.java index 066f95d..f223f19 100644 --- a/src/test/java/com/facebook/presto/plugin/tiledb/TestTileDBQueries.java +++ b/src/test/java/com/facebook/presto/plugin/tiledb/TestTileDBQueries.java @@ -115,6 +115,11 @@ public TestTileDBQueries() } } + private static QueryRunner createQueryRunner() throws Exception + { + return createTileDBQueryRunner(); + } + @AfterClass(alwaysRun = true) public final void destroy() throws TileDBError { @@ -126,11 +131,6 @@ public final void destroy() throws TileDBError } } - private static QueryRunner createQueryRunner() throws Exception - { - return createTileDBQueryRunner(); - } - @Test public void testCreate1DVector() { @@ -257,7 +257,7 @@ public void testCreateAllDataTypes() @Test public void testCreate1DVectorTinyInt() { - String arrayName = "test_create_tinyint"; + String arrayName = "test_create_table_tinyint"; // Tinyint dropArray(arrayName); create1DVectorTinyIntDimension(arrayName); @@ -459,7 +459,6 @@ public void testCreate1DVectorTimestamp() MaterializedResult desc = computeActual(format("DESC %s", arrayName)).toTestTypes(); - /* TODO: Verify that changing from timestamp(3) to timestamp is correct */ assertEquals(desc, MaterializedResult.resultBuilder(getQueryRunner().getDefaultSession(), VARCHAR, VARCHAR, VARCHAR, VARCHAR) .row("x", "timestamp", "", "Dimension") @@ -540,7 +539,6 @@ public void testCreate1DVectorYear() throws Exception createYearArray(arrayName); MaterializedResult desc = computeActual(format("DESC %s", arrayName)).toTestTypes(); - /* TODO: Verify that changing from timestamp(3) to timestamp is correct */ assertEquals(desc, MaterializedResult.resultBuilder(getQueryRunner().getDefaultSession(), VARCHAR, VARCHAR, VARCHAR, VARCHAR) .row("d1", "integer", "", "Dimension") @@ -562,7 +560,7 @@ public void testCreate1DVectorYear() throws Exception public void testCreate1DVectorBigInt() { // BigInt - String arrayName = "test_create_bigint"; + String arrayName = "test_create_table_bigint"; dropArray(arrayName); create1DVector(arrayName); @@ -615,19 +613,6 @@ public void testCreate1DVectorReal() getQueryRunner().execute(insertSql); String selectSql = format("SELECT * FROM %s ORDER BY x ASC", arrayName); - -// try { -// Array arr = new Array(new Context(), arrayName); -// Query q = new Query(arr, TILEDB_READ); -// q.setBuffer("x", new NativeArray(ctx, 1000, Datatype.TILEDB_FLOAT32)); -// q.setBuffer("a1", new NativeArray(ctx, 1000, Datatype.TILEDB_INT32)); -// q.submit(); -// System.out.println(q); -// } -// catch (Exception e) { -// System.out.println(); -// } - MaterializedResult selectResult = computeActual(selectSql); assertEquals(selectResult, MaterializedResult.resultBuilder(getQueryRunner().getDefaultSession(), REAL, INTEGER) .row((float) 0.0, 10) @@ -724,7 +709,8 @@ public void testCreateTableWithFilters() throws TileDBError public void testCreateTableEncrypted() throws Exception { // Integer - String arrayName = "test_create_encrypted"; + String arrayName = "test_create_table_encrypted"; + dropArray(arrayName); String encryptionKey = "0123456789abcdeF0123456789abcdeF"; create1DVectorStringEncrypted(arrayName, encryptionKey); @@ -767,7 +753,7 @@ public void testCreateTableEncrypted() throws Exception public void testTimeTraveling() throws Exception { // BigInt - String arrayName = "test_create_bigint"; + String arrayName = "test_time_traveling"; dropArray(arrayName); create1DVector(arrayName); @@ -859,7 +845,7 @@ public boolean accept(File dir, String name) public void testTimeTravelingEncrypted() throws Exception { // BigInt - String arrayName = "test_create_bigint"; + String arrayName = "test_time_traveling_encrypted"; String encryptionKey = "0123456789abcdeF0123456789abcdeF"; create1DVectorEncrypted(arrayName, encryptionKey); @@ -965,6 +951,7 @@ public boolean accept(File dir, String name) public void testInsert() { String arrayName = "test_insert"; + dropArray(arrayName); create1DVector(arrayName); String insertSql = format("INSERT INTO %s (x, a1) VALUES " + @@ -986,6 +973,7 @@ public void testInsert() public void testDimensionSlice() { String arrayName = "test_dim_slice"; + dropArray(arrayName); create1DVector(arrayName); String insertSql = format("INSERT INTO %s (x, a1) VALUES " + @@ -1486,11 +1474,11 @@ public void testWrite1DVectorNullableSparse() } @Test - public void testDenseWriteReadNullable() + public void testWriteReadNullable() { - String arrayName = "test_dense_write_read"; + String arrayName = "test_write_read"; dropArray(arrayName); - create1DVectorDense(arrayName); + create1DVectorNullable(arrayName); String insertSql = format("INSERT INTO %s (x, a1) VALUES " + "(1, 10), (2, 13), (3, null), (4, 15), (5, 19), (6, 10), (7, 5), (8, 1), (9, 7) ", arrayName); @@ -1561,34 +1549,34 @@ private void create1DVector(String arrayName) queryRunner.execute(createSql); } - private void create1D2AVector(String arrayName) + private void create1DVectorNullable(String arrayName) { QueryRunner queryRunner = getQueryRunner(); String createSql = format("CREATE TABLE %s(" + "x bigint WITH (dimension=true), " + - "a1 integer, " + - "a2 varchar, " + - "a3 real " + + "a1 integer WITH (nullable=true)" + ") WITH (uri='%s')", arrayName, arrayName); queryRunner.execute(createSql); } - private void create1DVectorNoAttribute(String arrayName) + private void create1D2AVector(String arrayName) { QueryRunner queryRunner = getQueryRunner(); String createSql = format("CREATE TABLE %s(" + - "x bigint WITH (dimension=true) " + + "x bigint WITH (dimension=true), " + + "a1 integer, " + + "a2 varchar, " + + "a3 real " + ") WITH (uri='%s')", arrayName, arrayName); queryRunner.execute(createSql); } - private void create1DVectorDense(String arrayName) + private void create1DVectorNoAttribute(String arrayName) { QueryRunner queryRunner = getQueryRunner(); String createSql = format("CREATE TABLE %s(" + - "x integer WITH (dimension=true, lower_bound=1, upper_bound=9, extent=2), " + - "a1 integer WITH (nullable=true)" + - ") WITH (uri='%s', type='DENSE')", arrayName, arrayName); + "x bigint WITH (dimension=true) " + + ") WITH (uri='%s')", arrayName, arrayName); queryRunner.execute(createSql); }