Merge branch 'master' of https://github.com/TileDB-Inc/TileDB-Presto

.github/workflows/github_actions.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ ubuntu-18.04, macos-10.15 ]
+        os: [ ubuntu-latest, macos-10.15 ]
 
     steps:
       - uses: actions/checkout@v2

README.md

@@ -138,46 +138,6 @@ See [docs/Limitations.md](docs/Limitations.md).
 When a multi-dimensional array is queried in Presto, the dimensions are converted
 to table columns for the result set. TileDB array attributes attributes are also returned as columns.
 
-### Dense Arrays
-
-Consider the following example 2D `4x2` dense array with `dim1` and `dim2`
-as the dimensions and a single attribute `a`:
-
-```
-+-------+-------+
-|       |       |
-|  a:1  |  a:2  |
-|       |       |
-+---------------+
-|       |       |
-|  a:3  |  a:4  |
-|       |       |
-+---------------+
-|       |       |
-|  a:5  |  a:6  |
-|       |       |
-+---------------+
-|       |       |
-|  a:7  |  a:8  |
-|       |       |
-+-------+-------+
-````
-
-When queried via Presto the results are mapped to the following table:
-
-```
- dim1 | dim2 | a
-------+------+---
-    1 |    1 | 1
-    1 |    2 | 2
-    2 |    1 | 3
-    2 |    2 | 4
-    3 |    1 | 5
-    3 |    2 | 6
-    4 |    1 | 7
-    4 |    2 | 8
-```
-
 ### Sparse Arrays
 
 A sparse array is materialized similarly to dense arrays. The following example

pom.xml

@@ -100,7 +100,7 @@
         <dependency>
             <groupId>io.tiledb</groupId>
             <artifactId>tiledb-java</artifactId>
-            <version>0.5.1</version>
+            <version>0.9.0</version>
         </dependency>
 
         <dependency>

src/main/java/com/facebook/presto/plugin/tiledb/TileDBMetadata.java

@@ -49,7 +49,6 @@
 import io.airlift.slice.Slice;
 import io.tiledb.java.api.Array;
 import io.tiledb.java.api.ArraySchema;
-import io.tiledb.java.api.ArrayType;
 import io.tiledb.java.api.Attribute;
 import io.tiledb.java.api.Context;
 import io.tiledb.java.api.Datatype;
@@ -91,7 +90,6 @@
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkState;
 import static io.airlift.slice.Slices.utf8Slice;
-import static io.tiledb.java.api.ArrayType.TILEDB_DENSE;
 import static io.tiledb.java.api.ArrayType.TILEDB_SPARSE;
 import static io.tiledb.java.api.Constants.TILEDB_VAR_NUM;
 import static io.tiledb.java.api.QueryType.TILEDB_READ;
@@ -537,29 +535,11 @@ public TileDBOutputTableHandle beginCreateArray(ConnectorSession session, Connec
                 encryptionKey = (String) properties.get(TileDBTableProperties.EncryptionKey);
             }
 
-            ArrayType arrayType;
-            // Get array type from table properties
-            String arrayTypeStr = ((String) properties.get(TileDBTableProperties.ArrayType)).toUpperCase();
-
-            // Set array type based on string value
-            if (arrayTypeStr.equals("DENSE")) {
-                arrayType = TILEDB_DENSE;
-            }
-            else if (arrayTypeStr.equals("SPARSE")) {
-                arrayType = TILEDB_SPARSE;
-            }
-            else {
-                throw new TileDBError("Invalid array type set, must be one of [DENSE, SPARSE]");
-            }
-
             // Create array schema
-            ArraySchema arraySchema = new ArraySchema(localCtx, arrayType);
+            ArraySchema arraySchema = new ArraySchema(localCtx, TILEDB_SPARSE);
             io.tiledb.java.api.Domain domain = new io.tiledb.java.api.Domain(localCtx);
 
-            // If we have a sparse array we need to set capacity
-            if (arrayType == TILEDB_SPARSE) {
-                arraySchema.setCapacity((long) properties.get(TileDBTableProperties.Capacity));
-            }
+            arraySchema.setCapacity((long) properties.get(TileDBTableProperties.Capacity));
 
             if (properties.containsKey(TileDBTableProperties.OffsetsFilterList)) {
                 String filters = TileDBTableProperties.getOffsetsFilterList(properties);

src/main/java/com/facebook/presto/plugin/tiledb/TileDBRecordCursor.java

@@ -275,7 +275,7 @@ private void initializeQuery(TileDBSplit split) throws TileDBError
         for (int i = 0; i < columnHandles.size(); i++) {
             columnIndexLookup.put(columnHandles.get(i).getColumnName(), i);
         }
-        HashMap<String, Pair<Integer, Integer>> estimations = new HashMap<>();
+        HashMap<String, Pair<Long, Long>> estimations = new HashMap<>();
         String name;
 
         // Build attribute array to avoid making calls to ArraySchema.getAttribute(string)
@@ -337,7 +337,7 @@ private void initializeQuery(TileDBSplit split) throws TileDBError
         totalNumRecordsUB = estimations.values().iterator().next().getSecond();
 
         // Build buffers for each column (attribute) in the query.
-        for (Map.Entry<String, Pair<Integer, Integer>> maxSize : estimations.entrySet()) {
+        for (Map.Entry<String, Pair<Long, Long>> maxSize : estimations.entrySet()) {
             String columnName = maxSize.getKey();
 
             // Check to see if column is in request list, if not don't set a buffer
@@ -356,7 +356,7 @@ private void initializeQuery(TileDBSplit split) throws TileDBError
     /**
      * Allocates a NativeArray buffer for the given attribute and adds it to the query object.
      */
-    private void initQueryBufferForField(String field, Pair<Integer, Integer> maxBufferElements) throws TileDBError
+    private void initQueryBufferForField(String field, Pair<Long, Long> maxBufferElements) throws TileDBError
     {
         Pair<Long, Long> timer = startTimer();
         boolean isAttribute = arraySchema.getAttributes().containsKey(field);

src/test/java/com/facebook/presto/plugin/tiledb/TestTileDBQueries.java

@@ -115,6 +115,11 @@ public TestTileDBQueries()
         }
     }
 
+    private static QueryRunner createQueryRunner() throws Exception
+    {
+        return createTileDBQueryRunner();
+    }
+
     @AfterClass(alwaysRun = true)
     public final void destroy() throws TileDBError
     {
@@ -126,11 +131,6 @@ public final void destroy() throws TileDBError
         }
     }
 
-    private static QueryRunner createQueryRunner() throws Exception
-    {
-        return createTileDBQueryRunner();
-    }
-
     @Test
     public void testCreate1DVector()
     {
@@ -257,7 +257,7 @@ public void testCreateAllDataTypes()
     @Test
     public void testCreate1DVectorTinyInt()
     {
-        String arrayName = "test_create_tinyint";
+        String arrayName = "test_create_table_tinyint";
         // Tinyint
         dropArray(arrayName);
         create1DVectorTinyIntDimension(arrayName);
@@ -459,7 +459,6 @@ public void testCreate1DVectorTimestamp()
 
         MaterializedResult desc = computeActual(format("DESC %s", arrayName)).toTestTypes();
 
-        /* TODO: Verify that changing from timestamp(3) to timestamp is correct */
         assertEquals(desc,
                 MaterializedResult.resultBuilder(getQueryRunner().getDefaultSession(), VARCHAR, VARCHAR, VARCHAR, VARCHAR)
                         .row("x", "timestamp", "", "Dimension")
@@ -540,7 +539,6 @@ public void testCreate1DVectorYear() throws Exception
         createYearArray(arrayName);
         MaterializedResult desc = computeActual(format("DESC %s", arrayName)).toTestTypes();
 
-        /* TODO: Verify that changing from timestamp(3) to timestamp is correct */
         assertEquals(desc,
                 MaterializedResult.resultBuilder(getQueryRunner().getDefaultSession(), VARCHAR, VARCHAR, VARCHAR, VARCHAR)
                         .row("d1", "integer", "", "Dimension")
@@ -562,7 +560,7 @@ public void testCreate1DVectorYear() throws Exception
     public void testCreate1DVectorBigInt()
     {
         // BigInt
-        String arrayName = "test_create_bigint";
+        String arrayName = "test_create_table_bigint";
         dropArray(arrayName);
         create1DVector(arrayName);
 
@@ -615,19 +613,6 @@ public void testCreate1DVectorReal()
         getQueryRunner().execute(insertSql);
 
         String selectSql = format("SELECT * FROM %s ORDER BY x ASC", arrayName);
-
-//        try {
-//            Array arr = new Array(new Context(), arrayName);
-//            Query q = new Query(arr, TILEDB_READ);
-//            q.setBuffer("x", new NativeArray(ctx, 1000, Datatype.TILEDB_FLOAT32));
-//            q.setBuffer("a1", new NativeArray(ctx, 1000, Datatype.TILEDB_INT32));
-//            q.submit();
-//            System.out.println(q);
-//        }
-//        catch (Exception e) {
-//            System.out.println();
-//        }
-
         MaterializedResult selectResult = computeActual(selectSql);
         assertEquals(selectResult, MaterializedResult.resultBuilder(getQueryRunner().getDefaultSession(), REAL, INTEGER)
                 .row((float) 0.0, 10)
@@ -724,7 +709,8 @@ public void testCreateTableWithFilters() throws TileDBError
     public void testCreateTableEncrypted() throws Exception
     {
         // Integer
-        String arrayName = "test_create_encrypted";
+        String arrayName = "test_create_table_encrypted";
+        dropArray(arrayName);
         String encryptionKey = "0123456789abcdeF0123456789abcdeF";
 
         create1DVectorStringEncrypted(arrayName, encryptionKey);
@@ -767,7 +753,7 @@ public void testCreateTableEncrypted() throws Exception
     public void testTimeTraveling() throws Exception
     {
         // BigInt
-        String arrayName = "test_create_bigint";
+        String arrayName = "test_time_traveling";
         dropArray(arrayName);
         create1DVector(arrayName);
 
@@ -859,7 +845,7 @@ public boolean accept(File dir, String name)
     public void testTimeTravelingEncrypted() throws Exception
     {
         // BigInt
-        String arrayName = "test_create_bigint";
+        String arrayName = "test_time_traveling_encrypted";
         String encryptionKey = "0123456789abcdeF0123456789abcdeF";
         create1DVectorEncrypted(arrayName, encryptionKey);
 
@@ -965,6 +951,7 @@ public boolean accept(File dir, String name)
     public void testInsert()
     {
         String arrayName = "test_insert";
+        dropArray(arrayName);
         create1DVector(arrayName);
 
         String insertSql = format("INSERT INTO %s (x, a1) VALUES " +
@@ -986,6 +973,7 @@ public void testInsert()
     public void testDimensionSlice()
     {
         String arrayName = "test_dim_slice";
+        dropArray(arrayName);
         create1DVector(arrayName);
 
         String insertSql = format("INSERT INTO %s (x, a1) VALUES " +
@@ -1486,11 +1474,11 @@ public void testWrite1DVectorNullableSparse()
     }
 
     @Test
-    public void testDenseWriteReadNullable()
+    public void testWriteReadNullable()
     {
-        String arrayName = "test_dense_write_read";
+        String arrayName = "test_write_read";
         dropArray(arrayName);
-        create1DVectorDense(arrayName);
+        create1DVectorNullable(arrayName);
 
         String insertSql = format("INSERT INTO %s (x, a1) VALUES " +
                 "(1, 10), (2, 13), (3, null), (4, 15), (5, 19), (6, 10), (7, 5), (8, 1), (9, 7) ", arrayName);
@@ -1561,34 +1549,34 @@ private void create1DVector(String arrayName)
         queryRunner.execute(createSql);
     }
 
-    private void create1D2AVector(String arrayName)
+    private void create1DVectorNullable(String arrayName)
     {
         QueryRunner queryRunner = getQueryRunner();
         String createSql = format("CREATE TABLE %s(" +
                 "x bigint WITH (dimension=true), " +
-                "a1 integer, " +
-                "a2 varchar, " +
-                "a3 real " +
+                "a1 integer WITH (nullable=true)" +
                 ") WITH (uri='%s')", arrayName, arrayName);
         queryRunner.execute(createSql);
     }
 
-    private void create1DVectorNoAttribute(String arrayName)
+    private void create1D2AVector(String arrayName)
     {
         QueryRunner queryRunner = getQueryRunner();
         String createSql = format("CREATE TABLE %s(" +
-                "x bigint WITH (dimension=true) " +
+                "x bigint WITH (dimension=true), " +
+                "a1 integer, " +
+                "a2 varchar, " +
+                "a3 real " +
                 ") WITH (uri='%s')", arrayName, arrayName);
         queryRunner.execute(createSql);
     }
 
-    private void create1DVectorDense(String arrayName)
+    private void create1DVectorNoAttribute(String arrayName)
     {
         QueryRunner queryRunner = getQueryRunner();
         String createSql = format("CREATE TABLE %s(" +
-                "x integer WITH (dimension=true, lower_bound=1, upper_bound=9, extent=2), " +
-                "a1 integer  WITH (nullable=true)" +
-                ") WITH (uri='%s', type='DENSE')", arrayName, arrayName);
+                "x bigint WITH (dimension=true) " +
+                ") WITH (uri='%s')", arrayName, arrayName);
         queryRunner.execute(createSql);
     }