Merge branch 'master' into twitter-master

pom.xml

@@ -10,7 +10,7 @@
 
     <groupId>com.facebook.presto</groupId>
     <artifactId>presto-root</artifactId>
-    <version>0.119-tw-0.1</version>
+    <version>0.122-SNAPSHOT</version>
     <packaging>pom</packaging>
 
     <name>presto-root</name>

presto-base-jdbc/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.119-tw-0.1</version>
+        <version>0.122-SNAPSHOT</version>
     </parent>
 
     <artifactId>presto-base-jdbc</artifactId>

presto-benchmark-driver/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.119-tw-0.1</version>
+        <version>0.122-SNAPSHOT</version>
     </parent>
 
     <artifactId>presto-benchmark-driver</artifactId>

presto-benchmark/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>presto-root</artifactId>
         <groupId>com.facebook.presto</groupId>
-        <version>0.119-tw-0.1</version>
+        <version>0.122-SNAPSHOT</version>
     </parent>
 
     <artifactId>presto-benchmark</artifactId>

presto-blackhole/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.119-tw-0.1</version>
+        <version>0.122-SNAPSHOT</version>
     </parent>
 
     <artifactId>presto-blackhole</artifactId>

presto-cassandra/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.119-tw-0.1</version>
+        <version>0.122-SNAPSHOT</version>
     </parent>
 
     <artifactId>presto-cassandra</artifactId>

presto-cli/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.119-tw-0.1</version>
+        <version>0.122-SNAPSHOT</version>
     </parent>
 
     <artifactId>presto-cli</artifactId>

presto-client/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.119-tw-0.1</version>
+        <version>0.122-SNAPSHOT</version>
     </parent>
 
     <artifactId>presto-client</artifactId>

presto-docs/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.119-tw-0.1</version>
+        <version>0.122-SNAPSHOT</version>
     </parent>
 
     <artifactId>presto-docs</artifactId>

presto-docs/src/main/sphinx/release.rst

@@ -5,6 +5,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.121
     release/release-0.120
     release/release-0.119
     release/release-0.118

presto-docs/src/main/sphinx/release/release-0.120.rst

@@ -1,10 +1,6 @@
 =============
 Release 0.120
 =============
+.. warning::
 
-General Changes
----------------
-
-* Fix regression that causes task scheduler to not retry requests in some cases.
-* Throttle task info refresher on errors.
-* Fix planning failure that prevented the use of large ``IN`` lists.
+   This release is broken and should not be used.

presto-docs/src/main/sphinx/release/release-0.121.rst

@@ -0,0 +1,11 @@
+=============
+Release 0.121
+=============
+
+General Changes
+---------------
+
+* Fix regression that causes task scheduler to not retry requests in some cases.
+* Throttle task info refresher on errors.
+* Fix planning failure that prevented the use of large ``IN`` lists.
+* Fix comparision of ``array<x>`` where ``x`` is a comparable, non-orderable type.

presto-example-http/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.119-tw-0.1</version>
+        <version>0.122-SNAPSHOT</version>
     </parent>
 
     <artifactId>presto-example-http</artifactId>

presto-hive-cdh4/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.119-tw-0.1</version>
+        <version>0.122-SNAPSHOT</version>
     </parent>
 
     <artifactId>presto-hive-cdh4</artifactId>

presto-hive-cdh5/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.119-tw-0.1</version>
+        <version>0.122-SNAPSHOT</version>
     </parent>
 
     <artifactId>presto-hive-cdh5</artifactId>

presto-hive-hadoop1/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.119-tw-0.1</version>
+        <version>0.122-SNAPSHOT</version>
     </parent>
 
     <artifactId>presto-hive-hadoop1</artifactId>

presto-hive-hadoop2/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.119-tw-0.1</version>
+        <version>0.122-SNAPSHOT</version>
     </parent>
 
     <artifactId>presto-hive-hadoop2</artifactId>

presto-hive/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.119-tw-0.1</version>
+        <version>0.122-SNAPSHOT</version>
     </parent>
 
     <artifactId>presto-hive</artifactId>

presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileFormats.java

@@ -29,6 +29,7 @@
 import com.facebook.presto.testing.MaterializedResult;
 import com.facebook.presto.testing.MaterializedRow;
 import com.facebook.presto.type.ArrayType;
+import com.facebook.presto.type.MapType;
 import com.facebook.presto.type.RowType;
 import com.facebook.presto.type.TypeRegistry;
 import com.google.common.base.Joiner;
@@ -176,7 +177,7 @@ public abstract class AbstractTestHiveFileFormats
                     getStandardMapObjectInspector(javaShortObjectInspector, javaShortObjectInspector),
                     ImmutableMap.of((short) 2, (short) 2),
                     mapBlockOf(BIGINT, BIGINT, 2, 2)))
-            .add(new TestColumn("t_map_null_key", getStandardMapObjectInspector(javaIntObjectInspector, javaIntObjectInspector), mapWithNullKey(), mapBlockOf(BIGINT, BIGINT, 2, 3)))
+            .add(new TestColumn("t_map_null_key", getStandardMapObjectInspector(javaIntObjectInspector, javaIntObjectInspector), asMap(null, 0, 2, 3), mapBlockOf(BIGINT, BIGINT, 2, 3)))
             .add(new TestColumn("t_map_int", getStandardMapObjectInspector(javaIntObjectInspector, javaIntObjectInspector), ImmutableMap.of(3, 3), mapBlockOf(BIGINT, BIGINT, 3, 3)))
             .add(new TestColumn("t_map_bigint", getStandardMapObjectInspector(javaLongObjectInspector, javaLongObjectInspector), ImmutableMap.of(4L, 4L), mapBlockOf(BIGINT, BIGINT, 4L, 4L)))
             .add(new TestColumn("t_map_float", getStandardMapObjectInspector(javaFloatObjectInspector, javaFloatObjectInspector), ImmutableMap.of(5.0f, 5.0f), mapBlockOf(DOUBLE, DOUBLE, 5.0f, 5.0f)))
@@ -228,6 +229,20 @@ public abstract class AbstractTestHiveFileFormats
                     mapBlockOf(VARCHAR, new ArrayType(new RowType(ImmutableList.of(BIGINT), Optional.empty())),
                             "test", arrayBlockOf(new RowType(ImmutableList.of(BIGINT), Optional.empty()), rowBlockOf(ImmutableList.of(BIGINT), 1)))
             ))
+            .add(new TestColumn("t_map_null_key_complex_value",
+                    getStandardMapObjectInspector(
+                            javaStringObjectInspector,
+                            getStandardMapObjectInspector(javaLongObjectInspector, javaBooleanObjectInspector)
+                    ),
+                    asMap(null, ImmutableMap.of(15L, true), "k", ImmutableMap.of(16L, false)),
+                    mapBlockOf(VARCHAR, new MapType(BIGINT, BOOLEAN), "k", mapBlockOf(BIGINT, BOOLEAN, 16L, false))))
+            .add(new TestColumn("t_map_null_key_complex_key_value",
+                    getStandardMapObjectInspector(
+                            getStandardListObjectInspector(javaStringObjectInspector),
+                            getStandardMapObjectInspector(javaLongObjectInspector, javaBooleanObjectInspector)
+                    ),
+                    asMap(null, ImmutableMap.of(15L, true), ImmutableList.of("k", "ka"), ImmutableMap.of(16L, false)),
+                    mapBlockOf(new ArrayType(VARCHAR), new MapType(BIGINT, BOOLEAN), arrayBlockOf(VARCHAR, "k", "ka"), mapBlockOf(BIGINT, BOOLEAN, 16L, false))))
             .add(new TestColumn("t_struct_nested", getStandardStructObjectInspector(ImmutableList.of("struct_field"),
                     ImmutableList.of(getStandardListObjectInspector(javaStringObjectInspector))), ImmutableList.of(ImmutableList.of("1", "2", "3")) , rowBlockOf(ImmutableList.of(new ArrayType(VARCHAR)), arrayBlockOf(VARCHAR, "1", "2", "3"))))
             .add(new TestColumn("t_struct_null", getStandardStructObjectInspector(ImmutableList.of("struct_field", "struct_field2"),
@@ -242,6 +257,14 @@ private static Map<Integer, Integer> mapWithNullKey()
         return map;
     }
 
+    private static <K, V> Map<K, V> asMap(K k1, V v1, K k2, V v2)
+    {
+        Map<K, V> map = new HashMap<>();
+        map.put(k1, v1);
+        map.put(k2, v2);
+        return map;
+    }
+
     protected List<HiveColumnHandle> getColumnHandles(List<TestColumn> testColumns)
     {
         List<HiveColumnHandle> columns = new ArrayList<>();

presto-hive/src/test/java/com/facebook/presto/hive/TestHiveFileFormats.java

@@ -112,14 +112,9 @@ public void setUp()
     public void testRCText()
             throws Exception
     {
-        List<TestColumn> testColumns = ImmutableList.copyOf(filter(TEST_COLUMNS, new Predicate<TestColumn>()
-        {
-            @Override
-            public boolean apply(TestColumn testColumn)
-            {
-                // TODO: This is a bug in the RC text reader
-                return !testColumn.getName().equals("t_struct_null");
-            }
+        List<TestColumn> testColumns = ImmutableList.copyOf(filter(TEST_COLUMNS, testColumn -> {
+            return !testColumn.getName().equals("t_struct_null") // TODO: This is a bug in the RC text reader
+                    && !testColumn.getName().equals("t_map_null_key_complex_key_value"); // RC file does not support complex type as key of a map
         }));
 
         HiveOutputFormat<?, ?> outputFormat = new RCFileOutputFormat();
@@ -162,15 +157,20 @@ public void testRcTextPageSource()
     public void testRCBinary()
             throws Exception
     {
+        List<TestColumn> testColumns = ImmutableList.copyOf(filter(TEST_COLUMNS, testColumn -> {
+            // RC file does not support complex type as key of a map
+            return !testColumn.getName().equals("t_map_null_key_complex_key_value");
+        }));
+
         HiveOutputFormat<?, ?> outputFormat = new RCFileOutputFormat();
         InputFormat<?, ?> inputFormat = new RCFileInputFormat<>();
         @SuppressWarnings("deprecation")
         SerDe serde = new LazyBinaryColumnarSerDe();
         File file = File.createTempFile("presto_test", "rc-binary");
         try {
-            FileSplit split = createTestFile(file.getAbsolutePath(), outputFormat, serde, null, TEST_COLUMNS, NUM_ROWS);
-            testCursorProvider(new ColumnarBinaryHiveRecordCursorProvider(), split, inputFormat, serde, TEST_COLUMNS, NUM_ROWS);
-            testCursorProvider(new GenericHiveRecordCursorProvider(), split, inputFormat, serde, TEST_COLUMNS, NUM_ROWS);
+            FileSplit split = createTestFile(file.getAbsolutePath(), outputFormat, serde, null, testColumns, NUM_ROWS);
+            testCursorProvider(new ColumnarBinaryHiveRecordCursorProvider(), split, inputFormat, serde, testColumns, NUM_ROWS);
+            testCursorProvider(new GenericHiveRecordCursorProvider(), split, inputFormat, serde, testColumns, NUM_ROWS);
         }
         finally {
             //noinspection ResultOfMethodCallIgnored
@@ -324,7 +324,7 @@ private static List<TestColumn> getTestColumnsSupportedByParquet()
         // TODO: empty arrays or maps with null keys don't seem to work
         // Parquet does not support DATE
         return TEST_COLUMNS.stream()
-                .filter(column -> !ImmutableSet.of("t_complex", "t_array_empty", "t_map_null_key")
+                .filter(column -> !ImmutableSet.of("t_array_empty", "t_map_null_key", "t_map_null_key_complex_value", "t_map_null_key_complex_key_value")
                         .contains(column.getName()))
                 .filter(column -> !hasType(column.getObjectInspector(), PrimitiveCategory.DATE))
                 .collect(toList());

presto-jdbc/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.119-tw-0.1</version>
+        <version>0.122-SNAPSHOT</version>
     </parent>
 
     <artifactId>presto-jdbc</artifactId>

presto-jmx/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.119-tw-0.1</version>
+        <version>0.122-SNAPSHOT</version>
     </parent>
 
     <artifactId>presto-jmx</artifactId>

presto-kafka/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.119-tw-0.1</version>
+        <version>0.122-SNAPSHOT</version>
     </parent>
 
     <artifactId>presto-kafka</artifactId>

presto-main/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.119-tw-0.1</version>
+        <version>0.122-SNAPSHOT</version>
     </parent>
 
     <artifactId>presto-main</artifactId>

presto-main/src/main/java/com/facebook/presto/security/AccessControlManager.java

@@ -53,7 +53,7 @@ public void addCatalogAccessControl(String catalogName, ConnectorAccessControl a
     @Override
     public void checkCanSetUser(Principal principal, String userName)
     {
-        requireNonNull(userName, "tableName is null");
+        requireNonNull(userName, "userName is null");
 
         for (SystemAccessControl accessControl : systemAccessControllers) {
             accessControl.checkCanSetUser(principal, userName);

presto-main/src/main/java/com/facebook/presto/type/ArrayType.java

@@ -65,7 +65,22 @@ public boolean isOrderable()
     @Override
     public boolean equalTo(Block leftBlock, int leftPosition, Block rightBlock, int rightPosition)
     {
-        return compareTo(leftBlock, leftPosition, rightBlock, rightPosition) == 0;
+        Block leftArray = leftBlock.getObject(leftPosition, Block.class);
+        Block rightArray = rightBlock.getObject(rightPosition, Block.class);
+
+        if (leftArray.getPositionCount() != rightArray.getPositionCount()) {
+            return false;
+        }
+
+        for (int i = 0; i < leftArray.getPositionCount(); i++) {
+            checkElementNotNull(leftArray.isNull(i), ARRAY_NULL_ELEMENT_MSG);
+            checkElementNotNull(rightArray.isNull(i), ARRAY_NULL_ELEMENT_MSG);
+            if (!elementType.equalTo(leftArray, i, rightArray, i)) {
+                return false;
+            }
+        }
+
+        return true;
     }
 
     @Override
@@ -83,6 +98,10 @@ public int hash(Block block, int position)
     @Override
     public int compareTo(Block leftBlock, int leftPosition, Block rightBlock, int rightPosition)
     {
+        if (!elementType.isOrderable()) {
+            throw new UnsupportedOperationException(getTypeSignature() + " type is not orderable");
+        }
+
         Block leftArray = leftBlock.getObject(leftPosition, Block.class);
         Block rightArray = rightBlock.getObject(rightPosition, Block.class);