From b070c0bae289926dbb0e72c83b1eb4fcc2e00843 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Fri, 11 Sep 2020 16:56:55 -0700 Subject: [PATCH 1/2] Cleanup warnings in AbstractTestIcebergSmoke --- .../iceberg/AbstractTestIcebergSmoke.java | 75 +++++++++---------- 1 file changed, 34 insertions(+), 41 deletions(-) diff --git a/presto-iceberg/src/test/java/io/prestosql/plugin/iceberg/AbstractTestIcebergSmoke.java b/presto-iceberg/src/test/java/io/prestosql/plugin/iceberg/AbstractTestIcebergSmoke.java index c01369460e07..6284ed48bc0d 100644 --- a/presto-iceberg/src/test/java/io/prestosql/plugin/iceberg/AbstractTestIcebergSmoke.java +++ b/presto-iceberg/src/test/java/io/prestosql/plugin/iceberg/AbstractTestIcebergSmoke.java @@ -31,7 +31,6 @@ import io.prestosql.testing.MaterializedResult; import io.prestosql.testing.MaterializedRow; import io.prestosql.testing.QueryRunner; -import io.prestosql.testing.assertions.Assert; import org.apache.iceberg.FileFormat; import org.intellij.lang.annotations.Language; import org.testng.annotations.Test; @@ -41,7 +40,6 @@ import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.stream.Collectors; import java.util.stream.IntStream; import static com.google.common.base.Preconditions.checkArgument; @@ -70,7 +68,7 @@ public abstract class AbstractTestIcebergSmoke private final FileFormat format; - public AbstractTestIcebergSmoke(FileFormat format) + protected AbstractTestIcebergSmoke(FileFormat format) { this.format = requireNonNull(format, "format is null"); } @@ -89,7 +87,7 @@ public void testShowCreateSchema() .matches("CREATE SCHEMA iceberg.tpch\n" + "AUTHORIZATION USER user\n" + "WITH \\(\n" + - " location = '.*/iceberg_data/tpch'\n" + + "\\s+location = '.*/iceberg_data/tpch'\n" + "\\)"); } @@ -109,7 +107,7 @@ public void testDescribeTable() .row("comment", "varchar", "", "") .build(); MaterializedResult actualColumns = computeActual("DESCRIBE orders"); - Assert.assertEquals(actualColumns, expectedColumns); + assertEquals(actualColumns, expectedColumns); } @Override @@ -218,12 +216,12 @@ private void testSelectOrPartitionedByTimestamp(boolean partitioned) String tableName = format("test_%s_by_timestamp", partitioned ? "partitioned" : "selected"); assertUpdate(format("CREATE TABLE %s (_timestamp timestamp(6)) %s", tableName, partitioned ? "WITH (partitioning = ARRAY['_timestamp'])" : "")); - String select1 = "SELECT TIMESTAMP '2017-05-01 10:12:34' _timestamp"; - assertUpdate(format("INSERT INTO %s ", tableName) + select1, 1); - String select2 = "SELECT TIMESTAMP '2017-10-01 10:12:34' _timestamp"; - assertUpdate(format("INSERT INTO %s " + select2, tableName), 1); - String select3 = "SELECT TIMESTAMP '2018-05-01 10:12:34' _timestamp"; - assertUpdate(format("INSERT INTO %s " + select3, tableName), 1); + @Language("SQL") String select1 = "SELECT TIMESTAMP '2017-05-01 10:12:34' _timestamp"; + @Language("SQL") String select2 = "SELECT TIMESTAMP '2017-10-01 10:12:34' _timestamp"; + @Language("SQL") String select3 = "SELECT TIMESTAMP '2018-05-01 10:12:34' _timestamp"; + assertUpdate(format("INSERT INTO %s %s", tableName, select1), 1); + assertUpdate(format("INSERT INTO %s %s", tableName, select2), 1); + assertUpdate(format("INSERT INTO %s %s", tableName, select3), 1); assertQuery(format("SELECT COUNT(*) from %s", tableName), "SELECT 3"); assertQuery(format("SELECT * from %s WHERE _timestamp = TIMESTAMP '2017-05-01 10:12:34'", tableName), select1); @@ -286,7 +284,7 @@ public void testCreatePartitionedTable() assertUpdate(format("INSERT INTO test_partitioned_table %s", select), 1); assertQuery("SELECT * FROM test_partitioned_table", select); - String selectAgain = "" + + @Language("SQL") String selectAgain = "" + "SELECT * FROM test_partitioned_table WHERE" + " 'foo' = _string" + " AND 456 = _integer" + @@ -396,8 +394,8 @@ public void testCreatePartitionedTableAs() " format = '%s',\n" + " partitioning = ARRAY['order_status','ship_priority','bucket(order_key, 9)']\n" + ")", - getSession().getCatalog().get(), - getSession().getSchema().get(), + getSession().getCatalog().orElseThrow(), + getSession().getSchema().orElseThrow(), "test_create_partitioned_table_as", format); @@ -430,11 +428,11 @@ public void testTableComments() "WITH (\n" + format(" format = '%s'\n", format) + ")"; - String createTableSql = format(createTableTemplate, "test table comment", format); + @Language("SQL") String createTableSql = format(createTableTemplate, "test table comment", format); assertUpdate(createTableSql); MaterializedResult resultOfCreate = computeActual("SHOW CREATE TABLE test_table_comments"); assertEquals(getOnlyElement(resultOfCreate.getOnlyColumnAsSet()), createTableSql); - String showCreateTable = "SHOW CREATE TABLE test_table_comments"; + @Language("SQL") String showCreateTable = "SHOW CREATE TABLE test_table_comments"; assertUpdate("COMMENT ON TABLE test_table_comments IS 'different test table comment'"); MaterializedResult resultOfCommentChange = computeActual(showCreateTable); @@ -593,12 +591,7 @@ private String getTablePropertiesString(String tableName) MaterializedResult showCreateTable = computeActual("SHOW CREATE TABLE " + tableName); String createTable = (String) getOnlyElement(showCreateTable.getOnlyColumnAsSet()); Matcher matcher = WITH_CLAUSE_EXTRACTER.matcher(createTable); - if (matcher.matches()) { - return matcher.group(1); - } - else { - return null; - } + return matcher.matches() ? matcher.group(1) : null; } @Test @@ -838,7 +831,7 @@ public void testTruncateTransform() assertUpdate("CREATE TABLE test_truncate_transform (d VARCHAR, b BIGINT) WITH (partitioning = ARRAY['truncate(d, 2)'])"); - String insertSql = "INSERT INTO test_truncate_transform VALUES" + + @Language("SQL") String insertSql = "INSERT INTO test_truncate_transform VALUES" + "('abcd', 1)," + "('abxy', 2)," + "('ab598', 3)," + @@ -868,7 +861,7 @@ public void testBucketTransform() String select = "SELECT d_bucket, row_count, d.min AS d_min, d.max AS d_max, b.min AS b_min, b.max AS b_max FROM \"test_bucket_transform$partitions\""; assertUpdate("CREATE TABLE test_bucket_transform (d VARCHAR, b BIGINT) WITH (partitioning = ARRAY['bucket(d, 2)'])"); - String insertSql = "INSERT INTO test_bucket_transform VALUES" + + @Language("SQL") String insertSql = "INSERT INTO test_bucket_transform VALUES" + "('abcd', 1)," + "('abxy', 2)," + "('ab598', 3)," + @@ -958,7 +951,7 @@ private void testInSet(int inCount) assertUpdate("CREATE TABLE test_in_set (col1 INTEGER, col2 BIGINT)"); assertUpdate(format("INSERT INTO test_in_set VALUES %s", values), inCount); // This proves that SELECTs with large IN phrases work correctly - MaterializedResult result = computeActual(format("SELECT col1 FROM test_in_set WHERE col1 IN (%s)", inList)); + computeActual(format("SELECT col1 FROM test_in_set WHERE col1 IN (%s)", inList)); dropTable("test_in_set"); } @@ -996,7 +989,7 @@ public void testBasicTableStatistics() private Double columnSizeForFormat(double size) { - return format == FileFormat.PARQUET ? size : null; + return format == PARQUET ? size : null; } @Test @@ -1032,11 +1025,11 @@ public void testMultipleColumnTableStatistics() assertUpdate(insertStart + " VALUES " + IntStream.rangeClosed(21, 25) .mapToObj(i -> format("(200, %d, DATE '2020-07-%d')", i, i)) - .collect(Collectors.joining(", ")), 5); + .collect(joining(", ")), 5); assertUpdate(insertStart + " VALUES " + IntStream.rangeClosed(26, 30) .mapToObj(i -> format("(NULL, %d, DATE '2020-06-%d')", i, i)) - .collect(Collectors.joining(", ")), 5); + .collect(joining(", ")), 5); result = computeActual("SHOW STATS FOR " + tableName); @@ -1065,10 +1058,10 @@ public void testPartitionedTableStatistics() assertEquals(result.getRowCount(), 3); MaterializedRow row0 = result.getMaterializedRows().get(0); - org.testng.Assert.assertEquals(row0.getField(0), "col1"); - org.testng.Assert.assertEquals(row0.getField(3), 0.0); - org.testng.Assert.assertEquals(row0.getField(5), "-10.0"); - org.testng.Assert.assertEquals(row0.getField(6), "100.0"); + assertEquals(row0.getField(0), "col1"); + assertEquals(row0.getField(3), 0.0); + assertEquals(row0.getField(5), "-10.0"); + assertEquals(row0.getField(6), "100.0"); MaterializedRow row1 = result.getMaterializedRows().get(1); assertEquals(row1.getField(0), "col2"); @@ -1081,11 +1074,11 @@ public void testPartitionedTableStatistics() assertUpdate(insertStart + " VALUES " + IntStream.rangeClosed(1, 5) .mapToObj(i -> format("(%d, 10)", i + 100)) - .collect(Collectors.joining(", ")), 5); + .collect(joining(", ")), 5); assertUpdate(insertStart + " VALUES " + IntStream.rangeClosed(6, 10) .mapToObj(i -> "(NULL, 10)") - .collect(Collectors.joining(", ")), 5); + .collect(joining(", ")), 5); result = computeActual("SHOW STATS FOR iceberg.tpch.test_partitioned_table_statistics"); assertEquals(result.getRowCount(), 3); @@ -1106,7 +1099,7 @@ public void testPartitionedTableStatistics() assertUpdate(insertStart + " VALUES " + IntStream.rangeClosed(6, 10) .mapToObj(i -> "(100, NULL)") - .collect(Collectors.joining(", ")), 5); + .collect(joining(", ")), 5); result = computeActual("SHOW STATS FOR iceberg.tpch.test_partitioned_table_statistics"); row0 = result.getMaterializedRows().get(0); @@ -1146,7 +1139,7 @@ public void testStatisticsConstraints() tableStatistics = getTableStatistics(tableName, constraint); assertEquals(tableStatistics.getRowCount().getValue(), 2.0); ColumnStatistics columnStatistics = getStatisticsForColumn(tableStatistics, "col1"); - assertEquals(columnStatistics.getRange().get(), new DoubleRange(3, 4)); + assertThat(columnStatistics.getRange()).hasValue(new DoubleRange(3, 4)); // This shows that Predicate only filters rows for partitioned columns. predicate = new TestRelationalNumberPredicate("col2", 102, i -> i >= 0); @@ -1154,7 +1147,7 @@ public void testStatisticsConstraints() tableStatistics = getTableStatistics(tableName, new Constraint(TupleDomain.all(), Optional.of(predicate), Optional.empty())); assertEquals(tableStatistics.getRowCount().getValue(), 4.0); columnStatistics = getStatisticsForColumn(tableStatistics, "col2"); - assertEquals(columnStatistics.getRange().get(), new DoubleRange(101, 104)); + assertThat(columnStatistics.getRange()).hasValue(new DoubleRange(101, 104)); dropTable(tableName); } @@ -1183,7 +1176,7 @@ public boolean test(Map nullableValues) if (object instanceof Long) { return comparePredicate.test(((Long) object).compareTo(comparand.longValue())); } - else if (object instanceof Double) { + if (object instanceof Double) { return comparePredicate.test(((Double) object).compareTo(comparand.doubleValue())); } throw new IllegalArgumentException(format("NullableValue is neither Long or Double, but %s", object)); @@ -1204,7 +1197,7 @@ private ColumnStatistics getStatisticsForColumn(TableStatistics tableStatistics, throw new IllegalArgumentException("TableStatistics did not contain column named " + columnName); } - private IcebergColumnHandle getColumnHandleFromStatistics(TableStatistics tableStatistics, String columnName) + private static IcebergColumnHandle getColumnHandleFromStatistics(TableStatistics tableStatistics, String columnName) { for (ColumnHandle columnHandle : tableStatistics.getColumnStatistics().keySet()) { IcebergColumnHandle handle = (IcebergColumnHandle) columnHandle; @@ -1220,7 +1213,7 @@ private ColumnStatistics checkColumnStatistics(ColumnStatistics statistics) assertNotNull(statistics, "statistics is null"); // Sadly, statistics.getDataSize().isUnknown() for columns in ORC files. See the TODO // in IcebergOrcFileWriter. - if (format != FileFormat.ORC) { + if (format != ORC) { assertFalse(statistics.getDataSize().isUnknown()); } assertFalse(statistics.getNullsFraction().isUnknown(), "statistics nulls fraction is unknown"); @@ -1262,7 +1255,7 @@ public void testCreateNestedPartitionedTable() assertUpdate(createTable); - String insertSql = "INSERT INTO test_nested_table_1 " + + @Language("SQL") String insertSql = "INSERT INTO test_nested_table_1 " + " select true, 1, array['uno', 'dos', 'tres'], BIGINT '1', REAL '1.0', DOUBLE '1.0', map(array[1,2,3,4], array['ek','don','teen','char'])," + " CAST(1.0 as DECIMAL(5,2))," + " 'one', VARBINARY 'binary0/1values',\n" + From fa0f7dbf6116816d50ae5798f3319907a6d533bb Mon Sep 17 00:00:00 2001 From: David Phillips Date: Tue, 15 Sep 2020 16:13:16 -0700 Subject: [PATCH 2/2] Match Iceberg transforms for negative epoch values --- .../plugin/iceberg/PartitionTransforms.java | 41 ++++- .../iceberg/AbstractTestIcebergSmoke.java | 150 ++++++++++++------ .../iceberg/TestPartitionTransforms.java | 60 +++---- 3 files changed, 165 insertions(+), 86 deletions(-) diff --git a/presto-iceberg/src/main/java/io/prestosql/plugin/iceberg/PartitionTransforms.java b/presto-iceberg/src/main/java/io/prestosql/plugin/iceberg/PartitionTransforms.java index 0906bb44c86a..d0250cd854b3 100644 --- a/presto-iceberg/src/main/java/io/prestosql/plugin/iceberg/PartitionTransforms.java +++ b/presto-iceberg/src/main/java/io/prestosql/plugin/iceberg/PartitionTransforms.java @@ -66,6 +66,8 @@ public final class PartitionTransforms private static final DateTimeField YEAR_FIELD = ISOChronology.getInstanceUTC().year(); private static final DateTimeField MONTH_FIELD = ISOChronology.getInstanceUTC().monthOfYear(); + private static final DateTimeField DAY_OF_YEAR_FIELD = ISOChronology.getInstanceUTC().dayOfYear(); + private static final DateTimeField DAY_OF_MONTH_FIELD = ISOChronology.getInstanceUTC().dayOfMonth(); private PartitionTransforms() {} @@ -493,27 +495,50 @@ private static Block truncateVarbinary(Block block, int max) } @VisibleForTesting - static long epochYear(long epochMillis) + static long epochYear(long epochMilli) { - return YEAR_FIELD.get(epochMillis) - 1970; + int epochYear = YEAR_FIELD.get(epochMilli) - 1970; + // Iceberg incorrectly handles negative epoch values + if ((epochMilli < 0) && ((DAY_OF_YEAR_FIELD.get(epochMilli) > 1) || !isMidnight(epochMilli))) { + epochYear++; + } + return epochYear; } @VisibleForTesting static long epochMonth(long epochMilli) { - long year = epochYear(epochMilli); + long year = YEAR_FIELD.get(epochMilli) - 1970; int month = MONTH_FIELD.get(epochMilli) - 1; - return (year * 12) + month; + long epochMonth = (year * 12) + month; + // Iceberg incorrectly handles negative epoch values + if ((epochMilli < 0) && ((DAY_OF_MONTH_FIELD.get(epochMilli) > 1) || !isMidnight(epochMilli))) { + epochMonth++; + } + return epochMonth; + } + + @VisibleForTesting + static long epochDay(long epochMilli) + { + long epochDay = floorDiv(epochMilli, MILLISECONDS_PER_DAY); + // Iceberg incorrectly handles negative epoch values + if ((epochMilli < 0) && !isMidnight(epochMilli)) { + epochDay++; + } + return epochDay; } - private static long epochDay(long epochMilli) + @VisibleForTesting + static long epochHour(long epochMilli) { - return floorDiv(epochMilli, MILLISECONDS_PER_DAY); + // Iceberg incorrectly handles negative epoch values + return epochMilli / MILLISECONDS_PER_HOUR; } - private static long epochHour(long epochMilli) + private static boolean isMidnight(long epochMilli) { - return floorDiv(epochMilli, MILLISECONDS_PER_HOUR); + return (epochMilli % MILLISECONDS_PER_DAY) == 0; } public static class ColumnTransform diff --git a/presto-iceberg/src/test/java/io/prestosql/plugin/iceberg/AbstractTestIcebergSmoke.java b/presto-iceberg/src/test/java/io/prestosql/plugin/iceberg/AbstractTestIcebergSmoke.java index 6284ed48bc0d..657db0161309 100644 --- a/presto-iceberg/src/test/java/io/prestosql/plugin/iceberg/AbstractTestIcebergSmoke.java +++ b/presto-iceberg/src/test/java/io/prestosql/plugin/iceberg/AbstractTestIcebergSmoke.java @@ -609,6 +609,10 @@ public void testHourTransform() assertUpdate("CREATE TABLE test_hour_transform (d TIMESTAMP(6), b BIGINT) WITH (partitioning = ARRAY['hour(d)'])"); @Language("SQL") String values = "VALUES " + + "(TIMESTAMP '1969-12-31 22:22:22.222222', 8)," + + "(TIMESTAMP '1969-12-31 23:33:11.456789', 9)," + + "(TIMESTAMP '1969-12-31 23:44:55.567890', 10)," + + "(TIMESTAMP '1970-01-01 00:55:44.765432', 11)," + "(TIMESTAMP '2015-01-01 10:01:23.123456', 1)," + "(TIMESTAMP '2015-01-01 10:10:02.987654', 2)," + "(TIMESTAMP '2015-01-01 10:55:00.456789', 3)," + @@ -616,15 +620,19 @@ public void testHourTransform() "(TIMESTAMP '2015-05-15 12:21:02.345678', 5)," + "(TIMESTAMP '2020-02-21 13:11:11.876543', 6)," + "(TIMESTAMP '2020-02-21 13:12:12.654321', 7)"; - assertUpdate("INSERT INTO test_hour_transform " + values, 7); + assertUpdate("INSERT INTO test_hour_transform " + values, 11); assertQuery("SELECT * FROM test_hour_transform", values); @Language("SQL") String expected = "VALUES " + + "(-1, 1, TIMESTAMP '1969-12-31 22:22:22.222222', TIMESTAMP '1969-12-31 22:22:22.222222', 8, 8), " + + "(0, 3, TIMESTAMP '1969-12-31 23:33:11.456789', TIMESTAMP '1970-01-01 00:55:44.765432', 9, 11), " + "(394474, 3, TIMESTAMP '2015-01-01 10:01:23.123456', TIMESTAMP '2015-01-01 10:55:00.456789', 1, 3), " + "(397692, 2, TIMESTAMP '2015-05-15 12:05:01.234567', TIMESTAMP '2015-05-15 12:21:02.345678', 4, 5), " + "(439525, 2, TIMESTAMP '2020-02-21 13:11:11.876543', TIMESTAMP '2020-02-21 13:12:12.654321', 6, 7)"; if (format == ORC) { expected = "VALUES " + + "(-1, 1, NULL, NULL, 8, 8), " + + "(0, 3, NULL, NULL, 9, 11), " + "(394474, 3, NULL, NULL, 1, 3), " + "(397692, 2, NULL, NULL, 4, 5), " + "(439525, 2, NULL, NULL, 6, 7)"; @@ -632,6 +640,8 @@ public void testHourTransform() assertQuery("SELECT d_hour, row_count, d.min, d.max, b.min, b.max FROM \"test_hour_transform$partitions\"", expected); + System.out.println(computeActual("SELECT * FROM \"test_hour_transform$files\"")); + dropTable("test_hour_transform"); } @@ -641,6 +651,8 @@ public void testDayTransformDate() assertUpdate("CREATE TABLE test_day_transform_date (d DATE, b BIGINT) WITH (partitioning = ARRAY['day(d)'])"); @Language("SQL") String values = "VALUES " + + "(DATE '1969-01-01', 10), " + + "(DATE '1969-12-31', 11), " + "(DATE '1970-01-01', 1), " + "(DATE '1970-03-04', 2), " + "(DATE '2015-01-01', 3), " + @@ -650,12 +662,14 @@ public void testDayTransformDate() "(DATE '2015-05-15', 7), " + "(DATE '2020-02-21', 8), " + "(DATE '2020-02-21', 9)"; - assertUpdate("INSERT INTO test_day_transform_date " + values, 9); + assertUpdate("INSERT INTO test_day_transform_date " + values, 11); assertQuery("SELECT * FROM test_day_transform_date", values); assertQuery( "SELECT d_day, row_count, d.min, d.max, b.min, b.max FROM \"test_day_transform_date$partitions\"", "VALUES " + + "(DATE '1969-01-01', 1, DATE '1969-01-01', DATE '1969-01-01', 10, 10), " + + "(DATE '1969-12-31', 1, DATE '1969-12-31', DATE '1969-12-31', 11, 11), " + "(DATE '1970-01-01', 1, DATE '1970-01-01', DATE '1970-01-01', 1, 1), " + "(DATE '1970-03-04', 1, DATE '1970-03-04', DATE '1970-03-04', 2, 2), " + "(DATE '2015-01-01', 1, DATE '2015-01-01', DATE '2015-01-01', 3, 3), " + @@ -672,6 +686,11 @@ public void testDayTransformTimestamp() assertUpdate("CREATE TABLE test_day_transform_timestamp (d TIMESTAMP(6), b BIGINT) WITH (partitioning = ARRAY['day(d)'])"); @Language("SQL") String values = "VALUES " + + "(TIMESTAMP '1969-12-25 15:13:12.876543', 8)," + + "(TIMESTAMP '1969-12-30 18:47:33.345678', 9)," + + "(TIMESTAMP '1969-12-31 00:00:00.000000', 10)," + + "(TIMESTAMP '1969-12-31 05:06:07.234567', 11)," + + "(TIMESTAMP '1970-01-01 12:03:08.456789', 12)," + "(TIMESTAMP '2015-01-01 10:01:23.123456', 1)," + "(TIMESTAMP '2015-01-01 11:10:02.987654', 2)," + "(TIMESTAMP '2015-01-01 12:55:00.456789', 3)," + @@ -679,15 +698,21 @@ public void testDayTransformTimestamp() "(TIMESTAMP '2015-05-15 14:21:02.345678', 5)," + "(TIMESTAMP '2020-02-21 15:11:11.876543', 6)," + "(TIMESTAMP '2020-02-21 16:12:12.654321', 7)"; - assertUpdate("INSERT INTO test_day_transform_timestamp " + values, 7); + assertUpdate("INSERT INTO test_day_transform_timestamp " + values, 12); assertQuery("SELECT * FROM test_day_transform_timestamp", values); @Language("SQL") String expected = "VALUES " + + "(DATE '1969-12-26', 1, TIMESTAMP '1969-12-25 15:13:12.876543', TIMESTAMP '1969-12-25 15:13:12.876543', 8, 8), " + + "(DATE '1969-12-31', 2, TIMESTAMP '1969-12-30 18:47:33.345678', TIMESTAMP '1969-12-31 00:00:00.000000', 9, 10), " + + "(DATE '1970-01-01', 2, TIMESTAMP '1969-12-31 05:06:07.234567', TIMESTAMP '1970-01-01 12:03:08.456789', 11, 12), " + "(DATE '2015-01-01', 3, TIMESTAMP '2015-01-01 10:01:23.123456', TIMESTAMP '2015-01-01 12:55:00.456789', 1, 3), " + "(DATE '2015-05-15', 2, TIMESTAMP '2015-05-15 13:05:01.234567', TIMESTAMP '2015-05-15 14:21:02.345678', 4, 5), " + "(DATE '2020-02-21', 2, TIMESTAMP '2020-02-21 15:11:11.876543', TIMESTAMP '2020-02-21 16:12:12.654321', 6, 7)"; if (format == ORC) { expected = "VALUES " + + "(DATE '1969-12-26', 1, NULL, NULL, 8, 8), " + + "(DATE '1969-12-31', 2, NULL, NULL, 9, 10), " + + "(DATE '1970-01-01', 2, NULL, NULL, 11, 12), " + "(DATE '2015-01-01', 3, NULL, NULL, 1, 3), " + "(DATE '2015-05-15', 2, NULL, NULL, 4, 5), " + "(DATE '2020-02-21', 2, NULL, NULL, 6, 7)"; @@ -704,29 +729,34 @@ public void testMonthTransformDate() assertUpdate("CREATE TABLE test_month_transform_date (d DATE, b BIGINT) WITH (partitioning = ARRAY['month(d)'])"); @Language("SQL") String values = "VALUES " + - "(DATE '1970-01-01', 1), " + - "(DATE '1970-05-13', 2), " + - "(DATE '1970-12-31', 3), " + - "(DATE '2020-01-01', 4), " + - "(DATE '2020-06-16', 5), " + - "(DATE '2020-06-28', 6), " + - "(DATE '2020-06-06', 7), " + - "(DATE '2020-07-18', 8), " + - "(DATE '2020-07-28', 9), " + - "(DATE '2020-12-31', 10)"; - assertUpdate("INSERT INTO test_month_transform_date " + values, 10); + "(DATE '1969-11-13', 1)," + + "(DATE '1969-12-01', 2)," + + "(DATE '1969-12-02', 3)," + + "(DATE '1969-12-31', 4)," + + "(DATE '1970-01-01', 5), " + + "(DATE '1970-05-13', 6), " + + "(DATE '1970-12-31', 7), " + + "(DATE '2020-01-01', 8), " + + "(DATE '2020-06-16', 9), " + + "(DATE '2020-06-28', 10), " + + "(DATE '2020-06-06', 11), " + + "(DATE '2020-07-18', 12), " + + "(DATE '2020-07-28', 13), " + + "(DATE '2020-12-31', 14)"; + assertUpdate("INSERT INTO test_month_transform_date " + values, 14); assertQuery("SELECT * FROM test_month_transform_date", values); assertQuery( "SELECT d_month, row_count, d.min, d.max, b.min, b.max FROM \"test_month_transform_date$partitions\"", "VALUES " + - "(0, 1, DATE '1970-01-01', DATE '1970-01-01', 1, 1), " + - "(4, 1, DATE '1970-05-13', DATE '1970-05-13', 2, 2), " + - "(11, 1, DATE '1970-12-31', DATE '1970-12-31', 3, 3), " + - "(600, 1, DATE '2020-01-01', DATE '2020-01-01', 4, 4), " + - "(605, 3, DATE '2020-06-06', DATE '2020-06-28', 5, 7), " + - "(606, 2, DATE '2020-07-18', DATE '2020-07-28', 8, 9), " + - "(611, 1, DATE '2020-12-31', DATE '2020-12-31', 10, 10)"); + "(-1, 2, DATE '1969-11-13', DATE '1969-12-01', 1, 2), " + + "(0, 3, DATE '1969-12-02', DATE '1970-01-01', 3, 5), " + + "(4, 1, DATE '1970-05-13', DATE '1970-05-13', 6, 6), " + + "(11, 1, DATE '1970-12-31', DATE '1970-12-31', 7, 7), " + + "(600, 1, DATE '2020-01-01', DATE '2020-01-01', 8, 8), " + + "(605, 3, DATE '2020-06-06', DATE '2020-06-28', 9, 11), " + + "(606, 2, DATE '2020-07-18', DATE '2020-07-28', 12, 13), " + + "(611, 1, DATE '2020-12-31', DATE '2020-12-31', 14, 14)"); dropTable("test_month_transform_date"); } @@ -737,6 +767,11 @@ public void testMonthTransformTimestamp() assertUpdate("CREATE TABLE test_month_transform_timestamp (d TIMESTAMP(6), b BIGINT) WITH (partitioning = ARRAY['month(d)'])"); @Language("SQL") String values = "VALUES " + + "(TIMESTAMP '1969-11-15 15:13:12.876543', 8)," + + "(TIMESTAMP '1969-11-19 18:47:33.345678', 9)," + + "(TIMESTAMP '1969-12-01 00:00:00.000000', 10)," + + "(TIMESTAMP '1969-12-01 05:06:07.234567', 11)," + + "(TIMESTAMP '1970-01-01 12:03:08.456789', 12)," + "(TIMESTAMP '2015-01-01 10:01:23.123456', 1)," + "(TIMESTAMP '2015-01-01 11:10:02.987654', 2)," + "(TIMESTAMP '2015-01-01 12:55:00.456789', 3)," + @@ -744,15 +779,19 @@ public void testMonthTransformTimestamp() "(TIMESTAMP '2015-05-15 14:21:02.345678', 5)," + "(TIMESTAMP '2020-02-21 15:11:11.876543', 6)," + "(TIMESTAMP '2020-02-21 16:12:12.654321', 7)"; - assertUpdate("INSERT INTO test_month_transform_timestamp " + values, 7); + assertUpdate("INSERT INTO test_month_transform_timestamp " + values, 12); assertQuery("SELECT * FROM test_month_transform_timestamp", values); @Language("SQL") String expected = "VALUES " + + "(-1, 3, TIMESTAMP '1969-11-15 15:13:12.876543', TIMESTAMP '1969-12-01 00:00:00.000000', 8, 10), " + + "(0, 2, TIMESTAMP '1969-12-01 05:06:07.234567', TIMESTAMP '1970-01-01 12:03:08.456789', 11, 12), " + "(540, 3, TIMESTAMP '2015-01-01 10:01:23.123456', TIMESTAMP '2015-01-01 12:55:00.456789', 1, 3), " + "(544, 2, TIMESTAMP '2015-05-15 13:05:01.234567', TIMESTAMP '2015-05-15 14:21:02.345678', 4, 5), " + "(601, 2, TIMESTAMP '2020-02-21 15:11:11.876543', TIMESTAMP '2020-02-21 16:12:12.654321', 6, 7)"; if (format == ORC) { expected = "VALUES " + + "(-1, 3, NULL, NULL, 8, 10), " + + "(0, 2, NULL, NULL, 11, 12), " + "(540, 3, NULL, NULL, 1, 3), " + "(544, 2, NULL, NULL, 4, 5), " + "(601, 2, NULL, NULL, 6, 7)"; @@ -769,25 +808,29 @@ public void testYearTransformDate() assertUpdate("CREATE TABLE test_year_transform_date (d DATE, b BIGINT) WITH (partitioning = ARRAY['year(d)'])"); @Language("SQL") String values = "VALUES " + - "(DATE '1970-01-01', 11), " + - "(DATE '1970-03-05', 12), " + - "(DATE '2015-01-01', 1), " + - "(DATE '2015-06-16', 2), " + - "(DATE '2015-07-28', 3), " + - "(DATE '2016-05-15', 4), " + - "(DATE '2016-06-06', 5), " + - "(DATE '2020-02-21', 6), " + - "(DATE '2020-11-10', 7)"; - assertUpdate("INSERT INTO test_year_transform_date " + values, 9); + "(DATE '1968-10-13', 1), " + + "(DATE '1969-01-01', 2), " + + "(DATE '1969-03-15', 3), " + + "(DATE '1970-01-01', 4), " + + "(DATE '1970-03-05', 5), " + + "(DATE '2015-01-01', 6), " + + "(DATE '2015-06-16', 7), " + + "(DATE '2015-07-28', 8), " + + "(DATE '2016-05-15', 9), " + + "(DATE '2016-06-06', 10), " + + "(DATE '2020-02-21', 11), " + + "(DATE '2020-11-10', 12)"; + assertUpdate("INSERT INTO test_year_transform_date " + values, 12); assertQuery("SELECT * FROM test_year_transform_date", values); assertQuery( "SELECT d_year, row_count, d.min, d.max, b.min, b.max FROM \"test_year_transform_date$partitions\"", "VALUES " + - "(0, 2, DATE '1970-01-01', DATE '1970-03-05', 11, 12), " + - "(45, 3, DATE '2015-01-01', DATE '2015-07-28', 1, 3), " + - "(46, 2, DATE '2016-05-15', DATE '2016-06-06', 4, 5), " + - "(50, 2, DATE '2020-02-21', DATE '2020-11-10', 6, 7)"); + "(-1, 2, DATE '1968-10-13', DATE '1969-01-01', 1, 2), " + + "(0, 3, DATE '1969-03-15', DATE '1970-03-05', 3, 5), " + + "(45, 3, DATE '2015-01-01', DATE '2015-07-28', 6, 8), " + + "(46, 2, DATE '2016-05-15', DATE '2016-06-06', 9, 10), " + + "(50, 2, DATE '2020-02-21', DATE '2020-11-10', 11, 12)"); dropTable("test_year_transform_date"); } @@ -798,25 +841,32 @@ public void testYearTransformTimestamp() assertUpdate("CREATE TABLE test_year_transform_timestamp (d TIMESTAMP(6), b BIGINT) WITH (partitioning = ARRAY['year(d)'])"); @Language("SQL") String values = "VALUES " + - "(TIMESTAMP '1970-01-01 10:01:23.123456', 1)," + - "(TIMESTAMP '1970-01-01 11:10:02.987654', 2)," + - "(TIMESTAMP '1970-01-01 12:55:00.456789', 3)," + - "(TIMESTAMP '2015-05-15 13:05:01.234567', 4)," + - "(TIMESTAMP '2015-05-15 14:21:02.345678', 5)," + - "(TIMESTAMP '2020-02-21 15:11:11.876543', 6)," + - "(TIMESTAMP '2020-02-21 16:12:12.654321', 7)"; - assertUpdate("INSERT INTO test_year_transform_timestamp " + values, 7); + "(TIMESTAMP '1968-03-15 15:13:12.876543', 1)," + + "(TIMESTAMP '1968-11-19 18:47:33.345678', 2)," + + "(TIMESTAMP '1969-01-01 00:00:00.000000', 3)," + + "(TIMESTAMP '1969-01-01 05:06:07.234567', 4)," + + "(TIMESTAMP '1970-01-18 12:03:08.456789', 5)," + + "(TIMESTAMP '1970-03-14 10:01:23.123456', 6)," + + "(TIMESTAMP '1970-08-19 11:10:02.987654', 7)," + + "(TIMESTAMP '1970-12-31 12:55:00.456789', 8)," + + "(TIMESTAMP '2015-05-15 13:05:01.234567', 9)," + + "(TIMESTAMP '2015-09-15 14:21:02.345678', 10)," + + "(TIMESTAMP '2020-02-21 15:11:11.876543', 11)," + + "(TIMESTAMP '2020-08-21 16:12:12.654321', 12)"; + assertUpdate("INSERT INTO test_year_transform_timestamp " + values, 12); assertQuery("SELECT * FROM test_year_transform_timestamp", values); @Language("SQL") String expected = "VALUES " + - "(0, 3, TIMESTAMP '1970-01-01 10:01:23.123456', TIMESTAMP '1970-01-01 12:55:00.456789', 1, 3), " + - "(45, 2, TIMESTAMP '2015-05-15 13:05:01.234567', TIMESTAMP '2015-05-15 14:21:02.345678', 4, 5), " + - "(50, 2, TIMESTAMP '2020-02-21 15:11:11.876543', TIMESTAMP '2020-02-21 16:12:12.654321', 6, 7)"; + "(-1, 3, TIMESTAMP '1968-03-15 15:13:12.876543', TIMESTAMP '1969-01-01 00:00:00.000000', 1, 3), " + + "(0, 5, TIMESTAMP '1969-01-01 05:06:07.234567', TIMESTAMP '1970-12-31 12:55:00.456789', 4, 8), " + + "(45, 2, TIMESTAMP '2015-05-15 13:05:01.234567', TIMESTAMP '2015-09-15 14:21:02.345678', 9, 10), " + + "(50, 2, TIMESTAMP '2020-02-21 15:11:11.876543', TIMESTAMP '2020-08-21 16:12:12.654321', 11, 12)"; if (format == ORC) { expected = "VALUES " + - "(0, 3, NULL, NULL, 1, 3), " + - "(45, 2, NULL, NULL, 4, 5), " + - "(50, 2, NULL, NULL, 6, 7)"; + "(-1, 3, NULL, NULL, 1, 3), " + + "(0, 5, NULL, NULL, 4, 8), " + + "(45, 2, NULL, NULL, 9, 10), " + + "(50, 2, NULL, NULL, 11, 12)"; } assertQuery("SELECT d_year, row_count, d.min, d.max, b.min, b.max FROM \"test_year_transform_timestamp$partitions\"", expected); diff --git a/presto-iceberg/src/test/java/io/prestosql/plugin/iceberg/TestPartitionTransforms.java b/presto-iceberg/src/test/java/io/prestosql/plugin/iceberg/TestPartitionTransforms.java index 067e44d745c6..5984aa48b8e2 100644 --- a/presto-iceberg/src/test/java/io/prestosql/plugin/iceberg/TestPartitionTransforms.java +++ b/presto-iceberg/src/test/java/io/prestosql/plugin/iceberg/TestPartitionTransforms.java @@ -20,15 +20,22 @@ import org.testng.annotations.Test; import java.time.LocalDateTime; +import java.time.LocalTime; import java.time.ZoneOffset; -import java.util.concurrent.TimeUnit; +import static io.prestosql.plugin.iceberg.PartitionTransforms.epochDay; +import static io.prestosql.plugin.iceberg.PartitionTransforms.epochHour; import static io.prestosql.plugin.iceberg.PartitionTransforms.epochMonth; import static io.prestosql.plugin.iceberg.PartitionTransforms.epochYear; +import static java.lang.Math.toIntExact; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.testng.Assert.assertEquals; public class TestPartitionTransforms { + private static final DateType ICEBERG_DATE = DateType.get(); + private static final TimestampType ICEBERG_TIMESTAMP = TimestampType.withoutZone(); + @Test public void testToStringMatchesSpecification() { @@ -42,35 +49,32 @@ public void testToStringMatchesSpecification() } @Test - public void testEpochYear() + public void testEpochTransforms() { - assertEquals(epochYear(dateMillis(1965, 1, 1)), -5); - assertEquals(epochYear(dateMillis(1965, 6, 3)), -5); - assertEquals(epochYear(dateMillis(1965, 12, 31)), -5); - assertEquals(epochYear(dateMillis(1969, 1, 1)), -1); - assertEquals(epochYear(dateMillis(1969, 7, 20)), -1); - assertEquals(epochYear(dateMillis(1969, 12, 31)), -1); - assertEquals(epochYear(dateMillis(1970, 1, 1)), 0); - assertEquals(epochYear(dateMillis(1970, 12, 31)), 0); - assertEquals(epochYear(dateMillis(2012, 8, 20)), 42); - } + long start = LocalDateTime.of(1965, 10, 1, 0, 0, 0).toEpochSecond(ZoneOffset.UTC); + long end = LocalDateTime.of(1974, 3, 1, 0, 0, 0).toEpochSecond(ZoneOffset.UTC); - @Test - public void testEpochMonth() - { - assertEquals(epochMonth(dateMillis(1965, 1, 1)), -60); - assertEquals(epochMonth(dateMillis(1965, 6, 3)), -55); - assertEquals(epochMonth(dateMillis(1965, 12, 31)), -49); - assertEquals(epochMonth(dateMillis(1969, 1, 1)), -12); - assertEquals(epochMonth(dateMillis(1969, 7, 20)), -6); - assertEquals(epochMonth(dateMillis(1969, 12, 31)), -1); - assertEquals(epochMonth(dateMillis(1970, 1, 1)), 0); - assertEquals(epochMonth(dateMillis(1970, 12, 31)), 11); - assertEquals(epochMonth(dateMillis(2012, 8, 20)), 511); - } + for (long epochSecond = start; epochSecond <= end; epochSecond += 1800) { + LocalDateTime time = LocalDateTime.ofEpochSecond(epochSecond, 0, ZoneOffset.UTC); - private static long dateMillis(int year, int month, int day) - { - return TimeUnit.SECONDS.toMillis(LocalDateTime.of(year, month, day, 0, 0, 0).toEpochSecond(ZoneOffset.UTC)); + long epochMilli = SECONDS.toMillis(epochSecond); + int actualYear = toIntExact(epochYear(epochMilli)); + int actualMonth = toIntExact(epochMonth(epochMilli)); + int actualDay = toIntExact(epochDay(epochMilli)); + int actualHour = toIntExact(epochHour(epochMilli)); + + if (time.toLocalTime().equals(LocalTime.MIDNIGHT)) { + int epochDay = toIntExact(time.toLocalDate().toEpochDay()); + assertEquals(actualYear, (int) Transforms.year(ICEBERG_DATE).apply(epochDay), time.toString()); + assertEquals(actualMonth, (int) Transforms.month(ICEBERG_DATE).apply(epochDay), time.toString()); + assertEquals(actualDay, (int) Transforms.day(ICEBERG_DATE).apply(epochDay), time.toString()); + } + + long epochMicro = SECONDS.toMicros(epochSecond); + assertEquals(actualYear, (int) Transforms.year(ICEBERG_TIMESTAMP).apply(epochMicro), time.toString()); + assertEquals(actualMonth, (int) Transforms.month(ICEBERG_TIMESTAMP).apply(epochMicro), time.toString()); + assertEquals(actualDay, (int) Transforms.day(ICEBERG_TIMESTAMP).apply(epochMicro), time.toString()); + assertEquals(actualHour, (int) Transforms.hour(ICEBERG_TIMESTAMP).apply(epochMicro), time.toString()); + } } }