Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Hive tests for trailing space in location #16407

Merged
merged 2 commits into from
Mar 14, 2023
Merged

Add Hive tests for trailing space in location #16407

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
24f1203
Extract method for obtaining table location
pajaks Mar 7, 2023
f2779e0
Add Hive tests for trailing space in location
pajaks Mar 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHiveFileSystem.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,11 @@
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.UncheckedIOException;
import java.util.Collection;
import java.util.List;
Expand Down Expand Up @@ -114,6 +118,7 @@
import static io.trino.testing.QueryAssertions.assertEqualsIgnoreOrder;
import static io.trino.testing.TestingPageSinkId.TESTING_PAGE_SINK_ID;
import static io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Locale.ENGLISH;
import static java.util.UUID.randomUUID;
import static java.util.concurrent.Executors.newCachedThreadPool;
Expand Down Expand Up @@ -488,6 +493,28 @@ public void testFileIteratorListing()
assertEqualsIgnoreOrder(shallowListing, ImmutableList.of(baseFile));
}

@Test
public void testDirectoryWithTrailingSpace()
throws Exception
{
Path basePath = new Path(getBasePath(), randomUUID().toString());
FileSystem fs = hdfsEnvironment.getFileSystem(TESTING_CONTEXT, basePath);
assertFalse(fs.exists(basePath));

Path path = new Path(new Path(basePath, "dir_with_space "), "foo.txt");
try (OutputStream outputStream = fs.create(path)) {
outputStream.write("test".getBytes(UTF_8));
}
assertTrue(fs.exists(path));

try (InputStream inputStream = fs.open(path)) {
String content = new BufferedReader(new InputStreamReader(inputStream, UTF_8)).readLine();
assertEquals(content, "test");
}

fs.delete(basePath, true);
}

@Test
public void testTableCreation()
throws Exception
Expand Down
13 changes: 9 additions & 4 deletions plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseHiveConnectorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5525,7 +5525,7 @@ public void testParquetColumnNameMappings()
"SELECT a FROM " + tableName + " WHERE b = 'b'",
"VALUES ('a')");

String tableLocation = (String) computeActual("SELECT DISTINCT regexp_replace(\"$path\", '/[^/]*$', '') FROM " + tableName).getOnlyValue();
ebyhr marked this conversation as resolved.
Show resolved Hide resolved
String tableLocation = getTableLocation(tableName);

// Reverse the table so that the Hive column ordering does not match the Parquet column ordering
String reversedTableName = "test_parquet_by_column_index_reversed";
Expand Down Expand Up @@ -5578,7 +5578,7 @@ public void testParquetWithMissingColumns()
singleColumnTableName));
assertUpdate(sessionUsingColumnIndex, "INSERT INTO " + singleColumnTableName + " VALUES ('a')", 1);

String tableLocation = (String) computeActual("SELECT DISTINCT regexp_replace(\"$path\", '/[^/]*$', '') FROM " + singleColumnTableName).getOnlyValue();
String tableLocation = getTableLocation(singleColumnTableName);
String multiColumnTableName = "test_parquet_missing_columns_two";
assertUpdate(sessionUsingColumnIndex, format(
"CREATE TABLE %s(" +
Expand Down Expand Up @@ -5630,7 +5630,7 @@ public void testParquetWithMissingNestedColumns()
missingNestedFieldsTableName));
assertUpdate(sessionUsingColumnIndex, "INSERT INTO " + missingNestedFieldsTableName + " VALUES (ARRAY[ROW(2)])", 1);

String tableLocation = (String) computeActual("SELECT DISTINCT regexp_replace(\"$path\", '/[^/]*$', '') FROM " + missingNestedFieldsTableName).getOnlyValue();
String tableLocation = getTableLocation(missingNestedFieldsTableName);
String missingNestedArrayTableName = "test_parquet_missing_nested_array";
assertUpdate(sessionUsingColumnIndex, format(
"CREATE TABLE %s(" +
Expand Down Expand Up @@ -7866,7 +7866,7 @@ private void testColumnPruning(Session session, HiveStorageFormat storageFormat)
"SELECT b, c.f1.g2, c.f3, d FROM " + tableName,
"VALUES ('ala', 873321, X'abcdef', 12345678)");

String tableLocation = (String) computeActual("SELECT DISTINCT regexp_replace(\"$path\", '/[^/]*$', '') FROM " + tableName).getOnlyValue();
String tableLocation = getTableLocation(tableName);

assertUpdate(session, format(
"CREATE TABLE %s(" +
Expand Down Expand Up @@ -8709,6 +8709,11 @@ private Session withTimestampPrecision(Session session, HiveTimestampPrecision p
.build();
}

private String getTableLocation(String tableName)
{
return (String) computeScalar("SELECT DISTINCT regexp_replace(\"$path\", '/[^/]*$', '') FROM " + tableName);
}

private static final class BucketedFilterTestSetup
{
private final String typeName;
Expand Down
33 changes: 33 additions & 0 deletions plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseTestHiveOnDataLake.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@
import static io.trino.testing.MaterializedResult.resultBuilder;
import static io.trino.testing.TestingNames.randomNameSuffix;
import static java.lang.String.format;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.time.temporal.ChronoUnit.DAYS;
import static java.time.temporal.ChronoUnit.MINUTES;
import static java.util.Objects.requireNonNull;
Expand Down Expand Up @@ -1624,6 +1625,33 @@ public void testAnalyzePartitionedTableWithCanonicalization()
assertUpdate("DROP TABLE " + getFullyQualifiedTestTableName(tableName));
}

@Test
public void testExternalLocationWithTrailingSpace()
{
String tableName = "test_external_location_with_trailing_space_" + randomNameSuffix();
String tableLocationDirWithTrailingSpace = tableName + " ";
String tableLocation = format("s3a://%s/%s/%s", bucketName, HIVE_TEST_SCHEMA, tableLocationDirWithTrailingSpace);

byte[] contents = "hello\u0001world\nbye\u0001world".getBytes(UTF_8);
String targetPath = format("%s/%s/test.txt", HIVE_TEST_SCHEMA, tableLocationDirWithTrailingSpace);
hiveMinioDataLake.getMinioClient().putObject(bucketName, contents, targetPath);

assertUpdate(format(
"CREATE TABLE %s (" +
" a varchar, " +
" b varchar) " +
"WITH (format='TEXTFILE', external_location='%s')",
tableName,
tableLocation));

assertQuery("SELECT a, b FROM " + tableName, "VALUES ('hello', 'world'), ('bye', 'world')");

String actualTableLocation = getTableLocation(tableName);
assertThat(actualTableLocation).isEqualTo(tableLocation);

assertUpdate("DROP TABLE " + tableName);
}

private void renamePartitionResourcesOutsideTrino(String tableName, String partitionColumn, String regionKey)
{
String partitionName = format("%s=%s", partitionColumn, regionKey);
Expand Down Expand Up @@ -1814,4 +1842,9 @@ private void addPartitions(
PartitionStatistics.empty()))
.collect(toImmutableList()));
}

private String getTableLocation(String tableName)
{
return (String) computeScalar("SELECT DISTINCT regexp_replace(\"$path\", '/[^/]*$', '') FROM " + tableName);
}
}