Skip to content

Commit

Permalink
Add Iceberg BaseTrinoCatalogTest
Browse files Browse the repository at this point in the history
  • Loading branch information
@alexjo2144
alexjo2144 committed Mar 14, 2022
1 parent 516c0a2 commit fb7124f
Show file tree
Hide file tree
Showing 2 changed files with 287 additions and 0 deletions.
188 changes: 188 additions & 0 deletions plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseTrinoCatalogTest.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,188 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.trino.plugin.iceberg;

import com.google.common.collect.ImmutableMap;
import io.airlift.log.Logger;
import io.trino.plugin.iceberg.catalog.TrinoCatalog;
import io.trino.spi.connector.SchemaTableName;
import io.trino.spi.security.PrincipalType;
import io.trino.spi.security.TrinoPrincipal;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.Schema;
import org.apache.iceberg.Table;
import org.apache.iceberg.types.Types;
import org.testng.annotations.Test;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Optional;

import static io.trino.plugin.iceberg.IcebergSchemaProperties.LOCATION_PROPERTY;
import static io.trino.plugin.iceberg.IcebergUtil.quotedTableName;
import static io.trino.testing.TestingConnectorSession.SESSION;
import static io.trino.testing.assertions.Assert.assertEquals;
import static io.trino.testing.sql.TestTable.randomTableSuffix;
import static org.assertj.core.api.Assertions.assertThat;
import static org.testng.Assert.assertNotEquals;

public abstract class BaseTrinoCatalogTest
{
private static final Logger LOG = Logger.get(BaseTrinoCatalogTest.class);

protected abstract TrinoCatalog createTrinoCatalog(boolean useUniqueTableLocations);

@Test
public void testCreateNamespaceWithLocation()
{
TrinoCatalog catalog = createTrinoCatalog(false);

String namespace = "test_create_namespace_with_location_" + randomTableSuffix();
catalog.createNamespace(SESSION, namespace, ImmutableMap.of(LOCATION_PROPERTY, "/a/path/"), new TrinoPrincipal(PrincipalType.USER, SESSION.getUser()));
assertThat(catalog.listNamespaces(SESSION)).contains(namespace);
assertEquals(catalog.loadNamespaceMetadata(SESSION, namespace), ImmutableMap.of(LOCATION_PROPERTY, "/a/path/"));
assertEquals(catalog.defaultTableLocation(SESSION, new SchemaTableName(namespace, "table")), "/a/path/table");
catalog.dropNamespace(SESSION, namespace);
assertThat(catalog.listNamespaces(SESSION)).doesNotContain(namespace);
}

@Test
public void testCreateTable()
throws IOException
{
TrinoCatalog catalog = createTrinoCatalog(false);
Path tmpDirectory = Files.createTempDirectory("iceberg_catalog_test_create_table_");
tmpDirectory.toFile().deleteOnExit();

String namespace = "test_create_table_" + randomTableSuffix();
String table = "tableName";
SchemaTableName schemaTableName = new SchemaTableName(namespace, table);
try {
catalog.createNamespace(SESSION, namespace, ImmutableMap.of(), new TrinoPrincipal(PrincipalType.USER, SESSION.getUser()));
catalog.newCreateTableTransaction(
SESSION,
schemaTableName,
new Schema(Types.NestedField.of(1, true, "col1", Types.LongType.get())),
PartitionSpec.unpartitioned(),
tmpDirectory.toAbsolutePath().toString(),
ImmutableMap.of())
.commitTransaction();
assertThat(catalog.listTables(SESSION, Optional.of(namespace))).contains(schemaTableName);
assertThat(catalog.listTables(SESSION, Optional.empty())).contains(schemaTableName);

Table icebergTable = catalog.loadTable(SESSION, schemaTableName);
assertEquals(icebergTable.name(), quotedTableName(schemaTableName));
assertEquals(icebergTable.schema().columns().size(), 1);
assertEquals(icebergTable.schema().columns().get(0).name(), "col1");
assertEquals(icebergTable.schema().columns().get(0).type(), Types.LongType.get());
assertEquals(icebergTable.location(), tmpDirectory.toAbsolutePath().toString());
assertEquals(icebergTable.properties(), ImmutableMap.of());

catalog.dropTable(SESSION, schemaTableName);
assertThat(catalog.listTables(SESSION, Optional.of(namespace))).doesNotContain(schemaTableName);
assertThat(catalog.listTables(SESSION, Optional.empty())).doesNotContain(schemaTableName);
}
finally {
try {
catalog.dropNamespace(SESSION, namespace);
}
catch (Exception e) {
LOG.warn("Failed to clean up namespace: %s", namespace);
}
}
}

@Test
public void testRenameTable()
throws IOException
{
TrinoCatalog catalog = createTrinoCatalog(false);
Path tmpDirectory = Files.createTempDirectory("iceberg_catalog_test_rename_table_");
tmpDirectory.toFile().deleteOnExit();

String namespace = "test_rename_table_" + randomTableSuffix();
String targetNamespace = "test_rename_table_" + randomTableSuffix();

String table = "tableName";
SchemaTableName sourceSchemaTableName = new SchemaTableName(namespace, table);
try {
catalog.createNamespace(SESSION, namespace, ImmutableMap.of(), new TrinoPrincipal(PrincipalType.USER, SESSION.getUser()));
catalog.createNamespace(SESSION, targetNamespace, ImmutableMap.of(), new TrinoPrincipal(PrincipalType.USER, SESSION.getUser()));
catalog.newCreateTableTransaction(
SESSION,
sourceSchemaTableName,
new Schema(Types.NestedField.of(1, true, "col1", Types.LongType.get())),
PartitionSpec.unpartitioned(),
tmpDirectory.toAbsolutePath().toString(),
ImmutableMap.of())
.commitTransaction();
assertThat(catalog.listTables(SESSION, Optional.of(namespace))).contains(sourceSchemaTableName);

// Rename within the same schema
SchemaTableName targetSchemaTableName = new SchemaTableName(sourceSchemaTableName.getSchemaName(), "newTableName");
catalog.renameTable(SESSION, sourceSchemaTableName, targetSchemaTableName);
assertThat(catalog.listTables(SESSION, Optional.of(namespace))).doesNotContain(sourceSchemaTableName);
assertThat(catalog.listTables(SESSION, Optional.of(namespace))).contains(targetSchemaTableName);

// Move to a different schema
sourceSchemaTableName = targetSchemaTableName;
targetSchemaTableName = new SchemaTableName(targetNamespace, sourceSchemaTableName.getTableName());
catalog.renameTable(SESSION, sourceSchemaTableName, targetSchemaTableName);
assertThat(catalog.listTables(SESSION, Optional.of(namespace))).doesNotContain(sourceSchemaTableName);
assertThat(catalog.listTables(SESSION, Optional.of(targetNamespace))).contains(targetSchemaTableName);

catalog.dropTable(SESSION, targetSchemaTableName);
}
finally {
try {
catalog.dropNamespace(SESSION, namespace);
catalog.dropNamespace(SESSION, targetNamespace);
}
catch (Exception e) {
LOG.warn("Failed to clean up namespaces: %s, %s", namespace, targetNamespace);
}
}
}

@Test
public void testUseUniqueTableLocations()
throws IOException
{
TrinoCatalog catalog = createTrinoCatalog(true);
Path tmpDirectory = Files.createTempDirectory("iceberg_catalog_test_rename_table_");
tmpDirectory.toFile().deleteOnExit();

String namespace = "test_unique_table_locations_" + randomTableSuffix();
String table = "tableName";
SchemaTableName schemaTableName = new SchemaTableName(namespace, table);
catalog.createNamespace(SESSION, namespace, ImmutableMap.of(LOCATION_PROPERTY, tmpDirectory.toString()), new TrinoPrincipal(PrincipalType.USER, SESSION.getUser()));
try {
String location1 = catalog.defaultTableLocation(SESSION, schemaTableName);
String location2 = catalog.defaultTableLocation(SESSION, schemaTableName);
assertNotEquals(location1, location2);

assertEquals(Path.of(location1).getParent(), tmpDirectory);
assertEquals(Path.of(location2).getParent(), tmpDirectory);
}
finally {
try {
catalog.dropNamespace(SESSION, namespace);
}
catch (Exception e) {
LOG.warn("Failed to clean up namespace: %s", namespace);
}
}
}
}
99 changes: 99 additions & 0 deletions plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestTrinoHiveCatalogTest.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.trino.plugin.iceberg;

import com.google.common.collect.ImmutableSet;
import io.trino.plugin.base.CatalogName;
import io.trino.plugin.hive.HdfsConfig;
import io.trino.plugin.hive.HdfsConfiguration;
import io.trino.plugin.hive.HdfsConfigurationInitializer;
import io.trino.plugin.hive.HdfsEnvironment;
import io.trino.plugin.hive.HiveHdfsConfiguration;
import io.trino.plugin.hive.authentication.NoHdfsAuthentication;
import io.trino.plugin.hive.metastore.HiveMetastore;
import io.trino.plugin.iceberg.catalog.TrinoCatalog;
import io.trino.plugin.iceberg.catalog.file.FileMetastoreTableOperationsProvider;
import io.trino.plugin.iceberg.catalog.hms.TrinoHiveCatalog;
import io.trino.spi.type.TestingTypeManager;
import org.testng.annotations.AfterClass;
import org.testng.annotations.Test;

import java.io.File;
import java.io.IOException;
import java.nio.file.Files;

import static com.google.common.io.MoreFiles.deleteRecursively;
import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE;
import static io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.memoizeMetastore;
import static io.trino.plugin.hive.metastore.file.FileHiveMetastore.createTestingFileHiveMetastore;
import static org.assertj.core.api.Assertions.assertThatThrownBy;

public class TestTrinoHiveCatalogTest
extends BaseTrinoCatalogTest
{
private final HiveMetastore metastore;
private final HdfsEnvironment hdfsEnvironment;
private final java.nio.file.Path tempDir;
private final File metastoreDir;

public TestTrinoHiveCatalogTest()
throws IOException
{
HdfsConfig config = new HdfsConfig();
HdfsConfiguration configuration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(config), ImmutableSet.of());
hdfsEnvironment = new HdfsEnvironment(configuration, config, new NoHdfsAuthentication());

tempDir = Files.createTempDirectory("test_trino_hive_catalog");
metastoreDir = tempDir.resolve("iceberg_data").toFile();
metastore = createTestingFileHiveMetastore(metastoreDir);
}

@AfterClass(alwaysRun = true)
public void tearDown()
throws IOException
{
deleteRecursively(tempDir, ALLOW_INSECURE);
}

@Override
protected TrinoCatalog createTrinoCatalog(boolean useUniqueTableLocations)
{
return new TrinoHiveCatalog(
new CatalogName("catalog"),
memoizeMetastore(metastore, 1000),
hdfsEnvironment,
new TestingTypeManager(),
new FileMetastoreTableOperationsProvider(new HdfsFileIoProvider(hdfsEnvironment)),
"trino-version",
useUniqueTableLocations,
false,
false);
}

@Override
@Test
public void testCreateNamespaceWithLocation()
{
assertThatThrownBy(super::testCreateNamespaceWithLocation)
.hasMessageContaining("Database cannot be created with a location set");
}

@Override
@Test
public void testUseUniqueTableLocations()
{
assertThatThrownBy(super::testCreateNamespaceWithLocation)
.hasMessageContaining("Database cannot be created with a location set");
}
}

0 comments on commit fb7124f

Please sign in to comment.