Skip to content

Commit

Permalink
Core: Propagate catalog-level properties to iceberg-views.
Browse files Browse the repository at this point in the history
  • Loading branch information
@nk1506
nk1506 committed Sep 17, 2024
1 parent d5b21d8 commit e05731b
Show file tree
Hide file tree
Showing 3 changed files with 140 additions and 26 deletions.
52 changes: 26 additions & 26 deletions core/src/main/java/org/apache/iceberg/BaseMetastoreCatalog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -254,34 +254,34 @@ private Transaction newReplaceTableTransaction(boolean orCreate) {
return Transactions.replaceTableTransaction(identifier.toString(), ops, metadata);
}
}
}

/**
* Get default table properties set at Catalog level through catalog properties.
*
* @return default table properties specified in catalog properties
*/
private Map<String, String> tableDefaultProperties() {
Map<String, String> tableDefaultProperties =
PropertyUtil.propertiesWithPrefix(properties(), CatalogProperties.TABLE_DEFAULT_PREFIX);
LOG.info(
"Table properties set at catalog level through catalog properties: {}",
tableDefaultProperties);
return tableDefaultProperties;
}
/**
* Get default table properties set at Catalog level through catalog properties.
*
* @return default table properties specified in catalog properties
*/
protected Map<String, String> tableDefaultProperties() {
Map<String, String> tableDefaultProperties =
PropertyUtil.propertiesWithPrefix(properties(), CatalogProperties.TABLE_DEFAULT_PREFIX);
LOG.info(
"Table properties set at catalog level through catalog properties: {}",
tableDefaultProperties);
return tableDefaultProperties;
}

/**
* Get table properties that are enforced at Catalog level through catalog properties.
*
* @return default table properties enforced through catalog properties
*/
private Map<String, String> tableOverrideProperties() {
Map<String, String> tableOverrideProperties =
PropertyUtil.propertiesWithPrefix(properties(), CatalogProperties.TABLE_OVERRIDE_PREFIX);
LOG.info(
"Table properties enforced at catalog level through catalog properties: {}",
tableOverrideProperties);
return tableOverrideProperties;
}
/**
* Get table properties that are enforced at Catalog level through catalog properties.
*
* @return default table properties enforced through catalog properties
*/
protected Map<String, String> tableOverrideProperties() {
Map<String, String> tableOverrideProperties =
PropertyUtil.propertiesWithPrefix(properties(), CatalogProperties.TABLE_OVERRIDE_PREFIX);
LOG.info(
"Table properties enforced at catalog level through catalog properties: {}",
tableOverrideProperties);
return tableOverrideProperties;
}

protected static String fullTableName(String catalogName, TableIdentifier identifier) {
Expand Down
3 changes: 3 additions & 0 deletions core/src/main/java/org/apache/iceberg/view/BaseMetastoreViewCatalog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ protected BaseViewBuilder(TableIdentifier identifier) {
Preconditions.checkArgument(
isValidIdentifier(identifier), "Invalid view identifier: %s", identifier);
this.identifier = identifier;
this.properties.putAll(tableDefaultProperties());
}

@Override
Expand Down Expand Up @@ -155,6 +156,7 @@ private View create(ViewOperations ops) {
Preconditions.checkState(
null != defaultNamespace, "Cannot create view without specifying a default namespace");

properties.putAll(tableOverrideProperties());
ViewVersion viewVersion =
ImmutableViewVersion.builder()
.versionId(1)
Expand Down Expand Up @@ -190,6 +192,7 @@ private View replace(ViewOperations ops) {
if (null == ops.current()) {
throw new NoSuchViewException("View does not exist: %s", identifier);
}
properties.putAll(tableOverrideProperties());

Preconditions.checkState(
!representations.isEmpty(), "Cannot replace view without specifying a query");
Expand Down
111 changes: 111 additions & 0 deletions core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@
import org.apache.iceberg.transforms.Transform;
import org.apache.iceberg.transforms.Transforms;
import org.apache.iceberg.types.Types;
import org.apache.iceberg.view.View;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
Expand Down Expand Up @@ -1129,6 +1130,116 @@ public void testCommitExceptionWithMessage() {
}
}

@Test
public void testTablePropsDefinedAtCatalogLevel() throws IOException {
TableIdentifier tableIdent = TableIdentifier.of("db", "ns1", "ns2", "tbl");
ImmutableMap<String, String> catalogProps =
ImmutableMap.of(
CatalogProperties.WAREHOUSE_LOCATION,
this.tableDir.toAbsolutePath().toString(),
CatalogProperties.URI,
"jdbc:sqlite:file::memory:?icebergDBV0",
JdbcUtil.SCHEMA_VERSION_PROPERTY,
JdbcUtil.SchemaVersion.V0.name(),
"table-default.key1",
"catalog-default-key1",
"table-default.key2",
"catalog-default-key2",
"table-default.key3",
"catalog-default-key3",
"table-override.key3",
"catalog-override-key3",
"table-override.key4",
"catalog-override-key4");
JdbcCatalog jdbcCatalog = new JdbcCatalog();
jdbcCatalog.setConf(conf);
jdbcCatalog.initialize("v0catalog", catalogProps);
Table table =
jdbcCatalog
.buildTable(tableIdent, SCHEMA)
.withProperties(null)
.withProperty("key2", "table-key2")
.withProperty("key3", "table-key3")
.withProperty("key5", "table-key5")
.create();

assertThat(table.properties().get("key1"))
.as("Table defaults set for the catalog must be added to the table properties.")
.isEqualTo("catalog-default-key1");
assertThat(table.properties().get("key2"))
.as("Table property must override table default properties set at catalog level.")
.isEqualTo("table-key2");
assertThat(table.properties().get("key3"))
.as(
"Table property override set at catalog level must override table default"
+ " properties set at catalog level and table property specified.")
.isEqualTo("catalog-override-key3");
assertThat(table.properties().get("key4"))
.as("Table override not in table props or defaults should be added to table properties")
.isEqualTo("catalog-override-key4");
assertThat(table.properties().get("key5"))
.as(
"Table properties without any catalog level default or override should be added to table"
+ " properties.")
.isEqualTo("table-key5");
}

@Test
public void testViewPropsDefinedAtCatalogLevel() throws IOException {
TableIdentifier viewIdent = TableIdentifier.of("db", "ns1");
ImmutableMap<String, String> catalogProps =
ImmutableMap.of(
CatalogProperties.WAREHOUSE_LOCATION,
this.tableDir.toAbsolutePath().toString(),
CatalogProperties.URI,
"jdbc:sqlite:file::memory:?icebergDBV0",
JdbcUtil.SCHEMA_VERSION_PROPERTY,
JdbcUtil.SchemaVersion.V1.name(),
"table-default.key1",
"catalog-default-key1",
"table-default.key2",
"catalog-default-key2",
"table-default.key3",
"catalog-default-key3",
"table-override.key3",
"catalog-override-key3",
"table-override.key4",
"catalog-override-key4");
JdbcCatalog jdbcCatalog = new JdbcCatalog();
jdbcCatalog.setConf(conf);
jdbcCatalog.initialize("v0catalog", catalogProps);
View view =
jdbcCatalog
.buildView(viewIdent)
.withQuery("spark", "SELECT * FROM t1")
.withSchema(SCHEMA)
.withDefaultNamespace(Namespace.of("db"))
.withProperty("key2", "view-key2")
.withProperty("key3", "view-key3")
.withProperty("key5", "view-key5")
.create();

assertThat(view.properties().get("key1"))
.as("Table defaults set for the catalog must be added to the view properties.")
.isEqualTo("catalog-default-key1");
assertThat(view.properties().get("key2"))
.as("View property must override table default properties set at catalog level.")
.isEqualTo("view-key2");
assertThat(view.properties().get("key3"))
.as(
"View property override set at catalog level must override table default"
+ " properties set at catalog level and table property specified.")
.isEqualTo("catalog-override-key3");
assertThat(view.properties().get("key4"))
.as("Table override not in table props or defaults should be added to view properties")
.isEqualTo("catalog-override-key4");
assertThat(view.properties().get("key5"))
.as(
"Table properties without any catalog level default or override should be added to view"
+ " properties.")
.isEqualTo("view-key5");
}

public static class CustomMetricsReporter implements MetricsReporter {
static final AtomicInteger COUNTER = new AtomicInteger(0);

Expand Down

0 comments on commit e05731b

Please sign in to comment.