From a0ed40f2b30f4c699d2ddd9c9867058302342e79 Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Mon, 2 Sep 2024 11:38:34 +0900 Subject: [PATCH] Core: Add support for view-default property in catalog --- core/src/main/java/org/apache/iceberg/CatalogProperties.java | 1 + .../java/org/apache/iceberg/rest/RESTSessionCatalog.java | 3 +++ .../org/apache/iceberg/view/BaseMetastoreViewCatalog.java | 4 ++++ .../org/apache/iceberg/inmemory/TestInMemoryViewCatalog.java | 5 ++++- .../java/org/apache/iceberg/jdbc/TestJdbcViewCatalog.java | 1 + .../java/org/apache/iceberg/rest/TestRESTViewCatalog.java | 5 ++++- .../test/java/org/apache/iceberg/view/ViewCatalogTests.java | 3 +++ docs/docs/spark-configuration.md | 2 ++ .../org/apache/iceberg/nessie/TestNessieViewCatalog.java | 4 +++- 9 files changed, 25 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/CatalogProperties.java b/core/src/main/java/org/apache/iceberg/CatalogProperties.java index b6fd990f0ac6..5261dc8207cd 100644 --- a/core/src/main/java/org/apache/iceberg/CatalogProperties.java +++ b/core/src/main/java/org/apache/iceberg/CatalogProperties.java @@ -29,6 +29,7 @@ private CatalogProperties() {} public static final String WAREHOUSE_LOCATION = "warehouse"; public static final String TABLE_DEFAULT_PREFIX = "table-default."; public static final String TABLE_OVERRIDE_PREFIX = "table-override."; + public static final String VIEW_DEFAULT_PREFIX = "view-default."; public static final String METRICS_REPORTER_IMPL = "metrics-reporter-impl"; /** diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java index 53ce45bb0a3f..d86328adb764 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java @@ -18,6 +18,8 @@ */ package org.apache.iceberg.rest; +import static org.apache.iceberg.util.PropertyUtil.propertiesWithPrefix; + import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.RemovalListener; @@ -1200,6 +1202,7 @@ private RESTViewBuilder(SessionContext context, TableIdentifier identifier) { checkViewIdentifierIsValid(identifier); this.identifier = identifier; this.context = context; + properties.putAll(propertiesWithPrefix(properties(), CatalogProperties.VIEW_DEFAULT_PREFIX)); } @Override diff --git a/core/src/main/java/org/apache/iceberg/view/BaseMetastoreViewCatalog.java b/core/src/main/java/org/apache/iceberg/view/BaseMetastoreViewCatalog.java index 6e2d6ff5e864..705b39e703cb 100644 --- a/core/src/main/java/org/apache/iceberg/view/BaseMetastoreViewCatalog.java +++ b/core/src/main/java/org/apache/iceberg/view/BaseMetastoreViewCatalog.java @@ -18,9 +18,12 @@ */ package org.apache.iceberg.view; +import static org.apache.iceberg.util.PropertyUtil.propertiesWithPrefix; + import java.util.List; import java.util.Map; import org.apache.iceberg.BaseMetastoreCatalog; +import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.EnvironmentContext; import org.apache.iceberg.Schema; import org.apache.iceberg.Transaction; @@ -79,6 +82,7 @@ protected BaseViewBuilder(TableIdentifier identifier) { Preconditions.checkArgument( isValidIdentifier(identifier), "Invalid view identifier: %s", identifier); this.identifier = identifier; + this.properties.putAll(propertiesWithPrefix(properties(), CatalogProperties.VIEW_DEFAULT_PREFIX)); } @Override diff --git a/core/src/test/java/org/apache/iceberg/inmemory/TestInMemoryViewCatalog.java b/core/src/test/java/org/apache/iceberg/inmemory/TestInMemoryViewCatalog.java index 76731f58a6be..acab44b648bd 100644 --- a/core/src/test/java/org/apache/iceberg/inmemory/TestInMemoryViewCatalog.java +++ b/core/src/test/java/org/apache/iceberg/inmemory/TestInMemoryViewCatalog.java @@ -18,6 +18,7 @@ */ package org.apache.iceberg.inmemory; +import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.view.ViewCatalogTests; @@ -29,7 +30,9 @@ public class TestInMemoryViewCatalog extends ViewCatalogTests { @BeforeEach public void before() { this.catalog = new InMemoryCatalog(); - this.catalog.initialize("in-memory-catalog", ImmutableMap.of()); + this.catalog.initialize("in-memory-catalog", ImmutableMap.builder() + .put(CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", "catalog-default-key1") + .build()); } @Override diff --git a/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcViewCatalog.java b/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcViewCatalog.java index a66532d90f63..dab623db9c0a 100644 --- a/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcViewCatalog.java +++ b/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcViewCatalog.java @@ -57,6 +57,7 @@ public void before() { properties.put(JdbcCatalog.PROPERTY_PREFIX + "password", "password"); properties.put(CatalogProperties.WAREHOUSE_LOCATION, tableDir.toAbsolutePath().toString()); properties.put(JdbcUtil.SCHEMA_VERSION_PROPERTY, JdbcUtil.SchemaVersion.V1.name()); + properties.put(CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", "catalog-default-key1"); catalog = new JdbcCatalog(); catalog.setConf(new Configuration()); diff --git a/core/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalog.java b/core/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalog.java index db0969620dc9..ced6e5ff6f8c 100644 --- a/core/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalog.java +++ b/core/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalog.java @@ -70,7 +70,10 @@ public void createCatalog() throws Exception { this.backendCatalog = new InMemoryCatalog(); this.backendCatalog.initialize( "in-memory", - ImmutableMap.of(CatalogProperties.WAREHOUSE_LOCATION, warehouse.getAbsolutePath())); + ImmutableMap.builder() + .put(CatalogProperties.WAREHOUSE_LOCATION, warehouse.getAbsolutePath()) + .put(CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", "catalog-default-key1") + .build()); RESTCatalogAdapter adaptor = new RESTCatalogAdapter(backendCatalog) { diff --git a/core/src/test/java/org/apache/iceberg/view/ViewCatalogTests.java b/core/src/test/java/org/apache/iceberg/view/ViewCatalogTests.java index b3765bb1eae7..5e1e3e076999 100644 --- a/core/src/test/java/org/apache/iceberg/view/ViewCatalogTests.java +++ b/core/src/test/java/org/apache/iceberg/view/ViewCatalogTests.java @@ -21,6 +21,7 @@ import static org.apache.iceberg.types.Types.NestedField.required; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.entry; import java.nio.file.Path; import java.nio.file.Paths; @@ -107,6 +108,7 @@ public void basicCreateView() { assertThat(view.currentVersion().operation()).isEqualTo("create"); assertThat(view.schemas()).hasSize(1).containsKey(0); assertThat(view.versions()).hasSize(1).containsExactly(view.currentVersion()); + assertThat(view.properties()).contains(entry("key1", "catalog-default-key1")); assertThat(view.currentVersion()) .isEqualTo( @@ -871,6 +873,7 @@ public void createOrReplaceView(boolean useCreateOrReplace) { assertThat(replacedView.name()).isEqualTo(ViewUtil.fullViewName(catalog().name(), identifier)); assertThat(((BaseView) replacedView).operations().current().metadataFileLocation()).isNotNull(); assertThat(replacedView.properties()) + .containsEntry("key1", "catalog-default-key1") .containsEntry("prop1", "val1") .containsEntry("prop2", "val2") .containsEntry("replacedProp1", "val1") diff --git a/docs/docs/spark-configuration.md b/docs/docs/spark-configuration.md index 5b281b19891a..072cf5259568 100644 --- a/docs/docs/spark-configuration.md +++ b/docs/docs/spark-configuration.md @@ -77,6 +77,8 @@ Both catalogs are configured using properties nested under the catalog name. Com | spark.sql.catalog._catalog-name_.cache.expiration-interval-ms | `30000` (30 seconds) | Duration after which cached catalog entries are expired; Only effective if `cache-enabled` is `true`. `-1` disables cache expiration and `0` disables caching entirely, irrespective of `cache-enabled`. Default is `30000` (30 seconds) | | spark.sql.catalog._catalog-name_.table-default._propertyKey_ | | Default Iceberg table property value for property key _propertyKey_, which will be set on tables created by this catalog if not overridden | | spark.sql.catalog._catalog-name_.table-override._propertyKey_ | | Enforced Iceberg table property value for property key _propertyKey_, which cannot be overridden by user | +| spark.sql.catalog._catalog-name_.view-default._propertyKey_ | | Default Iceberg view property value for property key _propertyKey_, which will be set on views created by this catalog if not overridden | +| spark.sql.catalog._catalog-name_.view-override._propertyKey_ | | Enforced Iceberg view property value for property key _propertyKey_, which cannot be overridden by user | | spark.sql.catalog._catalog-name_.use-nullable-query-schema | `true` or `false` | Whether to preserve fields' nullability when creating the table using CTAS and RTAS. If set to `true`, all fields will be marked as nullable. If set to `false`, fields' nullability will be preserved. The default value is `true`. Available in Spark 3.5 and above. | Additional properties can be found in common [catalog configuration](configuration.md#catalog-properties). diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieViewCatalog.java b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieViewCatalog.java index cb2c61e9dcb8..2fde8cb6f0cd 100644 --- a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieViewCatalog.java +++ b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieViewCatalog.java @@ -128,7 +128,9 @@ private NessieCatalog initNessieCatalog(String ref) { CatalogProperties.WAREHOUSE_LOCATION, temp.toUri().toString(), "client-api-version", - apiVersion == NessieApiVersion.V2 ? "2" : "1"); + apiVersion == NessieApiVersion.V2 ? "2" : "1", + CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", + "catalog-default-key1"); newCatalog.initialize("nessie", options); return newCatalog; }