Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add Privilege for Updating Maintenance Properties in Catalog #457

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
+216 −0
Open

Add Privilege for Updating Maintenance Properties in Catalog #457

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b565c2d
Add write maintenance privilege for catalog
Nov 18, 2024
3351ab8
Add logic for maintenance key removal
Nov 18, 2024
81919b5
Annotate the parameter to NotNull
Nov 18, 2024
52f6c7d
Make the semicolon in a separated line
Nov 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizableOperation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import static org.apache.polaris.core.entity.PolarisPrivilege.CATALOG_ROLE_MANAGE_GRANTS_ON_SECURABLE;
import static org.apache.polaris.core.entity.PolarisPrivilege.CATALOG_ROLE_READ_PROPERTIES;
import static org.apache.polaris.core.entity.PolarisPrivilege.CATALOG_ROLE_WRITE_PROPERTIES;
import static org.apache.polaris.core.entity.PolarisPrivilege.CATALOG_WRITE_MAINTENANCE_PROPERTIES;
import static org.apache.polaris.core.entity.PolarisPrivilege.CATALOG_WRITE_PROPERTIES;
import static org.apache.polaris.core.entity.PolarisPrivilege.NAMESPACE_CREATE;
import static org.apache.polaris.core.entity.PolarisPrivilege.NAMESPACE_DROP;
Expand Down Expand Up @@ -121,6 +122,7 @@ public enum PolarisAuthorizableOperation {
CREATE_CATALOG(CATALOG_CREATE),
GET_CATALOG(CATALOG_READ_PROPERTIES),
UPDATE_CATALOG(CATALOG_WRITE_PROPERTIES),
UPDATE_CATALOG_MAINTENANCE_PROPERTIES(CATALOG_WRITE_MAINTENANCE_PROPERTIES),
Copy link
Contributor

@dimas-b dimas-b Nov 18, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

tangential: This change LGTM, but I'm thinking about breaking down the operation enum into several enums with dedicated scope (operations within a catalog, catalog management (add/remote), permission management)... Just FYI :)

DELETE_CATALOG(CATALOG_DROP),
LIST_PRINCIPALS(PRINCIPAL_LIST),
CREATE_PRINCIPAL(PRINCIPAL_CREATE),
Expand Down
16 changes: 16 additions & 0 deletions polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import static org.apache.polaris.core.entity.PolarisPrivilege.CATALOG_ROLE_READ_PROPERTIES;
import static org.apache.polaris.core.entity.PolarisPrivilege.CATALOG_ROLE_USAGE;
import static org.apache.polaris.core.entity.PolarisPrivilege.CATALOG_ROLE_WRITE_PROPERTIES;
import static org.apache.polaris.core.entity.PolarisPrivilege.CATALOG_WRITE_MAINTENANCE_PROPERTIES;
import static org.apache.polaris.core.entity.PolarisPrivilege.CATALOG_WRITE_PROPERTIES;
import static org.apache.polaris.core.entity.PolarisPrivilege.NAMESPACE_CREATE;
import static org.apache.polaris.core.entity.PolarisPrivilege.NAMESPACE_DROP;
Expand Down Expand Up @@ -270,6 +271,7 @@ public class PolarisAuthorizerImpl implements PolarisAuthorizer {
CATALOG_MANAGE_METADATA,
CATALOG_READ_PROPERTIES,
CATALOG_WRITE_PROPERTIES,
CATALOG_WRITE_MAINTENANCE_PROPERTIES,
SERVICE_MANAGE_ACCESS));
SUPER_PRIVILEGES.putAll(
CATALOG_READ_PROPERTIES,
Expand All @@ -279,15 +281,29 @@ public class PolarisAuthorizerImpl implements PolarisAuthorizer {
CATALOG_MANAGE_METADATA,
CATALOG_READ_PROPERTIES,
CATALOG_WRITE_PROPERTIES,
CATALOG_WRITE_MAINTENANCE_PROPERTIES,
SERVICE_MANAGE_ACCESS));

// CATALOG_WRITE_MAINTENANCE_PROPERTIES is better than CATALOG_WRITE_PROPERTIES
SUPER_PRIVILEGES.putAll(
CATALOG_WRITE_PROPERTIES,
List.of(
CATALOG_FULL_METADATA,
CATALOG_MANAGE_CONTENT,
CATALOG_MANAGE_METADATA,
CATALOG_WRITE_PROPERTIES,
CATALOG_WRITE_MAINTENANCE_PROPERTIES,
SERVICE_MANAGE_ACCESS));

SUPER_PRIVILEGES.putAll(
CATALOG_WRITE_MAINTENANCE_PROPERTIES,
List.of(
CATALOG_FULL_METADATA,
CATALOG_MANAGE_CONTENT,
CATALOG_MANAGE_METADATA,
CATALOG_WRITE_MAINTENANCE_PROPERTIES,
SERVICE_MANAGE_ACCESS));

SUPER_PRIVILEGES.putAll(
CATALOG_FULL_METADATA, List.of(CATALOG_FULL_METADATA, SERVICE_MANAGE_ACCESS));

Expand Down
2 changes: 2 additions & 0 deletions polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@ public class PolarisEntityConstants {
public static final String PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_STATE =
"CREDENTIAL_ROTATION_REQUIRED";

public static final String MAINTENANCE_PREFIX = "polaris.maintenance.";

/**
* Name format of storage integration for polaris entity: {@code
* POLARIS_<catalog_id>_<entity_id>}. This name format gives us flexibility to switch to use
Expand Down
45 changes: 45 additions & 0 deletions polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisMaintenanceProperties.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.polaris.core.entity;

import static org.apache.polaris.core.entity.PolarisEntityConstants.MAINTENANCE_PREFIX;

public enum PolarisMaintenanceProperties {
COMPACTION(MAINTENANCE_PREFIX + "compaction"),
SNAPSHOT_RETENTION(MAINTENANCE_PREFIX + "snapshot_expiration");
Comment on lines +24 to +25
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will add more types, and validation in follow-up PRs.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you are referring to the "expire snapshots"? Shouldn't the enum be EXPIRE_SNAPSHOTS to align with iceberg interpretation -> https://iceberg.apache.org/docs/1.4.1/maintenance/#expire-snapshots

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, the enum name says RETENTION but the property has expiration – would be good to standardize.


private final String value;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the name value is misleading, it's rather the property name.


PolarisMaintenanceProperties(String value) {
this.value = value;
}

public String getValue() {
return value;
}

public static PolarisMaintenanceProperties fromValue(String value) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is the added value of this method, as opposed to PolarisMaintenanceProperties.valueOf()?

for (PolarisMaintenanceProperties property : PolarisMaintenanceProperties.values()) {
if (property.value.equals(value)) {
return property;
}
}
throw new IllegalArgumentException("Unknown value: " + value);
}
}
4 changes: 3 additions & 1 deletion polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisPrivilege.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,9 @@ public enum PolarisPrivilege {
CATALOG_ROLE_FULL_METADATA(67, PolarisEntityType.CATALOG_ROLE),
CATALOG_ROLE_MANAGE_GRANTS_ON_SECURABLE(68, PolarisEntityType.CATALOG_ROLE),
CATALOG_ROLE_MANAGE_GRANTS_FOR_GRANTEE(69, PolarisEntityType.CATALOG_ROLE),
;
flyrain marked this conversation as resolved.
Show resolved Hide resolved
TABLE_WRITE_MAINTENANCE_PROPERTIES(70, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.TABLE),
NAMESPACE_WRITE_MAINTENANCE_PROPERTIES(71, PolarisEntityType.NAMESPACE),
Comment on lines +105 to +106
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This PR focus on the catalog privilege only. The detailed implementation for table and namespace will be in follow-up PRs.

CATALOG_WRITE_MAINTENANCE_PROPERTIES(72, PolarisEntityType.CATALOG);

/**
* Full constructor
Expand Down
40 changes: 40 additions & 0 deletions polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,15 @@
*/
package org.apache.polaris.service.admin;

import static org.apache.polaris.core.auth.PolarisAuthorizableOperation.UPDATE_CATALOG_MAINTENANCE_PROPERTIES;
import static org.apache.polaris.core.entity.PolarisEntityConstants.MAINTENANCE_PREFIX;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
Expand Down Expand Up @@ -84,6 +88,7 @@
import org.apache.polaris.core.storage.azure.AzureStorageConfigurationInfo;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -657,6 +662,35 @@ private void validateUpdateCatalogDiffOrThrow(
}
}

@VisibleForTesting
static boolean maintenancePropertyChanged(
@NotNull Map<String, String> map1, @NotNull Map<String, String> map2) {
Set<String> addedKeys = new HashSet<>(map2.keySet());
addedKeys.removeAll(map1.keySet());
if (addedKeys.stream().anyMatch(key -> key.startsWith((MAINTENANCE_PREFIX)))) {
return true;
}

Set<String> removedKeys = new HashSet<>(map1.keySet());
removedKeys.removeAll(map2.keySet());
if (removedKeys.stream().anyMatch(key -> key.startsWith((MAINTENANCE_PREFIX)))) {
return true;
}

Set<String> commonKeys = new HashSet<>(map1.keySet());
commonKeys.retainAll(map2.keySet());

for (String key : commonKeys) {
if (!Objects.equals(map1.get(key), map2.get(key))) {
if (key.startsWith(MAINTENANCE_PREFIX)) {
return true;
}
}
}

return false;
}

public @NotNull CatalogEntity updateCatalog(String name, UpdateCatalogRequest updateRequest) {
PolarisAuthorizableOperation op = PolarisAuthorizableOperation.UPDATE_CATALOG;
authorizeBasicTopLevelEntityOperationOrThrow(op, name, PolarisEntityType.CATALOG);
Expand Down Expand Up @@ -699,6 +733,12 @@ private void validateUpdateCatalogDiffOrThrow(
}
CatalogEntity updatedEntity = updateBuilder.build();

if (maintenancePropertyChanged(
currentCatalogEntity.getPropertiesAsMap(), updatedEntity.getPropertiesAsMap())) {
authorizeBasicTopLevelEntityOperationOrThrow(
UPDATE_CATALOG_MAINTENANCE_PROPERTIES, name, PolarisEntityType.CATALOG);
}

validateUpdateCatalogDiffOrThrow(currentCatalogEntity, updatedEntity);

if (catalogOverlapsWithExistingCatalog(updatedEntity)) {
Expand Down
108 changes: 108 additions & 0 deletions ...-service/src/test/java/org/apache/polaris/service/admin/PolarisAdminServiceAuthzTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,13 @@
*/
package org.apache.polaris.service.admin;

import static org.apache.polaris.core.entity.PolarisEntityConstants.MAINTENANCE_PREFIX;
import static org.apache.polaris.core.entity.PolarisMaintenanceProperties.COMPACTION;
import static org.apache.polaris.service.admin.PolarisAdminService.maintenancePropertyChanged;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
Expand All @@ -33,6 +40,7 @@
import org.apache.polaris.core.entity.PrincipalEntity;
import org.apache.polaris.core.entity.PrincipalRoleEntity;
import org.assertj.core.api.Assertions;
import org.jetbrains.annotations.NotNull;
import org.junit.jupiter.api.Test;

public class PolarisAdminServiceAuthzTest extends PolarisAuthzTestBase {
Expand Down Expand Up @@ -211,6 +219,7 @@ public void testUpdateCatalogSufficientPrivileges() {
List.of(
PolarisPrivilege.SERVICE_MANAGE_ACCESS,
PolarisPrivilege.CATALOG_WRITE_PROPERTIES,
PolarisPrivilege.CATALOG_WRITE_MAINTENANCE_PROPERTIES,
PolarisPrivilege.CATALOG_FULL_METADATA,
PolarisPrivilege.CATALOG_MANAGE_METADATA,
PolarisPrivilege.CATALOG_MANAGE_CONTENT),
Expand Down Expand Up @@ -266,6 +275,61 @@ public void testUpdateCatalogInsufficientPrivileges() {
PRINCIPAL_ROLE1, privilege));
}

@Test
public void testUpdateCatalogSufficientPrivilegesForMaintenanceProperties() {
doTestSufficientPrivileges(
List.of(
PolarisPrivilege.CATALOG_WRITE_MAINTENANCE_PROPERTIES,
PolarisPrivilege.SERVICE_MANAGE_ACCESS,
PolarisPrivilege.CATALOG_FULL_METADATA,
PolarisPrivilege.CATALOG_MANAGE_METADATA,
PolarisPrivilege.CATALOG_MANAGE_CONTENT),
updateCatalogMaintenanceProperty(),
null,
(privilege) ->
adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege),
(privilege) ->
adminService.revokePrivilegeOnRootContainerFromPrincipalRole(
PRINCIPAL_ROLE1, privilege));
}

@Test
public void testUpdateCatalogInsufficientPrivilegesForMaintenanceProperties() {
doTestInsufficientPrivileges(
List.of(
PolarisPrivilege.NAMESPACE_FULL_METADATA,
PolarisPrivilege.TABLE_FULL_METADATA,
PolarisPrivilege.VIEW_FULL_METADATA,
PolarisPrivilege.VIEW_FULL_METADATA,
PolarisPrivilege.PRINCIPAL_FULL_METADATA,
PolarisPrivilege.PRINCIPAL_ROLE_FULL_METADATA,
PolarisPrivilege.CATALOG_READ_PROPERTIES,
PolarisPrivilege.CATALOG_LIST,
PolarisPrivilege.CATALOG_CREATE,
PolarisPrivilege.CATALOG_DROP,
PolarisPrivilege.CATALOG_ROLE_FULL_METADATA,
PolarisPrivilege.CATALOG_WRITE_PROPERTIES,
PolarisPrivilege.CATALOG_MANAGE_ACCESS),
updateCatalogMaintenanceProperty(),
(privilege) ->
adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege),
(privilege) ->
adminService.revokePrivilegeOnRootContainerFromPrincipalRole(
PRINCIPAL_ROLE1, privilege));
}

private @NotNull Runnable updateCatalogMaintenanceProperty() {
return () -> {
var entityVersion = newTestAdminService().getCatalog(CATALOG_NAME).getEntityVersion();
var updateRequest =
UpdateCatalogRequest.builder()
.setCurrentEntityVersion(entityVersion)
.setProperties(Map.of(COMPACTION.getValue(), "{}"))
.build();
newTestAdminService().updateCatalog(CATALOG_NAME, updateRequest);
};
}

@Test
public void testDeleteCatalogSufficientPrivileges() {
// Cleanup with PRINCIPAL_ROLE2
Expand Down Expand Up @@ -1837,4 +1901,48 @@ public void testRevokePrivilegeOnViewFromRoleInsufficientPrivileges() {
(privilege) ->
adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege));
}

@Test
public void testMaintenancePropertyChanged() {
// Scenario 1: Added Maintenance Key
Map<String, String> map1 = new HashMap<>();
Map<String, String> map2 = new HashMap<>();
map2.put(MAINTENANCE_PREFIX + "key1", "value1");

assertTrue(
maintenancePropertyChanged(map1, map2), "Should return true for added maintenance key");

// Scenario 2: Removed Maintenance Key
map1.put(MAINTENANCE_PREFIX + "key1", "value1");
map2.clear();

assertTrue(
maintenancePropertyChanged(map1, map2), "Should return true for removed maintenance key");

// Scenario 3: Changed Maintenance Key
map1.put(MAINTENANCE_PREFIX + "key1", "value1");
map2.put(MAINTENANCE_PREFIX + "key1", "value2");

assertTrue(
maintenancePropertyChanged(map1, map2), "Should return true for changed maintenance key");

// Scenario 4: No Maintenance Keys Involved
map1.clear();
map2.clear();
map1.put("other.key1", "value1");
map2.put("other.key1", "value1");

assertFalse(
maintenancePropertyChanged(map1, map2),
"Should return false when no maintenance keys are involved");

// Scenario 5: Added Non-Maintenance Key
map1.clear();
map2.clear();
map2.put("other.key1", "value1");

assertFalse(
maintenancePropertyChanged(map1, map2),
"Should return false for added non-maintenance key");
}
}