From 2ba362443e7151c232f5af0edb73465d1624fdcf Mon Sep 17 00:00:00 2001 From: saketa Date: Thu, 15 Aug 2024 19:25:34 -0700 Subject: [PATCH 1/4] HDDS-11227. Return required server default configurations. Use server default key provider to encrypt/decrypt keys from multiple OMs. --- .../apache/hadoop/ozone/OzoneConfigKeys.java | 7 +++ .../src/main/resources/ozone-default.xml | 13 ++++ .../ozone/client/protocol/ClientProtocol.java | 8 +++ .../hadoop/ozone/client/rpc/RpcClient.java | 27 ++++++++- .../apache/hadoop/ozone/FsServerDefaults.java | 59 +++++++++++++++++++ .../java/org/apache/hadoop/ozone/OmUtils.java | 1 + .../om/protocol/OzoneManagerProtocol.java | 9 +++ ...ManagerProtocolClientSideTranslatorPB.java | 19 ++++++ .../ozone/client/rpc/OzoneRpcClientTests.java | 6 ++ .../client/rpc/TestSecureOzoneRpcClient.java | 11 ++++ .../src/main/proto/OmClientProtocol.proto | 14 +++++ .../apache/hadoop/ozone/om/OzoneManager.java | 19 ++++++ .../OzoneManagerRequestHandler.java | 7 +++ .../ozone/client/ClientProtocolStub.java | 6 ++ 14 files changed, 204 insertions(+), 2 deletions(-) create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/FsServerDefaults.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index a1547a536b9..c61502ff4a8 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -583,6 +583,13 @@ public final class OzoneConfigKeys { public static final boolean OZONE_CLIENT_KEY_LATEST_VERSION_LOCATION_DEFAULT = true; + public static final String OZONE_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS = + "ozone.client.server-defaults.validity.period.ms"; + + public static final long + OZONE_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS_DEFAULT = + TimeUnit.HOURS.toMillis(1); // 1 hour + public static final String OZONE_FLEXIBLE_FQDN_RESOLUTION_ENABLED = "ozone.network.flexible.fqdn.resolution.enabled"; public static final boolean OZONE_FLEXIBLE_FQDN_RESOLUTION_ENABLED_DEFAULT = diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index ee060bf7ebe..20c1bed89be 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -3571,6 +3571,19 @@ + + ozone.client.server-defaults.validity.period.ms + OZONE, CLIENT, SECURITY + 3600000 + + The amount of milliseconds after which cached server defaults are updated. + + By default this parameter is set to 1 hour. + Support multiple time unit suffix(case insensitive). + If no time unit is specified then milliseconds is assumed. + + + ozone.scm.info.wait.duration OZONE, SCM, OM diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index 68812a7eb4d..80d8236a3d7 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.io.Text; +import org.apache.hadoop.ozone.FsServerDefaults; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.OzoneBucket; @@ -858,6 +859,13 @@ TenantUserList listUsersInTenant(String tenantId, String prefix) */ TenantStateList listTenant() throws IOException; + /** + * Get server default values for a number of configuration params. + * @return Default configuration from the server. + * @throws IOException + */ + FsServerDefaults getServerDefaults() throws IOException; + /** * Get KMS client provider. * @return KMS client provider. diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 8445f9e6954..336c4bed4c6 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -34,6 +34,7 @@ import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.Syncable; +import org.apache.hadoop.util.Time; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; @@ -61,6 +62,7 @@ import org.apache.hadoop.io.ByteBufferPool; import org.apache.hadoop.io.ElasticByteBufferPool; import org.apache.hadoop.io.Text; +import org.apache.hadoop.ozone.FsServerDefaults; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; @@ -178,6 +180,8 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_KEY_PROVIDER_CACHE_EXPIRY; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_REQUIRED_OM_VERSION_MIN_KEY; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS_DEFAULT; import static org.apache.hadoop.ozone.OzoneConsts.MAXIMUM_NUMBER_OF_PARTS_PER_UPLOAD; import static org.apache.hadoop.ozone.OzoneConsts.OLD_QUOTA_DEFAULT; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_MAXIMUM_ACCESS_ID_LENGTH; @@ -224,6 +228,9 @@ public class RpcClient implements ClientProtocol { private final ContainerClientMetrics clientMetrics; private final MemoizedSupplier writeExecutor; private final AtomicBoolean isS3GRequest = new AtomicBoolean(false); + private volatile FsServerDefaults serverDefaults; + private volatile long serverDefaultsLastUpdate; + private final long serverDefaultsValidityPeriod; /** * Creates RpcClient instance with the given configuration. @@ -330,6 +337,11 @@ public void onRemoval( .getInstance(byteBufferPool, ecReconstructExecutor); this.clientMetrics = ContainerClientMetrics.acquire(); + this.serverDefaultsValidityPeriod = conf.getTimeDuration( + OZONE_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS, + OZONE_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS_DEFAULT, + TimeUnit.MILLISECONDS); + TracingUtil.initTracing("client", conf); } @@ -2591,11 +2603,22 @@ public KeyProvider call() throws Exception { } } + @Override + public FsServerDefaults getServerDefaults() throws IOException { + long now = Time.monotonicNow(); + if ((serverDefaults == null) || + (now - serverDefaultsLastUpdate > serverDefaultsValidityPeriod)) { + serverDefaults = ozoneManagerClient.getServerDefaults(); + serverDefaultsLastUpdate = now; + } + assert serverDefaults != null; + return serverDefaults; + } + @Override public URI getKeyProviderUri() throws IOException { - // TODO: fix me to support kms instances for difference OMs return OzoneKMSUtil.getKeyProviderUri(ugi, - null, null, conf); + null, getServerDefaults().getKeyProviderUri(), conf); } @Override diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/FsServerDefaults.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/FsServerDefaults.java new file mode 100644 index 00000000000..bdc0c7d26e9 --- /dev/null +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/FsServerDefaults.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone; + +import org.apache.hadoop.hdds.annotation.InterfaceAudience; +import org.apache.hadoop.hdds.annotation.InterfaceStability; + +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.FsServerDefaultsProto; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.FsServerDefaultsProto.Builder; + + +/**************************************************** + * Provides server default configuration values to clients. + * + ****************************************************/ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class FsServerDefaults { + private String keyProviderUri; + + public FsServerDefaults() { + } + + public FsServerDefaults(String keyProviderUri) { + this.keyProviderUri = keyProviderUri; + } + + public String getKeyProviderUri() { + return keyProviderUri; + } + + public FsServerDefaultsProto getProtobuf() { + Builder builder = FsServerDefaultsProto.newBuilder(); + if (keyProviderUri != null) { + builder.setKeyProviderUri(keyProviderUri); + } + return builder.build(); + } + + public static FsServerDefaults getFromProtobuf( + FsServerDefaultsProto serverDefaults) { + return new FsServerDefaults(serverDefaults.getKeyProviderUri()); + } +} diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index bf27d7afb61..11f176362a6 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -274,6 +274,7 @@ public static boolean isReadOnly( case SetSafeMode: case PrintCompactionLogDag: case GetSnapshotInfo: + case GetServerDefaults: return true; case CreateVolume: case SetVolumeProperty: diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index 5f592663dad..86397cfd22a 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -27,6 +27,7 @@ import jakarta.annotation.Nonnull; import org.apache.hadoop.fs.SafeModeAction; import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; +import org.apache.hadoop.ozone.FsServerDefaults; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.om.IOmMetadataReader; import org.apache.hadoop.ozone.om.OMConfigKeys; @@ -1178,4 +1179,12 @@ void setTimes(OmKeyArgs keyArgs, long mtime, long atime) */ boolean setSafeMode(SafeModeAction action, boolean isChecked) throws IOException; + + /** + * Get server default configurations. + * + * @throws IOException + * @return FsServerDefaults some default configurations from server. + */ + FsServerDefaults getServerDefaults() throws IOException; } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 9a965b7c3d0..fb150e0464d 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -40,6 +40,7 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.CallerContext; import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.FsServerDefaults; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo; @@ -197,6 +198,8 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Authentication; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Secret; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SafeMode; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServerDefaultsRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServerDefaultsResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetAclRequest; @@ -2644,6 +2647,22 @@ public boolean setSafeMode(SafeModeAction action, boolean isChecked) return setSafeModeResponse.getResponse(); } + @Override + public FsServerDefaults getServerDefaults() + throws IOException { + ServerDefaultsRequest serverDefaultsRequest = + ServerDefaultsRequest.newBuilder().build(); + + OMRequest omRequest = createOMRequest(Type.GetServerDefaults) + .setServerDefaultsRequest(serverDefaultsRequest).build(); + + ServerDefaultsResponse serverDefaultsResponse = + handleError(submitRequest(omRequest)).getServerDefaultsResponse(); + + return FsServerDefaults.getFromProtobuf( + serverDefaultsResponse.getServerDefaults()); + } + private SafeMode toProtoBuf(SafeModeAction action) { switch (action) { case ENTER: diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java index 30597fc3e77..eb9f35f518c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java @@ -4908,6 +4908,12 @@ public void testParallelDeleteBucketAndCreateKey() throws IOException, assertThat(omSMLog.getOutput()).contains("Failed to write, Exception occurred"); } + @Test + public void testGetServerDefaults() throws IOException { + assertNotNull(getClient().getProxy().getServerDefaults()); + assertNull(getClient().getProxy().getServerDefaults().getKeyProviderUri()); + } + private static class OMRequestHandlerPauseInjector extends FaultInjector { private CountDownLatch ready; private CountDownLatch wait; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java index 773f6be966b..958a37380cf 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java @@ -99,6 +99,8 @@ */ class TestSecureOzoneRpcClient extends OzoneRpcClientTests { + private static String keyProviderUri = "kms://http@kms:9600/kms"; + @BeforeAll public static void init() throws Exception { File testDir = GenericTestUtils.getTestDir( @@ -120,6 +122,8 @@ public static void init() throws Exception { conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT, OMConfigKeys.OZONE_BUCKET_LAYOUT_OBJECT_STORE); conf.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true); + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, + keyProviderUri); MiniOzoneCluster.Builder builder = MiniOzoneCluster.newBuilder(conf) .setCertificateClient(certificateClientTest) .setSecretKeyClient(new SecretKeyTestClient()); @@ -433,6 +437,13 @@ public void testS3Auth() throws Exception { public void testZReadKeyWithUnhealthyContainerReplica() { } + @Test + public void testGetServerDefaults() throws IOException { + assertNotNull(getClient().getProxy().getServerDefaults()); + assertEquals(keyProviderUri, + getClient().getProxy().getServerDefaults().getKeyProviderUri()); + } + @AfterAll public static void shutdown() throws IOException { shutdownCluster(); diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 4bdfa97b935..9e0f729be40 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -149,6 +149,7 @@ enum Type { RenameSnapshot = 131; ListOpenFiles = 132; QuotaRepair = 133; + GetServerDefaults = 134; } enum SafeMode { @@ -287,6 +288,7 @@ message OMRequest { optional RenameSnapshotRequest RenameSnapshotRequest = 129; optional ListOpenFilesRequest ListOpenFilesRequest = 130; optional QuotaRepairRequest QuotaRepairRequest = 131; + optional ServerDefaultsRequest ServerDefaultsRequest = 132; } message OMResponse { @@ -412,6 +414,7 @@ message OMResponse { optional RenameSnapshotResponse RenameSnapshotResponse = 132; optional ListOpenFilesResponse ListOpenFilesResponse = 133; optional QuotaRepairResponse QuotaRepairResponse = 134; + optional ServerDefaultsResponse ServerDefaultsResponse = 135; } enum Status { @@ -2202,6 +2205,17 @@ message BucketQuotaCount { message QuotaRepairResponse { } +message ServerDefaultsRequest { +} + +message FsServerDefaultsProto { + optional string keyProviderUri = 1; +} + +message ServerDefaultsResponse { + required FsServerDefaultsProto serverDefaults = 1; +} + message OMLockDetailsProto { optional bool isLockAcquired = 1; optional uint64 waitLockNanos = 2; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 9ceb9bb96d1..9303e31870c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -26,6 +26,7 @@ import java.io.UncheckedIOException; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.URI; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -90,6 +91,7 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.Table.KeyValue; import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.ozone.FsServerDefaults; import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.audit.OMSystemAction; import org.apache.hadoop.ozone.om.helpers.LeaseKeyInfo; @@ -187,6 +189,7 @@ import org.apache.hadoop.ozone.om.protocolPB.OMAdminProtocolPB; import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB; import org.apache.hadoop.hdds.security.exception.OzoneSecurityException; +import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource; import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; @@ -434,6 +437,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl private List ratisReporterList = null; private KeyProviderCryptoExtension kmsProvider; + private FsServerDefaults serverDefaults; private final OMLayoutVersionManager versionManager; private final ReplicationConfigValidator replicationConfigValidator; @@ -651,6 +655,16 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption) kmsProvider = null; LOG.error("Fail to create Key Provider"); } + Configuration hadoopConfig = + LegacyHadoopConfigurationSource.asHadoopConfiguration(configuration); + URI keyProviderUri = KMSUtil.getKeyProviderUri( + hadoopConfig, + CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH); + if (keyProviderUri != null) { + serverDefaults = new FsServerDefaults(keyProviderUri.toString()); + } else { + serverDefaults = new FsServerDefaults(); + } if (secConfig.isSecurityEnabled()) { omComponent = OM_DAEMON + "-" + omId; HddsProtos.OzoneManagerDetailsProto omInfo = @@ -4768,6 +4782,11 @@ public boolean setSafeMode(SafeModeAction action, boolean isChecked) } } + @Override + public FsServerDefaults getServerDefaults() { + return serverDefaults; + } + /** * Write down Layout version of a finalized feature to DB on finalization. * @param lvm OMLayoutVersionManager diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index eb9410501fa..a5e94689aee 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -133,6 +133,7 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetSafeModeRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetSafeModeResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetS3VolumeContextResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServerDefaultsResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotDiffRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotDiffResponse; @@ -383,6 +384,12 @@ public OMResponse handleReadRequest(OMRequest request) { getSnapshotInfo(request.getSnapshotInfoRequest()); responseBuilder.setSnapshotInfoResponse(snapshotInfoResponse); break; + case GetServerDefaults: + responseBuilder.setServerDefaultsResponse( + ServerDefaultsResponse.newBuilder() + .setServerDefaults(impl.getServerDefaults().getProtobuf()) + .build()); + break; default: responseBuilder.setSuccess(false); responseBuilder.setMessage("Unrecognized Command Type: " + cmdType); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index 21c3f8358f4..0b0d7830651 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.io.Text; +import org.apache.hadoop.ozone.FsServerDefaults; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput; import org.apache.hadoop.ozone.client.io.OzoneInputStream; @@ -504,6 +505,11 @@ public TenantStateList listTenant() throws IOException { return null; } + @Override + public FsServerDefaults getServerDefaults() throws IOException { + return null; + } + @Override public KeyProvider getKeyProvider() throws IOException { return null; From 0f72911773b702123da14e76c0f918a232dfb5f7 Mon Sep 17 00:00:00 2001 From: saketa Date: Fri, 16 Aug 2024 16:07:22 -0700 Subject: [PATCH 2/4] HDDS-11227. Returning null from OM when key provider is not present. --- .../java/org/apache/hadoop/ozone/FsServerDefaults.java | 6 +++++- .../java/org/apache/hadoop/ozone/om/OzoneManager.java | 8 +++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/FsServerDefaults.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/FsServerDefaults.java index bdc0c7d26e9..e83ac99261f 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/FsServerDefaults.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/FsServerDefaults.java @@ -54,6 +54,10 @@ public FsServerDefaultsProto getProtobuf() { public static FsServerDefaults getFromProtobuf( FsServerDefaultsProto serverDefaults) { - return new FsServerDefaults(serverDefaults.getKeyProviderUri()); + String keyProviderUri = null; + if (serverDefaults.hasKeyProviderUri()) { + keyProviderUri = serverDefaults.getKeyProviderUri(); + } + return new FsServerDefaults(keyProviderUri); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 9303e31870c..6caa86f3c30 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -660,11 +660,9 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption) URI keyProviderUri = KMSUtil.getKeyProviderUri( hadoopConfig, CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH); - if (keyProviderUri != null) { - serverDefaults = new FsServerDefaults(keyProviderUri.toString()); - } else { - serverDefaults = new FsServerDefaults(); - } + String keyProviderUriStr = + (keyProviderUri != null) ? keyProviderUri.toString() : null; + serverDefaults = new FsServerDefaults(keyProviderUriStr); if (secConfig.isSecurityEnabled()) { omComponent = OM_DAEMON + "-" + omId; HddsProtos.OzoneManagerDetailsProto omInfo = From 1099b180db8d501d5613e1f7cc04299a54faa9e4 Mon Sep 17 00:00:00 2001 From: saketa Date: Tue, 20 Aug 2024 10:01:07 -0700 Subject: [PATCH 3/4] HDDS-11227. Extended org.apache.hadoop.fs.FileSystem --- .../hadoop/ozone/client/ObjectStore.java | 5 +++++ .../ozone/client/protocol/ClientProtocol.java | 4 ++-- .../hadoop/ozone/client/rpc/RpcClient.java | 6 ++--- ...faults.java => OzoneFsServerDefaults.java} | 22 ++++++++----------- .../om/protocol/OzoneManagerProtocol.java | 6 ++--- ...ManagerProtocolClientSideTranslatorPB.java | 6 ++--- .../apache/hadoop/ozone/om/OzoneManager.java | 8 +++---- .../fs/ozone/BasicOzoneClientAdapterImpl.java | 6 +++++ .../BasicRootedOzoneClientAdapterImpl.java | 6 +++++ .../hadoop/fs/ozone/OzoneClientAdapter.java | 3 +++ .../hadoop/fs/ozone/OzoneFileSystem.java | 6 +++++ .../fs/ozone/RootedOzoneFileSystem.java | 6 +++++ .../ozone/client/ClientProtocolStub.java | 4 ++-- 13 files changed, 58 insertions(+), 30 deletions(-) rename hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/{FsServerDefaults.java => OzoneFsServerDefaults.java} (78%) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java index e77f5580cca..65dce09cba1 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java @@ -34,6 +34,7 @@ import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.OzoneFsServerDefaults; import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -391,6 +392,10 @@ public void deleteVolume(String volumeName) throws IOException { proxy.deleteVolume(volumeName); } + public OzoneFsServerDefaults getServerDefaults() throws IOException { + return proxy.getServerDefaults(); + } + public KeyProvider getKeyProvider() throws IOException { return proxy.getKeyProvider(); } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index 80d8236a3d7..16211ebbb8e 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -31,8 +31,8 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.io.Text; -import org.apache.hadoop.ozone.FsServerDefaults; import org.apache.hadoop.ozone.OzoneAcl; +import org.apache.hadoop.ozone.OzoneFsServerDefaults; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneKey; @@ -864,7 +864,7 @@ TenantUserList listUsersInTenant(String tenantId, String prefix) * @return Default configuration from the server. * @throws IOException */ - FsServerDefaults getServerDefaults() throws IOException; + OzoneFsServerDefaults getServerDefaults() throws IOException; /** * Get KMS client provider. diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 336c4bed4c6..bfeb9c1e6c1 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -62,7 +62,7 @@ import org.apache.hadoop.io.ByteBufferPool; import org.apache.hadoop.io.ElasticByteBufferPool; import org.apache.hadoop.io.Text; -import org.apache.hadoop.ozone.FsServerDefaults; +import org.apache.hadoop.ozone.OzoneFsServerDefaults; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; @@ -228,7 +228,7 @@ public class RpcClient implements ClientProtocol { private final ContainerClientMetrics clientMetrics; private final MemoizedSupplier writeExecutor; private final AtomicBoolean isS3GRequest = new AtomicBoolean(false); - private volatile FsServerDefaults serverDefaults; + private volatile OzoneFsServerDefaults serverDefaults; private volatile long serverDefaultsLastUpdate; private final long serverDefaultsValidityPeriod; @@ -2604,7 +2604,7 @@ public KeyProvider call() throws Exception { } @Override - public FsServerDefaults getServerDefaults() throws IOException { + public OzoneFsServerDefaults getServerDefaults() throws IOException { long now = Time.monotonicNow(); if ((serverDefaults == null) || (now - serverDefaultsLastUpdate > serverDefaultsValidityPeriod)) { diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/FsServerDefaults.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OzoneFsServerDefaults.java similarity index 78% rename from hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/FsServerDefaults.java rename to hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OzoneFsServerDefaults.java index e83ac99261f..782fa88e8d2 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/FsServerDefaults.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OzoneFsServerDefaults.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.ozone; +import org.apache.hadoop.fs.FsServerDefaults; import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.annotation.InterfaceStability; @@ -30,34 +31,29 @@ ****************************************************/ @InterfaceAudience.Public @InterfaceStability.Evolving -public class FsServerDefaults { - private String keyProviderUri; +public class OzoneFsServerDefaults extends FsServerDefaults { - public FsServerDefaults() { + public OzoneFsServerDefaults() { } - public FsServerDefaults(String keyProviderUri) { - this.keyProviderUri = keyProviderUri; - } - - public String getKeyProviderUri() { - return keyProviderUri; + public OzoneFsServerDefaults(String keyProviderUri) { + super(0L, 0, 0, (short)0, 0, false, 0L, null, keyProviderUri); } public FsServerDefaultsProto getProtobuf() { Builder builder = FsServerDefaultsProto.newBuilder(); - if (keyProviderUri != null) { - builder.setKeyProviderUri(keyProviderUri); + if (getKeyProviderUri() != null) { + builder.setKeyProviderUri(getKeyProviderUri()); } return builder.build(); } - public static FsServerDefaults getFromProtobuf( + public static OzoneFsServerDefaults getFromProtobuf( FsServerDefaultsProto serverDefaults) { String keyProviderUri = null; if (serverDefaults.hasKeyProviderUri()) { keyProviderUri = serverDefaults.getKeyProviderUri(); } - return new FsServerDefaults(keyProviderUri); + return new OzoneFsServerDefaults(keyProviderUri); } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index 86397cfd22a..45922c107cb 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -27,8 +27,8 @@ import jakarta.annotation.Nonnull; import org.apache.hadoop.fs.SafeModeAction; import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; -import org.apache.hadoop.ozone.FsServerDefaults; import org.apache.hadoop.ozone.OzoneAcl; +import org.apache.hadoop.ozone.OzoneFsServerDefaults; import org.apache.hadoop.ozone.om.IOmMetadataReader; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.exceptions.OMException; @@ -1183,8 +1183,8 @@ boolean setSafeMode(SafeModeAction action, boolean isChecked) /** * Get server default configurations. * + * @return OzoneFsServerDefaults some default configurations from server. * @throws IOException - * @return FsServerDefaults some default configurations from server. */ - FsServerDefaults getServerDefaults() throws IOException; + OzoneFsServerDefaults getServerDefaults() throws IOException; } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index fb150e0464d..f70beed5f25 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -40,8 +40,8 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.CallerContext; import org.apache.hadoop.ozone.ClientVersion; -import org.apache.hadoop.ozone.FsServerDefaults; import org.apache.hadoop.ozone.OzoneAcl; +import org.apache.hadoop.ozone.OzoneFsServerDefaults; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.ErrorInfo; @@ -2648,7 +2648,7 @@ public boolean setSafeMode(SafeModeAction action, boolean isChecked) } @Override - public FsServerDefaults getServerDefaults() + public OzoneFsServerDefaults getServerDefaults() throws IOException { ServerDefaultsRequest serverDefaultsRequest = ServerDefaultsRequest.newBuilder().build(); @@ -2659,7 +2659,7 @@ public FsServerDefaults getServerDefaults() ServerDefaultsResponse serverDefaultsResponse = handleError(submitRequest(omRequest)).getServerDefaultsResponse(); - return FsServerDefaults.getFromProtobuf( + return OzoneFsServerDefaults.getFromProtobuf( serverDefaultsResponse.getServerDefaults()); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 6caa86f3c30..a514262cae2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -91,7 +91,7 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.Table.KeyValue; import org.apache.hadoop.hdds.utils.db.TableIterator; -import org.apache.hadoop.ozone.FsServerDefaults; +import org.apache.hadoop.ozone.OzoneFsServerDefaults; import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.audit.OMSystemAction; import org.apache.hadoop.ozone.om.helpers.LeaseKeyInfo; @@ -437,7 +437,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl private List ratisReporterList = null; private KeyProviderCryptoExtension kmsProvider; - private FsServerDefaults serverDefaults; + private OzoneFsServerDefaults serverDefaults; private final OMLayoutVersionManager versionManager; private final ReplicationConfigValidator replicationConfigValidator; @@ -662,7 +662,7 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption) CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH); String keyProviderUriStr = (keyProviderUri != null) ? keyProviderUri.toString() : null; - serverDefaults = new FsServerDefaults(keyProviderUriStr); + serverDefaults = new OzoneFsServerDefaults(keyProviderUriStr); if (secConfig.isSecurityEnabled()) { omComponent = OM_DAEMON + "-" + omId; HddsProtos.OzoneManagerDetailsProto omInfo = @@ -4781,7 +4781,7 @@ public boolean setSafeMode(SafeModeAction action, boolean isChecked) } @Override - public FsServerDefaults getServerDefaults() { + public OzoneFsServerDefaults getServerDefaults() { return serverDefaults; } diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java index 658685779e9..df8ece03486 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java @@ -55,6 +55,7 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.OzoneFsServerDefaults; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -463,6 +464,11 @@ public Token getDelegationToken(String renewer) } + @Override + public OzoneFsServerDefaults getServerDefaults() throws IOException { + return objectStore.getServerDefaults(); + } + @Override public KeyProvider getKeyProvider() throws IOException { return objectStore.getKeyProvider(); diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java index bb2725063e3..14c297d9f47 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java @@ -64,6 +64,7 @@ import org.apache.hadoop.ozone.OFSPath; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.OzoneFsServerDefaults; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -963,6 +964,11 @@ public ObjectStore getObjectStore() { return objectStore; } + @Override + public OzoneFsServerDefaults getServerDefaults() throws IOException { + return objectStore.getServerDefaults(); + } + @Override public KeyProvider getKeyProvider() throws IOException { return objectStore.getKeyProvider(); diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java index 7e78d6650ee..e468ac498c4 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.SafeModeAction; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; +import org.apache.hadoop.ozone.OzoneFsServerDefaults; import org.apache.hadoop.ozone.om.helpers.LeaseKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; @@ -74,6 +75,8 @@ List listStatus(String keyName, boolean recursive, Token getDelegationToken(String renewer) throws IOException; + + OzoneFsServerDefaults getServerDefaults() throws IOException; KeyProvider getKeyProvider() throws IOException; diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java index 65e3145e7d4..af176bf96b7 100644 --- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java +++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.annotation.InterfaceStability; import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.ozone.OzoneFsServerDefaults; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.LeaseKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; @@ -65,6 +66,11 @@ public OzoneFileSystem() { forceRecovery = Strings.isNullOrEmpty(force) ? false : Boolean.parseBoolean(force); } + @Override + public OzoneFsServerDefaults getServerDefaults() throws IOException { + return getAdapter().getServerDefaults(); + } + @Override public KeyProvider getKeyProvider() throws IOException { return getAdapter().getKeyProvider(); diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java index bed3505fe9b..989ae040823 100644 --- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hdds.annotation.InterfaceStability; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.tracing.TracingUtil; +import org.apache.hadoop.ozone.OzoneFsServerDefaults; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.LeaseKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; @@ -67,6 +68,11 @@ public RootedOzoneFileSystem() { forceRecovery = Strings.isNullOrEmpty(force) ? false : Boolean.parseBoolean(force); } + @Override + public OzoneFsServerDefaults getServerDefaults() throws IOException { + return getAdapter().getServerDefaults(); + } + @Override public KeyProvider getKeyProvider() throws IOException { return getAdapter().getKeyProvider(); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index 0b0d7830651..41876c6e245 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -27,7 +27,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.io.Text; -import org.apache.hadoop.ozone.FsServerDefaults; +import org.apache.hadoop.ozone.OzoneFsServerDefaults; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput; import org.apache.hadoop.ozone.client.io.OzoneInputStream; @@ -506,7 +506,7 @@ public TenantStateList listTenant() throws IOException { } @Override - public FsServerDefaults getServerDefaults() throws IOException { + public OzoneFsServerDefaults getServerDefaults() throws IOException { return null; } From 42b54ab2b1c921e2960d836e62131c44e06ad041 Mon Sep 17 00:00:00 2001 From: saketa Date: Tue, 20 Aug 2024 16:16:14 -0700 Subject: [PATCH 4/4] HDDS-11227. Moved getServerDefaults() to BasicRootedOzoneFileSystem and BasicOzoneFileSystem --- .../org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java | 6 ++++++ .../apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java | 6 ++++++ .../java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java | 6 ------ .../org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java | 6 ------ 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java index aabf17d086e..f25d9011475 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource; import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.OzoneFsServerDefaults; import org.apache.hadoop.ozone.client.io.SelectorOutputStream; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; @@ -865,6 +866,11 @@ public short getDefaultReplication() { return adapter.getDefaultReplication(); } + @Override + public OzoneFsServerDefaults getServerDefaults() throws IOException { + return adapter.getServerDefaults(); + } + @Override public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path[] srcs, Path dst) throws IOException { diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java index 55d8e37c469..eb346b5edc5 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java @@ -48,6 +48,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.ozone.OFSPath; import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.OzoneFsServerDefaults; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.io.SelectorOutputStream; @@ -1107,6 +1108,11 @@ public short getDefaultReplication() { return adapter.getDefaultReplication(); } + @Override + public OzoneFsServerDefaults getServerDefaults() throws IOException { + return adapter.getServerDefaults(); + } + @Override public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path[] srcs, Path dst) throws IOException { diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java index af176bf96b7..65e3145e7d4 100644 --- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java +++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java @@ -35,7 +35,6 @@ import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.annotation.InterfaceStability; import org.apache.hadoop.hdds.conf.ConfigurationSource; -import org.apache.hadoop.ozone.OzoneFsServerDefaults; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.LeaseKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; @@ -66,11 +65,6 @@ public OzoneFileSystem() { forceRecovery = Strings.isNullOrEmpty(force) ? false : Boolean.parseBoolean(force); } - @Override - public OzoneFsServerDefaults getServerDefaults() throws IOException { - return getAdapter().getServerDefaults(); - } - @Override public KeyProvider getKeyProvider() throws IOException { return getAdapter().getKeyProvider(); diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java index 989ae040823..bed3505fe9b 100644 --- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java @@ -32,7 +32,6 @@ import org.apache.hadoop.hdds.annotation.InterfaceStability; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.tracing.TracingUtil; -import org.apache.hadoop.ozone.OzoneFsServerDefaults; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.LeaseKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; @@ -68,11 +67,6 @@ public RootedOzoneFileSystem() { forceRecovery = Strings.isNullOrEmpty(force) ? false : Boolean.parseBoolean(force); } - @Override - public OzoneFsServerDefaults getServerDefaults() throws IOException { - return getAdapter().getServerDefaults(); - } - @Override public KeyProvider getKeyProvider() throws IOException { return getAdapter().getKeyProvider();