HDDS-11227. Use server default key provider to encrypt/decrypt keys from multiple OMs.

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java

@@ -583,6 +583,13 @@ public final class OzoneConfigKeys {
   public static final boolean OZONE_CLIENT_KEY_LATEST_VERSION_LOCATION_DEFAULT =
       true;
 
+  public static final String OZONE_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS =
+      "ozone.client.server-defaults.validity.period.ms";
+
+  public static final long
+      OZONE_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS_DEFAULT =
+      TimeUnit.HOURS.toMillis(1); // 1 hour
+
   public static final String OZONE_FLEXIBLE_FQDN_RESOLUTION_ENABLED =
           "ozone.network.flexible.fqdn.resolution.enabled";
   public static final boolean OZONE_FLEXIBLE_FQDN_RESOLUTION_ENABLED_DEFAULT =

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -3571,6 +3571,19 @@
     </description>
   </property>
 
+  <property>
+    <name>ozone.client.server-defaults.validity.period.ms</name>
+    <tag>OZONE, CLIENT, SECURITY</tag>
+    <value>3600000</value>
+    <description>
+      The amount of milliseconds after which cached server defaults are updated.
+
+      By default this parameter is set to 1 hour.
+      Support multiple time unit suffix(case insensitive).
+      If no time unit is specified then milliseconds is assumed.
+    </description>
+  </property>
+
   <property>
     <name>ozone.scm.info.wait.duration</name>
     <tag>OZONE, SCM, OM</tag>

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java

@@ -34,6 +34,7 @@
 import org.apache.hadoop.ozone.OmUtils;
 import org.apache.hadoop.ozone.OzoneAcl;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneFsServerDefaults;
 import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
 import org.apache.hadoop.ozone.om.helpers.BucketLayout;
@@ -391,6 +392,10 @@ public void deleteVolume(String volumeName) throws IOException {
     proxy.deleteVolume(volumeName);
   }
 
+  public OzoneFsServerDefaults getServerDefaults() throws IOException {
+    return proxy.getServerDefaults();
+  }
+
   public KeyProvider getKeyProvider() throws IOException {
     return proxy.getKeyProvider();
   }

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java

@@ -32,6 +32,7 @@
 import org.apache.hadoop.hdds.protocol.StorageType;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.ozone.OzoneAcl;
+import org.apache.hadoop.ozone.OzoneFsServerDefaults;
 import org.apache.hadoop.ozone.client.BucketArgs;
 import org.apache.hadoop.ozone.client.OzoneBucket;
 import org.apache.hadoop.ozone.client.OzoneKey;
@@ -858,6 +859,13 @@ TenantUserList listUsersInTenant(String tenantId, String prefix)
    */
   TenantStateList listTenant() throws IOException;
 
+  /**
+   * Get server default values for a number of configuration params.
+   * @return Default configuration from the server.
+   * @throws IOException
+   */
+  OzoneFsServerDefaults getServerDefaults() throws IOException;
+
   /**
    * Get KMS client provider.
    * @return KMS client provider.

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java

@@ -34,6 +34,7 @@
 import org.apache.hadoop.crypto.key.KeyProvider;
 import org.apache.hadoop.fs.FileEncryptionInfo;
 import org.apache.hadoop.fs.Syncable;
+import org.apache.hadoop.util.Time;
 import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
 import org.apache.hadoop.hdds.client.ECReplicationConfig;
 import org.apache.hadoop.hdds.client.ReplicationConfig;
@@ -61,6 +62,7 @@
 import org.apache.hadoop.io.ByteBufferPool;
 import org.apache.hadoop.io.ElasticByteBufferPool;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OzoneFsServerDefaults;
 import org.apache.hadoop.ozone.OzoneAcl;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.ozone.OzoneConsts;
@@ -178,6 +180,8 @@
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_KEY_PROVIDER_CACHE_EXPIRY;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_DEFAULT;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_REQUIRED_OM_VERSION_MIN_KEY;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS_DEFAULT;
 import static org.apache.hadoop.ozone.OzoneConsts.MAXIMUM_NUMBER_OF_PARTS_PER_UPLOAD;
 import static org.apache.hadoop.ozone.OzoneConsts.OLD_QUOTA_DEFAULT;
 import static org.apache.hadoop.ozone.OzoneConsts.OZONE_MAXIMUM_ACCESS_ID_LENGTH;
@@ -224,6 +228,9 @@ public class RpcClient implements ClientProtocol {
   private final ContainerClientMetrics clientMetrics;
   private final MemoizedSupplier<ExecutorService> writeExecutor;
   private final AtomicBoolean isS3GRequest = new AtomicBoolean(false);
+  private volatile OzoneFsServerDefaults serverDefaults;
+  private volatile long serverDefaultsLastUpdate;
+  private final long serverDefaultsValidityPeriod;
 
   /**
    * Creates RpcClient instance with the given configuration.
@@ -330,6 +337,11 @@ public void onRemoval(
         .getInstance(byteBufferPool, ecReconstructExecutor);
     this.clientMetrics = ContainerClientMetrics.acquire();
 
+    this.serverDefaultsValidityPeriod = conf.getTimeDuration(
+        OZONE_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS,
+        OZONE_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS_DEFAULT,
+        TimeUnit.MILLISECONDS);
+
     TracingUtil.initTracing("client", conf);
   }
 
@@ -2591,11 +2603,22 @@ public KeyProvider call() throws Exception {
     }
   }
 
+  @Override
+  public OzoneFsServerDefaults getServerDefaults() throws IOException {
+    long now = Time.monotonicNow();
+    if ((serverDefaults == null) ||
+        (now - serverDefaultsLastUpdate > serverDefaultsValidityPeriod)) {
+      serverDefaults = ozoneManagerClient.getServerDefaults();
+      serverDefaultsLastUpdate = now;
+    }
+    assert serverDefaults != null;
+    return serverDefaults;
+  }
+
   @Override
   public URI getKeyProviderUri() throws IOException {
-    // TODO: fix me to support kms instances for difference OMs
     return OzoneKMSUtil.getKeyProviderUri(ugi,
-        null, null, conf);
+        null, getServerDefaults().getKeyProviderUri(), conf);
   }
 
   @Override

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java

@@ -274,6 +274,7 @@ public static boolean isReadOnly(
     case SetSafeMode:
     case PrintCompactionLogDag:
     case GetSnapshotInfo:
+    case GetServerDefaults:
       return true;
     case CreateVolume:
     case SetVolumeProperty:

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OzoneFsServerDefaults.java

@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone;
+
+import org.apache.hadoop.fs.FsServerDefaults;
+import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.annotation.InterfaceStability;
+
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.FsServerDefaultsProto;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.FsServerDefaultsProto.Builder;
+
+
+/****************************************************
+ * Provides server default configuration values to clients.
+ *
+ ****************************************************/
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class OzoneFsServerDefaults extends FsServerDefaults {
+
+  public OzoneFsServerDefaults() {
+  }
+
+  public OzoneFsServerDefaults(String keyProviderUri) {
+    super(0L, 0, 0, (short)0, 0, false, 0L, null, keyProviderUri);
+  }
+
+  public FsServerDefaultsProto getProtobuf() {
+    Builder builder = FsServerDefaultsProto.newBuilder();
+    if (getKeyProviderUri() != null) {
+      builder.setKeyProviderUri(getKeyProviderUri());
+    }
+    return builder.build();
+  }
+
+  public static OzoneFsServerDefaults getFromProtobuf(
+      FsServerDefaultsProto serverDefaults) {
+    String keyProviderUri = null;
+    if (serverDefaults.hasKeyProviderUri()) {
+      keyProviderUri = serverDefaults.getKeyProviderUri();
+    }
+    return new OzoneFsServerDefaults(keyProviderUri);
+  }
+}

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java

@@ -28,6 +28,7 @@
 import org.apache.hadoop.fs.SafeModeAction;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
 import org.apache.hadoop.ozone.OzoneAcl;
+import org.apache.hadoop.ozone.OzoneFsServerDefaults;
 import org.apache.hadoop.ozone.om.IOmMetadataReader;
 import org.apache.hadoop.ozone.om.OMConfigKeys;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
@@ -1178,4 +1179,12 @@ void setTimes(OmKeyArgs keyArgs, long mtime, long atime)
    */
   boolean setSafeMode(SafeModeAction action, boolean isChecked)
       throws IOException;
+
+  /**
+   * Get server default configurations.
+   *
+   * @return OzoneFsServerDefaults some default configurations from server.
+   * @throws IOException
+   */
+  OzoneFsServerDefaults getServerDefaults() throws IOException;
 }

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java

@@ -41,6 +41,7 @@
 import org.apache.hadoop.ipc.CallerContext;
 import org.apache.hadoop.ozone.ClientVersion;
 import org.apache.hadoop.ozone.OzoneAcl;
+import org.apache.hadoop.ozone.OzoneFsServerDefaults;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
 import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.ErrorInfo;
@@ -197,6 +198,8 @@
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Authentication;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Secret;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SafeMode;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServerDefaultsRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServerDefaultsResponse;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListResponse;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetAclRequest;
@@ -2644,6 +2647,22 @@ public boolean setSafeMode(SafeModeAction action, boolean isChecked)
     return setSafeModeResponse.getResponse();
   }
 
+  @Override
+  public OzoneFsServerDefaults getServerDefaults()
+      throws IOException {
+    ServerDefaultsRequest serverDefaultsRequest =
+        ServerDefaultsRequest.newBuilder().build();
+
+    OMRequest omRequest = createOMRequest(Type.GetServerDefaults)
+        .setServerDefaultsRequest(serverDefaultsRequest).build();
+
+    ServerDefaultsResponse serverDefaultsResponse =
+        handleError(submitRequest(omRequest)).getServerDefaultsResponse();
+
+    return OzoneFsServerDefaults.getFromProtobuf(
+        serverDefaultsResponse.getServerDefaults());
+  }
+
   private SafeMode toProtoBuf(SafeModeAction action) {
     switch (action) {
     case ENTER:

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java

@@ -4908,6 +4908,12 @@ public void testParallelDeleteBucketAndCreateKey() throws IOException,
     assertThat(omSMLog.getOutput()).contains("Failed to write, Exception occurred");
   }
 
+  @Test
+  public void testGetServerDefaults() throws IOException {
+    assertNotNull(getClient().getProxy().getServerDefaults());
+    assertNull(getClient().getProxy().getServerDefaults().getKeyProviderUri());
+  }
+
   private static class OMRequestHandlerPauseInjector extends FaultInjector {
     private CountDownLatch ready;
     private CountDownLatch wait;

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java

@@ -99,6 +99,8 @@
  */
 class TestSecureOzoneRpcClient extends OzoneRpcClientTests {
 
+  private static String keyProviderUri = "kms://http@kms:9600/kms";
+
   @BeforeAll
   public static void init() throws Exception {
     File testDir = GenericTestUtils.getTestDir(
@@ -120,6 +122,8 @@ public static void init() throws Exception {
     conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
         OMConfigKeys.OZONE_BUCKET_LAYOUT_OBJECT_STORE);
     conf.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true);
+    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
+        keyProviderUri);
     MiniOzoneCluster.Builder builder = MiniOzoneCluster.newBuilder(conf)
         .setCertificateClient(certificateClientTest)
         .setSecretKeyClient(new SecretKeyTestClient());
@@ -433,6 +437,13 @@ public void testS3Auth() throws Exception {
   public void testZReadKeyWithUnhealthyContainerReplica() {
   }
 
+  @Test
+  public void testGetServerDefaults() throws IOException {
+    assertNotNull(getClient().getProxy().getServerDefaults());
+    assertEquals(keyProviderUri,
+        getClient().getProxy().getServerDefaults().getKeyProviderUri());
+  }
+
   @AfterAll
   public static void shutdown() throws IOException {
     shutdownCluster();

hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto

@@ -149,6 +149,7 @@ enum Type {
   RenameSnapshot = 131;
   ListOpenFiles = 132;
   QuotaRepair = 133;
+  GetServerDefaults = 134;
 }
 
 enum SafeMode {
@@ -287,6 +288,7 @@ message OMRequest {
   optional RenameSnapshotRequest            RenameSnapshotRequest          = 129;
   optional ListOpenFilesRequest             ListOpenFilesRequest           = 130;
   optional QuotaRepairRequest               QuotaRepairRequest             = 131;
+  optional ServerDefaultsRequest            ServerDefaultsRequest          = 132;
 }
 
 message OMResponse {
@@ -412,6 +414,7 @@ message OMResponse {
   optional RenameSnapshotResponse            RenameSnapshotResponse        = 132;
   optional ListOpenFilesResponse             ListOpenFilesResponse         = 133;
   optional QuotaRepairResponse            QuotaRepairResponse        = 134;
+  optional ServerDefaultsResponse            ServerDefaultsResponse        = 135;
 }
 
 enum Status {
@@ -2202,6 +2205,17 @@ message BucketQuotaCount {
 message QuotaRepairResponse {
 }
 
+message ServerDefaultsRequest {
+}
+
+message FsServerDefaultsProto {
+  optional string keyProviderUri = 1;
+}
+
+message ServerDefaultsResponse {
+  required FsServerDefaultsProto serverDefaults = 1;
+}
+
 message OMLockDetailsProto {
   optional bool isLockAcquired = 1;
   optional uint64 waitLockNanos    = 2;