HDDS-11371. Added error handling in getServerDefaults() and fixed acc…

…eptance tests.
apache · Sep 6, 2024 · eb9c9f4 · eb9c9f4
1 parent c611fc3
commit eb9c9f4
Show file tree

Hide file tree

Showing 11 changed files with 207 additions and 497 deletions.
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -2580,27 +2580,27 @@ public KeyProvider call() throws Exception {
 
   @Override
   public OzoneFsServerDefaults getServerDefaults() throws IOException {
+    if (omVersion.compareTo(OzoneManagerVersion.SERVER_DEFAULTS) < 0) {
+      return null;
+    }
     long now = Time.monotonicNow();
     if ((serverDefaults == null) ||
         (now - serverDefaultsLastUpdate > serverDefaultsValidityPeriod)) {
-      serverDefaults = ozoneManagerClient.getServerDefaults();
-      serverDefaultsLastUpdate = now;
+      try {
+        serverDefaults = ozoneManagerClient.getServerDefaults();
+        serverDefaultsLastUpdate = now;
+      } catch (Exception e) {
+        LOG.warn("Could not get server defaults from OM.", e);
+      }
     }
-    assert serverDefaults != null;
     return serverDefaults;
   }
 
   @Override
   public URI getKeyProviderUri() throws IOException {
-    if (omVersion.compareTo(OzoneManagerVersion.SERVER_DEFAULTS) >= 0) {
-      try {
-        return OzoneKMSUtil.getKeyProviderUri(ugi,
-            null, getServerDefaults().getKeyProviderUri(), conf);
-      } catch (Exception e) {
-        LOG.warn("Could not get key provider URI from OM.", e);
-      }
-    }
-    return OzoneKMSUtil.getKeyProviderUri(ugi, null, null, conf);
+    String keyProviderUri = (getServerDefaults() != null) ?
+        serverDefaults.getKeyProviderUri() : null;
+    return OzoneKMSUtil.getKeyProviderUri(ugi, null, keyProviderUri, conf);
   }
 
   @Override

diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/clients.yaml b/hadoop-ozone/dist/src/main/compose/xcompat/clients.yaml
@@ -21,41 +21,53 @@ services:
       - docker-config
     volumes:
       - ../..:/opt/ozone
+      - ../_keytabs:/etc/security/keytabs
+      - ./krb5.conf:/etc/krb5.conf
     command: ["sleep","1000000"]
   old_client_1_1_0:
     image: apache/ozone:1.1.0
     env_file:
       - docker-config
     volumes:
       - ../..:/opt/ozone
+      - ../_keytabs:/etc/security/keytabs
+      - ./krb5.conf:/etc/krb5.conf
     command: ["sleep","1000000"]
   old_client_1_2_1:
     image: apache/ozone:1.2.1
     env_file:
       - docker-config
     volumes:
       - ../..:/opt/ozone
+      - ../_keytabs:/etc/security/keytabs
+      - ./krb5.conf:/etc/krb5.conf
     command: ["sleep","1000000"]
   old_client_1_3_0:
     image: apache/ozone:1.3.0
     env_file:
       - docker-config
     volumes:
       - ../..:/opt/ozone
+      - ../_keytabs:/etc/security/keytabs
+      - ./krb5.conf:/etc/krb5.conf
     command: ["sleep","1000000"]
   old_client_1_4_0:
     image: apache/ozone:1.4.0
     env_file:
       - docker-config
     volumes:
       - ../..:/opt/ozone
+      - ../_keytabs:/etc/security/keytabs
+      - ./krb5.conf:/etc/krb5.conf
     command: ["sleep","1000000"]
   new_client:
     image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
     env_file:
       - docker-config
     volumes:
       - ../..:/opt/hadoop
+      - ../_keytabs:/etc/security/keytabs
+      - ./krb5.conf:/etc/krb5.conf
     environment:
       OZONE_OPTS:
     command: ["sleep","1000000"]
diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/docker-config b/hadoop-ozone/dist/src/main/compose/xcompat/docker-config
@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+CORE-SITE.XML_fs.defaultFS=ofs://om
+CORE-SITE.XML_fs.trash.interval=1
 CORE-SITE.XML_fs.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzoneFileSystem
 
 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
@@ -22,6 +24,7 @@ OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
 OZONE-SITE.XML_ozone.om.address=om
 OZONE-SITE.XML_ozone.om.http-address=om:9874
+OZONE-SITE.XML_ozone.scm.http-address=scm:9876
 OZONE-SITE.XML_ozone.recon.address=recon:9891
 OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
 OZONE-SITE.XML_ozone.server.default.replication=3
@@ -31,9 +34,98 @@ OZONE-SITE.XML_ozone.scm.container.size=1GB
 OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
 OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
 OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
 OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
+OZONE-SITE.XML_ozone.datanode.pipeline.limit=1
 OZONE-SITE.XML_recon.om.snapshot.task.interval.delay=1m
 OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
 OZONE-SITE.XML_ozone.default.bucket.layout=LEGACY
 OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
-no_proxy=om,recon,scm,s3g,kdc,localhost,127.0.0.1
+
+OZONE-SITE.XML_hdds.block.token.enabled=true
+OZONE-SITE.XML_hdds.container.token.enabled=true
+OZONE-SITE.XML_hdds.grpc.tls.enabled=true
+
+OZONE-SITE.XML_ozone.security.enabled=true
+OZONE-SITE.XML_ozone.acl.enabled=true
+OZONE-SITE.XML_ozone.acl.authorizer.class=org.apache.hadoop.ozone.security.acl.OzoneNativeAuthorizer
+OZONE-SITE.XML_ozone.administrators="testuser,recon,om"
+OZONE-SITE.XML_ozone.s3.administrators="testuser,recon,om"
+OZONE-SITE.XML_ozone.recon.administrators="testuser2"
+OZONE-SITE.XML_ozone.s3.administrators="testuser,s3g"
+
+HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
+HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
+CORE-SITE.XML_dfs.data.transfer.protection=authentication
+CORE-SITE.XML_hadoop.security.authentication=kerberos
+CORE-SITE.XML_hadoop.security.auth_to_local="DEFAULT"
+CORE-SITE.XML_hadoop.security.key.provider.path=kms://http@kms:9600/kms
+
+OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
+OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
+OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
+OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
+OZONE-SITE.XML_ozone.recon.kerberos.keytab.file=/etc/security/keytabs/recon.keytab
+OZONE-SITE.XML_ozone.recon.kerberos.principal=recon/recon@EXAMPLE.COM
+
+OZONE-SITE.XML_ozone.s3g.kerberos.keytab.file=/etc/security/keytabs/s3g.keytab
+OZONE-SITE.XML_ozone.s3g.kerberos.principal=s3g/s3g@EXAMPLE.COM
+
+OZONE-SITE.XML_ozone.httpfs.kerberos.keytab.file=/etc/security/keytabs/httpfs.keytab
+OZONE-SITE.XML_ozone.httpfs.kerberos.principal=httpfs/httpfs@EXAMPLE.COM
+
+HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/dn@EXAMPLE.COM
+HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
+HDFS-SITE.XML_dfs.datanode.kerberos.keytab.file=/etc/security/keytabs/dn.keytab
+HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/ozone@EXAMPLE.COM
+HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+
+OZONE-SITE.XML_ozone.security.http.kerberos.enabled=true
+OZONE-SITE.XML_ozone.s3g.secret.http.enabled=true
+OZONE-SITE.XML_ozone.http.filter.initializers=org.apache.hadoop.security.AuthenticationFilterInitializer
+
+OZONE-SITE.XML_ozone.om.http.auth.type=kerberos
+OZONE-SITE.XML_hdds.scm.http.auth.type=kerberos
+OZONE-SITE.XML_hdds.datanode.http.auth.type=kerberos
+OZONE-SITE.XML_ozone.s3g.http.auth.type=kerberos
+OZONE-SITE.XML_ozone.s3g.secret.http.auth.type=kerberos
+OZONE-SITE.XML_ozone.httpfs.http.auth.type=kerberos
+OZONE-SITE.XML_ozone.recon.http.auth.type=kerberos
+
+OZONE-SITE.XML_hdds.scm.http.auth.kerberos.principal=HTTP/scm@EXAMPLE.COM
+OZONE-SITE.XML_hdds.scm.http.auth.kerberos.keytab=/etc/security/keytabs/scm.keytab
+OZONE-SITE.XML_ozone.om.http.auth.kerberos.principal=HTTP/om@EXAMPLE.COM
+OZONE-SITE.XML_ozone.om.http.auth.kerberos.keytab=/etc/security/keytabs/om.keytab
+OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.principal=HTTP/dn@EXAMPLE.COM
+OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.keytab=/etc/security/keytabs/dn.keytab
+OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.keytab=/etc/security/keytabs/s3g.keytab
+OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.principal=HTTP/s3g@EXAMPLE.COM
+OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.keytab=/etc/security/keytabs/httpfs.keytab
+OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.principal=HTTP/httpfs@EXAMPLE.COM
+OZONE-SITE.XML_ozone.recon.http.auth.kerberos.principal=*
+OZONE-SITE.XML_ozone.recon.http.auth.kerberos.keytab=/etc/security/keytabs/recon.keytab
+
+CORE-SITE.XML_hadoop.http.authentication.simple.anonymous.allowed=false
+CORE-SITE.XML_hadoop.http.authentication.signature.secret.file=/etc/security/http_secret
+CORE-SITE.XML_hadoop.http.authentication.type=kerberos
+CORE-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/ozone@EXAMPLE.COM
+CORE-SITE.XML_hadoop.http.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+
+CORE-SITE.XML_hadoop.security.authorization=true
+HADOOP-POLICY.XML_ozone.om.security.client.protocol.acl=*
+HADOOP-POLICY.XML_hdds.security.client.datanode.container.protocol.acl=*
+HADOOP-POLICY.XML_hdds.security.client.scm.container.protocol.acl=*
+HADOOP-POLICY.XML_hdds.security.client.scm.block.protocol.acl=*
+HADOOP-POLICY.XML_hdds.security.client.scm.certificate.protocol.acl=*
+HADOOP-POLICY.XML_ozone.security.reconfigure.protocol.acl=*
+
+KMS-SITE.XML_hadoop.kms.proxyuser.s3g.users=*
+KMS-SITE.XML_hadoop.kms.proxyuser.s3g.groups=*
+KMS-SITE.XML_hadoop.kms.proxyuser.s3g.hosts=*
+
+OZONE_DATANODE_SECURE_USER=root
+JSVC_HOME=/usr/bin
+
+OZONE_LOG_DIR=/var/log/hadoop
+
+no_proxy=om,scm,recon,s3g,kdc,localhost,127.0.0.1
diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/new-cluster.yaml b/hadoop-ozone/dist/src/main/compose/xcompat/new-cluster.yaml
@@ -18,14 +18,39 @@
 x-new-config:
   &new-config
   image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
+  dns_search: .
   env_file:
     - docker-config
   volumes:
     - ../..:/opt/hadoop
+    - ../_keytabs:/etc/security/keytabs
+    - ./krb5.conf:/etc/krb5.conf
 
 services:
+  kdc:
+    image: ${OZONE_TESTKRB5_IMAGE}
+    hostname: kdc
+    dns_search: .
+    volumes:
+      - ../..:/opt/hadoop
+      - ../_keytabs:/etc/security/keytabs
+    command: [ "krb5kdc","-n" ]
+  kms:
+    image: apache/hadoop:${HADOOP_VERSION}
+    hostname: kms
+    dns_search: .
+    ports:
+      - 9600:9600
+    env_file:
+      - ./docker-config
+    environment:
+      HADOOP_CONF_DIR: /opt/hadoop/etc/hadoop
+    volumes:
+      - ../../libexec/transformation.py:/opt/transformation.py
+    command: [ "hadoop", "kms" ]
   datanode:
     <<: *new-config
+    hostname: dn
     ports:
       - 19864
       - 9882
@@ -34,32 +59,38 @@ services:
     command: ["ozone","datanode"]
   om:
     <<: *new-config
+    hostname: om
     environment:
       ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
-      OZONE_OPTS:
+      OZONE_OPTS: -Dcom.sun.net.ssl.checkRevocation=false
     ports:
       - 9874:9874
       - 9862:9862
     command: ["ozone","om"]
   recon:
     <<: *new-config
+    hostname: recon
     ports:
       - 9888:9888
     environment:
       OZONE_OPTS:
     command: ["ozone","recon"]
   s3g:
     <<: *new-config
+    hostname: s3g
     environment:
       OZONE_OPTS:
     ports:
       - 9878:9878
     command: ["ozone","s3g"]
   scm:
     <<: *new-config
+    hostname: scm
     ports:
       - 9876:9876
+      - 9860:9860
     environment:
       ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+      OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}"
       OZONE_OPTS:
     command: ["ozone","scm"]
diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/old-cluster.yaml b/hadoop-ozone/dist/src/main/compose/xcompat/old-cluster.yaml
@@ -18,14 +18,39 @@
 x-old-config:
   &old-config
   image: apache/ozone:${OZONE_VERSION}
+  dns_search: .
   env_file:
     - docker-config
   volumes:
     - ../..:/opt/ozone
+    - ../_keytabs:/etc/security/keytabs
+    - ./krb5.conf:/etc/krb5.conf
 
 services:
+  kdc:
+    image: ${OZONE_TESTKRB5_IMAGE}
+    hostname: kdc
+    dns_search: .
+    volumes:
+      - ../..:/opt/ozone
+      - ../_keytabs:/etc/security/keytabs
+    command: [ "krb5kdc","-n" ]
+  kms:
+    image: apache/hadoop:${HADOOP_VERSION}
+    hostname: kms
+    dns_search: .
+    ports:
+      - 9600:9600
+    env_file:
+      - ./docker-config
+    environment:
+      HADOOP_CONF_DIR: /opt/hadoop/etc/hadoop
+    volumes:
+      - ../../libexec/transformation.py:/opt/transformation.py
+    command: [ "hadoop", "kms" ]
   datanode:
     <<: *old-config
+    hostname: dn
     ports:
       - 19864
       - 9882
@@ -34,32 +59,38 @@ services:
     command: ["ozone","datanode"]
   om:
     <<: *old-config
+    hostname: om
     environment:
       ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+      OZONE_OPTS: -Dcom.sun.net.ssl.checkRevocation=false
       HADOOP_OPTS:
     ports:
       - 9874:9874
       - 9862:9862
     command: ["ozone","om"]
   recon:
     <<: *old-config
+    hostname: recon
     ports:
       - 9888:9888
     environment:
       HADOOP_OPTS:
     command: ["ozone","recon"]
   s3g:
     <<: *old-config
+    hostname: s3g
     environment:
       HADOOP_OPTS:
     ports:
       - 9878:9878
     command: ["ozone","s3g"]
   scm:
     <<: *old-config
+    hostname: scm
     ports:
       - 9876:9876
     environment:
       ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+      OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}"
       HADOOP_OPTS:
     command: ["ozone","scm"]