HDDS-11312. [hsync] Added upgrade tests

hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/1.5.0/callback.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source "$TEST_DIR"/testlib.sh
+
+with_this_version_pre_finalized() {
+  # New layout features were added in this version, so OM and SCM should be pre-finalized.
+  execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include pre-finalized upgrade/check-finalization.robot
+  # Test that HSync is disabled when pre-finalized.
+  execute_robot_test "$SCM" -N "${OUTPUT_NAME}-hsync" --include pre-finalized-hsync-tests hsync/upgrade-hsync-check.robot
+}
+
+with_this_version_finalized() {
+  execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot
+  execute_robot_test "$SCM" -N "${OUTPUT_NAME}-hsync" debug/ozone-debug-lease-recovery.robot
+}

hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-lease-recovery.robot

@@ -22,11 +22,20 @@ Test Timeout        5 minute
 Suite Setup         Create volume bucket and put key
 
 *** Variables ***
+${OMSERVICEID}
 ${VOLUME}           lease-recovery-volume
 ${BUCKET}           lease-recovery-bucket
 ${TESTFILE}         testfile22
 
 *** Keywords ***
+Get OM serviceId
+    ${confKey} =        Execute And Ignore Error        ozone getconf confKey ozone.om.service.ids
+    ${result} =         Evaluate                        "Configuration ozone.om.service.ids is missing" in """${confKey}"""
+    IF      ${result} == ${True}
+        Set Suite Variable  ${OMSERVICEID}         om
+    ELSE
+        Set Suite Variable  ${OMSERVICEID}         ${confKey}
+    END
 Create volume bucket and put key
     Execute                 ozone sh volume create /${VOLUME}
     Execute                 ozone sh bucket create /${VOLUME}/${BUCKET}
@@ -35,13 +44,15 @@ Create volume bucket and put key
 
 *** Test Cases ***
 Test ozone debug recover for o3fs
-    ${result} =              Execute Lease recovery cli    o3fs://${BUCKET}.${VOLUME}.om/${TESTFILE}
+    Get OM serviceId
+    ${result} =              Execute Lease recovery cli    o3fs://${BUCKET}.${VOLUME}.${OMSERVICEID}/${TESTFILE}
     Should Contain    ${result}   Lease recovery SUCCEEDED
-    ${result} =              Execute Lease recovery cli    o3fs://${BUCKET}.${VOLUME}.om/randomfile
+    ${result} =              Execute Lease recovery cli    o3fs://${BUCKET}.${VOLUME}.${OMSERVICEID}/randomfile
     Should Contain    ${result}    not found
 
 Test ozone debug recover for ofs
-    ${result} =              Execute Lease recovery cli    ofs://om/${VOLUME}/${BUCKET}/${TESTFILE}
+    Get OM serviceId
+    ${result} =              Execute Lease recovery cli    ofs://${OMSERVICEID}/${VOLUME}/${BUCKET}/${TESTFILE}
     Should Contain    ${result}   Lease recovery SUCCEEDED
-    ${result} =              Execute Lease recovery cli    ofs://om/${VOLUME}/${BUCKET}/randomfile
+    ${result} =              Execute Lease recovery cli    ofs://${OMSERVICEID}/${VOLUME}/${BUCKET}/randomfile
     Should Contain    ${result}    not found

hadoop-ozone/dist/src/main/smoketest/freon/hsync.robot

@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation       Test HSync via freon CLI.
+Library             OperatingSystem
+Library             String
+Library             BuiltIn
+Resource            ../ozone-lib/freon.robot
+Test Timeout        10 minutes
+Suite Setup         Get OM serviceId
+
+*** Variables ***
+${OMSERVICEID}
+${VOLUME}           hsync-volume
+${BUCKET}           hsync-bucket
+
+*** Keywords ***
+Get OM serviceId
+    ${confKey} =        Execute And Ignore Error        ozone getconf confKey ozone.om.service.ids
+    ${result} =         Evaluate                        "Configuration ozone.om.service.ids is missing" in """${confKey}"""
+    IF      ${result} == ${True}
+        Set Suite Variable  ${OMSERVICEID}         om
+    ELSE
+        Set Suite Variable  ${OMSERVICEID}         ${confKey}
+    END
+
+*** Test Cases ***
+Generate key by HSYNC
+    Freon DFSG    sync=HSYNC    path=ofs://${OMSERVICEID}/${VOLUME}/${BUCKET}
+
+Generate key by HFLUSH
+    Freon DFSG    sync=HFLUSH   path=ofs://${OMSERVICEID}/${VOLUME}/${BUCKET}

hadoop-ozone/dist/src/main/smoketest/hsync/upgrade-hsync-check.robot

@@ -0,0 +1,69 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation       Test HSync during upgrade
+Library             OperatingSystem
+Library             String
+Library             BuiltIn
+Resource            ../commonlib.robot
+Default Tags        pre-finalized-hsync-tests
+Suite Setup         Run Keyword if    '${SECURITY_ENABLED}' == 'true'    Kinit test user     testuser     testuser.keytab
+
+*** Variables ***
+${OMSERVICEID}
+${VOLUME}
+${BUCKET}
+${KEY}
+
+*** Keywords ***
+Get OM serviceId
+    ${confKey} =        Execute And Ignore Error        ozone getconf confKey ozone.om.service.ids
+    ${result} =         Evaluate                        "Configuration ozone.om.service.ids is missing" in """${confKey}"""
+    IF      ${result} == ${True}
+        Set Suite Variable  ${OMSERVICEID}         om
+    ELSE
+        Set Suite Variable  ${OMSERVICEID}         ${confKey}
+    END
+Create volume for upgrade test
+    ${random} =     Generate Random String  5  [LOWER]
+    ${volume} =     Set Variable        vol-${random}
+    ${result} =     Execute             ozone sh volume create /${volume}
+                    Should not contain  ${result}       Failed
+    Set Suite Variable      ${VOLUME}           ${volume}
+
+Create bucket for upgrade test
+    ${random} =     Generate Random String  5  [LOWER]
+    ${bucket} =     Set Variable        buc-${random}
+    ${result} =     Execute             ozone sh bucket create -l FILE_SYSTEM_OPTIMIZED /${volume}/${bucket}
+                    Should not contain  ${result}       Failed
+    Set Suite Variable      ${BUCKET}           ${bucket}
+
+Create key for upgrade test
+    ${random} =     Generate Random String  5  [LOWER]
+    ${key} =        Set Variable        key-${random}
+    ${result} =     Execute             ozone sh key put /${volume}/${bucket}/${key} /etc/hosts
+    Set Suite Variable      ${KEY}          ${key}
+
+*** Test Cases ***
+Test HSync Prior To Finalization
+    Get OM serviceId
+    Create volume for upgrade test
+    Create bucket for upgrade test
+    Create key for upgrade test
+    ${result} =     Execute and checkrc        ozone debug recover --path=ofs://${OMSERVICEID}/${VOLUME}/${BUCKET}/${KEY}    255
+                    Should contain  ${result}  It belongs to the layout feature HBASE_SUPPORT, whose layout version is 7
+    ${result} =     Execute and checkrc        ozone debug recover --path=o3fs://${BUCKET}.${VOLUME}.${OMSERVICEID}/${KEY}    255
+                    Should contain  ${result}  It belongs to the layout feature HBASE_SUPPORT, whose layout version is 7

hadoop-ozone/dist/src/main/smoketest/ozone-lib/freon.robot

@@ -67,3 +67,9 @@ Freon OMBR
     [arguments]    ${prefix}=ombg    ${n}=1    ${threads}=1    ${args}=${EMPTY}
     ${result} =        Execute          ozone freon ombr ${OM_HA_PARAM} -t ${threads} -n${n} -p ${prefix} ${args}
                        Should contain   ${result}   Successful executions: ${n}
+
+Freon DFSG
+    [arguments]    ${prefix}=dfsg    ${n}=1000    ${path}={EMPTY}    ${threads}=1    ${sync}=HSYNC    ${buffer}=1024    ${copy-buffer}=1024    ${size}=10240    ${args}=${EMPTY}
+    ${result} =    Execute    ozone freon dfsg -n ${n} --sync ${sync} -s ${size} --path ${path} --buffer ${buffer} --copy-buffer ${copy-buffer} -p ${prefix} -t ${threads} ${args}
+                   Should contain   ${result}   Successful executions: ${n}
+