Merge pull request #2935 from hjoliver/alt-run-dir-take-2

Support symlinked alternate run directories (cylc 8).
cylc · Aug 30, 2019 · 3918589 · 3918589
2 parents cd8b951 + 1a12d66
commit 3918589
Show file tree

Hide file tree

Showing 4 changed files with 201 additions and 80 deletions.
diff --git a/CHANGES.md b/CHANGES.md
@@ -36,6 +36,9 @@ Jinja filters were moved from its `Jinja2Filters` folder to within the
 [#3302](https://github.com/cylc/cylc-flow/pull/3302) - improve CLI
 task-globbing help.
 
+[#2935](https://github.com/cylc/cylc-flow/pull/2935) - support alternate run
+directories, particularly for sub-suites.
+
 [#3096](https://github.com/cylc/cylc-flow/pull/3096) - add colour to the
 Cylc CLI.
 

diff --git a/bin/cylc-register b/bin/cylc-register
@@ -67,16 +67,21 @@ def get_option_parser():
                 ("[PATH]", "Suite definition directory (defaults to $PWD)")])
 
     parser.add_option(
-        "--redirect", help="Allow an existing suite name and run directory "
-                           "to be used with another suite.",
+        "--redirect", help="Allow an existing suite name and run directory"
+                           " to be used with another suite.",
         action="store_true", default=False, dest="redirect")
 
+    parser.add_option(
+        "--run-dir", help="Symlink $HOME/cylc-run/REG to RUNDIR/REG.",
+        action="store", metavar="RUNDIR", default=None, dest="rundir")
+
     return parser
 
 
 @cli_function(get_option_parser)
 def main(parser, opts, reg=None, src=None):
-    SuiteSrvFilesManager().register(reg, src, redirect=opts.redirect)
+    SuiteSrvFilesManager().register(
+        reg, src, redirect=opts.redirect, rundir=opts.rundir)
 
 
 if __name__ == "__main__":

diff --git a/cylc/flow/suite_srv_files_mgr.py b/cylc/flow/suite_srv_files_mgr.py
@@ -385,7 +385,7 @@ def parse_suite_arg(self, options, arg):
                 name = os.path.basename(os.path.dirname(arg))
         return name, path
 
-    def register(self, reg=None, source=None, redirect=False):
+    def register(self, reg=None, source=None, redirect=False, rundir=None):
         """Register a suite, or renew its registration.
 
         Create suite service directory and symlink to suite source location.
@@ -430,7 +430,27 @@ def register(self, reg=None, source=None, redirect=False):
 
         # Create service dir if necessary.
         srv_d = self.get_suite_srv_dir(reg)
-        os.makedirs(srv_d, exist_ok=True)
+        if rundir is None:
+            os.makedirs(srv_d, exist_ok=True)
+        else:
+            suite_run_d, srv_d_name = os.path.split(srv_d)
+            alt_suite_run_d = os.path.join(rundir, reg)
+            alt_srv_d = os.path.join(rundir, reg, srv_d_name)
+            os.makedirs(alt_srv_d, exist_ok=True)
+            os.makedirs(os.path.dirname(suite_run_d), exist_ok=True)
+            if os.path.islink(suite_run_d) and not os.path.exists(suite_run_d):
+                # Remove a bad symlink.
+                os.unlink(suite_run_d)
+            if not os.path.exists(suite_run_d):
+                os.symlink(alt_suite_run_d, suite_run_d)
+            elif not os.path.islink(suite_run_d):
+                raise SuiteServiceFileError(
+                    f"Run directory '{suite_run_d}' already exists.")
+            elif alt_suite_run_d != os.readlink(suite_run_d):
+                target = os.readlink(suite_run_d)
+                raise SuiteServiceFileError(
+                    f"Symlink '{suite_run_d}' already points to {target}.")
+            # (else already the right symlink)
 
         # See if suite already has a source or not
         try:

diff --git a/tests/registration/00-simple.t b/tests/registration/00-simple.t
@@ -14,120 +14,213 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 #------------------------------------------------------------------------------
 # Test suite registration
 
+export RND_SUITE_NAME
+export RND_SUITE_SOURCE
+export RND_SUITE_RUNDIR
+export CYLC_RUN_DIR
+
+CYLC_RUN_DIR="$(cylc get-global-config --print-run-dir)"
+
+function make_rnd_suite() {
+    # Create a randomly-named suite source directory.
+    # Define its run directory.
+    RND_SUITE_NAME=x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)
+    RND_SUITE_SOURCE="$PWD/${RND_SUITE_NAME}"
+    mkdir -p "${RND_SUITE_SOURCE}"
+    touch "${RND_SUITE_SOURCE}/suite.rc"
+    RND_SUITE_RUNDIR="${CYLC_RUN_DIR}/${RND_SUITE_NAME}"
+}
+
+function purge_rnd_suite() {
+    # Remove the suite source created by make_rnd_suite().
+    # And remove its run-directory too.
+    RND_SUITE_SOURCE=${1:-$RND_SUITE_SOURCE}
+    RND_SUITE_RUNDIR=${2:-$RND_SUITE_RUNDIR}
+    rm -rf "${RND_SUITE_SOURCE}"
+    rm -rf "${RND_SUITE_RUNDIR}"
+}
+
 . "$(dirname "$0")/test_header"
-set_test_number 25
+set_test_number 37
 
-init_suite "${TEST_NAME_BASE}" <<'__SUITE_RC__'
-[meta]
-    title = the quick brown fox
-[scheduling]
-    [[graph]]
-        R1 = a => b => c
-[runtime]
-    [[a,b,c]]
-        script = true
-__SUITE_RC__
+# Use $SUITE_NAME and $SUITE_RUN_DIR defined by test_header
 
-# Unique suite run-dir prefix to avoid messing with real suites.
-PRE="cylctb-reg-${CYLC_TEST_TIME_INIT}"
+#------------------------------
+# Test fail no suite source dir
+TEST_NAME="${TEST_NAME_BASE}-nodir"
+make_rnd_suite
+rm -rf "${RND_SUITE_SOURCE}"
+run_fail "${TEST_NAME}" cylc register "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
+contains_ok "${TEST_NAME}.stderr" <<__ERR__
+SuiteServiceFileError: no suite.rc in ${RND_SUITE_SOURCE}
+__ERR__
+purge_rnd_suite
 
-# Test fail no suite.rc file.
-CYLC_RUN_DIR="$(cylc get-global --print-run-dir)"
-TEST_NAME="${TEST_NAME_BASE}-noreg"
-run_fail "${TEST_NAME}" cylc register "${SUITE_NAME}" "${PWD}/zilch"
+#---------------------------
+# Test fail no suite.rc file
+TEST_NAME="${TEST_NAME_BASE}-nodir"
+make_rnd_suite
+rm -f "${RND_SUITE_SOURCE}/suite.rc"
+run_fail "${TEST_NAME}" cylc register "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
 contains_ok "${TEST_NAME}.stderr" <<__ERR__
-SuiteServiceFileError: no suite.rc in ${PWD}/zilch
+SuiteServiceFileError: no suite.rc in ${RND_SUITE_SOURCE}
 __ERR__
+purge_rnd_suite
 
-CHEESE=${PRE}-cheese
+#-------------------------------------------------------
 # Test default name: "cylc reg" (suite in $PWD, no args)
-TEST_NAME="${TEST_NAME_BASE}-cheese"
-mkdir "${CHEESE}"
-cd "${CHEESE}" || exit 1
-touch 'suite.rc'
+TEST_NAME="${TEST_NAME_BASE}-pwd1"
+make_rnd_suite
+pushd "${RND_SUITE_SOURCE}" || exit 1
 run_ok "${TEST_NAME}" cylc register
 contains_ok "${TEST_NAME}.stdout" <<__OUT__
-REGISTERED ${CHEESE} -> ${PWD}
+REGISTERED $RND_SUITE_NAME -> ${RND_SUITE_SOURCE}
 __OUT__
-cd .. || exit 1
-rm -rf "${CYLC_RUN_DIR:?}/${CHEESE}"
-
-# Test default name: "cylc reg REG" (suite in $PWD)
-TEST_NAME="${TEST_NAME_BASE}-toast"
-cd "${CHEESE}" || exit 1
-TOAST="${PRE}-toast"
-run_ok "${TEST_NAME}" cylc register "${TOAST}"
+popd || exit 1
+purge_rnd_suite
+
+#--------------------------------------------------
+# Test default path: "cylc reg REG" (suite in $PWD)
+TEST_NAME="${TEST_NAME_BASE}-pwd2"
+make_rnd_suite
+pushd "${RND_SUITE_SOURCE}" || exit 1
+run_ok "${TEST_NAME}" cylc register "${RND_SUITE_NAME}"
 contains_ok "${TEST_NAME}.stdout" <<__OUT__
-REGISTERED ${TOAST} -> ${PWD}
+REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_SOURCE}
 __OUT__
-cd .. || exit 1
-rm -rf "${CYLC_RUN_DIR:?}/$TOAST"
+popd || exit 1
+purge_rnd_suite
 
+#-------------------------
 # Test "cylc reg REG PATH"
-TEST_NAME="${TEST_NAME_BASE}-bagels"
-BAGELS="${PRE}-bagels"
-run_ok "${TEST_NAME}" cylc register "${BAGELS}" "${CHEESE}"
+TEST_NAME="${TEST_NAME_BASE}-normal"
+make_rnd_suite
+run_ok "${TEST_NAME}" cylc register "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
 contains_ok "${TEST_NAME}.stdout" <<__OUT__
-REGISTERED ${BAGELS} -> ${PWD}/${CHEESE}
+REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_SOURCE}
 __OUT__
-rm -rf "${CYLC_RUN_DIR:?}/${BAGELS}"
-
-# Test "cylc reg REG ~/cylc-run/REG"
-TEST_NAME="${TEST_NAME_BASE}-onion"
-ONION="${PRE}-onion"
-mkdir -p "${CYLC_RUN_DIR}/${ONION}"
-cp -p "${PWD}/suite.rc" "${CYLC_RUN_DIR}/${ONION}/"
-run_ok "${TEST_NAME}" cylc register "${ONION}" "${CYLC_RUN_DIR}/${ONION}"
+purge_rnd_suite
+
+#--------------------------------------------------------------------
+# Test register existing run directory: "cylc reg REG ~/cylc-run/REG"
+TEST_NAME="${TEST_NAME_BASE}-reg-run-dir"
+make_rnd_suite
+mkdir -p "${RND_SUITE_RUNDIR}"
+cp "${RND_SUITE_SOURCE}/suite.rc" "${RND_SUITE_RUNDIR}"
+run_ok "${TEST_NAME}" cylc register "${RND_SUITE_NAME}" "${RND_SUITE_RUNDIR}"
 contains_ok "${TEST_NAME}.stdout" <<__OUT__
-REGISTERED ${ONION} -> ${CYLC_RUN_DIR}/${ONION}
+REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_RUNDIR}
 __OUT__
-SOURCE="$(readlink "${CYLC_RUN_DIR}/${ONION}/.service/source")"
+SOURCE="$(readlink "${RND_SUITE_RUNDIR}/.service/source")"
 run_ok "${TEST_NAME}-source" test '..' = "${SOURCE}"
 # Run it twice
-run_ok "${TEST_NAME}-2" cylc register "${ONION}" "${CYLC_RUN_DIR}/${ONION}"
+run_ok "${TEST_NAME}-2" cylc register "${RND_SUITE_NAME}" "${RND_SUITE_RUNDIR}"
 contains_ok "${TEST_NAME}-2.stdout" <<__OUT__
-REGISTERED ${ONION} -> ${CYLC_RUN_DIR}/${ONION}
+REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_RUNDIR}
 __OUT__
-SOURCE="$(readlink "${CYLC_RUN_DIR}/${ONION}/.service/source")"
-run_ok "${TEST_NAME}-2-source" test '..' = "${SOURCE}"
-rm -rf "${CYLC_RUN_DIR:?}/${ONION}"
+SOURCE="$(readlink "${RND_SUITE_RUNDIR}/.service/source")"
+run_ok "${TEST_NAME}-source" test '..' = "${SOURCE}"
+purge_rnd_suite
 
+#----------------------------------------------------------------
 # Test fail "cylc reg REG PATH" where REG already points to PATH2
-YOGHURT="${PRE}-YOGHURT"
-cp -r "${CHEESE}" "${YOGHURT}"
-TEST_NAME="${TEST_NAME_BASE}-cheese"
-run_ok "${TEST_NAME}" cylc register "${CHEESE}" "${CHEESE}"
-TEST_NAME="${TEST_NAME_BASE}-repurpose1"
-run_fail "${TEST_NAME}" cylc register "${CHEESE}" "${YOGHURT}"
+TEST_NAME="${TEST_NAME_BASE}-dup1"
+make_rnd_suite
+run_ok "${TEST_NAME}" cylc register "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
+RND_SUITE_NAME1="${RND_SUITE_NAME}"
+RND_SUITE_SOURCE1="${RND_SUITE_SOURCE}"
+RND_SUITE_RUNDIR1="${RND_SUITE_RUNDIR}"
+make_rnd_suite
+TEST_NAME="${TEST_NAME_BASE}-dup2"
+run_fail "${TEST_NAME}" cylc register "${RND_SUITE_NAME1}" "${RND_SUITE_SOURCE}"
 contains_ok "${TEST_NAME}.stderr" <<__ERR__
-SuiteServiceFileError: the name '${CHEESE}' already points to ${PWD}/${CHEESE}.
+SuiteServiceFileError: the name '${RND_SUITE_NAME1}' already points to ${RND_SUITE_SOURCE1}.
 Use --redirect to re-use an existing name and run directory.
 __ERR__
-
-# Test succeed "cylc reg REG PATH" where REG already points to PATH2
-TEST_NAME="${TEST_NAME_BASE}-repurpose2"
-cp -r "${CHEESE}" "${YOGHURT}"
-run_ok "${TEST_NAME}" cylc register --redirect "${CHEESE}" "${YOGHURT}"
+# Now force it
+TEST_NAME="${TEST_NAME_BASE}-dup3"
+run_ok "${TEST_NAME}" cylc register --redirect "${RND_SUITE_NAME1}" "${RND_SUITE_SOURCE}"
 sed -i 's/^\t//; s/^.* WARNING - /WARNING - /' "${TEST_NAME}.stderr"
 contains_ok "${TEST_NAME}.stderr" <<__ERR__
-WARNING - the name '${CHEESE}' points to ${PWD}/${CHEESE}.
-It will now be redirected to ${PWD}/${YOGHURT}.
-Files in the existing ${CHEESE} run directory will be overwritten.
+WARNING - the name '${RND_SUITE_NAME1}' points to ${RND_SUITE_SOURCE1}.
+It will now be redirected to ${RND_SUITE_SOURCE}.
+Files in the existing ${RND_SUITE_NAME1} run directory will be overwritten.
 __ERR__
 contains_ok "${TEST_NAME}.stdout" <<__OUT__
-REGISTERED ${CHEESE} -> ${PWD}/${YOGHURT}
+REGISTERED ${RND_SUITE_NAME1} -> ${RND_SUITE_SOURCE}
 __OUT__
-rm -rf "${CYLC_RUN_DIR:?}/${CHEESE}"
 
-run_ok "${TEST_NAME_BASE}-get-dir" cylc get-directory "${SUITE_NAME}"
+TEST_NAME="${TEST_NAME_BASE}-get-dir"
+run_ok "${TEST_NAME}" cylc get-directory "${RND_SUITE_NAME1}"
+contains_ok "${TEST_NAME}.stdout" <<__ERR__
+${RND_SUITE_SOURCE}
+__ERR__
+
+purge_rnd_suite
+purge_rnd_suite "${RND_SUITE_SOURCE1}" "${RND_SUITE_RUNDIR1}"
+
+#-----------------------
+# Test alternate run dir
+# 1. Normal case.
+TEST_NAME="${TEST_NAME_BASE}-alt-run-dir"
+make_rnd_suite
+ALT_RUN_DIR="${PWD}/alt"
+run_ok "${TEST_NAME}" \
+    cylc register --run-dir="${ALT_RUN_DIR}" "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
+contains_ok "${TEST_NAME}.stdout" <<__OUT__
+REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_SOURCE}
+__OUT__
+run_ok "${TEST_NAME}-check-link" test -L "${RND_SUITE_RUNDIR}"
+run_ok "${TEST_NAME}-rm-link" rm "${RND_SUITE_RUNDIR}"
+run_ok "${TEST_NAME}-rm-alt-run-dir" rm -r "${ALT_RUN_DIR}"
+purge_rnd_suite
+
+# 2. If reg already exists (as a directory).
+TEST_NAME="${TEST_NAME_BASE}-alt-exists1"
+make_rnd_suite
+ALT_RUN_DIR="${PWD}/alt"
+mkdir -p "${RND_SUITE_RUNDIR}"
+run_fail "${TEST_NAME}" \
+   cylc register --run-dir="${ALT_RUN_DIR}" "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
+contains_ok "${TEST_NAME}.stderr" <<__OUT__
+SuiteServiceFileError: Run directory '${RND_SUITE_RUNDIR}' already exists.
+__OUT__
+purge_rnd_suite
+
+# 3. If reg already exists (as a valid symlink).
+TEST_NAME="${TEST_NAME_BASE}-alt-exists2"
+make_rnd_suite
+ALT_RUN_DIR="${PWD}/alt"
+TDIR=$(mktemp -d)
+mkdir -p "$(dirname "${RND_SUITE_RUNDIR}")"
+ln -s "${TDIR}" "${RND_SUITE_RUNDIR}"
+run_fail "${TEST_NAME}" \
+    cylc register --run-dir="${ALT_RUN_DIR}" "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
+contains_ok "${TEST_NAME}.stderr" <<__OUT__
+SuiteServiceFileError: Symlink '${RND_SUITE_RUNDIR}' already points to ${TDIR}.
+__OUT__
+purge_rnd_suite
+rm -rf "${TDIR}"
+
+#-----------------------------------------------------------------------------
+# Now use a real suite
+
+init_suite "${TEST_NAME_BASE}" <<'__SUITE_RC__'
+[meta]
+    title = the quick brown fox
+[scheduling]
+    [[graph]]
+        R1 = a => b => c
+[runtime]
+    [[a,b,c]]
+        script = true
+__SUITE_RC__
 
-# necessary so the suite is being validated via the database not filepath
-cd .. || exit 1
 run_ok "${TEST_NAME_BASE}-val" cylc validate "${SUITE_NAME}"
-cd "${OLDPWD}" || exit 1
 
 run_ok "${TEST_NAME_BASE}-print" cylc print
 contains_ok "${TEST_NAME_BASE}-print.stdout" <<__OUT__