diff --git a/CHANGES.md b/CHANGES.md index edef92024d4..d73ed9d17d7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -36,6 +36,9 @@ Jinja filters were moved from its `Jinja2Filters` folder to within the [#3302](https://github.com/cylc/cylc-flow/pull/3302) - improve CLI task-globbing help. +[#2935](https://github.com/cylc/cylc-flow/pull/2935) - support alternate run +directories, particularly for sub-suites. + [#3096](https://github.com/cylc/cylc-flow/pull/3096) - add colour to the Cylc CLI. diff --git a/bin/cylc-register b/bin/cylc-register index 74b5addd870..0d2777ec2bc 100755 --- a/bin/cylc-register +++ b/bin/cylc-register @@ -67,16 +67,21 @@ def get_option_parser(): ("[PATH]", "Suite definition directory (defaults to $PWD)")]) parser.add_option( - "--redirect", help="Allow an existing suite name and run directory " - "to be used with another suite.", + "--redirect", help="Allow an existing suite name and run directory" + " to be used with another suite.", action="store_true", default=False, dest="redirect") + parser.add_option( + "--run-dir", help="Symlink $HOME/cylc-run/REG to RUNDIR/REG.", + action="store", metavar="RUNDIR", default=None, dest="rundir") + return parser @cli_function(get_option_parser) def main(parser, opts, reg=None, src=None): - SuiteSrvFilesManager().register(reg, src, redirect=opts.redirect) + SuiteSrvFilesManager().register( + reg, src, redirect=opts.redirect, rundir=opts.rundir) if __name__ == "__main__": diff --git a/cylc/flow/suite_srv_files_mgr.py b/cylc/flow/suite_srv_files_mgr.py index 876b3fb2735..88aad299aa6 100644 --- a/cylc/flow/suite_srv_files_mgr.py +++ b/cylc/flow/suite_srv_files_mgr.py @@ -385,7 +385,7 @@ def parse_suite_arg(self, options, arg): name = os.path.basename(os.path.dirname(arg)) return name, path - def register(self, reg=None, source=None, redirect=False): + def register(self, reg=None, source=None, redirect=False, rundir=None): """Register a suite, or renew its registration. Create suite service directory and symlink to suite source location. @@ -430,7 +430,27 @@ def register(self, reg=None, source=None, redirect=False): # Create service dir if necessary. srv_d = self.get_suite_srv_dir(reg) - os.makedirs(srv_d, exist_ok=True) + if rundir is None: + os.makedirs(srv_d, exist_ok=True) + else: + suite_run_d, srv_d_name = os.path.split(srv_d) + alt_suite_run_d = os.path.join(rundir, reg) + alt_srv_d = os.path.join(rundir, reg, srv_d_name) + os.makedirs(alt_srv_d, exist_ok=True) + os.makedirs(os.path.dirname(suite_run_d), exist_ok=True) + if os.path.islink(suite_run_d) and not os.path.exists(suite_run_d): + # Remove a bad symlink. + os.unlink(suite_run_d) + if not os.path.exists(suite_run_d): + os.symlink(alt_suite_run_d, suite_run_d) + elif not os.path.islink(suite_run_d): + raise SuiteServiceFileError( + f"Run directory '{suite_run_d}' already exists.") + elif alt_suite_run_d != os.readlink(suite_run_d): + target = os.readlink(suite_run_d) + raise SuiteServiceFileError( + f"Symlink '{suite_run_d}' already points to {target}.") + # (else already the right symlink) # See if suite already has a source or not try: diff --git a/tests/registration/00-simple.t b/tests/registration/00-simple.t index afe9c082940..a2c3946148e 100755 --- a/tests/registration/00-simple.t +++ b/tests/registration/00-simple.t @@ -14,120 +14,213 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . + #------------------------------------------------------------------------------ # Test suite registration +export RND_SUITE_NAME +export RND_SUITE_SOURCE +export RND_SUITE_RUNDIR +export CYLC_RUN_DIR + +CYLC_RUN_DIR="$(cylc get-global-config --print-run-dir)" + +function make_rnd_suite() { + # Create a randomly-named suite source directory. + # Define its run directory. + RND_SUITE_NAME=x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6) + RND_SUITE_SOURCE="$PWD/${RND_SUITE_NAME}" + mkdir -p "${RND_SUITE_SOURCE}" + touch "${RND_SUITE_SOURCE}/suite.rc" + RND_SUITE_RUNDIR="${CYLC_RUN_DIR}/${RND_SUITE_NAME}" +} + +function purge_rnd_suite() { + # Remove the suite source created by make_rnd_suite(). + # And remove its run-directory too. + RND_SUITE_SOURCE=${1:-$RND_SUITE_SOURCE} + RND_SUITE_RUNDIR=${2:-$RND_SUITE_RUNDIR} + rm -rf "${RND_SUITE_SOURCE}" + rm -rf "${RND_SUITE_RUNDIR}" +} + . "$(dirname "$0")/test_header" -set_test_number 25 +set_test_number 37 -init_suite "${TEST_NAME_BASE}" <<'__SUITE_RC__' -[meta] - title = the quick brown fox -[scheduling] - [[graph]] - R1 = a => b => c -[runtime] - [[a,b,c]] - script = true -__SUITE_RC__ +# Use $SUITE_NAME and $SUITE_RUN_DIR defined by test_header -# Unique suite run-dir prefix to avoid messing with real suites. -PRE="cylctb-reg-${CYLC_TEST_TIME_INIT}" +#------------------------------ +# Test fail no suite source dir +TEST_NAME="${TEST_NAME_BASE}-nodir" +make_rnd_suite +rm -rf "${RND_SUITE_SOURCE}" +run_fail "${TEST_NAME}" cylc register "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}" +contains_ok "${TEST_NAME}.stderr" <<__ERR__ +SuiteServiceFileError: no suite.rc in ${RND_SUITE_SOURCE} +__ERR__ +purge_rnd_suite -# Test fail no suite.rc file. -CYLC_RUN_DIR="$(cylc get-global --print-run-dir)" -TEST_NAME="${TEST_NAME_BASE}-noreg" -run_fail "${TEST_NAME}" cylc register "${SUITE_NAME}" "${PWD}/zilch" +#--------------------------- +# Test fail no suite.rc file +TEST_NAME="${TEST_NAME_BASE}-nodir" +make_rnd_suite +rm -f "${RND_SUITE_SOURCE}/suite.rc" +run_fail "${TEST_NAME}" cylc register "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}" contains_ok "${TEST_NAME}.stderr" <<__ERR__ -SuiteServiceFileError: no suite.rc in ${PWD}/zilch +SuiteServiceFileError: no suite.rc in ${RND_SUITE_SOURCE} __ERR__ +purge_rnd_suite -CHEESE=${PRE}-cheese +#------------------------------------------------------- # Test default name: "cylc reg" (suite in $PWD, no args) -TEST_NAME="${TEST_NAME_BASE}-cheese" -mkdir "${CHEESE}" -cd "${CHEESE}" || exit 1 -touch 'suite.rc' +TEST_NAME="${TEST_NAME_BASE}-pwd1" +make_rnd_suite +pushd "${RND_SUITE_SOURCE}" || exit 1 run_ok "${TEST_NAME}" cylc register contains_ok "${TEST_NAME}.stdout" <<__OUT__ -REGISTERED ${CHEESE} -> ${PWD} +REGISTERED $RND_SUITE_NAME -> ${RND_SUITE_SOURCE} __OUT__ -cd .. || exit 1 -rm -rf "${CYLC_RUN_DIR:?}/${CHEESE}" - -# Test default name: "cylc reg REG" (suite in $PWD) -TEST_NAME="${TEST_NAME_BASE}-toast" -cd "${CHEESE}" || exit 1 -TOAST="${PRE}-toast" -run_ok "${TEST_NAME}" cylc register "${TOAST}" +popd || exit 1 +purge_rnd_suite + +#-------------------------------------------------- +# Test default path: "cylc reg REG" (suite in $PWD) +TEST_NAME="${TEST_NAME_BASE}-pwd2" +make_rnd_suite +pushd "${RND_SUITE_SOURCE}" || exit 1 +run_ok "${TEST_NAME}" cylc register "${RND_SUITE_NAME}" contains_ok "${TEST_NAME}.stdout" <<__OUT__ -REGISTERED ${TOAST} -> ${PWD} +REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_SOURCE} __OUT__ -cd .. || exit 1 -rm -rf "${CYLC_RUN_DIR:?}/$TOAST" +popd || exit 1 +purge_rnd_suite +#------------------------- # Test "cylc reg REG PATH" -TEST_NAME="${TEST_NAME_BASE}-bagels" -BAGELS="${PRE}-bagels" -run_ok "${TEST_NAME}" cylc register "${BAGELS}" "${CHEESE}" +TEST_NAME="${TEST_NAME_BASE}-normal" +make_rnd_suite +run_ok "${TEST_NAME}" cylc register "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}" contains_ok "${TEST_NAME}.stdout" <<__OUT__ -REGISTERED ${BAGELS} -> ${PWD}/${CHEESE} +REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_SOURCE} __OUT__ -rm -rf "${CYLC_RUN_DIR:?}/${BAGELS}" - -# Test "cylc reg REG ~/cylc-run/REG" -TEST_NAME="${TEST_NAME_BASE}-onion" -ONION="${PRE}-onion" -mkdir -p "${CYLC_RUN_DIR}/${ONION}" -cp -p "${PWD}/suite.rc" "${CYLC_RUN_DIR}/${ONION}/" -run_ok "${TEST_NAME}" cylc register "${ONION}" "${CYLC_RUN_DIR}/${ONION}" +purge_rnd_suite + +#-------------------------------------------------------------------- +# Test register existing run directory: "cylc reg REG ~/cylc-run/REG" +TEST_NAME="${TEST_NAME_BASE}-reg-run-dir" +make_rnd_suite +mkdir -p "${RND_SUITE_RUNDIR}" +cp "${RND_SUITE_SOURCE}/suite.rc" "${RND_SUITE_RUNDIR}" +run_ok "${TEST_NAME}" cylc register "${RND_SUITE_NAME}" "${RND_SUITE_RUNDIR}" contains_ok "${TEST_NAME}.stdout" <<__OUT__ -REGISTERED ${ONION} -> ${CYLC_RUN_DIR}/${ONION} +REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_RUNDIR} __OUT__ -SOURCE="$(readlink "${CYLC_RUN_DIR}/${ONION}/.service/source")" +SOURCE="$(readlink "${RND_SUITE_RUNDIR}/.service/source")" run_ok "${TEST_NAME}-source" test '..' = "${SOURCE}" # Run it twice -run_ok "${TEST_NAME}-2" cylc register "${ONION}" "${CYLC_RUN_DIR}/${ONION}" +run_ok "${TEST_NAME}-2" cylc register "${RND_SUITE_NAME}" "${RND_SUITE_RUNDIR}" contains_ok "${TEST_NAME}-2.stdout" <<__OUT__ -REGISTERED ${ONION} -> ${CYLC_RUN_DIR}/${ONION} +REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_RUNDIR} __OUT__ -SOURCE="$(readlink "${CYLC_RUN_DIR}/${ONION}/.service/source")" -run_ok "${TEST_NAME}-2-source" test '..' = "${SOURCE}" -rm -rf "${CYLC_RUN_DIR:?}/${ONION}" +SOURCE="$(readlink "${RND_SUITE_RUNDIR}/.service/source")" +run_ok "${TEST_NAME}-source" test '..' = "${SOURCE}" +purge_rnd_suite +#---------------------------------------------------------------- # Test fail "cylc reg REG PATH" where REG already points to PATH2 -YOGHURT="${PRE}-YOGHURT" -cp -r "${CHEESE}" "${YOGHURT}" -TEST_NAME="${TEST_NAME_BASE}-cheese" -run_ok "${TEST_NAME}" cylc register "${CHEESE}" "${CHEESE}" -TEST_NAME="${TEST_NAME_BASE}-repurpose1" -run_fail "${TEST_NAME}" cylc register "${CHEESE}" "${YOGHURT}" +TEST_NAME="${TEST_NAME_BASE}-dup1" +make_rnd_suite +run_ok "${TEST_NAME}" cylc register "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}" +RND_SUITE_NAME1="${RND_SUITE_NAME}" +RND_SUITE_SOURCE1="${RND_SUITE_SOURCE}" +RND_SUITE_RUNDIR1="${RND_SUITE_RUNDIR}" +make_rnd_suite +TEST_NAME="${TEST_NAME_BASE}-dup2" +run_fail "${TEST_NAME}" cylc register "${RND_SUITE_NAME1}" "${RND_SUITE_SOURCE}" contains_ok "${TEST_NAME}.stderr" <<__ERR__ -SuiteServiceFileError: the name '${CHEESE}' already points to ${PWD}/${CHEESE}. +SuiteServiceFileError: the name '${RND_SUITE_NAME1}' already points to ${RND_SUITE_SOURCE1}. Use --redirect to re-use an existing name and run directory. __ERR__ - -# Test succeed "cylc reg REG PATH" where REG already points to PATH2 -TEST_NAME="${TEST_NAME_BASE}-repurpose2" -cp -r "${CHEESE}" "${YOGHURT}" -run_ok "${TEST_NAME}" cylc register --redirect "${CHEESE}" "${YOGHURT}" +# Now force it +TEST_NAME="${TEST_NAME_BASE}-dup3" +run_ok "${TEST_NAME}" cylc register --redirect "${RND_SUITE_NAME1}" "${RND_SUITE_SOURCE}" sed -i 's/^\t//; s/^.* WARNING - /WARNING - /' "${TEST_NAME}.stderr" contains_ok "${TEST_NAME}.stderr" <<__ERR__ -WARNING - the name '${CHEESE}' points to ${PWD}/${CHEESE}. -It will now be redirected to ${PWD}/${YOGHURT}. -Files in the existing ${CHEESE} run directory will be overwritten. +WARNING - the name '${RND_SUITE_NAME1}' points to ${RND_SUITE_SOURCE1}. +It will now be redirected to ${RND_SUITE_SOURCE}. +Files in the existing ${RND_SUITE_NAME1} run directory will be overwritten. __ERR__ contains_ok "${TEST_NAME}.stdout" <<__OUT__ -REGISTERED ${CHEESE} -> ${PWD}/${YOGHURT} +REGISTERED ${RND_SUITE_NAME1} -> ${RND_SUITE_SOURCE} __OUT__ -rm -rf "${CYLC_RUN_DIR:?}/${CHEESE}" -run_ok "${TEST_NAME_BASE}-get-dir" cylc get-directory "${SUITE_NAME}" +TEST_NAME="${TEST_NAME_BASE}-get-dir" +run_ok "${TEST_NAME}" cylc get-directory "${RND_SUITE_NAME1}" +contains_ok "${TEST_NAME}.stdout" <<__ERR__ +${RND_SUITE_SOURCE} +__ERR__ + +purge_rnd_suite +purge_rnd_suite "${RND_SUITE_SOURCE1}" "${RND_SUITE_RUNDIR1}" + +#----------------------- +# Test alternate run dir +# 1. Normal case. +TEST_NAME="${TEST_NAME_BASE}-alt-run-dir" +make_rnd_suite +ALT_RUN_DIR="${PWD}/alt" +run_ok "${TEST_NAME}" \ + cylc register --run-dir="${ALT_RUN_DIR}" "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}" +contains_ok "${TEST_NAME}.stdout" <<__OUT__ +REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_SOURCE} +__OUT__ +run_ok "${TEST_NAME}-check-link" test -L "${RND_SUITE_RUNDIR}" +run_ok "${TEST_NAME}-rm-link" rm "${RND_SUITE_RUNDIR}" +run_ok "${TEST_NAME}-rm-alt-run-dir" rm -r "${ALT_RUN_DIR}" +purge_rnd_suite + +# 2. If reg already exists (as a directory). +TEST_NAME="${TEST_NAME_BASE}-alt-exists1" +make_rnd_suite +ALT_RUN_DIR="${PWD}/alt" +mkdir -p "${RND_SUITE_RUNDIR}" +run_fail "${TEST_NAME}" \ + cylc register --run-dir="${ALT_RUN_DIR}" "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}" +contains_ok "${TEST_NAME}.stderr" <<__OUT__ +SuiteServiceFileError: Run directory '${RND_SUITE_RUNDIR}' already exists. +__OUT__ +purge_rnd_suite + +# 3. If reg already exists (as a valid symlink). +TEST_NAME="${TEST_NAME_BASE}-alt-exists2" +make_rnd_suite +ALT_RUN_DIR="${PWD}/alt" +TDIR=$(mktemp -d) +mkdir -p "$(dirname "${RND_SUITE_RUNDIR}")" +ln -s "${TDIR}" "${RND_SUITE_RUNDIR}" +run_fail "${TEST_NAME}" \ + cylc register --run-dir="${ALT_RUN_DIR}" "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}" +contains_ok "${TEST_NAME}.stderr" <<__OUT__ +SuiteServiceFileError: Symlink '${RND_SUITE_RUNDIR}' already points to ${TDIR}. +__OUT__ +purge_rnd_suite +rm -rf "${TDIR}" + +#----------------------------------------------------------------------------- +# Now use a real suite + +init_suite "${TEST_NAME_BASE}" <<'__SUITE_RC__' +[meta] + title = the quick brown fox +[scheduling] + [[graph]] + R1 = a => b => c +[runtime] + [[a,b,c]] + script = true +__SUITE_RC__ -# necessary so the suite is being validated via the database not filepath -cd .. || exit 1 run_ok "${TEST_NAME_BASE}-val" cylc validate "${SUITE_NAME}" -cd "${OLDPWD}" || exit 1 run_ok "${TEST_NAME_BASE}-print" cylc print contains_ok "${TEST_NAME_BASE}-print.stdout" <<__OUT__