diff --git a/docker/start_example.sh b/docker/start_example.sh index 82694533..213ceaff 100755 --- a/docker/start_example.sh +++ b/docker/start_example.sh @@ -6,7 +6,7 @@ action() { # valid example? if [ ! -d "${example_dir}" ]; then - 2>&1 echo "'${example_name}' is not a valid law example" + >&2 echo "'${example_name}' is not a valid law example" return "1" fi export LAW_DOCKER_EXAMPLE="${example_name}" diff --git a/law.cfg.example b/law.cfg.example index 68b68978..2255da8e 100644 --- a/law.cfg.example +++ b/law.cfg.example @@ -775,7 +775,7 @@ ; all jobs files to be present as long as jobs are running, or in case you want to debug the files ; later on. ; Type: boolean -; Default: True +; Default: False ; --- Options of contrib packages diff --git a/law/cli/cli.py b/law/cli/cli.py index 43124cf0..c24e6af2 100644 --- a/law/cli/cli.py +++ b/law/cli/cli.py @@ -12,7 +12,7 @@ import law -progs = ["run", "index", "config", "software", "completion", "location"] +progs = ["run", "index", "config", "software", "completion", "location", "quickstart"] def run(argv=None): diff --git a/law/cli/completion.sh b/law/cli/completion.sh index 35e790a1..0d7f76f7 100755 --- a/law/cli/completion.sh +++ b/law/cli/completion.sh @@ -26,8 +26,7 @@ _law_complete() { local index_file="${LAW_INDEX_FILE:-${law_home}/index}" # common parameters - local common_params="run index config software completion location --help --version" - local common_run_params="workers assistant local-scheduler scheduler-host scheduler-port log-level help" + local common_params="run index config software completion location quickstart --help --version" # the current word local cur="${COMP_WORDS[COMP_CWORD]}" @@ -62,6 +61,7 @@ _law_complete() { local task_family="${COMP_WORDS[2]}" # complete parameters of the root task and if parameters were found, stop + local common_run_params="workers assistant local-scheduler scheduler-host scheduler-port log-level help" local inp="${cur##-}" inp="${inp##-}" COMPREPLY=( $( compgen -W "$( _law_grep_Po "[^\:]+\:${task_family}\:\K.+" "${index_file}" ) ${common_run_params}" -P "--" -- "${inp}" ) ) @@ -195,6 +195,15 @@ _law_complete() { inp="${inp##-}" COMPREPLY=( $( compgen -W "$( echo ${words} )" -P "--" -- "${inp}" ) $( compgen -W "$( echo ${contribs} )" -- "${inp}" ) ) return "0" + + # complete the "quickstart" subcommand + elif [ "${sub_cmd}" = "quickstart" ]; then + local words="help directory no-tasks no-config no-setup" + local inp="${cur##-}" + inp="${inp##-}" + COMPREPLY=( $( compgen -W "$( echo ${words} )" -P "--" -- "${inp}" ) ) + return "0" + fi } diff --git a/law/cli/quickstart.py b/law/cli/quickstart.py new file mode 100644 index 00000000..e8195a0c --- /dev/null +++ b/law/cli/quickstart.py @@ -0,0 +1,76 @@ +# coding: utf-8 + +""" +"law quickstart" cli subprogram. +""" + + +import os +import shutil + +from law.config import Config +from law.util import law_src_path + + +_cfg = Config.instance() + + +def setup_parser(sub_parsers): + """ + Sets up the command line parser for the *quickstart* subprogram and adds it to *sub_parsers*. + """ + parser = sub_parsers.add_parser( + "quickstart", + prog="law quickstart", + description="Quickstart to create a minimal project structure and law configuration.", + ) + + parser.add_argument( + "--directory", + "-d", + help="the directory where the quickstart files are created; default: current directory", + ) + parser.add_argument( + "--no-tasks", + action="store_true", + help="skip creating tasks", + ) + parser.add_argument( + "--no-config", + action="store_true", + help="skip creating the law.cfg file", + ) + parser.add_argument( + "--no-setup", + action="store_true", + help="skip creating the setup.sh file", + ) + + +def execute(args): + """ + Executes the *quickstart* subprogram with parsed commandline *args*. + """ + # get the quickstart directory + qs_dir = law_src_path("templates", "quickstart") + + # prepare the directory if it does not exist yet + out_dir = os.path.normpath(os.path.abspath(args.directory)) + if not os.path.exists(out_dir): + os.makedirs(out_dir) + + # copy tasks + if not args.no_tasks: + dst = os.path.join(out_dir, "my_package") + shutil.copytree(os.path.join(qs_dir, "my_package"), dst) + print("created {}".format(dst)) + + # copy config + if not args.no_config: + shutil.copy2(os.path.join(qs_dir, "law.cfg"), out_dir) + print("created {}".format(os.path.join(out_dir, "law.cfg"))) + + # copy setup + if not args.no_setup: + shutil.copy2(os.path.join(qs_dir, "setup.sh"), out_dir) + print("created {}".format(os.path.join(out_dir, "setup.sh"))) diff --git a/law/config.py b/law/config.py index 012a3383..9abdc12c 100644 --- a/law/config.py +++ b/law/config.py @@ -127,7 +127,7 @@ def __str__(self): "job": { "job_file_dir": os.getenv("LAW_JOB_FILE_DIR") or tempfile.gettempdir(), "job_file_dir_mkdtemp": True, - "job_file_dir_cleanup": True, + "job_file_dir_cleanup": False, }, "notifications": { "mail_recipient": None, diff --git a/law/contrib/cms/crab/crab_wrapper.sh b/law/contrib/cms/crab/crab_wrapper.sh index f78fc370..96dc189d 100644 --- a/law/contrib/cms/crab/crab_wrapper.sh +++ b/law/contrib/cms/crab/crab_wrapper.sh @@ -21,7 +21,7 @@ action() { # get the job number export LAW_CRAB_JOB_NUMBER="$( ls -1 | grep -Po "jobReport\.json\.\K\d+" | head -n 1 )" if [ -z "${LAW_CRAB_JOB_NUMBER}" ]; then - 2>&1 echo "could not determine crab job number" + >&2 echo "could not determine crab job number" return "1" fi echo "running ${this_file_base} for job number ${LAW_CRAB_JOB_NUMBER}" @@ -29,7 +29,7 @@ action() { # get the comma-separated list of input files export LAW_CRAB_INPUT_FILES="$( python -c "from PSet import process; print(','.join(list(getattr(process.source, 'fileNames', []))))" )" if [ -z "${LAW_CRAB_INPUT_FILES}" ]; then - 2>&1 echo "could not determine crab input files" + >&2 echo "could not determine crab input files" # do not consider this an error for now # return "2" fi diff --git a/law/templates/quickstart/law.cfg b/law/templates/quickstart/law.cfg new file mode 100644 index 00000000..84dd173a --- /dev/null +++ b/law/templates/quickstart/law.cfg @@ -0,0 +1,41 @@ +; law configuration example +; for more info, see https://law.readthedocs.io/en/latest/config.html + +[modules] +; the task modules that should be scanned by "law index" +my_package.tasks + + +[logging] +; log levels mapped to python modules +law: INFO +luigi-interface: INFO +gfal2: WARNING + + +[luigi_core] +; luigi core settings +local_scheduler: True +scheduler_host: 127.0.0.1 +scheduler_port: 8080 +parallel_scheduling: False +no_lock: True +log_level: INFO + + +[luigi_scheduler] +; luigi scheduler settings +record_task_history: False +remove_delay: 86400 +retry_delay: 30 +worker_disconnect_delay: 30 + + +[luigi_worker] +; luigi worker settings +ping_interval: 20 +wait_interval: 20 +check_unfulfilled_deps: False +cache_task_completion: True +keep_alive: True +force_multiprocessing: False diff --git a/law/templates/quickstart/my_package/__init__.py b/law/templates/quickstart/my_package/__init__.py new file mode 100644 index 00000000..57d631c3 --- /dev/null +++ b/law/templates/quickstart/my_package/__init__.py @@ -0,0 +1 @@ +# coding: utf-8 diff --git a/law/templates/quickstart/my_package/tasks.py b/law/templates/quickstart/my_package/tasks.py new file mode 100644 index 00000000..42dffc95 --- /dev/null +++ b/law/templates/quickstart/my_package/tasks.py @@ -0,0 +1,16 @@ +# coding: utf-8 + +""" +Location of tasks. +""" + +import law + + +class MyTask(law.Task): + + def output(self): + return law.LocalFileTarget("$QS_DATA/output.txt") + + def run(self): + self.output().dump("output of {!r}".format(self)) diff --git a/law/templates/quickstart/setup.sh b/law/templates/quickstart/setup.sh new file mode 100644 index 00000000..89d089c8 --- /dev/null +++ b/law/templates/quickstart/setup.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Setup script for the quickstart template. Custom environment variables are prefixed with "QS_". + +action() { + # local variables + local shell_is_zsh="$( [ -z "${ZSH_VERSION}" ] && echo "false" || echo "true" )" + local this_file="$( ${shell_is_zsh} && echo "${(%):-%x}" || echo "${BASH_SOURCE[0]}" )" + local this_dir="$( cd "$( dirname "${this_file}" )" && pwd )" + + # global variables + export QS_BASE="${this_dir}" + export QS_DATA="${QS_BASE}/data" + export PYTHONPATH="${QS_BASE}:${PYTHONPATH}" + export LAW_HOME="${QS_BASE}/.law" + export LAW_CONFIG_FILE="${QS_BASE}/law.cfg" + + # detect law + if ! type type &> /dev/null; then + >&2 echo "law not found, please adjust PATH and PYTHONPATH or 'pip install law'" + else + source "$( law completion )" "" + fi +} +action