-
Notifications
You must be signed in to change notification settings - Fork 41
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Draft for htcondor group submission.
- Loading branch information
Showing
10 changed files
with
503 additions
and
103 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,155 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| # Wrapper script that is to be configured as htcondor's main executable file | ||
|
|
||
| htcondor_wrapper() { | ||
| # helper to select the correct python executable | ||
| _law_python() { | ||
| command -v python &> /dev/null && python "$@" || python3 "$@" | ||
| } | ||
|
|
||
| # | ||
| # detect variables | ||
| # | ||
|
|
||
| local shell_is_zsh="$( [ -z "${ZSH_VERSION}" ] && echo "false" || echo "true" )" | ||
| local this_file="$( ${shell_is_zsh} && echo "${(%):-%x}" || echo "${BASH_SOURCE[0]}" )" | ||
| local this_file_base="$( basename "${this_file}" )" | ||
|
|
||
| # get the job number | ||
| export LAW_HTCONDOR_JOB_NUMBER="${LAW_HTCONDOR_JOB_PROCESS}" | ||
| if [ -z "${LAW_HTCONDOR_JOB_NUMBER}" ]; then | ||
| >&2 echo "could not determine htcondor job number" | ||
| return "1" | ||
| fi | ||
| # htcondor process numbers start at 0, law job numbers at 1, so increment | ||
| ((LAW_HTCONDOR_JOB_NUMBER++)) | ||
| echo "running ${this_file_base} for job number ${LAW_HTCONDOR_JOB_NUMBER}" | ||
|
|
||
|
|
||
| # | ||
| # job argument definitons, depending on LAW_HTCONDOR_JOB_NUMBER | ||
| # | ||
|
|
||
| # definition | ||
| local htcondor_job_arguments_map | ||
| declare -A htcondor_job_arguments_map | ||
| htcondor_job_arguments_map=( | ||
| {{htcondor_job_arguments_map}} | ||
| ) | ||
|
|
||
| # pick | ||
| local htcondor_job_arguments="${htcondor_job_arguments_map[${LAW_HTCONDOR_JOB_NUMBER}]}" | ||
| if [ -z "${htcondor_job_arguments}" ]; then | ||
| >&2 echo "empty htcondor job arguments for LAW_HTCONDOR_JOB_NUMBER ${LAW_HTCONDOR_JOB_NUMBER}" | ||
| return "3" | ||
| fi | ||
|
|
||
|
|
||
| # | ||
| # variable rendering | ||
| # | ||
|
|
||
| # check variables | ||
| local render_variables="{{render_variables}}" | ||
| if [ -z "${render_variables}" ]; then | ||
| >&2 echo "empty render variables" | ||
| return "4" | ||
| fi | ||
|
|
||
| # decode | ||
| render_variables="$( echo "${render_variables}" | base64 --decode )" | ||
|
|
||
| # check files to render | ||
| local input_files_render=( {{input_files_render}} ) | ||
| if [ "${#input_files_render[@]}" == "0" ]; then | ||
| >&2 echo "received empty input files for rendering for LAW_HTCONDOR_JOB_NUMBER ${LAW_HTCONDOR_JOB_NUMBER}" | ||
| return "5" | ||
| fi | ||
|
|
||
| # render files | ||
| local input_file_render | ||
| for input_file_render in ${input_files_render[@]}; do | ||
| # skip if the file refers to _this_ one | ||
| local input_file_render_base="$( basename "${input_file_render}" )" | ||
| [ "${input_file_render_base}" = "${this_file_base}" ] && continue | ||
| # render | ||
| echo "render ${input_file_render}" | ||
| _law_python -c "\ | ||
| import re;\ | ||
| repl = ${render_variables};\ | ||
| repl['input_files_render'] = '';\ | ||
| repl['file_postfix'] = '${file_postfix}' or repl.get('file_postfix', '');\ | ||
| repl['log_file'] = '${log_file}' or repl.get('log_file', '');\ | ||
| content = open('${input_file_render}', 'r').read();\ | ||
| content = re.sub(r'\{\{(\w+)\}\}', lambda m: repl.get(m.group(1), ''), content);\ | ||
| open('${input_file_render_base}', 'w').write(content);\ | ||
| " | ||
| local render_ret="$?" | ||
| # handle rendering errors | ||
| if [ "${render_ret}" != "0" ]; then | ||
| >&2 echo "input file rendering failed with code ${render_ret}" | ||
| return "6" | ||
| fi | ||
| done | ||
|
|
||
|
|
||
| # | ||
| # run the actual job file | ||
| # | ||
|
|
||
| # check the job file | ||
| local job_file="{{job_file}}" | ||
| if [ ! -f "${job_file}" ]; then | ||
| >&2 echo "job file '${job_file}' does not exist" | ||
| return "7" | ||
| fi | ||
|
|
||
| # helper to print a banner | ||
| banner() { | ||
| local msg="$1" | ||
|
|
||
| echo | ||
| echo "================================================================================" | ||
| echo "=== ${msg}" | ||
| echo "================================================================================" | ||
| echo | ||
| } | ||
|
|
||
| # debugging: print its contents | ||
| # echo "=== content of job file '${job_file}'" | ||
| # echo | ||
| # cat "${job_file}" | ||
| # echo | ||
| # echo "=== end of job file content" | ||
|
|
||
| # run it | ||
| banner "Start of law job" | ||
|
|
||
| local job_ret | ||
| bash "${job_file}" ${htcondor_job_arguments} | ||
| job_ret="$?" | ||
|
|
||
| banner "End of law job" | ||
|
|
||
| return "${job_ret}" | ||
| } | ||
|
|
||
| action() { | ||
| # arguments: file_postfix, log_file | ||
| local file_postfix="$1" | ||
| local log_file="$2" | ||
|
|
||
| if [ -z "${log_file}" ]; then | ||
| htcondor_wrapper "$@" | ||
| elif command -v tee &> /dev/null; then | ||
| set -o pipefail | ||
| echo "---" >> "${log_file}" | ||
| htcondor_wrapper "$@" 2>&1 | tee -a "${log_file}" | ||
| else | ||
| echo "---" >> "${log_file}" | ||
| htcondor_wrapper "$@" &>> "${log_file}" | ||
| fi | ||
| } | ||
|
|
||
| action "$@" |
Oops, something went wrong.