From c863435c618a05ad993e03e560ced440cbb67946 Mon Sep 17 00:00:00 2001 From: tvoigtlaender Date: Tue, 17 Dec 2024 16:11:46 +0100 Subject: [PATCH] Add curly brackets to all bash variables to prevent issues with uncertain variable names --- processor/setup_law_remote.sh | 10 ++-- processor/tasks/scripts/compile_crown.sh | 52 ++++++++-------- .../tasks/scripts/compile_crown_friends.sh | 60 +++++++++---------- processor/tasks/scripts/compile_crown_lib.sh | 38 ++++++------ scripts/os-version.sh | 6 +- scripts/python-formatting.sh | 12 ++-- setup.sh | 32 +++++----- 7 files changed, 105 insertions(+), 105 deletions(-) diff --git a/processor/setup_law_remote.sh b/processor/setup_law_remote.sh index 7a06e7c3..84502bfe 100644 --- a/processor/setup_law_remote.sh +++ b/processor/setup_law_remote.sh @@ -2,10 +2,10 @@ action() { _addpy() { - [ ! -z "$1" ] && export PYTHONPATH="$1:${PYTHONPATH}" && echo "Add $1 to PYTHONPATH" + [ ! -z "${1}" ] && export PYTHONPATH="${1}:${PYTHONPATH}" && echo "Add ${1} to PYTHONPATH" } _addbin() { - [ ! -z "$1" ] && export PATH="$1:${PATH}" && echo "Add $1 to PATH" + [ ! -z "${1}" ] && export PATH="${1}:${PATH}" && echo "Add ${1} to PATH" } SPAWNPOINT=$(pwd) @@ -80,10 +80,10 @@ action() { export ANALYSIS_DATA_PATH=$(pwd) - # start a luigid scheduler using $LUIGIPORT + # start a luigid scheduler using ${LUIGIPORT} if [[ ! -z "${LUIGIPORT}" ]]; then - echo "Starting luigid scheduler on port $LUIGIPORT" - luigid --background --logdir logs --state-path luigid_state.pickle --port=$LUIGIPORT + echo "Starting luigid scheduler on port ${LUIGIPORT}" + luigid --background --logdir logs --state-path luigid_state.pickle --port=${LUIGIPORT} fi } diff --git a/processor/tasks/scripts/compile_crown.sh b/processor/tasks/scripts/compile_crown.sh index dee52d20..d978426b 100644 --- a/processor/tasks/scripts/compile_crown.sh +++ b/processor/tasks/scripts/compile_crown.sh @@ -1,23 +1,23 @@ #! /bin/bash -CROWNFOLDER=$1 -ANALYSIS=$2 -CONFIG=$3 -SAMPLES=$4 -ERAS=$5 -SCOPE=$6 -SHIFTS=$7 -INSTALLDIR=$8 -BUILDDIR=$9 +CROWNFOLDER=${1} +ANALYSIS=${2} +CONFIG=${3} +SAMPLES=${4} +ERAS=${5} +SCOPE=${6} +SHIFTS=${7} +INSTALLDIR=${8} +BUILDDIR=${9} TARBALLNAME=${10} EXECUTALBE_THREADS=${11} # setup with analysis clone if needed set -o pipefail set -e -source $ANALYSIS_PATH/CROWN/init.sh $ANALYSIS +source ${ANALYSIS_PATH}/CROWN/init.sh ${ANALYSIS} # remove conda prefix from $PATH so cmakes uses the LCG stack python and not the conda one if [[ ! -z "${CONDA_PREFIX}" ]]; then - PATH=$(echo $PATH | sed "s@$CONDA_PREFIX@@g") - # PATH=$(echo $PATH | sed 's%/cvmfs/etp.kit.edu/[^:]*:%%g') + PATH=$(echo ${PATH} | sed "s@${CONDA_PREFIX}@@g") + # PATH=$(echo ${PATH} | sed 's%/cvmfs/etp.kit.edu/[^:]*:%%g') CONDA_PYTHON_EXE="" CONDA_EXE="" CONDA_PREFIX="" @@ -25,29 +25,29 @@ fi # use a fourth of the machine for compiling THREADS_AVAILABLE=$(grep -c ^processor /proc/cpuinfo) THREADS=$(( THREADS_AVAILABLE / 4 )) -echo "Using $THREADS threads for the compilation" +echo "Using ${THREADS} threads for the compilation" which cmake -if cmake $CROWNFOLDER \ - -DANALYSIS=$ANALYSIS \ - -DCONFIG=$CONFIG \ - -DSAMPLES=$SAMPLES \ - -DERAS=$ERAS \ - -DSCOPES=$SCOPE \ - -DSHIFTS=$SHIFTS \ - -DTHREADS=$EXECUTALBE_THREADS \ - -DINSTALLDIR=$INSTALLDIR \ +if cmake ${CROWNFOLDER} \ + -DANALYSIS=${ANALYSIS} \ + -DCONFIG=${CONFIG} \ + -DSAMPLES=${SAMPLES} \ + -DERAS=${ERAS} \ + -DSCOPES=${SCOPE} \ + -DSHIFTS=${SHIFTS} \ + -DTHREADS=${EXECUTALBE_THREADS} \ + -DINSTALLDIR=${INSTALLDIR} \ -DPRODUCTION=True \ - -B$BUILDDIR 2>&1 |tee $BUILDDIR/cmake.log; then + -B${BUILDDIR} 2>&1 |tee ${BUILDDIR}/cmake.log; then echo "CMake finished successfully" else echo "-------------------------------------------------------------------------" - echo "CMake failed, check the log file $BUILDDIR/cmake.log for more information" + echo "CMake failed, check the log file ${BUILDDIR}/cmake.log for more information" echo "-------------------------------------------------------------------------" sleep 0.1 # wait for the log file to be written exit 1 fi -cd $BUILDDIR +cd ${BUILDDIR} echo "Finished preparing the compilation and starting to compile" -make install -j $THREADS 2>&1 |tee $BUILDDIR/build.log +make install -j ${THREADS} 2>&1 |tee ${BUILDDIR}/build.log echo "Finished the compilation" diff --git a/processor/tasks/scripts/compile_crown_friends.sh b/processor/tasks/scripts/compile_crown_friends.sh index 68ed35c2..3ed4c411 100644 --- a/processor/tasks/scripts/compile_crown_friends.sh +++ b/processor/tasks/scripts/compile_crown_friends.sh @@ -1,23 +1,23 @@ #! /bin/bash -CROWNFOLDER=$1 -ANALYSIS=$2 -CONFIG=$3 -SAMPLES=$4 -ERAS=$5 -SCOPE=$6 -SHIFTS=$7 -INSTALLDIR=$8 -BUILDDIR=$9 +CROWNFOLDER=${1} +ANALYSIS=${2} +CONFIG=${3} +SAMPLES=${4} +ERAS=${5} +SCOPE=${6} +SHIFTS=${7} +INSTALLDIR=${8} +BUILDDIR=${9} TARBALLNAME=${10} QUANTITIESMAP=${11} # setup with analysis clone if needed set -o pipefail set -e -source $ANALYSIS_PATH/CROWN/init.sh $ANALYSIS -# remove conda prefix from $PATH so cmakes uses the LCG stack python and not the conda one +source ${ANALYSIS_PATH}/CROWN/init.sh ${ANALYSIS} +# remove conda prefix from ${PATH} so cmakes uses the LCG stack python and not the conda one if [[ ! -z "${CONDA_PREFIX}" ]]; then - PATH=$(echo $PATH | sed "s@$CONDA_PREFIX@@g") - # PATH=$(echo $PATH | sed 's%/cvmfs/etp.kit.edu/[^:]*:%%g') + PATH=$(echo ${PATH} | sed "s@${CONDA_PREFIX}@@g") + # PATH=$(echo ${PATH} | sed 's%/cvmfs/etp.kit.edu/[^:]*:%%g') CONDA_PYTHON_EXE="" CONDA_EXE="" CONDA_PREFIX="" @@ -26,34 +26,34 @@ fi THREADS_AVAILABLE=$(grep -c ^processor /proc/cpuinfo) # THREADS=$(( THREADS_AVAILABLE / 4 )) THREADS=2 -echo "Using $THREADS threads for the compilation" +echo "Using ${THREADS} threads for the compilation" which cmake -if cmake $CROWNFOLDER \ - -DANALYSIS=$ANALYSIS \ - -DCONFIG=$CONFIG \ - -DSAMPLES=$SAMPLES \ - -DERAS=$ERAS \ - -DSCOPES=$SCOPE \ - -DSHIFTS=$SHIFTS \ - -DINSTALLDIR=$INSTALLDIR \ +if cmake ${CROWNFOLDER} \ + -DANALYSIS=${ANALYSIS} \ + -DCONFIG=${CONFIG} \ + -DSAMPLES=${SAMPLES} \ + -DERAS=${ERAS} \ + -DSCOPES=${SCOPE} \ + -DSHIFTS=${SHIFTS} \ + -DINSTALLDIR=${INSTALLDIR} \ -DPRODUCTION=True \ -DFRIENDS=true \ - -DQUANTITIESMAP=$QUANTITIESMAP \ - -B$BUILDDIR 2>&1 | tee $BUILDDIR/cmake.log; then + -DQUANTITIESMAP=${QUANTITIESMAP} \ + -B${BUILDDIR} 2>&1 | tee ${BUILDDIR}/cmake.log; then echo "CMake finished successfully" else echo "-------------------------------------------------------------------------" - echo "CMake failed, check the log file $BUILDDIR/cmake.log for more information" + echo "CMake failed, check the log file ${BUILDDIR}/cmake.log for more information" echo "-------------------------------------------------------------------------" sleep 0.1 # wait for the log file to be written exit 1 fi -cd $BUILDDIR +cd ${BUILDDIR} echo "Finished preparing the compilation and starting to compile" -make install -j $THREADS 2>&1 |tee $BUILDDIR/build.log +make install -j ${THREADS} 2>&1 |tee ${BUILDDIR}/build.log echo "Finished the compilation and starting to make the *.tar.gz archive" -cd $INSTALLDIR -touch $TARBALLNAME -tar -czvf $TARBALLNAME --exclude=$TARBALLNAME . +cd ${INSTALLDIR} +touch ${TARBALLNAME} +tar -czvf ${TARBALLNAME} --exclude=${TARBALLNAME} . diff --git a/processor/tasks/scripts/compile_crown_lib.sh b/processor/tasks/scripts/compile_crown_lib.sh index d1033f1c..867e6729 100644 --- a/processor/tasks/scripts/compile_crown_lib.sh +++ b/processor/tasks/scripts/compile_crown_lib.sh @@ -1,20 +1,20 @@ #! /bin/bash -CROWNFOLDER=$1 -INSTALLDIR=$2 -BUILDDIR=$3 -ANALYSIS=$4 +CROWNFOLDER=${1} +INSTALLDIR=${2} +BUILDDIR=${3} +ANALYSIS=${4} -echo "Crown folder: $CROWNFOLDER" -echo "Install dir: $INSTALLDIR" -echo "Build dir: $BUILDDIR" +echo "Crown folder: ${CROWNFOLDER}" +echo "Install dir: ${INSTALLDIR}" +echo "Build dir: ${BUILDDIR}" # setup with analysis clone if needed set -o pipefail set -e -source $ANALYSIS_PATH/CROWN/init.sh $ANALYSIS -# remove conda prefix from $PATH so cmakes uses the LCG stack python and not the conda one +source ${ANALYSIS_PATH}/CROWN/init.sh ${ANALYSIS} +# remove conda prefix from ${PATH} so cmakes uses the LCG stack python and not the conda one if [[ ! -z "${CONDA_PREFIX}" ]]; then - PATH=$(echo $PATH | sed "s@$CONDA_PREFIX@@g") - # PATH=$(echo $PATH | sed 's%/cvmfs/etp.kit.edu/[^:]*:%%g') + PATH=$(echo ${PATH} | sed "s@${CONDA_PREFIX}@@g") + # PATH=$(echo ${PATH} | sed 's%/cvmfs/etp.kit.edu/[^:]*:%%g') CONDA_PYTHON_EXE="" CONDA_EXE="" CONDA_PREFIX="" @@ -22,23 +22,23 @@ fi # use a fourth of the machine for compiling THREADS_AVAILABLE=$(grep -c ^processor /proc/cpuinfo) THREADS=$(( THREADS_AVAILABLE / 4 )) -echo "Using $THREADS threads for the compilation" +echo "Using ${THREADS} threads for the compilation" which cmake -if cmake $CROWNFOLDER \ +if cmake ${CROWNFOLDER} \ -DBUILD_CROWNLIB_ONLY=ON \ - -DINSTALLDIR=$INSTALLDIR \ - -DANALYSIS=$ANALYSIS \ - -B$BUILDDIR 2>&1 |tee $BUILDDIR/cmake.log; then + -DINSTALLDIR=${INSTALLDIR} \ + -DANALYSIS=${ANALYSIS} \ + -B${BUILDDIR} 2>&1 |tee ${BUILDDIR}/cmake.log; then echo "CMake finished successfully" else echo "-------------------------------------------------------------------------" - echo "CMake failed, check the log file $BUILDDIR/cmake.log for more information" + echo "CMake failed, check the log file ${BUILDDIR}/cmake.log for more information" echo "-------------------------------------------------------------------------" sleep 0.1 # wait for the log file to be written exit 1 fi -cd $BUILDDIR +cd ${BUILDDIR} echo "Finished preparing the compilation and starting to compile" -make install -j $THREADS 2>&1 |tee $BUILDDIR/build.log +make install -j ${THREADS} 2>&1 |tee ${BUILDDIR}/build.log echo "Finished the compilation crownlib build successfully" \ No newline at end of file diff --git a/scripts/os-version.sh b/scripts/os-version.sh index d789c570..69c8fef7 100644 --- a/scripts/os-version.sh +++ b/scripts/os-version.sh @@ -3,8 +3,8 @@ if ! command -v lsb_release &> /dev/null then source /etc/os-release - distro=$NAME - os_version=$VERSION_ID + distro=${NAME} + os_version=${VERSION_ID} else distro=$(lsb_release -i | cut -f2) os_version=$(lsb_release -r | cut -f2) @@ -12,4 +12,4 @@ fi distro=${distro//[[:space:]]/} distro="${distro//Linux/}" distro="${distro//linux/}" -#echo "Trying to run Kingmaker on $distro Version $os_version" +#echo "Trying to run Kingmaker on ${distro} Version ${os_version}" diff --git a/scripts/python-formatting.sh b/scripts/python-formatting.sh index 912e8de3..740e6d51 100644 --- a/scripts/python-formatting.sh +++ b/scripts/python-formatting.sh @@ -5,16 +5,16 @@ FOUND_ISSUE=0 for FILENAME in $(find . -path ./law -prune -o -name "*.py"); do # only run the check if the filename ends with .py - if [[ $FILENAME == *.py ]]; then - echo "Checking $FILENAME" - black --check $FILENAME + if [[ ${FILENAME} == *.py ]]; then + echo "Checking ${FILENAME}" + black --check ${FILENAME} RETURN_VALUE=$? - if [ $RETURN_VALUE -ne 0 ] + if [ ${RETURN_VALUE} -ne 0 ] then - black --diff $FILENAME 2> /dev/null + black --diff ${FILENAME} 2> /dev/null FOUND_ISSUE=1 fi fi done -exit $FOUND_ISSUE +exit ${FOUND_ISSUE} diff --git a/setup.sh b/setup.sh index 4b1e93a4..a1abb1cf 100755 --- a/setup.sh +++ b/setup.sh @@ -5,11 +5,11 @@ # Second argument sets alternative conda environment directory _addpy() { - [ ! -z "$1" ] && export PYTHONPATH="$1:${PYTHONPATH}" + [ ! -z "${1}" ] && export PYTHONPATH="${1}:${PYTHONPATH}" } _addbin() { - [ ! -z "$1" ] && export PATH="$1:${PATH}" + [ ! -z "${1}" ] && export PATH="${1}:${PATH}" } action() { @@ -39,15 +39,15 @@ action() { # Check if current OS is supported source ${BASE_DIR}/scripts/os-version.sh local VALID_OS="False" - if [[ "$distro" == "CentOS" ]]; then + if [[ "${distro}" == "CentOS" ]]; then if [[ ${os_version:0:1} == "7" ]]; then VALID_OS="True" fi - elif [[ "$distro" == "RedHatEnterprise" || "$distro" == "Alma" || "$distro" == "Rocky" ]]; then + elif [[ "${distro}" == "RedHatEnterprise" || "${distro}" == "Alma" || "${distro}" == "Rocky" ]]; then if [[ ${os_version:0:1} == "9" ]]; then VALID_OS="True" fi - elif [[ "$distro" == "Ubuntu" ]]; then + elif [[ "${distro}" == "Ubuntu" ]]; then if [[ ${os_version:0:2} == "22" ]]; then VALID_OS="True" fi @@ -56,11 +56,11 @@ action() { echo "Kingmaker not support on ${distro} ${os_version}" return 1 else - echo "Running Kingmaker on $distro Version $os_version on $(hostname) from dir ${BASE_DIR}" + echo "Running Kingmaker on ${distro} Version ${os_version} on $(hostname) from dir ${BASE_DIR}" fi # Workflow to be set up - ANA_NAME_GIVEN=$1 + ANA_NAME_GIVEN=${1} # List of available workflows ANA_LIST=("KingMaker" "GPU_example" "ML_train") @@ -103,7 +103,7 @@ action() { if [[ "${PARSED_ENVS_STATUS}" -eq "1" ]]; then IFS='@' read -ra ADDR <<< "${PARSED_ENVS}" for i in "${ADDR[@]}"; do - echo $i + echo ${i} done echo "Parsing of required envs failed with the above error." return 1 @@ -118,8 +118,8 @@ action() { # 2. Use dir from file if none provided # 3. Use local /cvmfs installation if available # 4. Use dir of setup script if neither provided - if [[ ! -z $2 ]]; then - ENV_PATH="$(realpath $2)" + if [[ ! -z ${2} ]]; then + ENV_PATH="$(realpath ${2})" elif [[ -f "${BASE_DIR}/environment.location" ]]; then ENV_PATH="$(tail -n 1 ${BASE_DIR}/environment.location)" elif [[ -d "/cvmfs/etp.kit.edu/LAW_envs/miniforge/envs/${STARTING_ENV}_${IMAGE_HASH}" ]]; then @@ -129,7 +129,7 @@ action() { fi echo "Using environments from ${ENV_PATH}/miniforge." # Save env location to file if provided - if [[ ! -z $2 ]]; then + if [[ ! -z ${2} ]]; then echo saving environment path to file for future setups. echo "### This file contains the environment location that was provided when the setup was last run ###" > ${BASE_DIR}/environment.location echo "${ENV_PATH}" >> ${BASE_DIR}/environment.location @@ -137,7 +137,7 @@ action() { # Remember the current value of VOMS_USERCONF to overwrite after conda source. # This is necessary as conda installs a seperate voms version without the relevant configs. - # Use primary default. Secondary default at $HOME/.voms/vomses has to be manually set. + # Use primary default. Secondary default at ${HOME}/.voms/vomses has to be manually set. INITIAL_VOMS_USERCONF=${VOMS_USERCONF:-"/etc/vomses"} # Try to install env via miniforge @@ -236,7 +236,7 @@ action() { if [[ "${LOCAL_SCHEDULER_STATUS}" -eq "1" ]]; then IFS='@' read -ra ADDR <<< "${LOCAL_SCHEDULER}" for i in "${ADDR[@]}"; do - echo $i + echo ${i} done echo "Parsing of required scheduler setting failed with the above error." return 1 @@ -255,12 +255,12 @@ action() { echo "Starting Luigi scheduler... using a random port" while export LUIGIPORT=$(shuf -n 1 -i 49152-65535) - netstat -atun | grep -q "$LUIGIPORT" + netstat -atun | grep -q "${LUIGIPORT}" do continue done - luigid --background --logdir logs --state-path luigid_state.pickle --port=$LUIGIPORT - echo "Luigi scheduler started on port $LUIGIPORT, setting LUIGIPORT to $LUIGIPORT" + luigid --background --logdir logs --state-path luigid_state.pickle --port=${LUIGIPORT} + echo "Luigi scheduler started on port ${LUIGIPORT}, setting LUIGIPORT to ${LUIGIPORT}" else # first get the (first) PID export LUIGIPID=$(pgrep -u ${USER} -f luigid | head -n 1)