[WIP] Project subdirectories in dev.eessi.io install path and EESSI-extend integration

EESSI-install-software.sh

@@ -135,6 +135,7 @@ else
       # Make sure EESSI_PREFIX and EESSI_OS_TYPE are set
       source $TOPDIR/init/minimal_eessi_env
 
+      echo ">> Creating directory for software subdirectory override: ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
       # make sure directory exists (since it's expected by init/eessi_environment_variables when using archdetect)
       mkdir -p ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
   )
@@ -206,6 +207,7 @@ export PYTHONPYCACHEPREFIX=$TMPDIR/pycache
 # - .lmod/lmodrc.lua
 # - .lmod/SitePackage.lua
 _eessi_software_path=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
+echo "_eessi_software_path: ${_eessi_software_path}"  
 _lmod_cfg_dir=${_eessi_software_path}/.lmod
 _lmod_rc_file=${_lmod_cfg_dir}/lmodrc.lua
 if [ ! -f ${_lmod_rc_file} ]; then
@@ -224,8 +226,8 @@ fi
 # order is important: these are needed to install a full CUDA SDK in host_injections
 # for now, this just reinstalls all scripts. Note the most elegant, but works
 
-# Only run install_scripts.sh if not dev.eessi.io for security
-if [[ "${EESSI_CVMFS_REPO}" != /cvmfs/dev.eessi.io ]]; then
+# Only run install_scripts.sh if not in dev.eessi.io for security
+if [[ -z ${EESSI_DEV_PROJECT} ]]; then
     ${TOPDIR}/install_scripts.sh --prefix ${EESSI_PREFIX}
 fi
 
@@ -250,7 +252,7 @@ export EESSI_CVMFS_INSTALL=1
 # NOTE 3, we have to set a default for EASYBUILD_INSTALLPATH here in cases the
 #   EESSI-extend module itself needs to be installed.
 export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
-source load_eessi_extend_module.sh ${EESSI_VERSION}
+source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION}
 
 # Install full CUDA SDK and cu* libraries in host_injections
 # Hardcode this for now, see if it works
@@ -292,12 +294,14 @@ if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then
 fi
 
 # If in dev.eessi.io, allow building on top of software.eessi.io
-if [[ "${EESSI_CVMFS_REPO}" == /cvmfs/dev.eessi.io ]]; then
+if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then
     module use /cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all
+    # Overwrite EASYBUILD_INSTALLPATH to point to dev.eessi.io
+    EASYBUILD_INSTALLPATH=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
+    echo ">> \$EASYBUILD_INSTALLPATH overwritten to ${EASYBUILD_INSTALLPATH}"
+    module use $EASYBUILD_INSTALLPATH/modules/all
 fi
 
-module use $EASYBUILD_INSTALLPATH/modules/all
-
 if [[ -z ${MODULEPATH} ]]; then
     fatal_error "Failed to set up \$MODULEPATH?!"
 else

bot/build.sh

@@ -284,15 +284,19 @@ timestamp=$(date +%s)
 source $software_layer_dir/init/eessi_defaults
 export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${timestamp})
 
+# Export EESSI_DEV_PROJECT to use it (if needed) when making tarball
+echo "bot/build.sh: EESSI_DEV_PROJECT='${EESSI_DEV_PROJECT}'"
+export EESSI_DEV_PROJECT=${EESSI_DEV_PROJECT}
+
 # value of first parameter to create_tarball.sh - TMP_IN_CONTAINER - needs to be
 # synchronised with setting of TMP_IN_CONTAINER in eessi_container.sh
 # TODO should we make this a configurable parameter of eessi_container.sh using
 # /tmp as default?
 TMP_IN_CONTAINER=/tmp
 echo "Executing command to create tarball:"
 echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}"
-echo "                     -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET}\" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
+echo "                     -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET}\" \"${EESSI_DEV_PROJECT}\" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
 $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \
-                     -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET}" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
+                     -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET}" "${EESSI_DEV_PROJECT}" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
 
 exit 0

bot/check-build.sh

@@ -480,6 +480,12 @@ if [[ ! -z ${TARBALL} ]]; then
     accelerator=$(cfg_get_value "architecture" "accelerator")
     prefix="${repo_version}/software/${os_type}/${software_subdir}"
 
+    # if building for dev.eessi.io is set, the prefix is different
+    # it needs the project name in the path
+    if [ ! -z ${EESSI_DEV_PROJECT} ]; then
+        prefix="${repo_version}/${EESSI_DEV_PROJECT}/software/${os_type}/${software_subdir}"
+    fi
+
     # if we build for an accelerator, the prefix is different
     if [[ ! -z ${accelerator} ]]; then
       prefix="${prefix}/accel/${accelerator}"

create_tarball.sh

@@ -4,15 +4,16 @@ set -e
 
 base_dir=$(dirname $(realpath $0))
 
-if [ $# -ne 5 ]; then
-    echo "ERROR: Usage: $0 <EESSI tmp dir (example: /tmp/$USER/EESSI)> <version (example: 2023.06)> <CPU arch subdir (example: x86_64/amd/zen2)> <accelerator subdir (example: nvidia/cc80)> <path to tarball>" >&2
+if [ $# -ne 6 ]; then
+    echo "ERROR: Usage: $0 <EESSI tmp dir (example: /tmp/$USER/EESSI)> <version (example: 2023.06)> <CPU arch subdir (example: x86_64/amd/zen2)> <accelerator subdir (example: nvidia/cc80)> <dev project> (example: ESPREsSo) <path to tarball>" >&2
     exit 1
 fi
 eessi_tmpdir=$1
 eessi_version=$2
 cpu_arch_subdir=$3
 accel_subdir=$4
-target_tgz=$5
+eessi_dev_project=$5
+target_tgz=$6
 
 tmpdir=`mktemp -d`
 echo ">> tmpdir: $tmpdir"
@@ -21,7 +22,18 @@ os="linux"
 source ${base_dir}/init/eessi_defaults
 cvmfs_repo=${EESSI_CVMFS_REPO}
 
-software_dir="${cvmfs_repo}/versions/${eessi_version}/software/${os}/${cpu_arch_subdir}"
+# install to $eessi_version/$eessi_dev_project if defined, otherwise install
+# to $eessi_version
+install_prefix_dir=${eessi_version}${eessi_dev_project:+/$eessi_dev_project}
+echo "Setting Install prefix directory to: ${install_prefix_dir}"
+
+if [ -z ${install_prefix_dir} ]; then
+    echo "\${install_prefix_dir} is empty?!" >&2
+    exit 2
+fi
+
+# software_dir includes $eessi_dev_project if defined, otherwise install omit
+software_dir="${cvmfs_repo}/versions/${eessi_version}${eessi_dev_project:+/$eessi_dev_project}/software/${os}/${cpu_arch_subdir}"
 if [ ! -d ${software_dir} ]; then
     echo "Software directory ${software_dir} does not exist?!" >&2
     exit 2
@@ -62,18 +74,18 @@ fi
 # consider both CPU-only and accelerator subdirectories
 for subdir in ${cpu_arch_subdir} ${cpu_arch_subdir}/accel/${accel_subdir}; do
 
-    if [ -d ${eessi_version}/software/${os}/${subdir}/modules ]; then
+    if [ -d ${install_prefix_dir}/software/${os}/${subdir}/modules ]; then
         # module files
-        find ${eessi_version}/software/${os}/${subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} || true  # Make sure we don't exit because of set -e if grep doesn't return a match
+        find ${install_prefix_dir}/software/${os}/${subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} || true  # Make sure we don't exit because of set -e if grep doesn't return a match
         # module symlinks
-        find ${eessi_version}/software/${os}/${subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} || true  # Make sure we don't exit because of set -e if grep doesn't return a match
+        find ${install_prefix_dir}/software/${os}/${subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} || true  # Make sure we don't exit because of set -e if grep doesn't return a match
         # module files and symlinks
-        find ${eessi_version}/software/${os}/${subdir}/modules/all -type f -o -type l \
+        find ${install_prefix_dir}/software/${os}/${subdir}/modules/all -type f -o -type l \
             | grep -v '/\.wh\.' | grep -v '/\.modulerc\.lua' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \
             >> ${module_files_list}
     fi
 
-    if [ -d ${eessi_version}/software/${os}/${subdir}/software -a -r ${module_files_list} ]; then
+    if [ -d ${install_prefix_dir}/software/${os}/${subdir}/software -a -r ${module_files_list} ]; then
         # installation directories but only those for which module files were created
         # Note, we assume that module names (as defined by 'PACKAGE_NAME/VERSION.lua'
         # using EasyBuild's standard module naming scheme) match the name of the
@@ -82,7 +94,7 @@ for subdir in ${cpu_arch_subdir} ${cpu_arch_subdir}/accel/${accel_subdir}; do
         # installation directories), the procedure will likely not work.
         for package_version in $(cat ${module_files_list}); do
             echo "handling ${package_version}"
-            ls -d ${eessi_version}/software/${os}/${subdir}/software/${package_version} \
+            ls -d ${install_prefix_dir}/software/${os}/${subdir}/software/${package_version} \
                 | grep -v '/\.wh\.' >> ${files_list} || true  # Make sure we don't exit because of set -e if grep doesn't return a match
         done
     fi

init/minimal_eessi_env

@@ -8,7 +8,17 @@ EESSI_INIT_DIR_PATH=$(dirname $(readlink -f $BASH_SOURCE))
 #   script takes *_OVERRIDEs into account
 source ${EESSI_INIT_DIR_PATH}/eessi_defaults
 
-export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION
+# If $EESSI_DEV_PROJECT is defined, then we are building for dev.eessi.io
+# and add a subdirectory to $EESSI_PREFIX
+if [ -z ${EESSI_DEV_PROJECT} ]; then
+    export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION/
+else
+    echo "\$EESSI_DEV_PROJECT: ${EESSI_DEV_PROJECT}; building for dev.eessi.io"
+    export EESSI_PREFIX=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/
+    # Set EESSI_PREFIX to the dev project path for EESSI-extend
+    export EESSI_CVMFS_INSTALL=$EESSI_PREFIX
+fi
+
 
 if [[ $(uname -s) == 'Linux' ]]; then
     export EESSI_OS_TYPE='linux'

run_in_compat_layer_env.sh

@@ -35,6 +35,9 @@ fi
 if [ ! -z ${EESSI_CVMFS_REPO_OVERRIDE} ]; then
     INPUT="export EESSI_CVMFS_REPO_OVERRIDE=${EESSI_CVMFS_REPO_OVERRIDE}; ${INPUT}"
 fi
+if [ ! -z ${EESSI_DEV_PROJECT} ]; then
+    INPUT="export EESSI_DEV_PROJECT=${EESSI_DEV_PROJECT}; ${INPUT}"
+fi
 if [ ! -z ${EESSI_VERSION_OVERRIDE} ]; then
     INPUT="export EESSI_VERSION_OVERRIDE=${EESSI_VERSION_OVERRIDE}; ${INPUT}"
 fi