Skip to content

Commit

Permalink
Combine CIs
Browse files Browse the repository at this point in the history
  • Loading branch information
@danielabdi-noaa
danielabdi-noaa committed Sep 25, 2022
1 parent 582a549 commit 10cc04b
Show file tree
Hide file tree
Showing 8 changed files with 159 additions and 232 deletions.
4 changes: 2 additions & 2 deletions .cicd/Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ pipeline {

post {
success {
sh 'tar --create --gzip --verbose --file "${WORKSPACE}/${BUILD_NAME}.tgz" bin include lib share'
sh 'tar --create --gzip --verbose --file "${WORKSPACE}/${BUILD_NAME}.tgz" install_${env.SRW_COMPILER}'
s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'woc-epic-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.BUILD_NAME}.tgz", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'woc-epic-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "build/srw_build-${env.SRW_PLATFORM}-${env.SRW_COMPILER}.log", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
}
}
Expand All @@ -140,7 +140,7 @@ pipeline {
}

environment {
SRW_WE2E_EXPERIMENT_BASE_DIR = "${env.WORKSPACE}/experiments"
SRW_WE2E_EXPERIMENT_BASE_DIR = "${env.WORKSPACE}/experiments_${env.SRW_COMPILER}"
}

steps {
Expand Down
20 changes: 8 additions & 12 deletions .cicd/scripts/srw_build.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,17 +24,13 @@ else
platform="${SRW_PLATFORM}"
fi

build_dir="${workspace}/build"
# Build and install
cd ${workspace}/test
./build.sh ${platform} ${SRW_COMPILER}
cd -

# Set build related environment variables and load required modules.
source "${workspace}/etc/lmod-setup.sh" "${platform}"
module use "${workspace}/modulefiles"
module load "build_${platform}_${SRW_COMPILER}"
# Create combined log file for upload to s3
build_dir="${workspace}/build_${SRW_COMPILER}"
cat ${build_dir}/log.cmake ${build_dir}/log.make \
>${build_dir}/srw_build-${platform}-${SRW_COMPILER}.log

# Compile SRW application and install to repository root.
mkdir "${build_dir}"
pushd "${build_dir}"
build_log_file="${build_dir}/srw_build-${platform}-${SRW_COMPILER}.log"
cmake -DCMAKE_INSTALL_PREFIX="${workspace}" "${workspace}" | tee "${build_log_file}"
make -j "${MAKE_JOBS}" | tee --append "${build_log_file}"
popd
154 changes: 11 additions & 143 deletions .cicd/scripts/srw_test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,115 +25,21 @@ else
platform="${SRW_PLATFORM}"
fi

declare we2e_experiment_base_dir
if [[ -n "${SRW_WE2E_EXPERIMENT_BASE_DIR}" ]]; then
we2e_experiment_base_dir="${SRW_WE2E_EXPERIMENT_BASE_DIR}"
else
we2e_experiment_base_dir="${workspace}/experiments"
fi

# Test directories
we2e_experiment_base_dir="${workspace}/../expt_dirs"
we2e_test_dir="${workspace}/tests/WE2E"

we2e_test_file="${we2e_test_dir}/experiments.txt"

# The fundamental set of end-to-end tests to run.
declare -a we2e_fundamental_tests
we2e_fundamental_tests=('grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
'grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta'
'grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR'
'grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR'
'grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta'
'grid_RRFS_CONUScompact_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
'grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR'
'grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR'
'grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta'
'grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
'grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_HRRR'
'grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta'
'nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR'
'community_ensemble_2mems'
'custom_ESGgrid'
'deactivate_tasks'
'inline_post'
'nco_ensemble'
'specify_DOT_OR_USCORE'
'specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE'
'specify_RESTART_INTERVAL'
'specify_template_filenames')

if [[ "${platform}" != 'gaea' && "${platform}" != 'noaacloud' ]]; then
we2e_fundamental_tests+=('MET_ensemble_verification'
'MET_verification'
'pregen_grid_orog_sfc_climo')
fi

# The comprehensive set of end-to-end tests to run.
declare -a we2e_comprehensive_tests
we2e_comprehensive_tests=('community_ensemble_008mems'
'custom_GFDLgrid'
'custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_FALSE'
'custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_TRUE'
'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200'
'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019101818'
'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022518'
'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022600'
'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2021010100'
'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio'
'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019061200'
'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019101818'
'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022518'
'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022600'
'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2021010100'
'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2021062000'
'get_from_HPSS_ics_GSMGFS_lbcs_GSMGFS'
'get_from_HPSS_ics_HRRR_lbcs_RAP'
'get_from_HPSS_ics_RAP_lbcs_RAP'
'get_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio'
'grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
'grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
'grid_RRFS_AK_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
'grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
'grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2'
'grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
'grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR'
'grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta'
'grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp'
'grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional'
'grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2'
'grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR'
'grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_2017_gfdlmp'
'grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2'
'grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v16'
'grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km'
'grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2'
'grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
'grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR'
'grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta'
'grid_RRFS_CONUScompact_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
'grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR'
'grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta'
'grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta'
'grid_RRFS_CONUScompact_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
'grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2'
'grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR'
'grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta'
'grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta'
'grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta'
'grid_RRFS_SUBCONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
'grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2'
'nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
'nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km')

declare -a we2e_tests
we2e_tests=("${we2e_fundamental_tests[@]}")
# Run the end-to-end tests.
if "${SRW_WE2E_COMPREHENSIVE_TESTS}"; then
we2e_tests+=("${we2e_comprehensive_tests[@]}")

# Add additional tests for Hera.
if [[ "${platform}" == 'hera' ]]; then
we2e_tests+=('specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS')
fi
test_type="comprehensive"
else
test_type="fundamental"
fi
${we2e_test_dir}/setup_WE2E_tests.sh ${platform} ${SRW_PROJECT} ${SRW_COMPILER} ${test_type}

# Allow the tests to start before checking for status.
# TODO: Create a parameter that sets the initial start delay.
sleep 180

# Parses the test log for the status of a specific test.
function workflow_status() {
Expand Down Expand Up @@ -188,44 +94,6 @@ function get_results() {
done
}

# Verify that there is a non-zero sized weather model executable.
[[ -s "${workspace}/bin/ufs_model" ]] || [[ -s "${workspace}/bin/NEMS.exe" ]]

# Set test related environment variables and load required modules.
source "${workspace}/etc/lmod-setup.sh" "${platform}"
module use "${workspace}/modulefiles"
module load "build_${platform}_${SRW_COMPILER}"
module load "wflow_${platform}"

if [[ "${platform}" == 'cheyenne' ]]; then
export PATH="/glade/p/ral/jntp/UFS_CAM/ncar_pylib_20200427/bin:${PATH}"
else
if [[ "${platform}" == 'noaacloud' && -z "${PROJ_LIB-}" ]]; then
PROJ_LIB=''
fi

conda activate regional_workflow
fi

# Create the experiments/tests base directory.
mkdir "${we2e_experiment_base_dir}"

# Generate the experiments/tests file.
for test in "${we2e_tests[@]}"; do
echo "${test}" >> "${we2e_test_file}"
done

# Run the end-to-end tests.
"${we2e_test_dir}/run_WE2E_tests.sh" \
tests_file="${we2e_test_file}" \
machine="${platform}" \
account="${SRW_PROJECT}" \
expt_basedir="${we2e_experiment_base_dir}" \
compiler="${SRW_COMPILER}"

# Allow the tests to start before checking for status.
# TODO: Create a parameter that sets the initial start delay.
sleep 180

# Wait for all tests to complete.
while check_progress; do
Expand Down
129 changes: 60 additions & 69 deletions test/build.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
#
# Usage: see function usage below
#
# Examples: ./build.sh hera >& test.out &
# Examples: ./build.sh hera intel all>& test.out &
#
set -eux # Uncomment for debugging
#=======================================================================
Expand All @@ -16,15 +16,14 @@ fail() { echo -e "\n$1\n" >> ${TEST_OUTPUT} && exit 1; }

function usage() {
echo
echo "Usage: $0 machine"
echo "Usage: $0 machine [compiler] [all/default]"
echo
exit 1
}

machines=( hera jet cheyenne orion wcoss2 gaea odin singularity macos noaacloud )

[[ $# -eq 2 ]] && usage

[[ $# -gt 4 ]] && usage

#-----------------------------------------------------------------------
# Set some directories
Expand All @@ -41,6 +40,10 @@ printf "PLATFORM(MACHINE)=${PLATFORM}\n" >&2

machine=$(echo "${MACHINE}" | tr '[A-Z]' '[a-z]') # scripts in sorc need lower case machine name

compiler=${2:-"intel"}

components=${3:-"default"}

#-----------------------------------------------------------------------
# Check that machine is valid
#-----------------------------------------------------------------------
Expand All @@ -51,18 +54,6 @@ else
exit 1
fi

#-----------------------------------------------------------------------
# Set compilers to be tested depending on machine
#-----------------------------------------------------------------------
if [ "${machine}" == "cheyenne" ] ; then
compilers=( intel gnu )
elif [ "${machine}" == "macos" ] || [ "${machine}" == "singularity" ] ; then
compilers=( gnu )
else
compilers=( intel )
fi


build_it=0 # Set to 1 to skip build (for testing pass/fail criteria)
#-----------------------------------------------------------------------
# Create the output file if it doesn't exist
Expand Down Expand Up @@ -99,63 +90,63 @@ declare -a executables_created=( chgres_cube \
vcoord_gen )


#-----------------------------------------------------------------------
# Array of all optional GSI executables built
#-----------------------------------------------------------------------
declare -a executables_gsi_created=( enkf.x \
gsi.x \
nc_diag_cat.x \
ncdiag_cat_serial.x \
test_nc_unlimdims.x )

#-----------------------------------------------------------------------
# Array of all optional rrfs_utl executables built
#-----------------------------------------------------------------------
declare -a executables_rrfs_utl_created=( adjust_soiltq.exe \
check_imssnow_fv3lam.exe \
fv3lam_nonvarcldana.exe \
gen_annual_maxmin_GVF.exe \
gen_cs.exe \
gen_ensmean_recenter.exe \
lakesurgery.exe \
process_imssnow_fv3lam.exe \
process_larccld.exe \
process_Lightning.exe \
process_metarcld.exe \
process_NSSL_mosaic.exe \
process_updatesst.exe \
ref2tten.exe \
update_bc.exe \
update_GVF.exe \
update_ice.exe \
use_raphrrr_sfc.exe )
if [ $components = "all" ]; then
#-----------------------------------------------------------------------
# Array of all optional GSI executables built
#-----------------------------------------------------------------------
executables_created+=( enkf.x \
gsi.x \
nc_diag_cat.x \
ncdiag_cat_serial.x \
test_nc_unlimdims.x )

#-----------------------------------------------------------------------
# Array of all optional rrfs_utl executables built
#-----------------------------------------------------------------------
executables_created=( adjust_soiltq.exe \
check_imssnow_fv3lam.exe \
fv3lam_nonvarcldana.exe \
gen_annual_maxmin_GVF.exe \
gen_cs.exe \
gen_ensmean_recenter.exe \
lakesurgery.exe \
process_imssnow_fv3lam.exe \
process_larccld.exe \
process_Lightning.exe \
process_metarcld.exe \
process_NSSL_mosaic.exe \
process_updatesst.exe \
ref2tten.exe \
update_bc.exe \
update_GVF.exe \
update_ice.exe \
use_raphrrr_sfc.exe )
fi

#-----------------------------------------------------------------------
# Set up the build environment and run the build script.
#-----------------------------------------------------------------------
for compiler in "${compilers[@]}"; do
BUILD_DIR=${TOP_DIR}/build_${compiler}
INSTALL_DIR=${TOP_DIR}/install_${compiler}
EXEC_DIR=${INSTALL_DIR}/exec
if [ $build_it -eq 0 ] ; then
./devbuild.sh --platform=${machine} --compiler=${compiler} --build-dir=${BUILD_DIR} --install-dir=${INSTALL_DIR} \
--remove all || fail "Build ${machine} ${compiler} FAILED"
fi # End of skip build for testing

#-----------------------------------------------------------------------
# check for existence of executables.
#-----------------------------------------------------------------------
n_fail=0
for file in "${executables_created[@]}" "${executables_gsi_created[@]}" "${executables_rrfs_utl_created[@]}" ; do
exec_file=${EXEC_DIR}/${file}
if [ -f ${exec_file} ]; then
echo "SUCCEED: ${compiler} executable file ${exec_file} exists" >> ${TEST_OUTPUT}
else
echo "FAIL: ${compiler} executable file ${exec_file} does NOT exist" >> ${TEST_OUTPUT}
let "n_fail=n_fail+1"
fi
done
done # End compiler loop
BUILD_DIR=${TOP_DIR}/build_${compiler}
INSTALL_DIR=${TOP_DIR}/install_${compiler}
EXEC_DIR=${INSTALL_DIR}/exec
if [ $build_it -eq 0 ] ; then
./devbuild.sh --platform=${machine} --compiler=${compiler} --build-dir=${BUILD_DIR} --install-dir=${INSTALL_DIR} \
--remove ${components} || fail "Build ${machine} ${compiler} FAILED"
fi # End of skip build for testing

#-----------------------------------------------------------------------
# check for existence of executables.
#-----------------------------------------------------------------------
n_fail=0
for file in "${executables_created[@]}" ; do
exec_file=${EXEC_DIR}/${file}
if [ -f ${exec_file} ]; then
echo "SUCCEED: ${compiler} executable file ${exec_file} exists" >> ${TEST_OUTPUT}
else
echo "FAIL: ${compiler} executable file ${exec_file} does NOT exist" >> ${TEST_OUTPUT}
let "n_fail=n_fail+1"
fi
done
#-----------------------------------------------------------------------
# Set message for output
#-----------------------------------------------------------------------
Expand Down
Loading

0 comments on commit 10cc04b

Please sign in to comment.