Skip to content

run-ss3-with-est

run-ss3-with-est #376

# Build SS3 and run test models with estimation and hessian
# Only runs manual run or if run-ss3-no-est.yml runs successfully
name: run-ss3-with-est
on:
workflow_dispatch:
# push:
# paths:
# - '**.tpl'
# branches:
# - main
workflow_run:
workflows: ["run-ss3-no-est"]
types:
- completed
# pull_request:
# types: ['opened', 'edited', 'reopened', 'synchronize', 'ready_for_review']
# paths:
# - '**.tpl'
# branches:
# - main
# Run fast running SS3 test models with estimation
jobs:
run-ss3-with-est:
if: ${{ github.event_name == 'workflow_dispatch' || (github.event.pull_request.draft == 'false' && github.event.workflow_run.conclusion == 'success') }}
runs-on: ubuntu-latest
env:
R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
RSPM: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest"
steps:
- name: Checkout ss3 repo
uses: actions/checkout@v4
- name: Checkout models repo
uses: actions/checkout@v4
with:
repository: 'nmfs-ost/ss3-test-models'
path: test-models-repo
- name: Update Ubuntu packages
run: |
sudo add-apt-repository ppa:ubuntu-toolchain-r/test
sudo apt-get update
sudo apt-get install --only-upgrade libstdc++6
- name: Setup R
uses: r-lib/actions/setup-r@v2
# - name: Get admb and put in path (not using docker), linux
# run: |
# wget https://github.com/admb-project/admb/releases/download/admb-13.2/admb-13.2-linux.zip
# sudo unzip admb-13.2-linux.zip -d /usr/local/bin
# sudo chmod 755 /usr/local/bin/admb-13.2/bin/admb
# echo "/usr/local/bin/admb-13.2/bin" >> $GITHUB_PATH
# - name: Build stock synthesis
# run: |
# rm -rf SS330
# mkdir SS330
# /bin/bash ./Make_SS_330_new.sh -b SS330
- name: Build stock synthesis with admb docker image
run: |
rm -rf SS330
rm -rf ss3_osx.tar
mkdir SS330
chmod 777 SS330
/bin/bash ./Make_SS_330_new.sh --admb docker -b SS330
- name: Move exes and R scripts to needed locations
run: |
mv test-models-repo/models test-models-repo/model_runs
mv SS330/ss3 test-models-repo/model_runs/ss3
- name: Change permissions on ss3 exes
run: sudo chmod a+x test-models-repo/model_runs/ss3
- name: Download R packages to run test models in parallel parallel
run: Rscript -e 'install.packages(c("parallely", "furrr", "future"))'
- name: Run test models
run: |
# Setup parallel
ncores <- parallelly::availableCores(omit = 1)
future::plan(future::multisession, workers = ncores)
# Get model names and file paths
mod_names <- list.dirs(file.path("test-models-repo", "model_runs"), full.names = FALSE, recursive = FALSE)
mod_paths <- list.dirs(file.path("test-models-repo", "model_runs"), full.names = TRUE, recursive = FALSE)
print(mod_names)
# Create function to run models with estimation
run_ss <- function(dir) {
wd <- getwd()
print(wd)
on.exit(system(paste0("cd ", wd)))
# Rename the reference files
file.rename(file.path(dir, "ss_summary.sso"),
file.path(dir, "ss_summary_ref.sso"))
file.rename(file.path(dir, "warning.sso"),
file.path(dir, "warning_ref.sso"))
file.copy(file.path(dir, "ss3.par"), file.path(dir, "ss3_ref.par"))
# Run the models with estimation and see if model finishes without error
message("running ss3 on ", basename(dir))
system(paste0("cd ", dir, " && ../ss3 -nox"))
model_ran <- file.exists(file.path(dir, "control.ss_new"))
return(model_ran)
}
# Run test models in parallel and print out error messages
mod_ran <- furrr::future_map(mod_paths, function(x){tryCatch(run_ss(x),
error = function(e) print(e))})
mod_errors <- mod_names[unlist(lapply(mod_ran, function(x) "simpleError" %in% class(x)))]
success <- TRUE
# Print list of models with errors if there are any
if(length(mod_errors) > 0) {
message("Model code with errors were: ", paste0(mod_errors, collapse = ", "),
". See error list above for more details.")
success <- FALSE
} else {
message("All code ran without error, but model runs may still have failed.")
}
# Print list of models that didn't run if there are any
mod_no_run <- mod_names[unlist(lapply(mod_ran, function(x) isFALSE(x)))] # false means model didn't run
if(length(mod_no_run) > 0) {
message("Models that didn't run are ", paste0(mod_no_run, collapse = ", "))
success <- FALSE
} else {
message("All models ran without error.")
}
# Determine if job fails or passes
if(success == FALSE) {
stop("Job failed due to code with errors or models that didn't run.")
} else {
message("All models successfully ran.")
}
shell: Rscript {0}
- name: Run comparison using compare.R file in test models repo
run: |
source("test-models-repo/.github/r_scripts/compare.R")
orig_wd <- getwd()
setwd("test-models-repo")
on.exit(orig_wd)
dir.create("run_R")
# Get model folder names
mod_fold <- file.path("model_runs")
mod_names <- list.dirs(mod_fold, full.names = FALSE, recursive = FALSE)
message("Will compare ref runs to new results for these models:")
print(mod_names)
message("Notable changes in total likelihood, max gradients, ",
" and number of warnings:")
compare_list <- vector(mode = "list", length = length(mod_names))
for(i in mod_names) {
pos <- which(mod_names == i)
sum_file <- file.path(mod_fold, i, "ss_summary.sso")
if (i == "Simple") {
file.copy(sum_file, file.path("run_R", paste0(i, "_ss_summary.sso")))
}
ref_sum_file <- file.path(mod_fold, i, "ss_summary_ref.sso")
par_file <- file.path(mod_fold, i, "ss3.par")
ref_par_file <- file.path(mod_fold, i, "ss3_ref.par")
warn_file <- file.path(mod_fold, i, "warning.sso")
ref_warn_file <- file.path(mod_fold, i, "warning_ref.sso")
fail_file <- file.path("run_R", "test_failed.csv")
compare_list[[pos]] <- compare_ss_runs(mod_name = i,
sum_file = sum_file, ref_sum_file = ref_sum_file,
par_file = par_file, ref_par_file = ref_par_file,
warn_file = warn_file, ref_warn_file = ref_warn_file,
hessian = TRUE,
new_file = NULL, fail_file = fail_file)
}
# Write out all model results
compare_df <- do.call("rbind", compare_list)
compare_df_print <- format(compare_df, digits = 6, nsmall = 3,
justify = "left")
message("See saved artifact all_results.csv for all compared values and their differences.")
# Write all model comparison results to csv
write.csv(compare_df_print, "run_R/all_results.csv", row.names = FALSE)
# Write all model changes to csv
message("See saved artifact all_changes.csv for only changed values (even if the threshold was too low to fail the job).")
filtered_df <- compare_df[compare_df$diff != 0, ]
filtered_df <- format(filtered_df, digits = 6, nsmall = 3,
justify = "left")
write.csv(filtered_df, "run_R/all_changes.csv", row.names = FALSE)
shell: Rscript {0}
- name: Determine results of test
run: cd test-models-repo && Rscript .github/r_scripts/check_failed.R
- name: Archive results
uses: actions/upload-artifact@main
if: always()
with:
name: 'result_textfiles'
path: test-models-repo/run_R/