run-ss3-with-est #355
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Build SS3 and run test models with estimation and hessian | |
| # Only runs manual run or if run-ss3-no-est.yml runs successfully | |
| name: run-ss3-with-est | |
| on: | |
| workflow_dispatch: | |
| # push: | |
| # paths: | |
| # - '**.tpl' | |
| # branches: | |
| # - main | |
| workflow_run: | |
| workflows: ["run-ss3-no-est"] | |
| types: | |
| - completed | |
| # pull_request: | |
| # types: ['opened', 'edited', 'reopened', 'synchronize', 'ready_for_review'] | |
| # paths: | |
| # - '**.tpl' | |
| # branches: | |
| # - main | |
| # Run fast running SS3 test models with estimation | |
| jobs: | |
| run-ss3-with-est: | |
| if: ${{ github.event_name == 'workflow_dispatch' || (github.event.pull_request.draft == 'false' && github.event.workflow_run.conclusion == 'success') }} | |
| runs-on: ubuntu-latest | |
| env: | |
| R_REMOTES_NO_ERRORS_FROM_WARNINGS: true | |
| RSPM: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest" | |
| steps: | |
| - name: Checkout ss3 repo | |
| uses: actions/checkout@v4 | |
| - name: Checkout models repo | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: 'nmfs-ost/ss3-test-models' | |
| path: test-models-repo | |
| - name: Update Ubuntu packages | |
| run: | | |
| sudo add-apt-repository ppa:ubuntu-toolchain-r/test | |
| sudo apt-get update | |
| sudo apt-get install --only-upgrade libstdc++6 | |
| - name: Setup R | |
| uses: r-lib/actions/setup-r@v2 | |
| # - name: Get admb and put in path (not using docker), linux | |
| # run: | | |
| # wget https://github.com/admb-project/admb/releases/download/admb-13.2/admb-13.2-linux.zip | |
| # sudo unzip admb-13.2-linux.zip -d /usr/local/bin | |
| # sudo chmod 755 /usr/local/bin/admb-13.2/bin/admb | |
| # echo "/usr/local/bin/admb-13.2/bin" >> $GITHUB_PATH | |
| # - name: Build stock synthesis | |
| # run: | | |
| # rm -rf SS330 | |
| # mkdir SS330 | |
| # /bin/bash ./Make_SS_330_new.sh -b SS330 | |
| - name: Build stock synthesis with admb docker image | |
| run: | | |
| rm -rf SS330 | |
| rm -rf ss3_osx.tar | |
| mkdir SS330 | |
| chmod 777 SS330 | |
| /bin/bash ./Make_SS_330_new.sh --admb docker -b SS330 | |
| - name: Move exes and R scripts to needed locations | |
| run: | | |
| mv test-models-repo/models test-models-repo/model_runs | |
| mv SS330/ss3 test-models-repo/model_runs/ss3 | |
| - name: Change permissions on ss3 exes | |
| run: sudo chmod a+x test-models-repo/model_runs/ss3 | |
| - name: Download R packages to run test models in parallel parallel | |
| run: Rscript -e 'install.packages(c("parallely", "furrr", "future"))' | |
| - name: Run test models | |
| run: | | |
| # Setup parallel | |
| ncores <- parallelly::availableCores(omit = 1) | |
| future::plan(future::multisession, workers = ncores) | |
| # Get model names and file paths | |
| mod_names <- list.dirs(file.path("test-models-repo", "model_runs"), full.names = FALSE, recursive = FALSE) | |
| mod_paths <- list.dirs(file.path("test-models-repo", "model_runs"), full.names = TRUE, recursive = FALSE) | |
| print(mod_names) | |
| # Create function to run models with estimation | |
| run_ss <- function(dir) { | |
| wd <- getwd() | |
| print(wd) | |
| on.exit(system(paste0("cd ", wd))) | |
| # Rename the reference files | |
| file.rename(file.path(dir, "ss_summary.sso"), | |
| file.path(dir, "ss_summary_ref.sso")) | |
| file.rename(file.path(dir, "warning.sso"), | |
| file.path(dir, "warning_ref.sso")) | |
| file.copy(file.path(dir, "ss3.par"), file.path(dir, "ss3_ref.par")) | |
| # Run the models with estimation and see if model finishes without error | |
| message("running ss3 on ", basename(dir)) | |
| system(paste0("cd ", dir, " && ../ss3 -nox")) | |
| model_ran <- file.exists(file.path(dir, "control.ss_new")) | |
| return(model_ran) | |
| } | |
| # Run test models in parallel and print out error messages | |
| mod_ran <- furrr::future_map(mod_paths, function(x){tryCatch(run_ss(x), | |
| error = function(e) print(e))}) | |
| mod_errors <- mod_names[unlist(lapply(mod_ran, function(x) "simpleError" %in% class(x)))] | |
| success <- TRUE | |
| # Print list of models with errors if there are any | |
| if(length(mod_errors) > 0) { | |
| message("Model code with errors were: ", paste0(mod_errors, collapse = ", "), | |
| ". See error list above for more details.") | |
| success <- FALSE | |
| } else { | |
| message("All code ran without error, but model runs may still have failed.") | |
| } | |
| # Print list of models that didn't run if there are any | |
| mod_no_run <- mod_names[unlist(lapply(mod_ran, function(x) isFALSE(x)))] # false means model didn't run | |
| if(length(mod_no_run) > 0) { | |
| message("Models that didn't run are ", paste0(mod_no_run, collapse = ", ")) | |
| success <- FALSE | |
| } else { | |
| message("All models ran without error.") | |
| } | |
| # Determine if job fails or passes | |
| if(success == FALSE) { | |
| stop("Job failed due to code with errors or models that didn't run.") | |
| } else { | |
| message("All models successfully ran.") | |
| } | |
| shell: Rscript {0} | |
| - name: Run comparison using compare.R file in test models repo | |
| run: | | |
| source("test-models-repo/.github/r_scripts/compare.R") | |
| orig_wd <- getwd() | |
| setwd("test-models-repo") | |
| on.exit(orig_wd) | |
| dir.create("run_R") | |
| # Get model folder names | |
| mod_fold <- file.path("model_runs") | |
| mod_names <- list.dirs(mod_fold, full.names = FALSE, recursive = FALSE) | |
| message("Will compare ref runs to new results for these models:") | |
| print(mod_names) | |
| message("Notable changes in total likelihood, max gradients, ", | |
| " and number of warnings:") | |
| compare_list <- vector(mode = "list", length = length(mod_names)) | |
| for(i in mod_names) { | |
| pos <- which(mod_names == i) | |
| sum_file <- file.path(mod_fold, i, "ss_summary.sso") | |
| if (i == "Simple") { | |
| file.copy(sum_file, file.path("run_R", paste0(i, "_ss_summary.sso"))) | |
| } | |
| ref_sum_file <- file.path(mod_fold, i, "ss_summary_ref.sso") | |
| par_file <- file.path(mod_fold, i, "ss3.par") | |
| ref_par_file <- file.path(mod_fold, i, "ss3_ref.par") | |
| warn_file <- file.path(mod_fold, i, "warning.sso") | |
| ref_warn_file <- file.path(mod_fold, i, "warning_ref.sso") | |
| fail_file <- file.path("run_R", "test_failed.csv") | |
| compare_list[[pos]] <- compare_ss_runs(mod_name = i, | |
| sum_file = sum_file, ref_sum_file = ref_sum_file, | |
| par_file = par_file, ref_par_file = ref_par_file, | |
| warn_file = warn_file, ref_warn_file = ref_warn_file, | |
| hessian = TRUE, | |
| new_file = NULL, fail_file = fail_file) | |
| } | |
| # Write out all model results | |
| compare_df <- do.call("rbind", compare_list) | |
| compare_df_print <- format(compare_df, digits = 6, nsmall = 3, | |
| justify = "left") | |
| message("See saved artifact all_results.csv for all compared values and their differences.") | |
| # Write all model comparison results to csv | |
| write.csv(compare_df_print, "run_R/all_results.csv", row.names = FALSE) | |
| # Write all model changes to csv | |
| message("See saved artifact all_changes.csv for only changed values (even if the threshold was too low to fail the job).") | |
| filtered_df <- compare_df[compare_df$diff != 0, ] | |
| filtered_df <- format(filtered_df, digits = 6, nsmall = 3, | |
| justify = "left") | |
| write.csv(filtered_df, "run_R/all_changes.csv", row.names = FALSE) | |
| shell: Rscript {0} | |
| - name: Determine results of test | |
| run: cd test-models-repo && Rscript .github/r_scripts/check_failed.R | |
| - name: Archive results | |
| uses: actions/upload-artifact@main | |
| if: always() | |
| with: | |
| name: 'result_textfiles' | |
| path: test-models-repo/run_R/ |