Another pull request from johnoel/stock-synthesis #99
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Build SS3 and run models with estimation and hessian | |
| name: run-ss3-with-est | |
| on: | |
| workflow_dispatch: | |
| push: | |
| paths: | |
| - '**.tpl' | |
| branches: | |
| - main | |
| pull_request: | |
| types: ['opened', 'edited', 'reopened', 'synchronize', 'ready_for_review'] | |
| paths: | |
| - '**.tpl' | |
| branches: | |
| - main | |
| # run fast running SS models with estimation | |
| jobs: | |
| run-ss3-with-est: | |
| if: github.event.pull_request.draft == false | |
| runs-on: ubuntu-latest | |
| env: | |
| R_REMOTES_NO_ERRORS_FROM_WARNINGS: true | |
| RSPM: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest" | |
| steps: | |
| - name: Checkout ss repo | |
| uses: actions/checkout@v3 | |
| - name: Checkout models repo | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: 'nmfs-stock-synthesis/test-models' | |
| path: test-models-repo | |
| - name: setup R | |
| uses: r-lib/actions/setup-r@v2 | |
| - name: Get admb and put in path, linux | |
| run: | | |
| wget https://github.com/admb-project/admb/releases/download/admb-13.1/admb-13.1-linux.zip | |
| sudo unzip admb-13.1-linux.zip -d /usr/local/bin | |
| sudo chmod 755 /usr/local/bin/admb-13.1/bin/admb | |
| echo "/usr/local/bin/admb-13.1/bin" >> $GITHUB_PATH | |
| - name: Build stock synthesis | |
| run: | | |
| rm -rf SS330 | |
| mkdir SS330 | |
| /bin/bash ./Make_SS_330_new.sh -b SS330 | |
| - name: move exes, scripts to needed locations | |
| run: | | |
| mv test-models-repo/models test-models-repo/model_runs | |
| mv SS330/ss test-models-repo/model_runs/ss | |
| - name: change permissions on ss exes | |
| run: chmod a+x test-models-repo/model_runs/ss | |
| - name: run models | |
| run: | | |
| mod_names <- list.dirs(file.path("test-models-repo", "model_runs"), full.names = FALSE, recursive = FALSE) | |
| mod_paths <- list.dirs(file.path("test-models-repo", "model_runs"), full.names = TRUE, recursive = FALSE) | |
| print(mod_names) | |
| run_ss <- function(dir) { | |
| wd <- getwd() | |
| print(wd) | |
| on.exit(system(paste0("cd ", wd))) | |
| # rename the reference files | |
| file.rename(file.path(dir, "ss_summary.sso"), | |
| file.path(dir, "ss_summary_ref.sso")) | |
| file.rename(file.path(dir, "warning.sso"), | |
| file.path(dir, "warning_ref.sso")) | |
| file.copy(file.path(dir, "ss.par"), file.path(dir, "ss_ref.par")) | |
| # run the models with estimation and see if model finishes without error | |
| message("running ss on ", basename(dir)) | |
| system(paste0("cd ", dir, " && ../ss -nox")) | |
| model_ran <- file.exists(file.path(dir, "control.ss_new")) | |
| return(model_ran) | |
| } | |
| mod_ran <- lapply(mod_paths, function(x) tryCatch(run_ss(x), | |
| error = function(e) print(e) | |
| )) | |
| mod_errors <- mod_names[unlist(lapply(mod_ran, function(x) "simpleError" %in% class(x)))] | |
| success <- TRUE | |
| if(length(mod_errors) > 0) { | |
| message("Model code with errors were: ", paste0(mod_errors, collapse = ", "), | |
| ". See error list above for more details.") | |
| success <- FALSE | |
| } else { | |
| message("All code ran without error, but model runs may still have failed.") | |
| } | |
| mod_no_run <- mod_names[unlist(lapply(mod_ran, function(x) isFALSE(x)))] # false means model didn't run | |
| if(length(mod_no_run) > 0) { | |
| message("Models that didn't run are ", paste0(mod_no_run, collapse = ", ")) | |
| success <- FALSE | |
| } else { | |
| message("All models ran without error.") | |
| } | |
| # determine if job fails or passes | |
| if(success == FALSE) { | |
| stop("Job failed due to code with errors or models that didn't run.") | |
| } else { | |
| message("All models successfully ran.") | |
| } | |
| shell: Rscript {0} | |
| - name: Run comparison | |
| run: | | |
| source("test-models-repo/.github/r_scripts/compare.R") | |
| orig_wd <- getwd() | |
| setwd("test-models-repo") | |
| on.exit(orig_wd) | |
| dir.create("run_R") | |
| # get model folder names | |
| mod_fold <- file.path("model_runs") | |
| mod_names <- list.dirs(mod_fold, full.names = FALSE, recursive = FALSE) | |
| message("Will compare ref runs to new results for these models:") | |
| print(mod_names) | |
| message("Notable changes in total likelihood, max gradients, ", | |
| " and number of warnings:") | |
| compare_list <- vector(mode = "list", length = length(mod_names)) | |
| for(i in mod_names) { | |
| pos <- which(mod_names == i) | |
| sum_file <- file.path(mod_fold, i, "ss_summary.sso") | |
| if (i == "Simple") { | |
| file.copy(sum_file, file.path("run_R", paste0(i, "_ss_summary.sso"))) | |
| } | |
| ref_sum_file <- file.path(mod_fold, i, "ss_summary_ref.sso") | |
| par_file <- file.path(mod_fold, i, "ss.par") | |
| ref_par_file <- file.path(mod_fold, i, "ss_ref.par") | |
| warn_file <- file.path(mod_fold, i, "warning.sso") | |
| ref_warn_file <- file.path(mod_fold, i, "warning_ref.sso") | |
| fail_file <- file.path("run_R", "test_failed.csv") | |
| compare_list[[pos]] <- compare_ss_runs(mod_name = i, | |
| sum_file = sum_file, ref_sum_file = ref_sum_file, | |
| par_file = par_file, ref_par_file = ref_par_file, | |
| warn_file = warn_file, ref_warn_file = ref_warn_file, | |
| hessian = TRUE, | |
| new_file = NULL, fail_file = fail_file) | |
| } | |
| # write out all model results | |
| compare_df <- do.call("rbind", compare_list) | |
| compare_df_print <- format(compare_df, digits = 6, nsmall = 3, | |
| justify = "left") | |
| message("see saved artifact all_results.csv for all compared values and their differences.") | |
| # write all model comparison results to csv | |
| write.csv(compare_df_print, "run_R/all_results.csv", row.names = FALSE) | |
| # write all | |
| message("see saved artifact all_changes.csv for only changed values (even if the threshold was too low to fail the job)") | |
| filtered_df <- compare_df[compare_df$diff != 0, ] | |
| filtered_df <- format(filtered_df, digits = 6, nsmall = 3, | |
| justify = "left") | |
| write.csv(filtered_df, "run_R/all_changes.csv", row.names = FALSE) | |
| shell: Rscript {0} | |
| - name: Determine results of test | |
| run: cd test-models-repo && Rscript .github/r_scripts/check_failed.R | |
| - name: Archive results | |
| uses: actions/upload-artifact@main | |
| if: always() | |
| with: | |
| name: 'result_textfiles' | |
| path: test-models-repo/run_R/ |