From 8b716d16a97f17cc27cc5348905dd141f910e745 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Fri, 15 Jul 2022 06:05:50 +0300 Subject: [PATCH 01/28] Initial support downloading and building under wsl --- R/install.R | 92 ++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 77 insertions(+), 15 deletions(-) diff --git a/R/install.R b/R/install.R index 6cf6d1634..0d763612b 100644 --- a/R/install.R +++ b/R/install.R @@ -81,7 +81,8 @@ install_cmdstan <- function(dir = NULL, version = NULL, release_url = NULL, cpp_options = list(), - check_toolchain = TRUE) { + check_toolchain = TRUE, + wsl = FALSE) { if (check_toolchain) { check_cmdstan_toolchain(fix = FALSE, quiet = quiet) } @@ -111,6 +112,7 @@ install_cmdstan <- function(dir = NULL, release_url <- paste0("https://github.com/stan-dev/cmdstan/releases/download/v", version, "/cmdstan-", version, cmdstan_arch_suffix(version), ".tar.gz") } + wsl_prefix <- ifelse(isTRUE(wsl), "wsl-", "") if (!is.null(release_url)) { if (!endsWith(release_url, ".tar.gz")) { stop(release_url, " is not a .tar.gz archive!", @@ -122,14 +124,14 @@ install_cmdstan <- function(dir = NULL, tar_name <- utils::tail(split_url[[1]], n = 1) cmdstan_ver <- substr(tar_name, 0, nchar(tar_name) - 7) tar_gz_file <- paste0(cmdstan_ver, ".tar.gz") - dir_cmdstan <- file.path(dir, cmdstan_ver) + dir_cmdstan <- file.path(dir, paste0(wsl_prefix, cmdstan_ver)) dest_file <- file.path(dir, tar_gz_file) } else { ver <- latest_released_version() message("* Latest CmdStan release is v", ver) cmdstan_ver <- paste0("cmdstan-", ver, cmdstan_arch_suffix(ver)) tar_gz_file <- paste0(cmdstan_ver, ".tar.gz") - dir_cmdstan <- file.path(dir, cmdstan_ver) + dir_cmdstan <- file.path(dir, paste0(wsl_prefix, cmdstan_ver)) message("* Installing CmdStan v", ver, " in ", dir_cmdstan) message("* Downloading ", tar_gz_file, " from GitHub...") download_url <- github_download_url(ver) @@ -171,7 +173,7 @@ install_cmdstan <- function(dir = NULL, append = TRUE ) } - if (is_rtools42_toolchain()) { + if (is_rtools42_toolchain() && !isTRUE(wsl)) { cmdstan_make_local( dir = dir_cmdstan, cpp_options = list( @@ -183,12 +185,12 @@ install_cmdstan <- function(dir = NULL, } message("* Building CmdStan binaries...") - build_log <- build_cmdstan(dir_cmdstan, cores, quiet, timeout) + build_log <- build_cmdstan(dir_cmdstan, cores, quiet, timeout, wsl) if (!build_status_ok(build_log, quiet = quiet)) { return(invisible(build_log)) } - example_log <- build_example(dir_cmdstan, cores, quiet, timeout) + example_log <- build_example(dir_cmdstan, cores, quiet, timeout, wsl) if (!build_status_ok(example_log, quiet = quiet)) { return(invisible(example_log)) } @@ -263,9 +265,11 @@ cmdstan_make_local <- function(dir = cmdstan_path(), #' Windows. The default is `FALSE`, in which case problems are only reported #' along with suggested fixes. #' -check_cmdstan_toolchain <- function(fix = FALSE, quiet = FALSE) { +check_cmdstan_toolchain <- function(fix = FALSE, quiet = FALSE, wsl = FALSE) { if (os_is_windows()) { - if (R.version$major >= "4") { + if (isTRUE(wsl)) { + check_wsl_toolchain() + } else if (R.version$major >= "4") { check_rtools4x_windows_toolchain(fix = fix, quiet = quiet) } else { check_rtools35_windows_toolchain(fix = fix, quiet = quiet) @@ -366,14 +370,19 @@ download_with_retries <- function(download_url, build_cmdstan <- function(dir, cores = getOption("mc.cores", 2), quiet = FALSE, - timeout) { + timeout, + wsl = FALSE) { translation_args <- NULL if (is_rosetta2()) { run_cmd <- "/usr/bin/arch" translation_args <- c("-arch", "arm64e", "make") + } else if (isTRUE(wsl)) { + run_cmd <- "wsl" + translation_args <- "make" } else { run_cmd <- make_cmd() } + withr::with_path( c( toolchain_PATH_env_var(), @@ -395,15 +404,23 @@ build_cmdstan <- function(dir, clean_cmdstan <- function(dir = cmdstan_path(), cores = getOption("mc.cores", 2), - quiet = FALSE) { + quiet = FALSE, + wsl = FALSE) { + translation_args <- NULL + if (isTRUE(wsl)) { + run_cmd <- "wsl" + translation_args <- "make" + } else { + run_cmd <- make_cmd() + } withr::with_path( c( toolchain_PATH_env_var(), tbb_path(dir = dir) ), processx::run( - make_cmd(), - args = c("clean-all"), + run_cmd, + args = c(translation_args, "clean-all"), wd = dir, echo_cmd = is_verbose_mode(), echo = !quiet || is_verbose_mode(), @@ -414,15 +431,22 @@ clean_cmdstan <- function(dir = cmdstan_path(), ) } -build_example <- function(dir, cores, quiet, timeout) { +build_example <- function(dir, cores, quiet, timeout, wsl = FALSE) { + translation_args <- NULL + if (isTRUE(wsl)) { + run_cmd <- "wsl" + translation_args <- "make" + } else { + run_cmd <- make_cmd() + } withr::with_path( c( toolchain_PATH_env_var(), tbb_path(dir = dir) ), processx::run( - make_cmd(), - args = c(paste0("-j", cores), cmdstan_ext(file.path("examples", "bernoulli", "bernoulli"))), + run_cmd, + args = c(translation_args, paste0("-j", cores), cmdstan_ext(file.path("examples", "bernoulli", "bernoulli"))), wd = dir, echo_cmd = is_verbose_mode(), echo = !quiet || is_verbose_mode(), @@ -499,6 +523,44 @@ install_toolchain <- function(quiet = FALSE) { invisible(NULL) } +check_wsl_toolchain <- function() { + wsl_inaccessible <- processx::run(command = "wsl", + args = "uname", + error_on_status = FALSE) + if (wsl_inaccessible$status) { + stop("\n", "A WSL distribution is not installed or is not accessible.", + "\n", "Please see the Microsoft documentation for guidance on installing WSL: ", + "\n", "https://docs.microsoft.com/en-us/windows/wsl/install", + call. = FALSE) + } + + make_not_present <- processx::run(command = "wsl", + args = "which make", + windows_verbatim_args = TRUE, + error_on_status = FALSE) + + gpp_not_present <- processx::run(command = "wsl", + args = "which g++", + windows_verbatim_args = TRUE, + error_on_status = FALSE) + + clangpp_not_present <- processx::run(command = "wsl", + args = "which clang++", + windows_verbatim_args = TRUE, + error_on_status = FALSE) + + if (make_not_present$status || (gpp_not_present$status + && clangpp_not_present$status)) { + stop("\n", "Your distribution is missing the needed utilities for compiling C++.", + "\n", "Please launch your WSL install them using the appropriate command:", + "\n", "Debian/Ubuntu: sudo apt-get install build-essential", + "\n", "Fedora: sudo dnf group install \"C Development Tools and Libraries\"", + "\n", "Arch: pacman -Sy base-devel", + call. = FALSE) + } + +} + check_rtools4x_windows_toolchain <- function(fix = FALSE, quiet = FALSE) { rtools_path <- rtools4x_home_path() rtools_version <- if (is_rtools42_toolchain()) "Rtools42" else "Rtools40" From d631a4e1f2a31c6612c431ccf4fd45085c2194c0 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Fri, 15 Jul 2022 06:38:10 +0300 Subject: [PATCH 02/28] Fix path ending for install/build --- R/install.R | 4 +++- R/utils.R | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/R/install.R b/R/install.R index 0d763612b..2b29e1ec0 100644 --- a/R/install.R +++ b/R/install.R @@ -446,7 +446,9 @@ build_example <- function(dir, cores, quiet, timeout, wsl = FALSE) { ), processx::run( run_cmd, - args = c(translation_args, paste0("-j", cores), cmdstan_ext(file.path("examples", "bernoulli", "bernoulli"))), + args = c(translation_args, paste0("-j", cores), + cmdstan_ext(file.path("examples", "bernoulli", "bernoulli"), + wsl)), wd = dir, echo_cmd = is_verbose_mode(), echo = !quiet || is_verbose_mode(), diff --git a/R/utils.R b/R/utils.R index a400dd33c..b6ca4e98f 100644 --- a/R/utils.R +++ b/R/utils.R @@ -109,8 +109,8 @@ repair_path <- function(path) { #' @param path If not `NULL` then a path to add the extension to. #' @return If `path` is `NULL` then `".exe"` on Windows and `""` otherwise. If #' `path` is not `NULL` then `.exe` is added as the extension on Windows. -cmdstan_ext <- function(path = NULL) { - ext <- if (os_is_windows()) ".exe" else "" +cmdstan_ext <- function(path = NULL, wsl = FALSE) { + ext <- if (os_is_windows() && !isTRUE(wsl)) ".exe" else "" if (is.null(path)) { return(ext) } From a4aa7172ef0eb2b311cf73d3f0fe35097c42c4df Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Fri, 15 Jul 2022 10:54:07 +0300 Subject: [PATCH 03/28] Working download and build --- R/install.R | 54 +++++++++++++++++++++-------------------------------- R/model.R | 29 ++++++++++++++++++---------- R/path.R | 18 +++++++++++++++++- R/run.R | 16 +++++++++------- R/utils.R | 40 +++++++++++++++++++++++++++++++++++---- 5 files changed, 102 insertions(+), 55 deletions(-) diff --git a/R/install.R b/R/install.R index 2b29e1ec0..d3c1abb4b 100644 --- a/R/install.R +++ b/R/install.R @@ -83,6 +83,11 @@ install_cmdstan <- function(dir = NULL, cpp_options = list(), check_toolchain = TRUE, wsl = FALSE) { + # Use environment variable to record WSL usage throughout install, + # post-installation will simply check for 'wsl-' prefix in cmdstan path + if (isTRUE(wsl)) { + Sys.setenv("CMDSTANR_USE_WSL" = 1) + } if (check_toolchain) { check_cmdstan_toolchain(fix = FALSE, quiet = quiet) } @@ -185,12 +190,12 @@ install_cmdstan <- function(dir = NULL, } message("* Building CmdStan binaries...") - build_log <- build_cmdstan(dir_cmdstan, cores, quiet, timeout, wsl) + build_log <- build_cmdstan(dir_cmdstan, cores, quiet, timeout) if (!build_status_ok(build_log, quiet = quiet)) { return(invisible(build_log)) } - example_log <- build_example(dir_cmdstan, cores, quiet, timeout, wsl) + example_log <- build_example(dir_cmdstan, cores, quiet, timeout) if (!build_status_ok(example_log, quiet = quiet)) { return(invisible(example_log)) } @@ -265,9 +270,9 @@ cmdstan_make_local <- function(dir = cmdstan_path(), #' Windows. The default is `FALSE`, in which case problems are only reported #' along with suggested fixes. #' -check_cmdstan_toolchain <- function(fix = FALSE, quiet = FALSE, wsl = FALSE) { +check_cmdstan_toolchain <- function(fix = FALSE, quiet = FALSE) { if (os_is_windows()) { - if (isTRUE(wsl)) { + if (os_is_wsl()) { check_wsl_toolchain() } else if (R.version$major >= "4") { check_rtools4x_windows_toolchain(fix = fix, quiet = quiet) @@ -370,13 +375,12 @@ download_with_retries <- function(download_url, build_cmdstan <- function(dir, cores = getOption("mc.cores", 2), quiet = FALSE, - timeout, - wsl = FALSE) { + timeout) { translation_args <- NULL if (is_rosetta2()) { run_cmd <- "/usr/bin/arch" translation_args <- c("-arch", "arm64e", "make") - } else if (isTRUE(wsl)) { + } else if (os_is_wsl()) { run_cmd <- "wsl" translation_args <- "make" } else { @@ -404,22 +408,15 @@ build_cmdstan <- function(dir, clean_cmdstan <- function(dir = cmdstan_path(), cores = getOption("mc.cores", 2), - quiet = FALSE, - wsl = FALSE) { - translation_args <- NULL - if (isTRUE(wsl)) { - run_cmd <- "wsl" - translation_args <- "make" - } else { - run_cmd <- make_cmd() - } + quiet = FALSE) { + translation_args <- ifelse(os_is_wsl(), "make", NULL) withr::with_path( c( toolchain_PATH_env_var(), tbb_path(dir = dir) ), processx::run( - run_cmd, + make_cmd(), args = c(translation_args, "clean-all"), wd = dir, echo_cmd = is_verbose_mode(), @@ -431,24 +428,17 @@ clean_cmdstan <- function(dir = cmdstan_path(), ) } -build_example <- function(dir, cores, quiet, timeout, wsl = FALSE) { - translation_args <- NULL - if (isTRUE(wsl)) { - run_cmd <- "wsl" - translation_args <- "make" - } else { - run_cmd <- make_cmd() - } +build_example <- function(dir, cores, quiet, timeout) { + translation_args <- ifelse(os_is_wsl(), "make", NULL) withr::with_path( c( toolchain_PATH_env_var(), tbb_path(dir = dir) ), processx::run( - run_cmd, + make_cmd(), args = c(translation_args, paste0("-j", cores), - cmdstan_ext(file.path("examples", "bernoulli", "bernoulli"), - wsl)), + cmdstan_ext(file.path("examples", "bernoulli", "bernoulli"))), wd = dir, echo_cmd = is_verbose_mode(), echo = !quiet || is_verbose_mode(), @@ -537,17 +527,15 @@ check_wsl_toolchain <- function() { } make_not_present <- processx::run(command = "wsl", - args = "which make", - windows_verbatim_args = TRUE, + args = c("which", "make"), error_on_status = FALSE) gpp_not_present <- processx::run(command = "wsl", - args = "which g++", - windows_verbatim_args = TRUE, + args = c("which", "g++"), error_on_status = FALSE) clangpp_not_present <- processx::run(command = "wsl", - args = "which clang++", + args = c("which", "clang++"), windows_verbatim_args = TRUE, error_on_status = FALSE) diff --git a/R/model.R b/R/model.R index 18b1a9a72..7441bcd52 100644 --- a/R/model.R +++ b/R/model.R @@ -243,7 +243,7 @@ CmdStanModel <- R6::R6Class( } } if (!is.null(exe_file)) { - ext <- if (os_is_windows()) "exe" else "" + ext <- if (os_is_windows() && !os_is_wsl()) "exe" else "" private$exe_file_ <- repair_path(absolute_path(exe_file)) if (is.null(stan_file)) { checkmate::assert_file_exists(private$exe_file_, access = "r", extension = ext) @@ -508,7 +508,7 @@ compile <- function(quiet = TRUE, file.copy(self$stan_file(), temp_stan_file, overwrite = TRUE) temp_file_no_ext <- strip_ext(temp_stan_file) tmp_exe <- cmdstan_ext(temp_file_no_ext) # adds .exe on Windows - if (os_is_windows()) { + if (os_is_windows() && !os_is_wsl()) { tmp_exe <- utils::shortPathName(tmp_exe) } private$hpp_file_ <- paste0(temp_file_no_ext, ".hpp") @@ -554,7 +554,9 @@ compile <- function(quiet = TRUE, ), run_log <- processx::run( command = make_cmd(), - args = c(tmp_exe, + args = c( + ifelse(os_is_wsl(), "make", ""), + tmp_exe, cpp_options_to_compile_flags(cpp_options), stancflags_val), wd = cmdstan_path(), @@ -581,7 +583,7 @@ compile <- function(quiet = TRUE, ) } if (os_is_macos()) { - if (R.version$arch == "aarch64" + if (R.version$arch == "aarch64" && grepl("but the current translation unit is being compiled for target", x)) { warning( "The C++ compiler has errored due to incompatibility between the x86 and ", @@ -653,7 +655,10 @@ variables <- function() { } assert_stan_file_exists(self$stan_file()) if (is.null(private$variables_) && file.exists(self$stan_file())) { - private$variables_ <- model_variables(self$stan_file(), self$include_paths(), allow_undefined = private$using_user_header_) + private$variables_ <- model_variables( + self$stan_file(), + self$include_paths(), + allow_undefined = private$using_user_header_) } private$variables_ } @@ -761,7 +766,8 @@ check_syntax <- function(pedantic = FALSE, ), run_log <- processx::run( command = stanc_cmd(), - args = c(self$stan_file(), stanc_built_options, stancflags_val), + args = c(ifelse(os_is_wsl(), "bin/stanc", ""), + self$stan_file(), stanc_built_options, stancflags_val), wd = cmdstan_path(), echo = is_verbose_mode(), echo_cmd = is_verbose_mode(), @@ -903,7 +909,8 @@ format <- function(overwrite_file = FALSE, ), run_log <- processx::run( command = stanc_cmd(), - args = c(self$stan_file(), stanc_built_options, stancflags_val), + args = c(ifelse(os_is_wsl(), "bin/stanc", ""), + self$stan_file(), stanc_built_options, stancflags_val), wd = cmdstan_path(), echo = is_verbose_mode(), echo_cmd = is_verbose_mode(), @@ -1780,9 +1787,11 @@ model_variables <- function(stan_file, include_paths = NULL, allow_undefined = F allow_undefined_arg <- NULL } out_file <- tempfile(fileext = ".json") + stan_file <- stan_file run_log <- processx::run( command = stanc_cmd(), - args = c(stan_file, "--info", include_paths_stanc3_args(include_paths), allow_undefined_arg), + args = c(ifelse(os_is_wsl(), "bin/stanc", ""), + stan_file, "--info", include_paths_stanc3_args(include_paths), allow_undefined_arg), wd = cmdstan_path(), echo = FALSE, echo_cmd = FALSE, @@ -1810,8 +1819,8 @@ model_compile_info <- function(exe_file) { tbb_path() ), ret <- processx::run( - command = exe_file, - args = c("info"), + command = ifelse(os_is_wsl(), "wsl", exe_file), + args = c(ifelse(os_is_wsl(), exe_file, NULL),"info"), error_on_status = FALSE ) ) diff --git a/R/path.R b/R/path.R index d0ae7109f..7a12398b5 100644 --- a/R/path.R +++ b/R/path.R @@ -145,8 +145,24 @@ cmdstan_default_path <- function(old = FALSE, dir = NULL) { cmdstan_installs <- list.dirs(path = installs_path, recursive = FALSE, full.names = FALSE) } if (length(cmdstan_installs) > 0) { + wsl_installs <- grep("^wsl-cmdstan-", cmdstan_installs, value = TRUE) + cmdstan_installs <- cmdstan_installs[!grepl("wsl-", cmdstan_installs)] cmdstan_installs <- grep("^cmdstan-", cmdstan_installs, value = TRUE) - latest_cmdstan <- sort(cmdstan_installs, decreasing = TRUE)[1] + if (length(wsl_installs) > 0) { + wsl_installs_trim <- gsub("wsl-", "", wsl_installs, fixed = TRUE) + wsl_latest <- sort(wsl_installs_trim, decreasing = TRUE)[1] + if (length(cmdstan_installs) > 0) { + non_wsl_latest <- sort(cmdstan_installs, decreasing = TRUE)[1] + latest_cmdstan <- ifelse(wsl_latest > non_wsl_latest + || wsl_latest == non_wsl_latest, + grep(wsl_latest, wsl_installs, value = TRUE), + non_wsl_latest) + } else { + latest_cmdstan <- grep(wsl_latest, wsl_installs, value = TRUE) + } + } else { + latest_cmdstan <- sort(cmdstan_installs, decreasing = TRUE)[1] + } if (is_release_candidate(latest_cmdstan)) { non_rc_path <- strsplit(latest_cmdstan, "-rc")[[1]][1] if (dir.exists(file.path(installs_path, non_rc_path))) { diff --git a/R/run.R b/R/run.R index cb703663a..db092c1be 100644 --- a/R/run.R +++ b/R/run.R @@ -232,8 +232,9 @@ CmdStanRun <- R6::R6Class( tbb_path() ), run_log <- processx::run( - command = target_exe, - args = c(self$output_files(include_failed = FALSE), flags), + command = ifelse(os_is_wsl(), "wsl", target_exe), + args = c(ifelse(os_is_wsl(), target_exe, NULL), + self$output_files(include_failed = FALSE), flags), wd = cmdstan_path(), echo = TRUE, echo_cmd = is_verbose_mode(), @@ -302,7 +303,7 @@ check_target_exe <- function(exe) { ), run_log <- processx::run( command = make_cmd(), - args = exe, + args = ifelse(os_is_wsl(), "make", exe), wd = cmdstan_path(), echo_cmd = TRUE, echo = TRUE, @@ -508,8 +509,9 @@ CmdStanRun$set("private", name = "run_variational_", value = .run_other) tbb_path() ), ret <- processx::run( - command = self$command(), - args = self$command_args()[[1]], + command = ifelse(os_is_wsl(), "wsl", self$command()), + args = c(ifelse(os_is_wsl(), self$command(), NULL), + self$command_args()[[1]]), wd = dirname(self$exe_file()), stderr = stderr_file, stdout = stdout_file, @@ -624,8 +626,8 @@ CmdStanProcs <- R6::R6Class( tbb_path() ), private$processes_[[id]] <- processx::process$new( - command = command, - args = args, + command = ifelse(os_is_wsl(), "wsl", command), + args = c(ifelse(os_is_wsl(), paste0("./", command), NULL), args), wd = wd, stdout = "|", stderr = "|", diff --git a/R/utils.R b/R/utils.R index b6ca4e98f..b935c3d12 100644 --- a/R/utils.R +++ b/R/utils.R @@ -41,6 +41,16 @@ os_is_windows <- function() { isTRUE(.Platform$OS.type == "windows") } +os_is_wsl <- function() { + # Avoid errors if checking cmdstan_path before it has been set + wsl_in_path <- tryCatch({ + grepl("wsl-cmdstan", cmdstan_path()) + }, error = function(e) { + FALSE + }) + os_is_windows() && (wsl_in_path || Sys.getenv("CMDSTANR_USE_WSL") == 1) +} + os_is_macos <- function() { isTRUE(Sys.info()[["sysname"]] == "Darwin") } @@ -68,7 +78,9 @@ is_rosetta2 <- function() { # Returns the type of make command to use to compile depending on the OS make_cmd <- function() { - if (os_is_windows()) { + if (os_is_wsl()) { + "wsl" + } else if (os_is_windows()) { "mingw32-make.exe" } else { "make" @@ -77,7 +89,9 @@ make_cmd <- function() { # Returns the stanc exe path depending on the OS stanc_cmd <- function() { - if (os_is_windows()) { + if (os_is_wsl()) { + "wsl" + } else if (os_is_windows()) { "bin/stanc.exe" } else { "bin/stanc" @@ -109,8 +123,8 @@ repair_path <- function(path) { #' @param path If not `NULL` then a path to add the extension to. #' @return If `path` is `NULL` then `".exe"` on Windows and `""` otherwise. If #' `path` is not `NULL` then `.exe` is added as the extension on Windows. -cmdstan_ext <- function(path = NULL, wsl = FALSE) { - ext <- if (os_is_windows() && !isTRUE(wsl)) ".exe" else "" +cmdstan_ext <- function(path = NULL) { + ext <- if (os_is_windows() && !os_is_wsl()) ".exe" else "" if (is.null(path)) { return(ext) } @@ -139,7 +153,25 @@ strip_ext <- function(file) { } absolute_path <- Vectorize(.absolute_path, USE.NAMES = FALSE) +# When providing the model path to WSL, it needs to be in reference to the +# to Windows mount point (/mnt/drive-letter) within the WSL install: +# e.g., C:/Users/... -> /mnt/c/Users/... +.wsl_path_compat <- function(path) { + path_already_safe <- grepl("^/mnt/", path) + if (os_is_wsl() && !isTRUE(path_already_safe) && !is.na(path)) { + path <- normalizePath(path) + # Need forward-slashes for WSL + path <- gsub("\\\\", "/", path) + drive_letter <- tolower(strtrim(path, 1)) + path <- gsub(paste0(drive_letter, ":"), + paste0("/mnt/", drive_letter), + path, + ignore.case = TRUE) + } + path +} +wsl_path_compat <- Vectorize(.wsl_path_compat, USE.NAMES = FALSE) # read, write, and copy files -------------------------------------------- From 7348384704a47222eed9f2a0b3cf430ed3285e37 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Fri, 15 Jul 2022 11:31:30 +0300 Subject: [PATCH 04/28] Fix compatibility with non-wsl --- R/install.R | 19 ++++++++++++------- R/model.R | 35 ++++++++++++++++++----------------- R/run.R | 25 ++++++++++++++++--------- R/utils.R | 14 ++++++++++++-- 4 files changed, 58 insertions(+), 35 deletions(-) diff --git a/R/install.R b/R/install.R index d3c1abb4b..9ab5592cf 100644 --- a/R/install.R +++ b/R/install.R @@ -178,7 +178,7 @@ install_cmdstan <- function(dir = NULL, append = TRUE ) } - if (is_rtools42_toolchain() && !isTRUE(wsl)) { + if (is_rtools42_toolchain() && !os_is_wsl()) { cmdstan_make_local( dir = dir_cmdstan, cpp_options = list( @@ -211,6 +211,10 @@ install_cmdstan <- function(dir = NULL, "\nrebuild_cmdstan(cores = ...)" ) } + + if (isTRUE(wsl)) { + Sys.unsetenv("CMDSTANR_USE_WSL") + } } @@ -409,7 +413,6 @@ build_cmdstan <- function(dir, clean_cmdstan <- function(dir = cmdstan_path(), cores = getOption("mc.cores", 2), quiet = FALSE) { - translation_args <- ifelse(os_is_wsl(), "make", NULL) withr::with_path( c( toolchain_PATH_env_var(), @@ -417,7 +420,8 @@ clean_cmdstan <- function(dir = cmdstan_path(), ), processx::run( make_cmd(), - args = c(translation_args, "clean-all"), + args = wsl_args(command = "make", + args = "clean_all"), wd = dir, echo_cmd = is_verbose_mode(), echo = !quiet || is_verbose_mode(), @@ -429,7 +433,6 @@ clean_cmdstan <- function(dir = cmdstan_path(), } build_example <- function(dir, cores, quiet, timeout) { - translation_args <- ifelse(os_is_wsl(), "make", NULL) withr::with_path( c( toolchain_PATH_env_var(), @@ -437,8 +440,11 @@ build_example <- function(dir, cores, quiet, timeout) { ), processx::run( make_cmd(), - args = c(translation_args, paste0("-j", cores), - cmdstan_ext(file.path("examples", "bernoulli", "bernoulli"))), + args = wsl_args(command = "make", + args = c(paste0("-j", cores), + cmdstan_ext(file.path("examples", + "bernoulli", + "bernoulli")))), wd = dir, echo_cmd = is_verbose_mode(), echo = !quiet || is_verbose_mode(), @@ -548,7 +554,6 @@ check_wsl_toolchain <- function() { "\n", "Arch: pacman -Sy base-devel", call. = FALSE) } - } check_rtools4x_windows_toolchain <- function(fix = FALSE, quiet = FALSE) { diff --git a/R/model.R b/R/model.R index 7441bcd52..5646bad59 100644 --- a/R/model.R +++ b/R/model.R @@ -554,11 +554,10 @@ compile <- function(quiet = TRUE, ), run_log <- processx::run( command = make_cmd(), - args = c( - ifelse(os_is_wsl(), "make", ""), - tmp_exe, - cpp_options_to_compile_flags(cpp_options), - stancflags_val), + args = wsl_args(command = "make", + args = c(tmp_exe, + cpp_options_to_compile_flags(cpp_options), + stancflags_val)), wd = cmdstan_path(), echo = !quiet || is_verbose_mode(), echo_cmd = is_verbose_mode(), @@ -655,10 +654,7 @@ variables <- function() { } assert_stan_file_exists(self$stan_file()) if (is.null(private$variables_) && file.exists(self$stan_file())) { - private$variables_ <- model_variables( - self$stan_file(), - self$include_paths(), - allow_undefined = private$using_user_header_) + private$variables_ <- model_variables(self$stan_file(), self$include_paths(), allow_undefined = private$using_user_header_) } private$variables_ } @@ -766,8 +762,9 @@ check_syntax <- function(pedantic = FALSE, ), run_log <- processx::run( command = stanc_cmd(), - args = c(ifelse(os_is_wsl(), "bin/stanc", ""), - self$stan_file(), stanc_built_options, stancflags_val), + args = wsl_args(command = "bin/stanc", + args = c(self$stan_file(), stanc_built_options, + stancflags_val)), wd = cmdstan_path(), echo = is_verbose_mode(), echo_cmd = is_verbose_mode(), @@ -909,8 +906,8 @@ format <- function(overwrite_file = FALSE, ), run_log <- processx::run( command = stanc_cmd(), - args = c(ifelse(os_is_wsl(), "bin/stanc", ""), - self$stan_file(), stanc_built_options, stancflags_val), + args = wsl_args(command = "bin/stanc", + args = c(self$stan_file(), stanc_built_options, stancflags_val)), wd = cmdstan_path(), echo = is_verbose_mode(), echo_cmd = is_verbose_mode(), @@ -1790,8 +1787,11 @@ model_variables <- function(stan_file, include_paths = NULL, allow_undefined = F stan_file <- stan_file run_log <- processx::run( command = stanc_cmd(), - args = c(ifelse(os_is_wsl(), "bin/stanc", ""), - stan_file, "--info", include_paths_stanc3_args(include_paths), allow_undefined_arg), + args = wsl_args( + command = "bin/stanc", + args = c(stan_file, "--info", include_paths_stanc3_args(include_paths), + allow_undefined_arg) + ), wd = cmdstan_path(), echo = FALSE, echo_cmd = FALSE, @@ -1819,8 +1819,9 @@ model_compile_info <- function(exe_file) { tbb_path() ), ret <- processx::run( - command = ifelse(os_is_wsl(), "wsl", exe_file), - args = c(ifelse(os_is_wsl(), exe_file, NULL),"info"), + command = wsl_command(exe_file), + args = wsl_args(command = exe_file, + args = "info"), error_on_status = FALSE ) ) diff --git a/R/run.R b/R/run.R index db092c1be..221b3bfd0 100644 --- a/R/run.R +++ b/R/run.R @@ -232,9 +232,11 @@ CmdStanRun <- R6::R6Class( tbb_path() ), run_log <- processx::run( - command = ifelse(os_is_wsl(), "wsl", target_exe), - args = c(ifelse(os_is_wsl(), target_exe, NULL), - self$output_files(include_failed = FALSE), flags), + command = wsl_command(target_exe), + args = wsl_args( + command = target_exe, + args = c(self$output_files(include_failed = FALSE), flags) + ), wd = cmdstan_path(), echo = TRUE, echo_cmd = is_verbose_mode(), @@ -303,7 +305,7 @@ check_target_exe <- function(exe) { ), run_log <- processx::run( command = make_cmd(), - args = ifelse(os_is_wsl(), "make", exe), + args = wsl_args(command = "make", args = exe), wd = cmdstan_path(), echo_cmd = TRUE, echo = TRUE, @@ -509,9 +511,11 @@ CmdStanRun$set("private", name = "run_variational_", value = .run_other) tbb_path() ), ret <- processx::run( - command = ifelse(os_is_wsl(), "wsl", self$command()), - args = c(ifelse(os_is_wsl(), self$command(), NULL), - self$command_args()[[1]]), + command = wsl_command(self$command()), + args = wsl_args( + command = self$command(), + args = self$command_args()[[1]] + ), wd = dirname(self$exe_file()), stderr = stderr_file, stdout = stdout_file, @@ -626,8 +630,11 @@ CmdStanProcs <- R6::R6Class( tbb_path() ), private$processes_[[id]] <- processx::process$new( - command = ifelse(os_is_wsl(), "wsl", command), - args = c(ifelse(os_is_wsl(), paste0("./", command), NULL), args), + command = wsl_command(command), + args = wsl_args( + command = paste0("./", command), + args = args + ), wd = wd, stdout = "|", stderr = "|", diff --git a/R/utils.R b/R/utils.R index b935c3d12..6b6738c37 100644 --- a/R/utils.R +++ b/R/utils.R @@ -156,7 +156,7 @@ absolute_path <- Vectorize(.absolute_path, USE.NAMES = FALSE) # When providing the model path to WSL, it needs to be in reference to the # to Windows mount point (/mnt/drive-letter) within the WSL install: # e.g., C:/Users/... -> /mnt/c/Users/... -.wsl_path_compat <- function(path) { +wsl_path_compat <- function(path) { path_already_safe <- grepl("^/mnt/", path) if (os_is_wsl() && !isTRUE(path_already_safe) && !is.na(path)) { path <- normalizePath(path) @@ -171,7 +171,17 @@ absolute_path <- Vectorize(.absolute_path, USE.NAMES = FALSE) path } -wsl_path_compat <- Vectorize(.wsl_path_compat, USE.NAMES = FALSE) +wsl_args <- function(command, args) { + if (os_is_wsl()) { + c(command, args) + } else { + args + } +} + +wsl_command <- function(command) { + ifelse(os_is_wsl(), "wsl", command) +} # read, write, and copy files -------------------------------------------- From ac896676e78d4f8d79d428215c27f3c641a95286 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Fri, 15 Jul 2022 14:01:00 +0300 Subject: [PATCH 05/28] Initial full working example --- R/args.R | 10 +++++----- R/model.R | 4 ++-- R/run.R | 4 ++-- R/utils.R | 26 +++++++++++++++++--------- 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/R/args.R b/R/args.R index 5900f6f31..e58dfb6e2 100644 --- a/R/args.R +++ b/R/args.R @@ -142,12 +142,12 @@ CmdStanArgs <- R6::R6Class( } if (!is.null(self$data_file)) { - args$data <- c("data", paste0("file=", self$data_file)) + args$data <- c("data", paste0("file=", wsl_path_compat(self$data_file))) } - args$output <- c("output", paste0("file=", output_file)) + args$output <- c("output", paste0("file=", wsl_path_compat(output_file))) if (!is.null(latent_dynamics_file)) { - args$output <- c(args$output, paste0("diagnostic_file=", latent_dynamics_file)) + args$output <- c(args$output, paste0("diagnostic_file=", wsl_path_compat(latent_dynamics_file))) } if (!is.null(self$refresh)) { args$output <- c(args$output, paste0("refresh=", self$refresh)) @@ -158,7 +158,7 @@ CmdStanArgs <- R6::R6Class( } if (!is.null(profile_file)) { - args$output <- c(args$output, paste0("profile_file=", profile_file)) + args$output <- c(args$output, paste0("profile_file=", wsl_path_compat(profile_file))) } if (!is.null(self$opencl_ids)) { args$opencl <- c("opencl", paste0("platform=", self$opencl_ids[1]), paste0("device=", self$opencl_ids[2])) @@ -167,7 +167,7 @@ CmdStanArgs <- R6::R6Class( self$method_args$compose(idx, args) }, command = function() { - paste0(if (!os_is_windows()) "./", basename(self$exe_file)) + paste0(if (!os_is_windows() || os_is_wsl()) "./", basename(self$exe_file)) } ) ) diff --git a/R/model.R b/R/model.R index 5646bad59..663b9d2eb 100644 --- a/R/model.R +++ b/R/model.R @@ -555,7 +555,7 @@ compile <- function(quiet = TRUE, run_log <- processx::run( command = make_cmd(), args = wsl_args(command = "make", - args = c(tmp_exe, + args = c(wsl_path_compat(tmp_exe), cpp_options_to_compile_flags(cpp_options), stancflags_val)), wd = cmdstan_path(), @@ -1789,7 +1789,7 @@ model_variables <- function(stan_file, include_paths = NULL, allow_undefined = F command = stanc_cmd(), args = wsl_args( command = "bin/stanc", - args = c(stan_file, "--info", include_paths_stanc3_args(include_paths), + args = c(wsl_path_compat(stan_file), "--info", include_paths_stanc3_args(include_paths), allow_undefined_arg) ), wd = cmdstan_path(), diff --git a/R/run.R b/R/run.R index 221b3bfd0..7e6c6f549 100644 --- a/R/run.R +++ b/R/run.R @@ -632,7 +632,7 @@ CmdStanProcs <- R6::R6Class( private$processes_[[id]] <- processx::process$new( command = wsl_command(command), args = wsl_args( - command = paste0("./", command), + command = command, args = args ), wd = wd, @@ -1079,4 +1079,4 @@ tbb_path <- function(dir = NULL) { path_to_TBB <- file.path(dir, "stan", "lib", "stan_math", "lib", "tbb") } path_to_TBB -} \ No newline at end of file +} diff --git a/R/utils.R b/R/utils.R index 6b6738c37..7a84e2497 100644 --- a/R/utils.R +++ b/R/utils.R @@ -157,16 +157,24 @@ absolute_path <- Vectorize(.absolute_path, USE.NAMES = FALSE) # to Windows mount point (/mnt/drive-letter) within the WSL install: # e.g., C:/Users/... -> /mnt/c/Users/... wsl_path_compat <- function(path) { - path_already_safe <- grepl("^/mnt/", path) + path_already_safe <- grepl("/mnt/", path) if (os_is_wsl() && !isTRUE(path_already_safe) && !is.na(path)) { - path <- normalizePath(path) - # Need forward-slashes for WSL - path <- gsub("\\\\", "/", path) - drive_letter <- tolower(strtrim(path, 1)) - path <- gsub(paste0(drive_letter, ":"), - paste0("/mnt/", drive_letter), - path, - ignore.case = TRUE) + abs_path <- repair_path(path) + # Special handling for arguments to be passed to stanc3 + if(grepl("file=", path, fixed = TRUE)) { + abs_path <- gsub("file=", "", abs_path) + } + trim_lead_whitespace <- gsub("^\\s*", "", abs_path) + drive_letter <- tolower(strtrim(trim_lead_whitespace, 1)) + trim_lead_whitespace <- gsub(paste0(drive_letter, ":"), + paste0("/mnt/", drive_letter), + trim_lead_whitespace, + ignore.case = TRUE) + if(grepl("file=", path, fixed = TRUE)) { + path <- (paste0("file=", trim_lead_whitespace)) + } else { + path <- trim_lead_whitespace + } } path } From 973f684cf423d3cff4b172ec8d4b652351f9390b Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Fri, 15 Jul 2022 15:04:20 +0300 Subject: [PATCH 06/28] Simplify wsl run, trial GHA workflow --- .github/workflows/R-CMD-check-wsl.yaml | 97 +++++++++++++++++++++++ .github/workflows/R-CMD-check.yaml | 2 +- .github/workflows/Test-coverage.yaml | 2 +- R/install.R | 27 +++---- R/model.R | 37 ++++----- R/run.R | 34 ++++---- R/utils.R | 45 +++++------ tests/testthat/helper-envvars-and-paths.R | 4 +- 8 files changed, 160 insertions(+), 88 deletions(-) create mode 100644 .github/workflows/R-CMD-check-wsl.yaml diff --git a/.github/workflows/R-CMD-check-wsl.yaml b/.github/workflows/R-CMD-check-wsl.yaml new file mode 100644 index 000000000..10ae00275 --- /dev/null +++ b/.github/workflows/R-CMD-check-wsl.yaml @@ -0,0 +1,97 @@ +--- +# Github Actions workflow to check CmdStanR +# yamllint disable rule:line-length + +name: Unit tests + +'on': + push: + branches: + - wsl-support + pull_request: + branches: + - master + +jobs: + R-CMD-check: + if: "! contains(github.event.head_commit.message, '[ci skip]')" + runs-on: windows-latest + + name: WSL1 Backend + + env: + R_REMOTES_NO_ERRORS_FROM_WARNINGS: true + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + NOT_CRAN: true + + steps: + - name: cmdstan env vars + run: | + echo "CMDSTAN_PATH=${HOME}/.cmdstan" >> $GITHUB_ENV + shell: bash + + - uses: n1hility/cancel-previous-runs@v2 + with: + token: ${{ secrets.GITHUB_TOKEN }} + if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/master'" + + - uses: actions/checkout@v3 + + - uses: r-lib/actions/setup-r@v2.2.3 + with: + r-version: 'release' + rtools-version: '42' + - uses: r-lib/actions/setup-pandoc@v1 + + - name: Query dependencies + run: | + install.packages('remotes') + saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2) + writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version") + shell: Rscript {0} + + - name: Install dependencies + run: | + remotes::install_deps(dependencies = TRUE) + remotes::install_cran("rcmdcheck") + remotes::install_local(path = ".") + install.packages("curl") + shell: Rscript {0} + + - uses: Vampire/setup-wsl@v1 + with: + distribution: Ubuntu-22.04 + use-cache: 'true' + set-as-default: 'true' + additional-packages: | + build-essential + + - name: Install cmdstan + run: | + cmdstanr::install_cmdstan(cores = 2, wsl = TRUE) + shell: Rscript {0} + + - name: Session info + run: | + options(width = 100) + pkgs <- installed.packages()[, "Package"] + sessioninfo::session_info(pkgs, include_base = TRUE) + shell: Rscript {0} + + - name: Check + env: + _R_CHECK_CRAN_INCOMING_: false + run: rcmdcheck::rcmdcheck(args = c("--no-manual", "--as-cran"), error_on = "warning", check_dir = "check") + shell: Rscript {0} + + - name: Show testthat output + if: always() + run: find check -name 'testthat.Rout*' -exec cat '{}' \; || true + shell: bash + + - name: Upload check results + if: failure() + uses: actions/upload-artifact@v3 + with: + name: ${{ runner.os }}-r${{ matrix.config.r }}-results + path: check diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 0b5af44e4..feb714230 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -7,7 +7,7 @@ name: Unit tests 'on': push: branches: - - master + - wsl-support pull_request: branches: - master diff --git a/.github/workflows/Test-coverage.yaml b/.github/workflows/Test-coverage.yaml index efffffa56..44e34d86d 100644 --- a/.github/workflows/Test-coverage.yaml +++ b/.github/workflows/Test-coverage.yaml @@ -7,7 +7,7 @@ name: Test coverage 'on': push: branches: - - master + - wsl-support pull_request: branches: - master diff --git a/R/install.R b/R/install.R index 9ab5592cf..77479d717 100644 --- a/R/install.R +++ b/R/install.R @@ -384,9 +384,6 @@ build_cmdstan <- function(dir, if (is_rosetta2()) { run_cmd <- "/usr/bin/arch" translation_args <- c("-arch", "arm64e", "make") - } else if (os_is_wsl()) { - run_cmd <- "wsl" - translation_args <- "make" } else { run_cmd <- make_cmd() } @@ -396,8 +393,8 @@ build_cmdstan <- function(dir, toolchain_PATH_env_var(), tbb_path(dir = dir) ), - processx::run( - run_cmd, + wsl_compatible_run( + command = run_cmd, args = c(translation_args, paste0("-j", cores), "build"), wd = dir, echo_cmd = is_verbose_mode(), @@ -418,10 +415,9 @@ clean_cmdstan <- function(dir = cmdstan_path(), toolchain_PATH_env_var(), tbb_path(dir = dir) ), - processx::run( - make_cmd(), - args = wsl_args(command = "make", - args = "clean_all"), + wsl_compatible_run( + command = make_cmd(), + args = "clean_all", wd = dir, echo_cmd = is_verbose_mode(), echo = !quiet || is_verbose_mode(), @@ -438,13 +434,10 @@ build_example <- function(dir, cores, quiet, timeout) { toolchain_PATH_env_var(), tbb_path(dir = dir) ), - processx::run( - make_cmd(), - args = wsl_args(command = "make", - args = c(paste0("-j", cores), - cmdstan_ext(file.path("examples", - "bernoulli", - "bernoulli")))), + wsl_compatible_run( + command = make_cmd(), + args = c(paste0("-j", cores), + cmdstan_ext(file.path("examples", "bernoulli", "bernoulli"))), wd = dir, echo_cmd = is_verbose_mode(), echo = !quiet || is_verbose_mode(), @@ -548,7 +541,7 @@ check_wsl_toolchain <- function() { if (make_not_present$status || (gpp_not_present$status && clangpp_not_present$status)) { stop("\n", "Your distribution is missing the needed utilities for compiling C++.", - "\n", "Please launch your WSL install them using the appropriate command:", + "\n", "Please launch your WSL and install them using the appropriate command:", "\n", "Debian/Ubuntu: sudo apt-get install build-essential", "\n", "Fedora: sudo dnf group install \"C Development Tools and Libraries\"", "\n", "Arch: pacman -Sy base-devel", diff --git a/R/model.R b/R/model.R index 663b9d2eb..00b34a155 100644 --- a/R/model.R +++ b/R/model.R @@ -552,12 +552,11 @@ compile <- function(quiet = TRUE, toolchain_PATH_env_var(), tbb_path() ), - run_log <- processx::run( + run_log <- wsl_compatible_run( command = make_cmd(), - args = wsl_args(command = "make", - args = c(wsl_path_compat(tmp_exe), - cpp_options_to_compile_flags(cpp_options), - stancflags_val)), + args = c(wsl_path_compat(tmp_exe), + cpp_options_to_compile_flags(cpp_options), + stancflags_val), wd = cmdstan_path(), echo = !quiet || is_verbose_mode(), echo_cmd = is_verbose_mode(), @@ -760,11 +759,9 @@ check_syntax <- function(pedantic = FALSE, toolchain_PATH_env_var(), tbb_path() ), - run_log <- processx::run( + run_log <- wsl_compatible_run( command = stanc_cmd(), - args = wsl_args(command = "bin/stanc", - args = c(self$stan_file(), stanc_built_options, - stancflags_val)), + args = c(self$stan_file(), stanc_built_options, stancflags_val), wd = cmdstan_path(), echo = is_verbose_mode(), echo_cmd = is_verbose_mode(), @@ -904,10 +901,10 @@ format <- function(overwrite_file = FALSE, toolchain_PATH_env_var(), tbb_path() ), - run_log <- processx::run( + run_log <- wsl_compatible_run( command = stanc_cmd(), - args = wsl_args(command = "bin/stanc", - args = c(self$stan_file(), stanc_built_options, stancflags_val)), + args = c(wsl_path_compat(self$stan_file()), stanc_built_options, + stancflags_val), wd = cmdstan_path(), echo = is_verbose_mode(), echo_cmd = is_verbose_mode(), @@ -1785,13 +1782,10 @@ model_variables <- function(stan_file, include_paths = NULL, allow_undefined = F } out_file <- tempfile(fileext = ".json") stan_file <- stan_file - run_log <- processx::run( + run_log <- wsl_compatible_run( command = stanc_cmd(), - args = wsl_args( - command = "bin/stanc", - args = c(wsl_path_compat(stan_file), "--info", include_paths_stanc3_args(include_paths), - allow_undefined_arg) - ), + args = c(wsl_path_compat(stan_file), "--info", include_paths_stanc3_args(include_paths), + allow_undefined_arg), wd = cmdstan_path(), echo = FALSE, echo_cmd = FALSE, @@ -1818,10 +1812,9 @@ model_compile_info <- function(exe_file) { toolchain_PATH_env_var(), tbb_path() ), - ret <- processx::run( - command = wsl_command(exe_file), - args = wsl_args(command = exe_file, - args = "info"), + ret <- wsl_compatible_run( + command = exe_file, + args = "info", error_on_status = FALSE ) ) diff --git a/R/run.R b/R/run.R index 7e6c6f549..f20c9f413 100644 --- a/R/run.R +++ b/R/run.R @@ -231,12 +231,12 @@ CmdStanRun <- R6::R6Class( toolchain_PATH_env_var(), tbb_path() ), - run_log <- processx::run( - command = wsl_command(target_exe), - args = wsl_args( - command = target_exe, - args = c(self$output_files(include_failed = FALSE), flags) - ), + run_log <- wsl_compatible_run( + command = target_exe, + args = c( + sapply(self$output_files(include_failed = FALSE), + wsl_path_compat), + flags), wd = cmdstan_path(), echo = TRUE, echo_cmd = is_verbose_mode(), @@ -303,9 +303,9 @@ check_target_exe <- function(exe) { toolchain_PATH_env_var(), tbb_path() ), - run_log <- processx::run( + run_log <- wsl_compatible_run( command = make_cmd(), - args = wsl_args(command = "make", args = exe), + args = exe, wd = cmdstan_path(), echo_cmd = TRUE, echo = TRUE, @@ -510,12 +510,9 @@ CmdStanRun$set("private", name = "run_variational_", value = .run_other) toolchain_PATH_env_var(), tbb_path() ), - ret <- processx::run( - command = wsl_command(self$command()), - args = wsl_args( - command = self$command(), - args = self$command_args()[[1]] - ), + ret <- wsl_compatible_run( + command = self$command(), + args = self$command_args()[[1]], wd = dirname(self$exe_file()), stderr = stderr_file, stdout = stdout_file, @@ -629,12 +626,9 @@ CmdStanProcs <- R6::R6Class( toolchain_PATH_env_var(), tbb_path() ), - private$processes_[[id]] <- processx::process$new( - command = wsl_command(command), - args = wsl_args( - command = command, - args = args - ), + private$processes_[[id]] <- wsl_compatible_process_new( + command = command, + args = args, wd = wd, stdout = "|", stderr = "|", diff --git a/R/utils.R b/R/utils.R index 7a84e2497..a043b9f12 100644 --- a/R/utils.R +++ b/R/utils.R @@ -78,9 +78,7 @@ is_rosetta2 <- function() { # Returns the type of make command to use to compile depending on the OS make_cmd <- function() { - if (os_is_wsl()) { - "wsl" - } else if (os_is_windows()) { + if (os_is_windows() && !os_is_wsl()) { "mingw32-make.exe" } else { "make" @@ -89,9 +87,7 @@ make_cmd <- function() { # Returns the stanc exe path depending on the OS stanc_cmd <- function() { - if (os_is_wsl()) { - "wsl" - } else if (os_is_windows()) { + if (os_is_windows() && !os_is_wsl()) { "bin/stanc.exe" } else { "bin/stanc" @@ -160,35 +156,34 @@ wsl_path_compat <- function(path) { path_already_safe <- grepl("/mnt/", path) if (os_is_wsl() && !isTRUE(path_already_safe) && !is.na(path)) { abs_path <- repair_path(path) - # Special handling for arguments to be passed to stanc3 - if(grepl("file=", path, fixed = TRUE)) { - abs_path <- gsub("file=", "", abs_path) - } trim_lead_whitespace <- gsub("^\\s*", "", abs_path) drive_letter <- tolower(strtrim(trim_lead_whitespace, 1)) - trim_lead_whitespace <- gsub(paste0(drive_letter, ":"), - paste0("/mnt/", drive_letter), - trim_lead_whitespace, - ignore.case = TRUE) - if(grepl("file=", path, fixed = TRUE)) { - path <- (paste0("file=", trim_lead_whitespace)) - } else { - path <- trim_lead_whitespace - } + path <- gsub(paste0(drive_letter, ":"), + paste0("/mnt/", drive_letter), + trim_lead_whitespace, + ignore.case = TRUE) } path } -wsl_args <- function(command, args) { +wsl_compatible_run <- function(...) { + run_args <- list(...) if (os_is_wsl()) { - c(command, args) - } else { - args + command <- run_args$command + run_args$command <- "wsl" + run_args$args <- c(command, run_args$args) } + do.call(processx::run, run_args) } -wsl_command <- function(command) { - ifelse(os_is_wsl(), "wsl", command) +wsl_compatible_process_new <- function(...) { + run_args <- list(...) + if (os_is_wsl()) { + command <- run_args$command + run_args$command <- "wsl" + run_args$args <- c(command, run_args$args) + } + do.call(processx::process$new, run_args) } # read, write, and copy files -------------------------------------------- diff --git a/tests/testthat/helper-envvars-and-paths.R b/tests/testthat/helper-envvars-and-paths.R index 7a18165d4..66283716b 100644 --- a/tests/testthat/helper-envvars-and-paths.R +++ b/tests/testthat/helper-envvars-and-paths.R @@ -8,8 +8,8 @@ on_ci <- function() { mpi_toolchain_present <- function() { tryCatch( - processx::run(command = "mpicxx", args = "--version")$status == 0 && - processx::run(command = "mpiexec", args = "--version")$status == 0, + wsl_compatible_run(command = "mpicxx", args = "--version")$status == 0 && + wsl_compatible_run(command = "mpiexec", args = "--version")$status == 0, error=function(cond) { FALSE } From afa95cc6f788e69697fcc3c3ebfb7eb5563d903a Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Fri, 15 Jul 2022 15:48:48 +0300 Subject: [PATCH 07/28] Fix syntax checking failure --- .github/workflows/R-CMD-check-wsl.yaml | 5 ++--- R/install.R | 2 ++ R/model.R | 2 +- man/install_cmdstan.Rd | 6 +++++- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/R-CMD-check-wsl.yaml b/.github/workflows/R-CMD-check-wsl.yaml index 10ae00275..1a505290a 100644 --- a/.github/workflows/R-CMD-check-wsl.yaml +++ b/.github/workflows/R-CMD-check-wsl.yaml @@ -2,7 +2,7 @@ # Github Actions workflow to check CmdStanR # yamllint disable rule:line-length -name: Unit tests +name: Unit tests - WSL Backend 'on': push: @@ -63,8 +63,7 @@ jobs: distribution: Ubuntu-22.04 use-cache: 'true' set-as-default: 'true' - additional-packages: | - build-essential + additional-packages: build-essential - name: Install cmdstan run: | diff --git a/R/install.R b/R/install.R index 77479d717..01b94e771 100644 --- a/R/install.R +++ b/R/install.R @@ -57,6 +57,8 @@ #' @param check_toolchain (logical) Should `install_cmdstan()` attempt to check #' that the required toolchain is installed and properly configured. The #' default is `TRUE`. +#' @param wsl (logical) Should CmdStan be installed and run through the Windows +#' Subsystem for Linux (WSL). The default is `FALSE`. #' #' @examples #' \dontrun{ diff --git a/R/model.R b/R/model.R index 00b34a155..2645d67e5 100644 --- a/R/model.R +++ b/R/model.R @@ -761,7 +761,7 @@ check_syntax <- function(pedantic = FALSE, ), run_log <- wsl_compatible_run( command = stanc_cmd(), - args = c(self$stan_file(), stanc_built_options, stancflags_val), + args = c(wsl_path_compat(self$stan_file()), stanc_built_options, stancflags_val), wd = cmdstan_path(), echo = is_verbose_mode(), echo_cmd = is_verbose_mode(), diff --git a/man/install_cmdstan.Rd b/man/install_cmdstan.Rd index edbd419ae..84b0f2964 100644 --- a/man/install_cmdstan.Rd +++ b/man/install_cmdstan.Rd @@ -16,7 +16,8 @@ install_cmdstan( version = NULL, release_url = NULL, cpp_options = list(), - check_toolchain = TRUE + check_toolchain = TRUE, + wsl = FALSE ) rebuild_cmdstan( @@ -73,6 +74,9 @@ the use of clang for compilation.} that the required toolchain is installed and properly configured. The default is \code{TRUE}.} +\item{wsl}{(logical) Should CmdStan be installed and run through the Windows +Subsystem for Linux (WSL). The default is \code{FALSE}.} + \item{append}{(logical) For \code{cmdstan_make_local()}, should the listed makefile flags be appended to the end of the existing \code{make/local} file? The default is \code{TRUE}. If \code{FALSE} the file is overwritten.} From 26e78e34dddfe2f162d4fcc68f00dcc7af63c1dc Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Fri, 15 Jul 2022 19:33:47 +0300 Subject: [PATCH 08/28] First pass of fixes --- .github/workflows/R-CMD-check-wsl.yaml | 2 +- R/args.R | 5 ++++- R/model.R | 1 + R/run.R | 16 ++++++++++++++++ R/utils.R | 5 +++++ tests/testthat/test-model-compile.R | 4 ++-- tests/testthat/test-path.R | 2 +- 7 files changed, 30 insertions(+), 5 deletions(-) diff --git a/.github/workflows/R-CMD-check-wsl.yaml b/.github/workflows/R-CMD-check-wsl.yaml index 1a505290a..e3f1a76af 100644 --- a/.github/workflows/R-CMD-check-wsl.yaml +++ b/.github/workflows/R-CMD-check-wsl.yaml @@ -67,7 +67,7 @@ jobs: - name: Install cmdstan run: | - cmdstanr::install_cmdstan(cores = 2, wsl = TRUE) + cmdstanr::install_cmdstan(cores = 2, wsl = TRUE, overwrite = TRUE) shell: Rscript {0} - name: Session info diff --git a/R/args.R b/R/args.R index e58dfb6e2..fb842c476 100644 --- a/R/args.R +++ b/R/args.R @@ -138,7 +138,7 @@ CmdStanArgs <- R6::R6Class( } if (!is.null(self$init)) { - args$init <- paste0("init=", self$init[idx]) + args$init <- paste0("init=", wsl_path_compat(self$init[idx])) } if (!is.null(self$data_file)) { @@ -1019,6 +1019,9 @@ available_metrics <- function() { #' @param idx Chain id (only applicable for MCMC). compose_arg <- function(self, arg_name, cmdstan_arg_name = NULL, idx = NULL) { val <- self[[arg_name]] + if (os_is_wsl() && (arg_name %in% c("fitted_params", "metric_file"))) { + val <- sapply(val, wsl_path_compat) + } cmdstan_arg_name <- cmdstan_arg_name %||% arg_name if (is.null(val)) { diff --git a/R/model.R b/R/model.R index 2645d67e5..395ba8fb7 100644 --- a/R/model.R +++ b/R/model.R @@ -1761,6 +1761,7 @@ include_paths_stanc3_args <- function(include_paths = NULL) { if (!is.null(include_paths)) { checkmate::assert_directory_exists(include_paths, access = "r") include_paths <- absolute_path(include_paths) + include_paths <- sapply(include_paths, wsl_path_compat) paths_w_space <- grep(" ", include_paths) include_paths[paths_w_space] <- paste0("'", include_paths[paths_w_space], "'") include_paths <- paste0(include_paths, collapse = ",") diff --git a/R/run.R b/R/run.R index f20c9f413..9b6cb3996 100644 --- a/R/run.R +++ b/R/run.R @@ -353,6 +353,10 @@ check_target_exe <- function(exe) { } else { cat(paste0(start_msg, ", with ", procs$threads_per_proc(), " thread(s) per chain...\n\n")) Sys.setenv("STAN_NUM_THREADS" = as.integer(procs$threads_per_proc())) + # Windows environment variables have to be explicitly exported to WSL + if (os_is_wsl()) { + Sys.setenv("WSLENV"="STAN_NUM_THREADS/u") + } } start_time <- Sys.time() chains <- procs$proc_ids() @@ -412,6 +416,10 @@ CmdStanRun$set("private", name = "run_sample_", value = .run_sample) } else { cat(paste0(start_msg, ", with ", procs$threads_per_proc(), " thread(s) per chain...\n\n")) Sys.setenv("STAN_NUM_THREADS" = as.integer(procs$threads_per_proc())) + # Windows environment variables have to be explicitly exported to WSL + if (os_is_wsl()) { + Sys.setenv("WSLENV"="STAN_NUM_THREADS/u") + } } start_time <- Sys.time() chains <- procs$proc_ids() @@ -454,6 +462,10 @@ CmdStanRun$set("private", name = "run_generate_quantities_", value = .run_genera procs <- self$procs if (!is.null(procs$threads_per_proc())) { Sys.setenv("STAN_NUM_THREADS" = as.integer(procs$threads_per_proc())) + # Windows environment variables have to be explicitly exported to WSL + if (os_is_wsl()) { + Sys.setenv("WSLENV"="STAN_NUM_THREADS/u") + } } start_time <- Sys.time() id <- 1 @@ -501,6 +513,10 @@ CmdStanRun$set("private", name = "run_variational_", value = .run_other) procs <- self$procs if (!is.null(procs$threads_per_proc())) { Sys.setenv("STAN_NUM_THREADS" = as.integer(procs$threads_per_proc())) + # Windows environment variables have to be explicitly exported to WSL + if (os_is_wsl()) { + Sys.setenv("WSLENV"="STAN_NUM_THREADS/u") + } } stdout_file <- tempfile() stderr_file <- tempfile() diff --git a/R/utils.R b/R/utils.R index a043b9f12..bb28f2aab 100644 --- a/R/utils.R +++ b/R/utils.R @@ -47,6 +47,8 @@ os_is_wsl <- function() { grepl("wsl-cmdstan", cmdstan_path()) }, error = function(e) { FALSE + }, warning = function(e) { + FALSE }) os_is_windows() && (wsl_in_path || Sys.getenv("CMDSTANR_USE_WSL") == 1) } @@ -153,6 +155,9 @@ absolute_path <- Vectorize(.absolute_path, USE.NAMES = FALSE) # to Windows mount point (/mnt/drive-letter) within the WSL install: # e.g., C:/Users/... -> /mnt/c/Users/... wsl_path_compat <- function(path) { + if (!is.character(path)) { + path + } path_already_safe <- grepl("/mnt/", path) if (os_is_wsl() && !isTRUE(path_already_safe) && !is.na(path)) { abs_path <- repair_path(path) diff --git a/tests/testthat/test-model-compile.R b/tests/testthat/test-model-compile.R index b2ba0ab29..54a0c904d 100644 --- a/tests/testthat/test-model-compile.R +++ b/tests/testthat/test-model-compile.R @@ -73,7 +73,7 @@ test_that("compile() method works with spaces in path", { expect_interactive_message(mod_spaces$compile(), "Compiling Stan program...") file.remove(stan_model_with_spaces) file.remove(exe) - file.remove(dir_with_spaces) + unlink(dir_with_spaces, recursive = TRUE) }) test_that("compile() method overwrites binaries", { @@ -581,7 +581,7 @@ test_that("cmdstan_model errors with no args ", { }) test_that("cmdstan_model works with user_header", { - skip_if(os_is_macos() | os_is_windows()) + skip_if(os_is_macos() | os_is_windows()) tmpfile <- tempfile(fileext = ".hpp") hpp <- " diff --git a/tests/testthat/test-path.R b/tests/testthat/test-path.R index 190f48d3b..daed18ae3 100644 --- a/tests/testthat/test-path.R +++ b/tests/testthat/test-path.R @@ -90,7 +90,7 @@ test_that("Warning message is thrown if can't detect version number", { }) test_that("cmdstan_ext() works", { - if (os_is_windows()) { + if (os_is_windows() && !os_is_wsl()) { expect_identical(cmdstan_ext(), ".exe") expect_identical(cmdstan_ext("path"), "path.exe") } else { From 02ff40cb88daac84959286a5e692721ed34b6756 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Fri, 15 Jul 2022 21:20:35 +0300 Subject: [PATCH 09/28] Second pass of fixes --- R/args.R | 7 ++++--- tests/testthat/test-install.R | 22 +++++++++++++--------- tests/testthat/test-model-sample.R | 6 +++--- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/R/args.R b/R/args.R index fb842c476..5402d7671 100644 --- a/R/args.R +++ b/R/args.R @@ -1019,14 +1019,15 @@ available_metrics <- function() { #' @param idx Chain id (only applicable for MCMC). compose_arg <- function(self, arg_name, cmdstan_arg_name = NULL, idx = NULL) { val <- self[[arg_name]] - if (os_is_wsl() && (arg_name %in% c("fitted_params", "metric_file"))) { - val <- sapply(val, wsl_path_compat) - } cmdstan_arg_name <- cmdstan_arg_name %||% arg_name if (is.null(val)) { return(NULL) } + + if (os_is_wsl() && (arg_name %in% c("fitted_params", "metric_file"))) { + val <- sapply(val, wsl_path_compat) + } if (!is.null(idx) && length(val) >= idx) { val <- val[idx] } diff --git a/tests/testthat/test-install.R b/tests/testthat/test-install.R index a09020a09..f2964cc98 100644 --- a/tests/testthat/test-install.R +++ b/tests/testthat/test-install.R @@ -1,5 +1,7 @@ context("install") +wsl_prefix <- ifelse(os_is_wsl(), "wsl-", "") + cmdstan_test_tarball_url <- Sys.getenv("CMDSTAN_TEST_TARBALL_URL") if (!nzchar(cmdstan_test_tarball_url)) { cmdstan_test_tarball_url <- NULL @@ -25,13 +27,13 @@ test_that("install_cmdstan() successfully installs cmdstan", { test_that("install_cmdstan() errors if installation already exists", { install_dir <- cmdstan_default_install_path() - dir <- file.path(install_dir, "cmdstan-2.23.0") + dir <- file.path(install_dir, paste0(wsl_prefix, "cmdstan-2.23.0")) if (!dir.exists(dir)) { dir.create(dir) } expect_warning( install_cmdstan(dir = install_dir, overwrite = FALSE, - version = "2.23.0"), + version = "2.23.0", wsl = os_is_wsl()), "An installation already exists", fixed = TRUE ) @@ -44,19 +46,19 @@ test_that("install_cmdstan() errors if it times out", { dir <- tempdir(check = TRUE) } ver <- latest_released_version() - dir_exists <- dir.exists(file.path(dir, paste0("cmdstan-",ver))) + dir_exists <- dir.exists(file.path(dir, paste0(wsl_prefix, "cmdstan-",ver))) # with quiet=TRUE expect_warning( expect_message( install_cmdstan(dir = dir, timeout = 1, quiet = TRUE, overwrite = dir_exists, - release_url = cmdstan_test_tarball_url), + release_url = cmdstan_test_tarball_url, wsl = os_is_wsl()), if (dir_exists) "* Removing the existing installation" else "* * Installing CmdStan from https://github.com", fixed = TRUE ), "increasing the value of the 'timeout' argument and running again with 'quiet=FALSE'", fixed = TRUE ) - dir_exists <- dir.exists(file.path(dir, paste0("cmdstan-",ver))) + dir_exists <- dir.exists(file.path(dir, paste0(wsl_prefix,"cmdstan-",ver))) # with quiet=FALSE expect_warning( expect_message( @@ -72,15 +74,16 @@ test_that("install_cmdstan() errors if it times out", { test_that("install_cmdstan() errors if invalid version or URL", { expect_error( - install_cmdstan(version = "2.23.2"), + install_cmdstan(version = "2.23.2", wsl = os_is_wsl()), "Download of CmdStan failed. Please check if the supplied version number is valid." ) expect_error( - install_cmdstan(release_url = "https://github.com/stan-dev/cmdstan/releases/download/v2.23.2/cmdstan-2.23.2.tar.gz"), + install_cmdstan(release_url = "https://github.com/stan-dev/cmdstan/releases/download/v2.23.2/cmdstan-2.23.2.tar.gz", + wsl = os_is_wsl()), "Download of CmdStan failed. Please check if the supplied release URL is valid." ) expect_error( - install_cmdstan(release_url = "https://github.com/stan-dev/cmdstan/releases/tag/v2.24.0"), + install_cmdstan(release_url = "https://github.com/stan-dev/cmdstan/releases/tag/v2.24.0", wsl = os_is_wsl()), "cmdstanr supports installing from .tar.gz archives only" ) }) @@ -95,7 +98,8 @@ test_that("install_cmdstan() works with version and release_url", { expect_message( expect_output( install_cmdstan(dir = dir, overwrite = TRUE, cores = 4, - release_url = "https://github.com/stan-dev/cmdstan/releases/download/v2.26.1/cmdstan-2.26.1.tar.gz"), + release_url = "https://github.com/stan-dev/cmdstan/releases/download/v2.26.1/cmdstan-2.26.1.tar.gz", + wsl = os_is_wsl()), "Compiling, linking C++ code", fixed = TRUE ), diff --git a/tests/testthat/test-model-sample.R b/tests/testthat/test-model-sample.R index ee6098ce9..60ecbd01a 100644 --- a/tests/testthat/test-model-sample.R +++ b/tests/testthat/test-model-sample.R @@ -246,7 +246,7 @@ test_that("print statements in transformed data work", { }' )) - out <- capture.output(fit <- mod$sample(iter_warmup = 1, iter_sampling = 1, chains = 1)) + out <- capture.output(fit <- mod$sample(iter_warmup = 1, iter_sampling = 3, chains = 1)) expect_true(any(grepl("*N = 2*", out))) }) @@ -276,7 +276,7 @@ test_that("seed works for multi chain sampling", { f <- write_stan_file(m, basename = "rngs.stan") mod <- cmdstan_model(f) utils::capture.output( - fit_sample <- mod$sample(chains = 2, iter_sampling = 1, iter_warmup = 100, seed = 2) + fit_sample <- mod$sample(chains = 2, iter_sampling = 3, iter_warmup = 100, seed = 2) ) chain_tdata_1 <- posterior::subset_draws(fit_sample$draws("tdata"), chain = 1) chain_tdata_2 <- posterior::subset_draws(fit_sample$draws("tdata"), chain = 2) @@ -286,7 +286,7 @@ test_that("seed works for multi chain sampling", { expect_false(all(chain_tdata_1 == chain_tdata_2)) utils::capture.output( - fit_sample <- mod$sample(chains = 2, iter_sampling = 1, iter_warmup = 100, + fit_sample <- mod$sample(chains = 2, iter_sampling = 3, iter_warmup = 100, seed = c(1, 2)) ) chain_tdata_1 <- posterior::subset_draws(fit_sample$draws("tdata"), chain = 1) From 21e53804c138dcf1b7f13316f51a813eb58fb245 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Sat, 16 Jul 2022 02:28:53 +0300 Subject: [PATCH 10/28] Third pass --- R/args.R | 2 +- R/csv.R | 7 ++++++ R/model.R | 2 +- R/utils.R | 38 ++++++++++++++++++++--------- tests/testthat/test-model-compile.R | 8 ++++-- 5 files changed, 41 insertions(+), 16 deletions(-) diff --git a/R/args.R b/R/args.R index 5402d7671..04b0bf8c0 100644 --- a/R/args.R +++ b/R/args.R @@ -1025,7 +1025,7 @@ compose_arg <- function(self, arg_name, cmdstan_arg_name = NULL, idx = NULL) { return(NULL) } - if (os_is_wsl() && (arg_name %in% c("fitted_params", "metric_file"))) { + if (os_is_wsl() && (arg_name %in% c("metric_file"))) { val <- sapply(val, wsl_path_compat) } if (!is.null(idx) && length(val) >= idx) { diff --git a/R/csv.R b/R/csv.R index e8e956871..2d89f7ac4 100644 --- a/R/csv.R +++ b/R/csv.R @@ -734,6 +734,13 @@ read_csv_metadata <- function(csv_file) { if (length(gradients) > 0) { csv_file_info$gradients <- gradients } + if (os_is_wsl()) { + csv_file_info$init <- wsl_path_compat(csv_file_info$init, revert = TRUE) + csv_file_info$profile_file <- wsl_path_compat(csv_file_info$profile_file, + revert = TRUE) + csv_file_info$fitted_params <- wsl_path_compat(csv_file_info$fitted_params, + revert = TRUE) + } csv_file_info } diff --git a/R/model.R b/R/model.R index 395ba8fb7..53551cb5b 100644 --- a/R/model.R +++ b/R/model.R @@ -731,7 +731,7 @@ check_syntax <- function(pedantic = FALSE, } temp_hpp_file <- tempfile(pattern = "model-", fileext = ".hpp") - stanc_options[["o"]] <- temp_hpp_file + stanc_options[["o"]] <- wsl_path_compat(temp_hpp_file) if (pedantic) { stanc_options[["warn-pedantic"]] <- TRUE diff --git a/R/utils.R b/R/utils.R index bb28f2aab..641192731 100644 --- a/R/utils.R +++ b/R/utils.R @@ -91,6 +91,8 @@ make_cmd <- function() { stanc_cmd <- function() { if (os_is_windows() && !os_is_wsl()) { "bin/stanc.exe" + } else if (os_is_wsl()) { + "./bin/stanc" } else { "bin/stanc" } @@ -154,19 +156,29 @@ absolute_path <- Vectorize(.absolute_path, USE.NAMES = FALSE) # When providing the model path to WSL, it needs to be in reference to the # to Windows mount point (/mnt/drive-letter) within the WSL install: # e.g., C:/Users/... -> /mnt/c/Users/... -wsl_path_compat <- function(path) { - if (!is.character(path)) { - path +wsl_path_compat <- function(path = NULL, revert = FALSE) { + if (!is.character(path) || is.null(path)) { + return(path) } - path_already_safe <- grepl("/mnt/", path) - if (os_is_wsl() && !isTRUE(path_already_safe) && !is.na(path)) { - abs_path <- repair_path(path) - trim_lead_whitespace <- gsub("^\\s*", "", abs_path) - drive_letter <- tolower(strtrim(trim_lead_whitespace, 1)) - path <- gsub(paste0(drive_letter, ":"), - paste0("/mnt/", drive_letter), - trim_lead_whitespace, - ignore.case = TRUE) + if (revert) { + if (!grepl("^/mnt/", path)) { + return(path) + } + strip_mnt <- gsub("^/mnt/", "", path) + drive_letter <- strtrim(strip_mnt, 1) + path <- gsub(paste0("^/mnt/", drive_letter), + paste0(toupper(drive_letter), ":"), + path) + } else { + path_already_safe <- grepl("^/mnt/", path) + if (os_is_wsl() && !isTRUE(path_already_safe) && !is.na(path)) { + abs_path <- repair_path(path) + drive_letter <- tolower(strtrim(abs_path, 1)) + path <- gsub(paste0(drive_letter, ":"), + paste0("/mnt/", drive_letter), + abs_path, + ignore.case = TRUE) + } } path } @@ -176,7 +188,9 @@ wsl_compatible_run <- function(...) { if (os_is_wsl()) { command <- run_args$command run_args$command <- "wsl" + run_args$args[1] <- paste0("'", run_args$args[1], "'") run_args$args <- c(command, run_args$args) + run_args$windows_verbatim_args <- TRUE } do.call(processx::run, run_args) } diff --git a/tests/testthat/test-model-compile.R b/tests/testthat/test-model-compile.R index 54a0c904d..748d38806 100644 --- a/tests/testthat/test-model-compile.R +++ b/tests/testthat/test-model-compile.R @@ -473,16 +473,20 @@ test_that("include_paths_stanc3_args() works", { dir.create(path_1) } path_1 <- repair_path(path_1) - expect_equal(include_paths_stanc3_args(path_1), paste0("--include-paths=", path_1)) + path_1_compare <- ifelse(os_is_wsl(), wsl_path_compat(path_1), path_1) + expect_equal( + include_paths_stanc3_args(path_1), + paste0("--include-paths=", path_1_compare)) path_2 <- file.path(tempdir(), "folder2") if (!dir.exists(path_2)) { dir.create(path_2) } path_2 <- repair_path(path_2) + path_2_compare <- ifelse(os_is_wsl(), wsl_path_compat(path_2), path_2) expect_equal( include_paths_stanc3_args(c(path_1, path_2)), c( - paste0("--include-paths=", path_1, ",", path_2) + paste0("--include-paths=", path_1_compare, ",", path_2_compare) ) ) }) From 620e8cec24058619f66cd28c5eaa35d8054f43af Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Sat, 16 Jul 2022 15:30:00 +0300 Subject: [PATCH 11/28] Fourth pass --- .github/workflows/R-CMD-check-wsl.yaml | 4 +++- R/args.R | 2 +- R/model.R | 6 +++--- R/run.R | 2 +- R/utils.R | 11 +++++------ tests/testthat/test-install.R | 12 ++++++++---- tests/testthat/test-model-compile.R | 8 ++++---- 7 files changed, 25 insertions(+), 20 deletions(-) diff --git a/.github/workflows/R-CMD-check-wsl.yaml b/.github/workflows/R-CMD-check-wsl.yaml index e3f1a76af..2779ec5a6 100644 --- a/.github/workflows/R-CMD-check-wsl.yaml +++ b/.github/workflows/R-CMD-check-wsl.yaml @@ -63,7 +63,9 @@ jobs: distribution: Ubuntu-22.04 use-cache: 'true' set-as-default: 'true' - additional-packages: build-essential + additional-packages: | + build-essential + libopenmpi-dev - name: Install cmdstan run: | diff --git a/R/args.R b/R/args.R index 04b0bf8c0..bd2e59a14 100644 --- a/R/args.R +++ b/R/args.R @@ -1025,7 +1025,7 @@ compose_arg <- function(self, arg_name, cmdstan_arg_name = NULL, idx = NULL) { return(NULL) } - if (os_is_wsl() && (arg_name %in% c("metric_file"))) { + if (os_is_wsl() && (arg_name %in% c("metric_file", "fitted_params"))) { val <- sapply(val, wsl_path_compat) } if (!is.null(idx) && length(val) >= idx) { diff --git a/R/model.R b/R/model.R index 53551cb5b..81d53101c 100644 --- a/R/model.R +++ b/R/model.R @@ -523,14 +523,14 @@ compile <- function(quiet = TRUE, stanc_options[["use-opencl"]] <- TRUE } if (!is.null(user_header)) { - cpp_options[["USER_HEADER"]] <- user_header + cpp_options[["USER_HEADER"]] <- wsl_path_compat(user_header) stanc_options[["allow-undefined"]] <- TRUE } if (!is.null(cpp_options[["USER_HEADER"]])) { - cpp_options[["USER_HEADER"]] <- absolute_path(cpp_options[["USER_HEADER"]]) + cpp_options[["USER_HEADER"]] <- wsl_path_compat(absolute_path(cpp_options[["USER_HEADER"]])) } if (!is.null(cpp_options[["user_header"]])) { - cpp_options[["user_header"]] <- absolute_path(cpp_options[["user_header"]]) + cpp_options[["user_header"]] <- wsl_path_compat(absolute_path(cpp_options[["user_header"]])) } if (is.null(stanc_options[["name"]])) { stanc_options[["name"]] <- paste0(self$model_name(), "_model") diff --git a/R/run.R b/R/run.R index 9b6cb3996..a95bf35ae 100644 --- a/R/run.R +++ b/R/run.R @@ -322,7 +322,7 @@ check_target_exe <- function(exe) { if (is.null(mpi_args)) { mpi_args <- list() } - mpi_args[["exe"]] <- self$exe_file() + mpi_args[["exe"]] <- wsl_path_compat(self$exe_file()) } if (procs$num_procs() == 1) { start_msg <- "Running MCMC with 1 chain" diff --git a/R/utils.R b/R/utils.R index 641192731..2b5e9feec 100644 --- a/R/utils.R +++ b/R/utils.R @@ -91,14 +91,11 @@ make_cmd <- function() { stanc_cmd <- function() { if (os_is_windows() && !os_is_wsl()) { "bin/stanc.exe" - } else if (os_is_wsl()) { - "./bin/stanc" } else { "bin/stanc" } } - # paths and extensions ---------------------------------------------------- # Replace `\\` with `/` in a path @@ -157,7 +154,7 @@ absolute_path <- Vectorize(.absolute_path, USE.NAMES = FALSE) # to Windows mount point (/mnt/drive-letter) within the WSL install: # e.g., C:/Users/... -> /mnt/c/Users/... wsl_path_compat <- function(path = NULL, revert = FALSE) { - if (!is.character(path) || is.null(path)) { + if (!is.character(path) || is.null(path) || !os_is_wsl()) { return(path) } if (revert) { @@ -188,9 +185,11 @@ wsl_compatible_run <- function(...) { if (os_is_wsl()) { command <- run_args$command run_args$command <- "wsl" - run_args$args[1] <- paste0("'", run_args$args[1], "'") + if (grepl("stanc", command)) { + run_args$args[1] <- paste0("'", run_args$args[1], "'") + run_args$windows_verbatim_args <- TRUE + } run_args$args <- c(command, run_args$args) - run_args$windows_verbatim_args <- TRUE } do.call(processx::run, run_args) } diff --git a/tests/testthat/test-install.R b/tests/testthat/test-install.R index f2964cc98..39192033d 100644 --- a/tests/testthat/test-install.R +++ b/tests/testthat/test-install.R @@ -16,7 +16,8 @@ test_that("install_cmdstan() successfully installs cmdstan", { expect_message( expect_output( install_cmdstan(dir = dir, cores = 2, quiet = FALSE, overwrite = TRUE, - release_url = cmdstan_test_tarball_url), + release_url = cmdstan_test_tarball_url, + wsl = os_is_wsl()), "Compiling, linking C++ code", fixed = TRUE ), @@ -63,7 +64,8 @@ test_that("install_cmdstan() errors if it times out", { expect_warning( expect_message( install_cmdstan(dir = dir, timeout = 1, quiet = FALSE, overwrite = dir_exists, - release_url = cmdstan_test_tarball_url), + release_url = cmdstan_test_tarball_url, + wsl = os_is_wsl()), if (dir_exists) "* Removing the existing installation" else "* * Installing CmdStan from https://github.com", fixed = TRUE ), @@ -112,7 +114,8 @@ test_that("install_cmdstan() works with version and release_url", { install_cmdstan(dir = dir, overwrite = TRUE, cores = 4, version = "2.27.0", # the URL is intentionally invalid to test that the version has higher priority - release_url = "https://github.com/stan-dev/cmdstan/releases/download/v2.27.3/cmdstan-2.27.3.tar.gz"), + release_url = "https://github.com/stan-dev/cmdstan/releases/download/v2.27.3/cmdstan-2.27.3.tar.gz", + wsl = os_is_wsl()), "Compiling, linking C++ code", fixed = TRUE ), @@ -122,7 +125,7 @@ test_that("install_cmdstan() works with version and release_url", { "version and release_url shouldn't both be specified", fixed = TRUE ) - expect_true(dir.exists(file.path(dir, "cmdstan-2.27.0"))) + expect_true(dir.exists(file.path(dir, paste0(wsl_prefix, "cmdstan-2.27.0")))) set_cmdstan_path(cmdstan_default_path()) }) @@ -152,6 +155,7 @@ test_that("toolchain checks on Unix work", { test_that("toolchain checks on Windows with RTools 3.5 work", { skip_if_not(os_is_windows()) + skip_if(os_is_wsl()) skip_if(R.Version()$major > "3") path_backup <- Sys.getenv("PATH") diff --git a/tests/testthat/test-model-compile.R b/tests/testthat/test-model-compile.R index 748d38806..c77271c6a 100644 --- a/tests/testthat/test-model-compile.R +++ b/tests/testthat/test-model-compile.R @@ -117,7 +117,7 @@ test_that("compilation works with include_paths", { test_that("name in STANCFLAGS is set correctly", { out <- utils::capture.output(mod$compile(quiet = FALSE, force_recompile = TRUE)) - if(os_is_windows()) { + if(os_is_windows() && !os_is_wsl()) { out_no_name <- "bin/stanc.exe --name='bernoulli_model' --o" out_name <- "bin/stanc.exe --name='bernoulli2_model' --o" } else { @@ -428,7 +428,7 @@ test_that("check_syntax() works with pedantic=TRUE", { }) test_that("compiliation errors if folder with the model name exists", { - skip_if(os_is_windows()) + skip_if(os_is_windows() && !os_is_wsl()) model_code <- " parameters { real y; @@ -449,7 +449,7 @@ test_that("compiliation errors if folder with the model name exists", { cmdstan_model(stan_file), "There is a subfolder matching the model name in the same folder as the model! Please remove or rename the subfolder and try again." ) - file.remove(exe) + unlink(exe, recursive = TRUE) }) test_that("cpp_options_to_compile_flags() works", { @@ -585,7 +585,7 @@ test_that("cmdstan_model errors with no args ", { }) test_that("cmdstan_model works with user_header", { - skip_if(os_is_macos() | os_is_windows()) + skip_if(os_is_macos() | (os_is_windows() && !os_is_wsl())) tmpfile <- tempfile(fileext = ".hpp") hpp <- " From a3a1c887e1236f76c42f7fdf1d7c50fc78c38e49 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Sat, 16 Jul 2022 16:54:39 +0300 Subject: [PATCH 12/28] Test wsl gzip fix --- .github/workflows/R-CMD-check-wsl.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/R-CMD-check-wsl.yaml b/.github/workflows/R-CMD-check-wsl.yaml index 2779ec5a6..5fbdeb93f 100644 --- a/.github/workflows/R-CMD-check-wsl.yaml +++ b/.github/workflows/R-CMD-check-wsl.yaml @@ -63,9 +63,11 @@ jobs: distribution: Ubuntu-22.04 use-cache: 'true' set-as-default: 'true' - additional-packages: | - build-essential - libopenmpi-dev + - name: WSL gzip version fix + run: | + sudo apt-get install -y gzip=1.10-4ubuntu1 + sudo apt-mark hold gzip + sudo apt-get install -y build-essential libopenmpi-dev - name: Install cmdstan run: | From 32fd9cc346b3bca0d3ae29465bb54763f39cb7b1 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Sat, 16 Jul 2022 17:07:26 +0300 Subject: [PATCH 13/28] Update wsl shell --- .github/workflows/R-CMD-check-wsl.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/R-CMD-check-wsl.yaml b/.github/workflows/R-CMD-check-wsl.yaml index 5fbdeb93f..03afd4dc7 100644 --- a/.github/workflows/R-CMD-check-wsl.yaml +++ b/.github/workflows/R-CMD-check-wsl.yaml @@ -65,9 +65,11 @@ jobs: set-as-default: 'true' - name: WSL gzip version fix run: | + sudo apt-get update sudo apt-get install -y gzip=1.10-4ubuntu1 sudo apt-mark hold gzip sudo apt-get install -y build-essential libopenmpi-dev + shell: : wsl-bash {0} - name: Install cmdstan run: | From d95b5d3c137203fbf134390c46518ae49df3e021 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Sat, 16 Jul 2022 17:08:12 +0300 Subject: [PATCH 14/28] Update wsl shell --- .github/workflows/R-CMD-check-wsl.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/R-CMD-check-wsl.yaml b/.github/workflows/R-CMD-check-wsl.yaml index 03afd4dc7..3df8b7013 100644 --- a/.github/workflows/R-CMD-check-wsl.yaml +++ b/.github/workflows/R-CMD-check-wsl.yaml @@ -69,7 +69,7 @@ jobs: sudo apt-get install -y gzip=1.10-4ubuntu1 sudo apt-mark hold gzip sudo apt-get install -y build-essential libopenmpi-dev - shell: : wsl-bash {0} + shell: wsl-bash {0} - name: Install cmdstan run: | From d9ebb71f3b8ddc436039ed5c2f9c2c7f2fa8aa55 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Sat, 16 Jul 2022 17:25:13 +0300 Subject: [PATCH 15/28] Test wsl gzip fix --- .github/workflows/R-CMD-check-wsl.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/R-CMD-check-wsl.yaml b/.github/workflows/R-CMD-check-wsl.yaml index 3df8b7013..96423e75d 100644 --- a/.github/workflows/R-CMD-check-wsl.yaml +++ b/.github/workflows/R-CMD-check-wsl.yaml @@ -63,11 +63,12 @@ jobs: distribution: Ubuntu-22.04 use-cache: 'true' set-as-default: 'true' - - name: WSL gzip version fix + - name: Install WSL Dependencies run: | + # Bugfix for current gzip under WSLv1: + # https://github.com/microsoft/WSL/issues/8219#issuecomment-1110508016 + echo -en '\x10' | sudo dd of=/usr/bin/gzip count=1 bs=1 conv=notrunc seek=$((0x189)) sudo apt-get update - sudo apt-get install -y gzip=1.10-4ubuntu1 - sudo apt-mark hold gzip sudo apt-get install -y build-essential libopenmpi-dev shell: wsl-bash {0} From c652398a4839053211426e9def336009addbe2be Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Sat, 16 Jul 2022 17:39:58 +0300 Subject: [PATCH 16/28] Fix getting exe info --- R/model.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/model.R b/R/model.R index 81d53101c..352d2d20d 100644 --- a/R/model.R +++ b/R/model.R @@ -1814,7 +1814,7 @@ model_compile_info <- function(exe_file) { tbb_path() ), ret <- wsl_compatible_run( - command = exe_file, + command = wsl_path_compat(exe_file), args = "info", error_on_status = FALSE ) From 9defd7de65b3b62f462f44f254b86a0ab70f651e Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Sat, 16 Jul 2022 18:52:09 +0300 Subject: [PATCH 17/28] mpi verbose --- .github/workflows/R-CMD-check-wsl.yaml | 2 +- .github/workflows/mpi.yaml | 122 +++++++++++++++++++++++++ R/args.R | 12 +-- R/csv.R | 6 +- R/install.R | 3 +- R/model.R | 20 ++-- R/run.R | 4 +- R/utils.R | 2 +- tests/testthat/test-model-compile.R | 4 +- tests/testthat/test-model-sample.R | 6 +- 10 files changed, 151 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/mpi.yaml diff --git a/.github/workflows/R-CMD-check-wsl.yaml b/.github/workflows/R-CMD-check-wsl.yaml index 96423e75d..1d3c5ac5b 100644 --- a/.github/workflows/R-CMD-check-wsl.yaml +++ b/.github/workflows/R-CMD-check-wsl.yaml @@ -99,5 +99,5 @@ jobs: if: failure() uses: actions/upload-artifact@v3 with: - name: ${{ runner.os }}-r${{ matrix.config.r }}-results + name: wsl-backend-results path: check diff --git a/.github/workflows/mpi.yaml b/.github/workflows/mpi.yaml new file mode 100644 index 000000000..e20bb2f6d --- /dev/null +++ b/.github/workflows/mpi.yaml @@ -0,0 +1,122 @@ +--- +# Github Actions workflow to check CmdStanR +# yamllint disable rule:line-length + +name: Unit tests - WSL Backend + +'on': + push: + branches: + - wsl-support + pull_request: + branches: + - master + +jobs: + R-CMD-check: + if: "! contains(github.event.head_commit.message, '[ci skip]')" + runs-on: windows-latest + + name: WSL1 Backend + + env: + R_REMOTES_NO_ERRORS_FROM_WARNINGS: true + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + NOT_CRAN: true + + steps: + - name: cmdstan env vars + run: | + echo "CMDSTAN_PATH=${HOME}/.cmdstan" >> $GITHUB_ENV + shell: bash + + - uses: n1hility/cancel-previous-runs@v2 + with: + token: ${{ secrets.GITHUB_TOKEN }} + if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/master'" + + - uses: actions/checkout@v3 + + - uses: r-lib/actions/setup-r@v2.2.3 + with: + r-version: 'release' + rtools-version: '42' + - uses: r-lib/actions/setup-pandoc@v1 + + - name: Query dependencies + run: | + install.packages('remotes') + saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2) + writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version") + shell: Rscript {0} + + - name: Install dependencies + run: | + remotes::install_deps(dependencies = TRUE) + remotes::install_cran("rcmdcheck") + remotes::install_local(path = ".") + install.packages("curl") + shell: Rscript {0} + + - uses: Vampire/setup-wsl@v1 + with: + distribution: Ubuntu-22.04 + use-cache: 'true' + set-as-default: 'true' + - name: Install WSL Dependencies + run: | + # Bugfix for current gzip under WSLv1: + # https://github.com/microsoft/WSL/issues/8219#issuecomment-1110508016 + echo -en '\x10' | sudo dd of=/usr/bin/gzip count=1 bs=1 conv=notrunc seek=$((0x189)) + sudo apt-get update + sudo apt-get install -y build-essential libopenmpi-dev + shell: wsl-bash {0} + + - name: Install cmdstan + run: | + cmdstanr::install_cmdstan(cores = 2, wsl = TRUE, overwrite = TRUE) + shell: Rscript {0} + + - name: Session info + run: | + options(width = 100) + pkgs <- installed.packages()[, "Package"] + sessioninfo::session_info(pkgs, include_base = TRUE) + shell: Rscript {0} + + - name: Check + env: + _R_CHECK_CRAN_INCOMING_: false + run: | + options("cmdstanr_verbose"=TRUE) + mpi_file <- write_stan_file(" + functions { + vector test(vector beta, vector theta, real[] x, int[] y) { + return theta; + } + } + transformed data { + vector[4] a; + vector[5] b[4] = {[1,1,1,1,1]', [2,2,2,2,2]', [3,3,3,3,3]', [4,4,4,4,4]'}; + real x[4,4]; + int y[4,4]; + } + parameters { + real beta; + } + model { + beta ~ std_normal(); + } + generated quantities { + vector[20] c = map_rect(test, a, b, x, y); + } + ") + tbb_cxx_type <- "gcc" + cpp_options = list(cxx="mpicxx", stan_mpi = TRUE, tbb_cxx_type=tbb_cxx_type) + mod_mpi <- cmdstan_model(mpi_file, cpp_options = cpp_options) + + f <- mod_mpi$sample_mpi(chains = 1, mpi_args = list("n" = 1)) + + f$metadata()$mpi_enable + + shell: Rscript {0} diff --git a/R/args.R b/R/args.R index bd2e59a14..bb311ff5f 100644 --- a/R/args.R +++ b/R/args.R @@ -138,16 +138,16 @@ CmdStanArgs <- R6::R6Class( } if (!is.null(self$init)) { - args$init <- paste0("init=", wsl_path_compat(self$init[idx])) + args$init <- paste0("init=", wsl_safe_path(self$init[idx])) } if (!is.null(self$data_file)) { - args$data <- c("data", paste0("file=", wsl_path_compat(self$data_file))) + args$data <- c("data", paste0("file=", wsl_safe_path(self$data_file))) } - args$output <- c("output", paste0("file=", wsl_path_compat(output_file))) + args$output <- c("output", paste0("file=", wsl_safe_path(output_file))) if (!is.null(latent_dynamics_file)) { - args$output <- c(args$output, paste0("diagnostic_file=", wsl_path_compat(latent_dynamics_file))) + args$output <- c(args$output, paste0("diagnostic_file=", wsl_safe_path(latent_dynamics_file))) } if (!is.null(self$refresh)) { args$output <- c(args$output, paste0("refresh=", self$refresh)) @@ -158,7 +158,7 @@ CmdStanArgs <- R6::R6Class( } if (!is.null(profile_file)) { - args$output <- c(args$output, paste0("profile_file=", wsl_path_compat(profile_file))) + args$output <- c(args$output, paste0("profile_file=", wsl_safe_path(profile_file))) } if (!is.null(self$opencl_ids)) { args$opencl <- c("opencl", paste0("platform=", self$opencl_ids[1]), paste0("device=", self$opencl_ids[2])) @@ -1026,7 +1026,7 @@ compose_arg <- function(self, arg_name, cmdstan_arg_name = NULL, idx = NULL) { } if (os_is_wsl() && (arg_name %in% c("metric_file", "fitted_params"))) { - val <- sapply(val, wsl_path_compat) + val <- sapply(val, wsl_safe_path) } if (!is.null(idx) && length(val) >= idx) { val <- val[idx] diff --git a/R/csv.R b/R/csv.R index 2d89f7ac4..c7a9007a5 100644 --- a/R/csv.R +++ b/R/csv.R @@ -735,10 +735,10 @@ read_csv_metadata <- function(csv_file) { csv_file_info$gradients <- gradients } if (os_is_wsl()) { - csv_file_info$init <- wsl_path_compat(csv_file_info$init, revert = TRUE) - csv_file_info$profile_file <- wsl_path_compat(csv_file_info$profile_file, + csv_file_info$init <- wsl_safe_path(csv_file_info$init, revert = TRUE) + csv_file_info$profile_file <- wsl_safe_path(csv_file_info$profile_file, revert = TRUE) - csv_file_info$fitted_params <- wsl_path_compat(csv_file_info$fitted_params, + csv_file_info$fitted_params <- wsl_safe_path(csv_file_info$fitted_params, revert = TRUE) } csv_file_info diff --git a/R/install.R b/R/install.R index 01b94e771..15fa0394a 100644 --- a/R/install.R +++ b/R/install.R @@ -389,7 +389,6 @@ build_cmdstan <- function(dir, } else { run_cmd <- make_cmd() } - withr::with_path( c( toolchain_PATH_env_var(), @@ -419,7 +418,7 @@ clean_cmdstan <- function(dir = cmdstan_path(), ), wsl_compatible_run( command = make_cmd(), - args = "clean_all", + args = "clean-all", wd = dir, echo_cmd = is_verbose_mode(), echo = !quiet || is_verbose_mode(), diff --git a/R/model.R b/R/model.R index 352d2d20d..fbb4177dc 100644 --- a/R/model.R +++ b/R/model.R @@ -523,14 +523,14 @@ compile <- function(quiet = TRUE, stanc_options[["use-opencl"]] <- TRUE } if (!is.null(user_header)) { - cpp_options[["USER_HEADER"]] <- wsl_path_compat(user_header) + cpp_options[["USER_HEADER"]] <- wsl_safe_path(user_header) stanc_options[["allow-undefined"]] <- TRUE } if (!is.null(cpp_options[["USER_HEADER"]])) { - cpp_options[["USER_HEADER"]] <- wsl_path_compat(absolute_path(cpp_options[["USER_HEADER"]])) + cpp_options[["USER_HEADER"]] <- wsl_safe_path(absolute_path(cpp_options[["USER_HEADER"]])) } if (!is.null(cpp_options[["user_header"]])) { - cpp_options[["user_header"]] <- wsl_path_compat(absolute_path(cpp_options[["user_header"]])) + cpp_options[["user_header"]] <- wsl_safe_path(absolute_path(cpp_options[["user_header"]])) } if (is.null(stanc_options[["name"]])) { stanc_options[["name"]] <- paste0(self$model_name(), "_model") @@ -554,7 +554,7 @@ compile <- function(quiet = TRUE, ), run_log <- wsl_compatible_run( command = make_cmd(), - args = c(wsl_path_compat(tmp_exe), + args = c(wsl_safe_path(tmp_exe), cpp_options_to_compile_flags(cpp_options), stancflags_val), wd = cmdstan_path(), @@ -731,7 +731,7 @@ check_syntax <- function(pedantic = FALSE, } temp_hpp_file <- tempfile(pattern = "model-", fileext = ".hpp") - stanc_options[["o"]] <- wsl_path_compat(temp_hpp_file) + stanc_options[["o"]] <- wsl_safe_path(temp_hpp_file) if (pedantic) { stanc_options[["warn-pedantic"]] <- TRUE @@ -761,7 +761,7 @@ check_syntax <- function(pedantic = FALSE, ), run_log <- wsl_compatible_run( command = stanc_cmd(), - args = c(wsl_path_compat(self$stan_file()), stanc_built_options, stancflags_val), + args = c(wsl_safe_path(self$stan_file()), stanc_built_options, stancflags_val), wd = cmdstan_path(), echo = is_verbose_mode(), echo_cmd = is_verbose_mode(), @@ -903,7 +903,7 @@ format <- function(overwrite_file = FALSE, ), run_log <- wsl_compatible_run( command = stanc_cmd(), - args = c(wsl_path_compat(self$stan_file()), stanc_built_options, + args = c(wsl_safe_path(self$stan_file()), stanc_built_options, stancflags_val), wd = cmdstan_path(), echo = is_verbose_mode(), @@ -1761,7 +1761,7 @@ include_paths_stanc3_args <- function(include_paths = NULL) { if (!is.null(include_paths)) { checkmate::assert_directory_exists(include_paths, access = "r") include_paths <- absolute_path(include_paths) - include_paths <- sapply(include_paths, wsl_path_compat) + include_paths <- sapply(include_paths, wsl_safe_path) paths_w_space <- grep(" ", include_paths) include_paths[paths_w_space] <- paste0("'", include_paths[paths_w_space], "'") include_paths <- paste0(include_paths, collapse = ",") @@ -1785,7 +1785,7 @@ model_variables <- function(stan_file, include_paths = NULL, allow_undefined = F stan_file <- stan_file run_log <- wsl_compatible_run( command = stanc_cmd(), - args = c(wsl_path_compat(stan_file), "--info", include_paths_stanc3_args(include_paths), + args = c(wsl_safe_path(stan_file), "--info", include_paths_stanc3_args(include_paths), allow_undefined_arg), wd = cmdstan_path(), echo = FALSE, @@ -1814,7 +1814,7 @@ model_compile_info <- function(exe_file) { tbb_path() ), ret <- wsl_compatible_run( - command = wsl_path_compat(exe_file), + command = wsl_safe_path(exe_file), args = "info", error_on_status = FALSE ) diff --git a/R/run.R b/R/run.R index a95bf35ae..093cba7ee 100644 --- a/R/run.R +++ b/R/run.R @@ -235,7 +235,7 @@ CmdStanRun <- R6::R6Class( command = target_exe, args = c( sapply(self$output_files(include_failed = FALSE), - wsl_path_compat), + wsl_safe_path), flags), wd = cmdstan_path(), echo = TRUE, @@ -322,7 +322,7 @@ check_target_exe <- function(exe) { if (is.null(mpi_args)) { mpi_args <- list() } - mpi_args[["exe"]] <- wsl_path_compat(self$exe_file()) + mpi_args[["exe"]] <- wsl_safe_path(self$exe_file()) } if (procs$num_procs() == 1) { start_msg <- "Running MCMC with 1 chain" diff --git a/R/utils.R b/R/utils.R index 2b5e9feec..8a479eb40 100644 --- a/R/utils.R +++ b/R/utils.R @@ -153,7 +153,7 @@ absolute_path <- Vectorize(.absolute_path, USE.NAMES = FALSE) # When providing the model path to WSL, it needs to be in reference to the # to Windows mount point (/mnt/drive-letter) within the WSL install: # e.g., C:/Users/... -> /mnt/c/Users/... -wsl_path_compat <- function(path = NULL, revert = FALSE) { +wsl_safe_path <- function(path = NULL, revert = FALSE) { if (!is.character(path) || is.null(path) || !os_is_wsl()) { return(path) } diff --git a/tests/testthat/test-model-compile.R b/tests/testthat/test-model-compile.R index c77271c6a..e2bb85283 100644 --- a/tests/testthat/test-model-compile.R +++ b/tests/testthat/test-model-compile.R @@ -473,7 +473,7 @@ test_that("include_paths_stanc3_args() works", { dir.create(path_1) } path_1 <- repair_path(path_1) - path_1_compare <- ifelse(os_is_wsl(), wsl_path_compat(path_1), path_1) + path_1_compare <- ifelse(os_is_wsl(), wsl_safe_path(path_1), path_1) expect_equal( include_paths_stanc3_args(path_1), paste0("--include-paths=", path_1_compare)) @@ -482,7 +482,7 @@ test_that("include_paths_stanc3_args() works", { dir.create(path_2) } path_2 <- repair_path(path_2) - path_2_compare <- ifelse(os_is_wsl(), wsl_path_compat(path_2), path_2) + path_2_compare <- ifelse(os_is_wsl(), wsl_safe_path(path_2), path_2) expect_equal( include_paths_stanc3_args(c(path_1, path_2)), c( diff --git a/tests/testthat/test-model-sample.R b/tests/testthat/test-model-sample.R index 60ecbd01a..ee6098ce9 100644 --- a/tests/testthat/test-model-sample.R +++ b/tests/testthat/test-model-sample.R @@ -246,7 +246,7 @@ test_that("print statements in transformed data work", { }' )) - out <- capture.output(fit <- mod$sample(iter_warmup = 1, iter_sampling = 3, chains = 1)) + out <- capture.output(fit <- mod$sample(iter_warmup = 1, iter_sampling = 1, chains = 1)) expect_true(any(grepl("*N = 2*", out))) }) @@ -276,7 +276,7 @@ test_that("seed works for multi chain sampling", { f <- write_stan_file(m, basename = "rngs.stan") mod <- cmdstan_model(f) utils::capture.output( - fit_sample <- mod$sample(chains = 2, iter_sampling = 3, iter_warmup = 100, seed = 2) + fit_sample <- mod$sample(chains = 2, iter_sampling = 1, iter_warmup = 100, seed = 2) ) chain_tdata_1 <- posterior::subset_draws(fit_sample$draws("tdata"), chain = 1) chain_tdata_2 <- posterior::subset_draws(fit_sample$draws("tdata"), chain = 2) @@ -286,7 +286,7 @@ test_that("seed works for multi chain sampling", { expect_false(all(chain_tdata_1 == chain_tdata_2)) utils::capture.output( - fit_sample <- mod$sample(chains = 2, iter_sampling = 3, iter_warmup = 100, + fit_sample <- mod$sample(chains = 2, iter_sampling = 1, iter_warmup = 100, seed = c(1, 2)) ) chain_tdata_1 <- posterior::subset_draws(fit_sample$draws("tdata"), chain = 1) From 426d0fdf6184435eda8bddb4420d3afc5ceb69a0 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Sat, 16 Jul 2022 19:14:32 +0300 Subject: [PATCH 18/28] mpi verbose --- .github/workflows/mpi.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/mpi.yaml b/.github/workflows/mpi.yaml index e20bb2f6d..bcdd2c744 100644 --- a/.github/workflows/mpi.yaml +++ b/.github/workflows/mpi.yaml @@ -88,6 +88,7 @@ jobs: env: _R_CHECK_CRAN_INCOMING_: false run: | + library(cmdstanr) options("cmdstanr_verbose"=TRUE) mpi_file <- write_stan_file(" functions { From 3901b6bef02208ff5e01ee392aacc24838149b9d Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Sat, 16 Jul 2022 19:45:42 +0300 Subject: [PATCH 19/28] mpi verbose --- .github/workflows/mpi.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/mpi.yaml b/.github/workflows/mpi.yaml index bcdd2c744..8639551ec 100644 --- a/.github/workflows/mpi.yaml +++ b/.github/workflows/mpi.yaml @@ -116,6 +116,11 @@ jobs: cpp_options = list(cxx="mpicxx", stan_mpi = TRUE, tbb_cxx_type=tbb_cxx_type) mod_mpi <- cmdstan_model(mpi_file, cpp_options = cpp_options) + if (os_is_wsl()) { + # Default GHA WSL install runs as root, which MPI discourages + # Specify that this is safe to ignore for this test + Sys.setenv("WSLENV"="OMPI_ALLOW_RUN_AS_ROOT=1/u:OMPI_ALLOW_RUN_AS_ROOT=1/u") + } f <- mod_mpi$sample_mpi(chains = 1, mpi_args = list("n" = 1)) f$metadata()$mpi_enable From 91ce8242f5fd52649d1bf7c24d5af4507921d62d Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Sat, 16 Jul 2022 20:09:48 +0300 Subject: [PATCH 20/28] mpi verbose --- .github/workflows/mpi.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mpi.yaml b/.github/workflows/mpi.yaml index 8639551ec..40c56b37e 100644 --- a/.github/workflows/mpi.yaml +++ b/.github/workflows/mpi.yaml @@ -116,11 +116,11 @@ jobs: cpp_options = list(cxx="mpicxx", stan_mpi = TRUE, tbb_cxx_type=tbb_cxx_type) mod_mpi <- cmdstan_model(mpi_file, cpp_options = cpp_options) - if (os_is_wsl()) { + #if (os_is_wsl()) { # Default GHA WSL install runs as root, which MPI discourages # Specify that this is safe to ignore for this test Sys.setenv("WSLENV"="OMPI_ALLOW_RUN_AS_ROOT=1/u:OMPI_ALLOW_RUN_AS_ROOT=1/u") - } + #} f <- mod_mpi$sample_mpi(chains = 1, mpi_args = list("n" = 1)) f$metadata()$mpi_enable From 13d3564106edea309fb411dae2245aafb368eab9 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Sat, 16 Jul 2022 20:32:18 +0300 Subject: [PATCH 21/28] mpi verbose --- .github/workflows/mpi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mpi.yaml b/.github/workflows/mpi.yaml index 40c56b37e..698389794 100644 --- a/.github/workflows/mpi.yaml +++ b/.github/workflows/mpi.yaml @@ -119,7 +119,7 @@ jobs: #if (os_is_wsl()) { # Default GHA WSL install runs as root, which MPI discourages # Specify that this is safe to ignore for this test - Sys.setenv("WSLENV"="OMPI_ALLOW_RUN_AS_ROOT=1/u:OMPI_ALLOW_RUN_AS_ROOT=1/u") + Sys.setenv("WSLENV"="OMPI_ALLOW_RUN_AS_ROOT=1/u:OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1/u") #} f <- mod_mpi$sample_mpi(chains = 1, mpi_args = list("n" = 1)) From a3c0e7cf52b0c7de85710eda589ea37b2d9a5b3a Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Sat, 16 Jul 2022 21:03:47 +0300 Subject: [PATCH 22/28] mpi verbose --- .github/workflows/mpi.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/mpi.yaml b/.github/workflows/mpi.yaml index 698389794..71dd8df57 100644 --- a/.github/workflows/mpi.yaml +++ b/.github/workflows/mpi.yaml @@ -115,13 +115,15 @@ jobs: tbb_cxx_type <- "gcc" cpp_options = list(cxx="mpicxx", stan_mpi = TRUE, tbb_cxx_type=tbb_cxx_type) mod_mpi <- cmdstan_model(mpi_file, cpp_options = cpp_options) - - #if (os_is_wsl()) { + mpi_args <- list("n" = 1) + if (os_is_wsl()) { # Default GHA WSL install runs as root, which MPI discourages # Specify that this is safe to ignore for this test - Sys.setenv("WSLENV"="OMPI_ALLOW_RUN_AS_ROOT=1/u:OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1/u") - #} - f <- mod_mpi$sample_mpi(chains = 1, mpi_args = list("n" = 1)) + Sys.setenv(OMPI_ALLOW_RUN_AS_ROOT=1) + Sys.setenv(OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1) + Sys.setenv("WSLENV"="OMPI_ALLOW_RUN_AS_ROOT/u:OMPI_ALLOW_RUN_AS_ROOT_CONFIRM/u") + } + f <- mod_mpi$sample_mpi(chains = 1, mpi_args) f$metadata()$mpi_enable From f0e747916ec98d7714f34787ca3093ddce40cb6c Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Sat, 16 Jul 2022 21:04:03 +0300 Subject: [PATCH 23/28] mpi verbose --- .github/workflows/mpi.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mpi.yaml b/.github/workflows/mpi.yaml index 71dd8df57..bdab74594 100644 --- a/.github/workflows/mpi.yaml +++ b/.github/workflows/mpi.yaml @@ -119,8 +119,8 @@ jobs: if (os_is_wsl()) { # Default GHA WSL install runs as root, which MPI discourages # Specify that this is safe to ignore for this test - Sys.setenv(OMPI_ALLOW_RUN_AS_ROOT=1) - Sys.setenv(OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1) + Sys.setenv("OMPI_ALLOW_RUN_AS_ROOT"=1) + Sys.setenv("OMPI_ALLOW_RUN_AS_ROOT_CONFIRM"=1) Sys.setenv("WSLENV"="OMPI_ALLOW_RUN_AS_ROOT/u:OMPI_ALLOW_RUN_AS_ROOT_CONFIRM/u") } f <- mod_mpi$sample_mpi(chains = 1, mpi_args) From 22728e8828af9993027c64590c934f844071ee74 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Sat, 16 Jul 2022 21:26:45 +0300 Subject: [PATCH 24/28] mpi verbose --- .github/workflows/mpi.yaml | 7 +++---- tests/testthat/test-model-sample_mpi.R | 9 +++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/mpi.yaml b/.github/workflows/mpi.yaml index bdab74594..df8896909 100644 --- a/.github/workflows/mpi.yaml +++ b/.github/workflows/mpi.yaml @@ -115,15 +115,14 @@ jobs: tbb_cxx_type <- "gcc" cpp_options = list(cxx="mpicxx", stan_mpi = TRUE, tbb_cxx_type=tbb_cxx_type) mod_mpi <- cmdstan_model(mpi_file, cpp_options = cpp_options) - mpi_args <- list("n" = 1) - if (os_is_wsl()) { + # if (os_is_wsl()) { # Default GHA WSL install runs as root, which MPI discourages # Specify that this is safe to ignore for this test Sys.setenv("OMPI_ALLOW_RUN_AS_ROOT"=1) Sys.setenv("OMPI_ALLOW_RUN_AS_ROOT_CONFIRM"=1) Sys.setenv("WSLENV"="OMPI_ALLOW_RUN_AS_ROOT/u:OMPI_ALLOW_RUN_AS_ROOT_CONFIRM/u") - } - f <- mod_mpi$sample_mpi(chains = 1, mpi_args) + # } + f <- mod_mpi$sample_mpi(chains = 1, list("n" = 1)) f$metadata()$mpi_enable diff --git a/tests/testthat/test-model-sample_mpi.R b/tests/testthat/test-model-sample_mpi.R index e0b8ac8a5..6b348f2ed 100644 --- a/tests/testthat/test-model-sample_mpi.R +++ b/tests/testthat/test-model-sample_mpi.R @@ -31,6 +31,15 @@ test_that("sample_mpi() works", { } cpp_options = list(cxx="mpicxx", stan_mpi = TRUE, tbb_cxx_type=tbb_cxx_type) mod_mpi <- cmdstan_model(mpi_file, cpp_options = cpp_options) + + if (os_is_wsl()) { + # Default GHA WSL install runs as root, which MPI discourages + # Specify that this is safe to ignore for this test + Sys.setenv("OMPI_ALLOW_RUN_AS_ROOT"=1) + Sys.setenv("OMPI_ALLOW_RUN_AS_ROOT_CONFIRM"=1) + Sys.setenv("WSLENV"="OMPI_ALLOW_RUN_AS_ROOT/u:OMPI_ALLOW_RUN_AS_ROOT_CONFIRM/u") + } + utils::capture.output( f <- mod_mpi$sample_mpi(chains = 1, mpi_args = list("n" = 1)) ) From c2ef835e167727fbc7a946fe80e398165088f6a1 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Sat, 16 Jul 2022 22:28:31 +0300 Subject: [PATCH 25/28] Final check --- .github/workflows/R-CMD-check-wsl.yaml | 6 +- .github/workflows/Test-coverage.yaml | 2 +- .github/workflows/mpi.yaml | 129 ------------------------- R/csv.R | 6 +- R/install.R | 1 - R/model.R | 10 +- R/path.R | 5 + R/utils.R | 10 +- 8 files changed, 23 insertions(+), 146 deletions(-) delete mode 100644 .github/workflows/mpi.yaml diff --git a/.github/workflows/R-CMD-check-wsl.yaml b/.github/workflows/R-CMD-check-wsl.yaml index 1d3c5ac5b..71a42519b 100644 --- a/.github/workflows/R-CMD-check-wsl.yaml +++ b/.github/workflows/R-CMD-check-wsl.yaml @@ -13,11 +13,11 @@ name: Unit tests - WSL Backend - master jobs: - R-CMD-check: + WSL-R-CMD-check: if: "! contains(github.event.head_commit.message, '[ci skip]')" runs-on: windows-latest - name: WSL1 Backend + name: windows-latest-WSLv1 env: R_REMOTES_NO_ERRORS_FROM_WARNINGS: true @@ -65,7 +65,7 @@ jobs: set-as-default: 'true' - name: Install WSL Dependencies run: | - # Bugfix for current gzip under WSLv1: + # Bugfix for current gzip (for unpacking apt packages) under WSLv1: # https://github.com/microsoft/WSL/issues/8219#issuecomment-1110508016 echo -en '\x10' | sudo dd of=/usr/bin/gzip count=1 bs=1 conv=notrunc seek=$((0x189)) sudo apt-get update diff --git a/.github/workflows/Test-coverage.yaml b/.github/workflows/Test-coverage.yaml index 44e34d86d..efffffa56 100644 --- a/.github/workflows/Test-coverage.yaml +++ b/.github/workflows/Test-coverage.yaml @@ -7,7 +7,7 @@ name: Test coverage 'on': push: branches: - - wsl-support + - master pull_request: branches: - master diff --git a/.github/workflows/mpi.yaml b/.github/workflows/mpi.yaml deleted file mode 100644 index df8896909..000000000 --- a/.github/workflows/mpi.yaml +++ /dev/null @@ -1,129 +0,0 @@ ---- -# Github Actions workflow to check CmdStanR -# yamllint disable rule:line-length - -name: Unit tests - WSL Backend - -'on': - push: - branches: - - wsl-support - pull_request: - branches: - - master - -jobs: - R-CMD-check: - if: "! contains(github.event.head_commit.message, '[ci skip]')" - runs-on: windows-latest - - name: WSL1 Backend - - env: - R_REMOTES_NO_ERRORS_FROM_WARNINGS: true - GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - NOT_CRAN: true - - steps: - - name: cmdstan env vars - run: | - echo "CMDSTAN_PATH=${HOME}/.cmdstan" >> $GITHUB_ENV - shell: bash - - - uses: n1hility/cancel-previous-runs@v2 - with: - token: ${{ secrets.GITHUB_TOKEN }} - if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/master'" - - - uses: actions/checkout@v3 - - - uses: r-lib/actions/setup-r@v2.2.3 - with: - r-version: 'release' - rtools-version: '42' - - uses: r-lib/actions/setup-pandoc@v1 - - - name: Query dependencies - run: | - install.packages('remotes') - saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2) - writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version") - shell: Rscript {0} - - - name: Install dependencies - run: | - remotes::install_deps(dependencies = TRUE) - remotes::install_cran("rcmdcheck") - remotes::install_local(path = ".") - install.packages("curl") - shell: Rscript {0} - - - uses: Vampire/setup-wsl@v1 - with: - distribution: Ubuntu-22.04 - use-cache: 'true' - set-as-default: 'true' - - name: Install WSL Dependencies - run: | - # Bugfix for current gzip under WSLv1: - # https://github.com/microsoft/WSL/issues/8219#issuecomment-1110508016 - echo -en '\x10' | sudo dd of=/usr/bin/gzip count=1 bs=1 conv=notrunc seek=$((0x189)) - sudo apt-get update - sudo apt-get install -y build-essential libopenmpi-dev - shell: wsl-bash {0} - - - name: Install cmdstan - run: | - cmdstanr::install_cmdstan(cores = 2, wsl = TRUE, overwrite = TRUE) - shell: Rscript {0} - - - name: Session info - run: | - options(width = 100) - pkgs <- installed.packages()[, "Package"] - sessioninfo::session_info(pkgs, include_base = TRUE) - shell: Rscript {0} - - - name: Check - env: - _R_CHECK_CRAN_INCOMING_: false - run: | - library(cmdstanr) - options("cmdstanr_verbose"=TRUE) - mpi_file <- write_stan_file(" - functions { - vector test(vector beta, vector theta, real[] x, int[] y) { - return theta; - } - } - transformed data { - vector[4] a; - vector[5] b[4] = {[1,1,1,1,1]', [2,2,2,2,2]', [3,3,3,3,3]', [4,4,4,4,4]'}; - real x[4,4]; - int y[4,4]; - } - parameters { - real beta; - } - model { - beta ~ std_normal(); - } - generated quantities { - vector[20] c = map_rect(test, a, b, x, y); - } - ") - tbb_cxx_type <- "gcc" - cpp_options = list(cxx="mpicxx", stan_mpi = TRUE, tbb_cxx_type=tbb_cxx_type) - mod_mpi <- cmdstan_model(mpi_file, cpp_options = cpp_options) - # if (os_is_wsl()) { - # Default GHA WSL install runs as root, which MPI discourages - # Specify that this is safe to ignore for this test - Sys.setenv("OMPI_ALLOW_RUN_AS_ROOT"=1) - Sys.setenv("OMPI_ALLOW_RUN_AS_ROOT_CONFIRM"=1) - Sys.setenv("WSLENV"="OMPI_ALLOW_RUN_AS_ROOT/u:OMPI_ALLOW_RUN_AS_ROOT_CONFIRM/u") - # } - f <- mod_mpi$sample_mpi(chains = 1, list("n" = 1)) - - f$metadata()$mpi_enable - - shell: Rscript {0} diff --git a/R/csv.R b/R/csv.R index c7a9007a5..a1bb466dc 100644 --- a/R/csv.R +++ b/R/csv.R @@ -734,12 +734,14 @@ read_csv_metadata <- function(csv_file) { if (length(gradients) > 0) { csv_file_info$gradients <- gradients } + + # Revert any WSL-updated paths before returning the metadata if (os_is_wsl()) { csv_file_info$init <- wsl_safe_path(csv_file_info$init, revert = TRUE) csv_file_info$profile_file <- wsl_safe_path(csv_file_info$profile_file, - revert = TRUE) + revert = TRUE) csv_file_info$fitted_params <- wsl_safe_path(csv_file_info$fitted_params, - revert = TRUE) + revert = TRUE) } csv_file_info } diff --git a/R/install.R b/R/install.R index 15fa0394a..597b29eed 100644 --- a/R/install.R +++ b/R/install.R @@ -213,7 +213,6 @@ install_cmdstan <- function(dir = NULL, "\nrebuild_cmdstan(cores = ...)" ) } - if (isTRUE(wsl)) { Sys.unsetenv("CMDSTANR_USE_WSL") } diff --git a/R/model.R b/R/model.R index fbb4177dc..5346346b9 100644 --- a/R/model.R +++ b/R/model.R @@ -1760,8 +1760,7 @@ include_paths_stanc3_args <- function(include_paths = NULL) { stancflags <- NULL if (!is.null(include_paths)) { checkmate::assert_directory_exists(include_paths, access = "r") - include_paths <- absolute_path(include_paths) - include_paths <- sapply(include_paths, wsl_safe_path) + include_paths <- sapply(absolute_path(include_paths), wsl_safe_path) paths_w_space <- grep(" ", include_paths) include_paths[paths_w_space] <- paste0("'", include_paths[paths_w_space], "'") include_paths <- paste0(include_paths, collapse = ",") @@ -1782,11 +1781,12 @@ model_variables <- function(stan_file, include_paths = NULL, allow_undefined = F allow_undefined_arg <- NULL } out_file <- tempfile(fileext = ".json") - stan_file <- stan_file run_log <- wsl_compatible_run( command = stanc_cmd(), - args = c(wsl_safe_path(stan_file), "--info", include_paths_stanc3_args(include_paths), - allow_undefined_arg), + args = c(wsl_safe_path(stan_file), + "--info", + include_paths_stanc3_args(include_paths), + allow_undefined_arg), wd = cmdstan_path(), echo = FALSE, echo_cmd = FALSE, diff --git a/R/path.R b/R/path.R index 7a12398b5..823b84899 100644 --- a/R/path.R +++ b/R/path.R @@ -121,6 +121,11 @@ cmdstan_default_install_path <- function(old = FALSE) { #' Returns the path to the installation of CmdStan with the most recent release #' version. #' +#' For Windows systems with WSL CmdStan installs, if there are side-by-side WSL +#' and native installs with the same version then the WSL is preferred. +#' Otherwise, the most recent release is chosen, regardless of whether it is +#' native or WSL. +#' #' @export #' @keywords internal #' @param old See [cmdstan_default_install_path()]. diff --git a/R/utils.R b/R/utils.R index 8a479eb40..d79db030f 100644 --- a/R/utils.R +++ b/R/utils.R @@ -169,7 +169,7 @@ wsl_safe_path <- function(path = NULL, revert = FALSE) { } else { path_already_safe <- grepl("^/mnt/", path) if (os_is_wsl() && !isTRUE(path_already_safe) && !is.na(path)) { - abs_path <- repair_path(path) + abs_path <- repair_path(utils::shortPathName(path)) drive_letter <- tolower(strtrim(abs_path, 1)) path <- gsub(paste0(drive_letter, ":"), paste0("/mnt/", drive_letter), @@ -180,15 +180,15 @@ wsl_safe_path <- function(path = NULL, revert = FALSE) { path } +# Running commands through WSL requires using 'wsl' as the command with the +# intended command (e.g., stanc) as the first argument. This function acts as +# a wrapper around processx::run() to apply this change where necessary, and +# forward all other arguments wsl_compatible_run <- function(...) { run_args <- list(...) if (os_is_wsl()) { command <- run_args$command run_args$command <- "wsl" - if (grepl("stanc", command)) { - run_args$args[1] <- paste0("'", run_args$args[1], "'") - run_args$windows_verbatim_args <- TRUE - } run_args$args <- c(command, run_args$args) } do.call(processx::run, run_args) From b2bc04e12b66bf41a3b6945b65665ff4fe406560 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Sat, 16 Jul 2022 23:23:15 +0300 Subject: [PATCH 26/28] Fix short-path handling --- .github/workflows/R-CMD-check.yaml | 2 +- R/utils.R | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index feb714230..0b5af44e4 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -7,7 +7,7 @@ name: Unit tests 'on': push: branches: - - wsl-support + - master pull_request: branches: - master diff --git a/R/utils.R b/R/utils.R index d79db030f..4e064eb14 100644 --- a/R/utils.R +++ b/R/utils.R @@ -169,12 +169,15 @@ wsl_safe_path <- function(path = NULL, revert = FALSE) { } else { path_already_safe <- grepl("^/mnt/", path) if (os_is_wsl() && !isTRUE(path_already_safe) && !is.na(path)) { + base_file <- basename(path) + path <- dirname(path) abs_path <- repair_path(utils::shortPathName(path)) drive_letter <- tolower(strtrim(abs_path, 1)) path <- gsub(paste0(drive_letter, ":"), paste0("/mnt/", drive_letter), abs_path, ignore.case = TRUE) + path <- paste0(path, "/", base_file) } } path From 81bac3cc76716355720b7a92f4b23f0c84156842 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Sun, 17 Jul 2022 10:56:15 +0300 Subject: [PATCH 27/28] Workflow branch --- .github/workflows/R-CMD-check-wsl.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/R-CMD-check-wsl.yaml b/.github/workflows/R-CMD-check-wsl.yaml index 71a42519b..1e7e3c50b 100644 --- a/.github/workflows/R-CMD-check-wsl.yaml +++ b/.github/workflows/R-CMD-check-wsl.yaml @@ -7,7 +7,7 @@ name: Unit tests - WSL Backend 'on': push: branches: - - wsl-support + - master pull_request: branches: - master From 9f4e8a6b68593863b33a4a75430c2b77c853916a Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Sun, 17 Jul 2022 14:39:07 +0300 Subject: [PATCH 28/28] Review comments --- DESCRIPTION | 4 +++- R/install.R | 8 +++++++- R/path.R | 1 + R/utils.R | 10 +--------- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 12b2be772..36825ba09 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -13,7 +13,9 @@ Authors@R: person(given = "Mike", family = "Lawrence", role = "ctb"), person(given = c("William", "Michael"), family = "Landau", role = "ctb", email = "will.landau@gmail.com", comment = c(ORCID = "0000-0003-1878-3253")), - person(given = "Jacob", family = "Socolar", role = "ctb")) + person(given = "Jacob", family = "Socolar", role = "ctb"), + person(given = "Andrew", family = "Johnson", role = "ctb", + comment = c(ORCID = "0000-0001-7000-8065 "))) Description: A lightweight interface to 'Stan' . The 'CmdStanR' interface is an alternative to 'RStan' that calls the command line interface for compilation and running algorithms instead of interfacing diff --git a/R/install.R b/R/install.R index 597b29eed..9909df454 100644 --- a/R/install.R +++ b/R/install.R @@ -88,7 +88,13 @@ install_cmdstan <- function(dir = NULL, # Use environment variable to record WSL usage throughout install, # post-installation will simply check for 'wsl-' prefix in cmdstan path if (isTRUE(wsl)) { - Sys.setenv("CMDSTANR_USE_WSL" = 1) + if (!os_is_windows()) { + warning("wsl=TRUE is only available on Windows, and will be ignored!", + call. = FALSE) + wsl <- FALSE + } else { + Sys.setenv("CMDSTANR_USE_WSL" = 1) + } } if (check_toolchain) { check_cmdstan_toolchain(fix = FALSE, quiet = quiet) diff --git a/R/path.R b/R/path.R index 823b84899..a7a2ce252 100644 --- a/R/path.R +++ b/R/path.R @@ -45,6 +45,7 @@ set_cmdstan_path <- function(path = NULL) { path <- absolute_path(path) .cmdstanr$PATH <- path .cmdstanr$VERSION <- read_cmdstan_version(path) + .cmdstanr$WSL <- grepl("wsl-cmdstan", path) message("CmdStan path set to: ", path) } else { warning("Path not set. Can't find directory: ", path, call. = FALSE) diff --git a/R/utils.R b/R/utils.R index 4e064eb14..7071d0fcd 100644 --- a/R/utils.R +++ b/R/utils.R @@ -42,15 +42,7 @@ os_is_windows <- function() { } os_is_wsl <- function() { - # Avoid errors if checking cmdstan_path before it has been set - wsl_in_path <- tryCatch({ - grepl("wsl-cmdstan", cmdstan_path()) - }, error = function(e) { - FALSE - }, warning = function(e) { - FALSE - }) - os_is_windows() && (wsl_in_path || Sys.getenv("CMDSTANR_USE_WSL") == 1) + os_is_windows() && (isTRUE(.cmdstanr$WSL) || Sys.getenv("CMDSTANR_USE_WSL") == 1) } os_is_macos <- function() {