diff --git a/.github/workflows/R-CMD-check-wsl.yaml b/.github/workflows/R-CMD-check-wsl.yaml new file mode 100644 index 000000000..1e7e3c50b --- /dev/null +++ b/.github/workflows/R-CMD-check-wsl.yaml @@ -0,0 +1,103 @@ +--- +# Github Actions workflow to check CmdStanR +# yamllint disable rule:line-length + +name: Unit tests - WSL Backend + +'on': + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + WSL-R-CMD-check: + if: "! contains(github.event.head_commit.message, '[ci skip]')" + runs-on: windows-latest + + name: windows-latest-WSLv1 + + env: + R_REMOTES_NO_ERRORS_FROM_WARNINGS: true + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + NOT_CRAN: true + + steps: + - name: cmdstan env vars + run: | + echo "CMDSTAN_PATH=${HOME}/.cmdstan" >> $GITHUB_ENV + shell: bash + + - uses: n1hility/cancel-previous-runs@v2 + with: + token: ${{ secrets.GITHUB_TOKEN }} + if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/master'" + + - uses: actions/checkout@v3 + + - uses: r-lib/actions/setup-r@v2.2.3 + with: + r-version: 'release' + rtools-version: '42' + - uses: r-lib/actions/setup-pandoc@v1 + + - name: Query dependencies + run: | + install.packages('remotes') + saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2) + writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version") + shell: Rscript {0} + + - name: Install dependencies + run: | + remotes::install_deps(dependencies = TRUE) + remotes::install_cran("rcmdcheck") + remotes::install_local(path = ".") + install.packages("curl") + shell: Rscript {0} + + - uses: Vampire/setup-wsl@v1 + with: + distribution: Ubuntu-22.04 + use-cache: 'true' + set-as-default: 'true' + - name: Install WSL Dependencies + run: | + # Bugfix for current gzip (for unpacking apt packages) under WSLv1: + # https://github.com/microsoft/WSL/issues/8219#issuecomment-1110508016 + echo -en '\x10' | sudo dd of=/usr/bin/gzip count=1 bs=1 conv=notrunc seek=$((0x189)) + sudo apt-get update + sudo apt-get install -y build-essential libopenmpi-dev + shell: wsl-bash {0} + + - name: Install cmdstan + run: | + cmdstanr::install_cmdstan(cores = 2, wsl = TRUE, overwrite = TRUE) + shell: Rscript {0} + + - name: Session info + run: | + options(width = 100) + pkgs <- installed.packages()[, "Package"] + sessioninfo::session_info(pkgs, include_base = TRUE) + shell: Rscript {0} + + - name: Check + env: + _R_CHECK_CRAN_INCOMING_: false + run: rcmdcheck::rcmdcheck(args = c("--no-manual", "--as-cran"), error_on = "warning", check_dir = "check") + shell: Rscript {0} + + - name: Show testthat output + if: always() + run: find check -name 'testthat.Rout*' -exec cat '{}' \; || true + shell: bash + + - name: Upload check results + if: failure() + uses: actions/upload-artifact@v3 + with: + name: wsl-backend-results + path: check diff --git a/DESCRIPTION b/DESCRIPTION index 12b2be772..36825ba09 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -13,7 +13,9 @@ Authors@R: person(given = "Mike", family = "Lawrence", role = "ctb"), person(given = c("William", "Michael"), family = "Landau", role = "ctb", email = "will.landau@gmail.com", comment = c(ORCID = "0000-0003-1878-3253")), - person(given = "Jacob", family = "Socolar", role = "ctb")) + person(given = "Jacob", family = "Socolar", role = "ctb"), + person(given = "Andrew", family = "Johnson", role = "ctb", + comment = c(ORCID = "0000-0001-7000-8065 "))) Description: A lightweight interface to 'Stan' . The 'CmdStanR' interface is an alternative to 'RStan' that calls the command line interface for compilation and running algorithms instead of interfacing diff --git a/R/args.R b/R/args.R index 5900f6f31..bb311ff5f 100644 --- a/R/args.R +++ b/R/args.R @@ -138,16 +138,16 @@ CmdStanArgs <- R6::R6Class( } if (!is.null(self$init)) { - args$init <- paste0("init=", self$init[idx]) + args$init <- paste0("init=", wsl_safe_path(self$init[idx])) } if (!is.null(self$data_file)) { - args$data <- c("data", paste0("file=", self$data_file)) + args$data <- c("data", paste0("file=", wsl_safe_path(self$data_file))) } - args$output <- c("output", paste0("file=", output_file)) + args$output <- c("output", paste0("file=", wsl_safe_path(output_file))) if (!is.null(latent_dynamics_file)) { - args$output <- c(args$output, paste0("diagnostic_file=", latent_dynamics_file)) + args$output <- c(args$output, paste0("diagnostic_file=", wsl_safe_path(latent_dynamics_file))) } if (!is.null(self$refresh)) { args$output <- c(args$output, paste0("refresh=", self$refresh)) @@ -158,7 +158,7 @@ CmdStanArgs <- R6::R6Class( } if (!is.null(profile_file)) { - args$output <- c(args$output, paste0("profile_file=", profile_file)) + args$output <- c(args$output, paste0("profile_file=", wsl_safe_path(profile_file))) } if (!is.null(self$opencl_ids)) { args$opencl <- c("opencl", paste0("platform=", self$opencl_ids[1]), paste0("device=", self$opencl_ids[2])) @@ -167,7 +167,7 @@ CmdStanArgs <- R6::R6Class( self$method_args$compose(idx, args) }, command = function() { - paste0(if (!os_is_windows()) "./", basename(self$exe_file)) + paste0(if (!os_is_windows() || os_is_wsl()) "./", basename(self$exe_file)) } ) ) @@ -1024,6 +1024,10 @@ compose_arg <- function(self, arg_name, cmdstan_arg_name = NULL, idx = NULL) { if (is.null(val)) { return(NULL) } + + if (os_is_wsl() && (arg_name %in% c("metric_file", "fitted_params"))) { + val <- sapply(val, wsl_safe_path) + } if (!is.null(idx) && length(val) >= idx) { val <- val[idx] } diff --git a/R/csv.R b/R/csv.R index e8e956871..a1bb466dc 100644 --- a/R/csv.R +++ b/R/csv.R @@ -734,6 +734,15 @@ read_csv_metadata <- function(csv_file) { if (length(gradients) > 0) { csv_file_info$gradients <- gradients } + + # Revert any WSL-updated paths before returning the metadata + if (os_is_wsl()) { + csv_file_info$init <- wsl_safe_path(csv_file_info$init, revert = TRUE) + csv_file_info$profile_file <- wsl_safe_path(csv_file_info$profile_file, + revert = TRUE) + csv_file_info$fitted_params <- wsl_safe_path(csv_file_info$fitted_params, + revert = TRUE) + } csv_file_info } diff --git a/R/install.R b/R/install.R index 6cf6d1634..9909df454 100644 --- a/R/install.R +++ b/R/install.R @@ -57,6 +57,8 @@ #' @param check_toolchain (logical) Should `install_cmdstan()` attempt to check #' that the required toolchain is installed and properly configured. The #' default is `TRUE`. +#' @param wsl (logical) Should CmdStan be installed and run through the Windows +#' Subsystem for Linux (WSL). The default is `FALSE`. #' #' @examples #' \dontrun{ @@ -81,7 +83,19 @@ install_cmdstan <- function(dir = NULL, version = NULL, release_url = NULL, cpp_options = list(), - check_toolchain = TRUE) { + check_toolchain = TRUE, + wsl = FALSE) { + # Use environment variable to record WSL usage throughout install, + # post-installation will simply check for 'wsl-' prefix in cmdstan path + if (isTRUE(wsl)) { + if (!os_is_windows()) { + warning("wsl=TRUE is only available on Windows, and will be ignored!", + call. = FALSE) + wsl <- FALSE + } else { + Sys.setenv("CMDSTANR_USE_WSL" = 1) + } + } if (check_toolchain) { check_cmdstan_toolchain(fix = FALSE, quiet = quiet) } @@ -111,6 +125,7 @@ install_cmdstan <- function(dir = NULL, release_url <- paste0("https://github.com/stan-dev/cmdstan/releases/download/v", version, "/cmdstan-", version, cmdstan_arch_suffix(version), ".tar.gz") } + wsl_prefix <- ifelse(isTRUE(wsl), "wsl-", "") if (!is.null(release_url)) { if (!endsWith(release_url, ".tar.gz")) { stop(release_url, " is not a .tar.gz archive!", @@ -122,14 +137,14 @@ install_cmdstan <- function(dir = NULL, tar_name <- utils::tail(split_url[[1]], n = 1) cmdstan_ver <- substr(tar_name, 0, nchar(tar_name) - 7) tar_gz_file <- paste0(cmdstan_ver, ".tar.gz") - dir_cmdstan <- file.path(dir, cmdstan_ver) + dir_cmdstan <- file.path(dir, paste0(wsl_prefix, cmdstan_ver)) dest_file <- file.path(dir, tar_gz_file) } else { ver <- latest_released_version() message("* Latest CmdStan release is v", ver) cmdstan_ver <- paste0("cmdstan-", ver, cmdstan_arch_suffix(ver)) tar_gz_file <- paste0(cmdstan_ver, ".tar.gz") - dir_cmdstan <- file.path(dir, cmdstan_ver) + dir_cmdstan <- file.path(dir, paste0(wsl_prefix, cmdstan_ver)) message("* Installing CmdStan v", ver, " in ", dir_cmdstan) message("* Downloading ", tar_gz_file, " from GitHub...") download_url <- github_download_url(ver) @@ -171,7 +186,7 @@ install_cmdstan <- function(dir = NULL, append = TRUE ) } - if (is_rtools42_toolchain()) { + if (is_rtools42_toolchain() && !os_is_wsl()) { cmdstan_make_local( dir = dir_cmdstan, cpp_options = list( @@ -204,6 +219,9 @@ install_cmdstan <- function(dir = NULL, "\nrebuild_cmdstan(cores = ...)" ) } + if (isTRUE(wsl)) { + Sys.unsetenv("CMDSTANR_USE_WSL") + } } @@ -265,7 +283,9 @@ cmdstan_make_local <- function(dir = cmdstan_path(), #' check_cmdstan_toolchain <- function(fix = FALSE, quiet = FALSE) { if (os_is_windows()) { - if (R.version$major >= "4") { + if (os_is_wsl()) { + check_wsl_toolchain() + } else if (R.version$major >= "4") { check_rtools4x_windows_toolchain(fix = fix, quiet = quiet) } else { check_rtools35_windows_toolchain(fix = fix, quiet = quiet) @@ -379,8 +399,8 @@ build_cmdstan <- function(dir, toolchain_PATH_env_var(), tbb_path(dir = dir) ), - processx::run( - run_cmd, + wsl_compatible_run( + command = run_cmd, args = c(translation_args, paste0("-j", cores), "build"), wd = dir, echo_cmd = is_verbose_mode(), @@ -401,9 +421,9 @@ clean_cmdstan <- function(dir = cmdstan_path(), toolchain_PATH_env_var(), tbb_path(dir = dir) ), - processx::run( - make_cmd(), - args = c("clean-all"), + wsl_compatible_run( + command = make_cmd(), + args = "clean-all", wd = dir, echo_cmd = is_verbose_mode(), echo = !quiet || is_verbose_mode(), @@ -420,9 +440,10 @@ build_example <- function(dir, cores, quiet, timeout) { toolchain_PATH_env_var(), tbb_path(dir = dir) ), - processx::run( - make_cmd(), - args = c(paste0("-j", cores), cmdstan_ext(file.path("examples", "bernoulli", "bernoulli"))), + wsl_compatible_run( + command = make_cmd(), + args = c(paste0("-j", cores), + cmdstan_ext(file.path("examples", "bernoulli", "bernoulli"))), wd = dir, echo_cmd = is_verbose_mode(), echo = !quiet || is_verbose_mode(), @@ -499,6 +520,41 @@ install_toolchain <- function(quiet = FALSE) { invisible(NULL) } +check_wsl_toolchain <- function() { + wsl_inaccessible <- processx::run(command = "wsl", + args = "uname", + error_on_status = FALSE) + if (wsl_inaccessible$status) { + stop("\n", "A WSL distribution is not installed or is not accessible.", + "\n", "Please see the Microsoft documentation for guidance on installing WSL: ", + "\n", "https://docs.microsoft.com/en-us/windows/wsl/install", + call. = FALSE) + } + + make_not_present <- processx::run(command = "wsl", + args = c("which", "make"), + error_on_status = FALSE) + + gpp_not_present <- processx::run(command = "wsl", + args = c("which", "g++"), + error_on_status = FALSE) + + clangpp_not_present <- processx::run(command = "wsl", + args = c("which", "clang++"), + windows_verbatim_args = TRUE, + error_on_status = FALSE) + + if (make_not_present$status || (gpp_not_present$status + && clangpp_not_present$status)) { + stop("\n", "Your distribution is missing the needed utilities for compiling C++.", + "\n", "Please launch your WSL and install them using the appropriate command:", + "\n", "Debian/Ubuntu: sudo apt-get install build-essential", + "\n", "Fedora: sudo dnf group install \"C Development Tools and Libraries\"", + "\n", "Arch: pacman -Sy base-devel", + call. = FALSE) + } +} + check_rtools4x_windows_toolchain <- function(fix = FALSE, quiet = FALSE) { rtools_path <- rtools4x_home_path() rtools_version <- if (is_rtools42_toolchain()) "Rtools42" else "Rtools40" diff --git a/R/model.R b/R/model.R index 84bbb7bc9..5346346b9 100644 --- a/R/model.R +++ b/R/model.R @@ -243,7 +243,7 @@ CmdStanModel <- R6::R6Class( } } if (!is.null(exe_file)) { - ext <- if (os_is_windows()) "exe" else "" + ext <- if (os_is_windows() && !os_is_wsl()) "exe" else "" private$exe_file_ <- repair_path(absolute_path(exe_file)) if (is.null(stan_file)) { checkmate::assert_file_exists(private$exe_file_, access = "r", extension = ext) @@ -508,7 +508,7 @@ compile <- function(quiet = TRUE, file.copy(self$stan_file(), temp_stan_file, overwrite = TRUE) temp_file_no_ext <- strip_ext(temp_stan_file) tmp_exe <- cmdstan_ext(temp_file_no_ext) # adds .exe on Windows - if (os_is_windows()) { + if (os_is_windows() && !os_is_wsl()) { tmp_exe <- utils::shortPathName(tmp_exe) } private$hpp_file_ <- paste0(temp_file_no_ext, ".hpp") @@ -523,14 +523,14 @@ compile <- function(quiet = TRUE, stanc_options[["use-opencl"]] <- TRUE } if (!is.null(user_header)) { - cpp_options[["USER_HEADER"]] <- user_header + cpp_options[["USER_HEADER"]] <- wsl_safe_path(user_header) stanc_options[["allow-undefined"]] <- TRUE } if (!is.null(cpp_options[["USER_HEADER"]])) { - cpp_options[["USER_HEADER"]] <- absolute_path(cpp_options[["USER_HEADER"]]) + cpp_options[["USER_HEADER"]] <- wsl_safe_path(absolute_path(cpp_options[["USER_HEADER"]])) } if (!is.null(cpp_options[["user_header"]])) { - cpp_options[["user_header"]] <- absolute_path(cpp_options[["user_header"]]) + cpp_options[["user_header"]] <- wsl_safe_path(absolute_path(cpp_options[["user_header"]])) } if (is.null(stanc_options[["name"]])) { stanc_options[["name"]] <- paste0(self$model_name(), "_model") @@ -552,9 +552,9 @@ compile <- function(quiet = TRUE, toolchain_PATH_env_var(), tbb_path() ), - run_log <- processx::run( + run_log <- wsl_compatible_run( command = make_cmd(), - args = c(tmp_exe, + args = c(wsl_safe_path(tmp_exe), cpp_options_to_compile_flags(cpp_options), stancflags_val), wd = cmdstan_path(), @@ -731,7 +731,7 @@ check_syntax <- function(pedantic = FALSE, } temp_hpp_file <- tempfile(pattern = "model-", fileext = ".hpp") - stanc_options[["o"]] <- temp_hpp_file + stanc_options[["o"]] <- wsl_safe_path(temp_hpp_file) if (pedantic) { stanc_options[["warn-pedantic"]] <- TRUE @@ -759,9 +759,9 @@ check_syntax <- function(pedantic = FALSE, toolchain_PATH_env_var(), tbb_path() ), - run_log <- processx::run( + run_log <- wsl_compatible_run( command = stanc_cmd(), - args = c(self$stan_file(), stanc_built_options, stancflags_val), + args = c(wsl_safe_path(self$stan_file()), stanc_built_options, stancflags_val), wd = cmdstan_path(), echo = is_verbose_mode(), echo_cmd = is_verbose_mode(), @@ -901,9 +901,10 @@ format <- function(overwrite_file = FALSE, toolchain_PATH_env_var(), tbb_path() ), - run_log <- processx::run( + run_log <- wsl_compatible_run( command = stanc_cmd(), - args = c(self$stan_file(), stanc_built_options, stancflags_val), + args = c(wsl_safe_path(self$stan_file()), stanc_built_options, + stancflags_val), wd = cmdstan_path(), echo = is_verbose_mode(), echo_cmd = is_verbose_mode(), @@ -1759,7 +1760,7 @@ include_paths_stanc3_args <- function(include_paths = NULL) { stancflags <- NULL if (!is.null(include_paths)) { checkmate::assert_directory_exists(include_paths, access = "r") - include_paths <- absolute_path(include_paths) + include_paths <- sapply(absolute_path(include_paths), wsl_safe_path) paths_w_space <- grep(" ", include_paths) include_paths[paths_w_space] <- paste0("'", include_paths[paths_w_space], "'") include_paths <- paste0(include_paths, collapse = ",") @@ -1780,9 +1781,12 @@ model_variables <- function(stan_file, include_paths = NULL, allow_undefined = F allow_undefined_arg <- NULL } out_file <- tempfile(fileext = ".json") - run_log <- processx::run( + run_log <- wsl_compatible_run( command = stanc_cmd(), - args = c(stan_file, "--info", include_paths_stanc3_args(include_paths), allow_undefined_arg), + args = c(wsl_safe_path(stan_file), + "--info", + include_paths_stanc3_args(include_paths), + allow_undefined_arg), wd = cmdstan_path(), echo = FALSE, echo_cmd = FALSE, @@ -1809,9 +1813,9 @@ model_compile_info <- function(exe_file) { toolchain_PATH_env_var(), tbb_path() ), - ret <- processx::run( - command = exe_file, - args = c("info"), + ret <- wsl_compatible_run( + command = wsl_safe_path(exe_file), + args = "info", error_on_status = FALSE ) ) diff --git a/R/path.R b/R/path.R index d0ae7109f..a7a2ce252 100644 --- a/R/path.R +++ b/R/path.R @@ -45,6 +45,7 @@ set_cmdstan_path <- function(path = NULL) { path <- absolute_path(path) .cmdstanr$PATH <- path .cmdstanr$VERSION <- read_cmdstan_version(path) + .cmdstanr$WSL <- grepl("wsl-cmdstan", path) message("CmdStan path set to: ", path) } else { warning("Path not set. Can't find directory: ", path, call. = FALSE) @@ -121,6 +122,11 @@ cmdstan_default_install_path <- function(old = FALSE) { #' Returns the path to the installation of CmdStan with the most recent release #' version. #' +#' For Windows systems with WSL CmdStan installs, if there are side-by-side WSL +#' and native installs with the same version then the WSL is preferred. +#' Otherwise, the most recent release is chosen, regardless of whether it is +#' native or WSL. +#' #' @export #' @keywords internal #' @param old See [cmdstan_default_install_path()]. @@ -145,8 +151,24 @@ cmdstan_default_path <- function(old = FALSE, dir = NULL) { cmdstan_installs <- list.dirs(path = installs_path, recursive = FALSE, full.names = FALSE) } if (length(cmdstan_installs) > 0) { + wsl_installs <- grep("^wsl-cmdstan-", cmdstan_installs, value = TRUE) + cmdstan_installs <- cmdstan_installs[!grepl("wsl-", cmdstan_installs)] cmdstan_installs <- grep("^cmdstan-", cmdstan_installs, value = TRUE) - latest_cmdstan <- sort(cmdstan_installs, decreasing = TRUE)[1] + if (length(wsl_installs) > 0) { + wsl_installs_trim <- gsub("wsl-", "", wsl_installs, fixed = TRUE) + wsl_latest <- sort(wsl_installs_trim, decreasing = TRUE)[1] + if (length(cmdstan_installs) > 0) { + non_wsl_latest <- sort(cmdstan_installs, decreasing = TRUE)[1] + latest_cmdstan <- ifelse(wsl_latest > non_wsl_latest + || wsl_latest == non_wsl_latest, + grep(wsl_latest, wsl_installs, value = TRUE), + non_wsl_latest) + } else { + latest_cmdstan <- grep(wsl_latest, wsl_installs, value = TRUE) + } + } else { + latest_cmdstan <- sort(cmdstan_installs, decreasing = TRUE)[1] + } if (is_release_candidate(latest_cmdstan)) { non_rc_path <- strsplit(latest_cmdstan, "-rc")[[1]][1] if (dir.exists(file.path(installs_path, non_rc_path))) { diff --git a/R/run.R b/R/run.R index cb703663a..093cba7ee 100644 --- a/R/run.R +++ b/R/run.R @@ -231,9 +231,12 @@ CmdStanRun <- R6::R6Class( toolchain_PATH_env_var(), tbb_path() ), - run_log <- processx::run( + run_log <- wsl_compatible_run( command = target_exe, - args = c(self$output_files(include_failed = FALSE), flags), + args = c( + sapply(self$output_files(include_failed = FALSE), + wsl_safe_path), + flags), wd = cmdstan_path(), echo = TRUE, echo_cmd = is_verbose_mode(), @@ -300,7 +303,7 @@ check_target_exe <- function(exe) { toolchain_PATH_env_var(), tbb_path() ), - run_log <- processx::run( + run_log <- wsl_compatible_run( command = make_cmd(), args = exe, wd = cmdstan_path(), @@ -319,7 +322,7 @@ check_target_exe <- function(exe) { if (is.null(mpi_args)) { mpi_args <- list() } - mpi_args[["exe"]] <- self$exe_file() + mpi_args[["exe"]] <- wsl_safe_path(self$exe_file()) } if (procs$num_procs() == 1) { start_msg <- "Running MCMC with 1 chain" @@ -350,6 +353,10 @@ check_target_exe <- function(exe) { } else { cat(paste0(start_msg, ", with ", procs$threads_per_proc(), " thread(s) per chain...\n\n")) Sys.setenv("STAN_NUM_THREADS" = as.integer(procs$threads_per_proc())) + # Windows environment variables have to be explicitly exported to WSL + if (os_is_wsl()) { + Sys.setenv("WSLENV"="STAN_NUM_THREADS/u") + } } start_time <- Sys.time() chains <- procs$proc_ids() @@ -409,6 +416,10 @@ CmdStanRun$set("private", name = "run_sample_", value = .run_sample) } else { cat(paste0(start_msg, ", with ", procs$threads_per_proc(), " thread(s) per chain...\n\n")) Sys.setenv("STAN_NUM_THREADS" = as.integer(procs$threads_per_proc())) + # Windows environment variables have to be explicitly exported to WSL + if (os_is_wsl()) { + Sys.setenv("WSLENV"="STAN_NUM_THREADS/u") + } } start_time <- Sys.time() chains <- procs$proc_ids() @@ -451,6 +462,10 @@ CmdStanRun$set("private", name = "run_generate_quantities_", value = .run_genera procs <- self$procs if (!is.null(procs$threads_per_proc())) { Sys.setenv("STAN_NUM_THREADS" = as.integer(procs$threads_per_proc())) + # Windows environment variables have to be explicitly exported to WSL + if (os_is_wsl()) { + Sys.setenv("WSLENV"="STAN_NUM_THREADS/u") + } } start_time <- Sys.time() id <- 1 @@ -498,6 +513,10 @@ CmdStanRun$set("private", name = "run_variational_", value = .run_other) procs <- self$procs if (!is.null(procs$threads_per_proc())) { Sys.setenv("STAN_NUM_THREADS" = as.integer(procs$threads_per_proc())) + # Windows environment variables have to be explicitly exported to WSL + if (os_is_wsl()) { + Sys.setenv("WSLENV"="STAN_NUM_THREADS/u") + } } stdout_file <- tempfile() stderr_file <- tempfile() @@ -507,7 +526,7 @@ CmdStanRun$set("private", name = "run_variational_", value = .run_other) toolchain_PATH_env_var(), tbb_path() ), - ret <- processx::run( + ret <- wsl_compatible_run( command = self$command(), args = self$command_args()[[1]], wd = dirname(self$exe_file()), @@ -623,7 +642,7 @@ CmdStanProcs <- R6::R6Class( toolchain_PATH_env_var(), tbb_path() ), - private$processes_[[id]] <- processx::process$new( + private$processes_[[id]] <- wsl_compatible_process_new( command = command, args = args, wd = wd, @@ -1070,4 +1089,4 @@ tbb_path <- function(dir = NULL) { path_to_TBB <- file.path(dir, "stan", "lib", "stan_math", "lib", "tbb") } path_to_TBB -} \ No newline at end of file +} diff --git a/R/utils.R b/R/utils.R index a400dd33c..7071d0fcd 100644 --- a/R/utils.R +++ b/R/utils.R @@ -41,6 +41,10 @@ os_is_windows <- function() { isTRUE(.Platform$OS.type == "windows") } +os_is_wsl <- function() { + os_is_windows() && (isTRUE(.cmdstanr$WSL) || Sys.getenv("CMDSTANR_USE_WSL") == 1) +} + os_is_macos <- function() { isTRUE(Sys.info()[["sysname"]] == "Darwin") } @@ -68,7 +72,7 @@ is_rosetta2 <- function() { # Returns the type of make command to use to compile depending on the OS make_cmd <- function() { - if (os_is_windows()) { + if (os_is_windows() && !os_is_wsl()) { "mingw32-make.exe" } else { "make" @@ -77,14 +81,13 @@ make_cmd <- function() { # Returns the stanc exe path depending on the OS stanc_cmd <- function() { - if (os_is_windows()) { + if (os_is_windows() && !os_is_wsl()) { "bin/stanc.exe" } else { "bin/stanc" } } - # paths and extensions ---------------------------------------------------- # Replace `\\` with `/` in a path @@ -110,7 +113,7 @@ repair_path <- function(path) { #' @return If `path` is `NULL` then `".exe"` on Windows and `""` otherwise. If #' `path` is not `NULL` then `.exe` is added as the extension on Windows. cmdstan_ext <- function(path = NULL) { - ext <- if (os_is_windows()) ".exe" else "" + ext <- if (os_is_windows() && !os_is_wsl()) ".exe" else "" if (is.null(path)) { return(ext) } @@ -139,7 +142,62 @@ strip_ext <- function(file) { } absolute_path <- Vectorize(.absolute_path, USE.NAMES = FALSE) +# When providing the model path to WSL, it needs to be in reference to the +# to Windows mount point (/mnt/drive-letter) within the WSL install: +# e.g., C:/Users/... -> /mnt/c/Users/... +wsl_safe_path <- function(path = NULL, revert = FALSE) { + if (!is.character(path) || is.null(path) || !os_is_wsl()) { + return(path) + } + if (revert) { + if (!grepl("^/mnt/", path)) { + return(path) + } + strip_mnt <- gsub("^/mnt/", "", path) + drive_letter <- strtrim(strip_mnt, 1) + path <- gsub(paste0("^/mnt/", drive_letter), + paste0(toupper(drive_letter), ":"), + path) + } else { + path_already_safe <- grepl("^/mnt/", path) + if (os_is_wsl() && !isTRUE(path_already_safe) && !is.na(path)) { + base_file <- basename(path) + path <- dirname(path) + abs_path <- repair_path(utils::shortPathName(path)) + drive_letter <- tolower(strtrim(abs_path, 1)) + path <- gsub(paste0(drive_letter, ":"), + paste0("/mnt/", drive_letter), + abs_path, + ignore.case = TRUE) + path <- paste0(path, "/", base_file) + } + } + path +} + +# Running commands through WSL requires using 'wsl' as the command with the +# intended command (e.g., stanc) as the first argument. This function acts as +# a wrapper around processx::run() to apply this change where necessary, and +# forward all other arguments +wsl_compatible_run <- function(...) { + run_args <- list(...) + if (os_is_wsl()) { + command <- run_args$command + run_args$command <- "wsl" + run_args$args <- c(command, run_args$args) + } + do.call(processx::run, run_args) +} +wsl_compatible_process_new <- function(...) { + run_args <- list(...) + if (os_is_wsl()) { + command <- run_args$command + run_args$command <- "wsl" + run_args$args <- c(command, run_args$args) + } + do.call(processx::process$new, run_args) +} # read, write, and copy files -------------------------------------------- diff --git a/man/install_cmdstan.Rd b/man/install_cmdstan.Rd index edbd419ae..84b0f2964 100644 --- a/man/install_cmdstan.Rd +++ b/man/install_cmdstan.Rd @@ -16,7 +16,8 @@ install_cmdstan( version = NULL, release_url = NULL, cpp_options = list(), - check_toolchain = TRUE + check_toolchain = TRUE, + wsl = FALSE ) rebuild_cmdstan( @@ -73,6 +74,9 @@ the use of clang for compilation.} that the required toolchain is installed and properly configured. The default is \code{TRUE}.} +\item{wsl}{(logical) Should CmdStan be installed and run through the Windows +Subsystem for Linux (WSL). The default is \code{FALSE}.} + \item{append}{(logical) For \code{cmdstan_make_local()}, should the listed makefile flags be appended to the end of the existing \code{make/local} file? The default is \code{TRUE}. If \code{FALSE} the file is overwritten.} diff --git a/tests/testthat/helper-envvars-and-paths.R b/tests/testthat/helper-envvars-and-paths.R index 7a18165d4..66283716b 100644 --- a/tests/testthat/helper-envvars-and-paths.R +++ b/tests/testthat/helper-envvars-and-paths.R @@ -8,8 +8,8 @@ on_ci <- function() { mpi_toolchain_present <- function() { tryCatch( - processx::run(command = "mpicxx", args = "--version")$status == 0 && - processx::run(command = "mpiexec", args = "--version")$status == 0, + wsl_compatible_run(command = "mpicxx", args = "--version")$status == 0 && + wsl_compatible_run(command = "mpiexec", args = "--version")$status == 0, error=function(cond) { FALSE } diff --git a/tests/testthat/test-install.R b/tests/testthat/test-install.R index a09020a09..39192033d 100644 --- a/tests/testthat/test-install.R +++ b/tests/testthat/test-install.R @@ -1,5 +1,7 @@ context("install") +wsl_prefix <- ifelse(os_is_wsl(), "wsl-", "") + cmdstan_test_tarball_url <- Sys.getenv("CMDSTAN_TEST_TARBALL_URL") if (!nzchar(cmdstan_test_tarball_url)) { cmdstan_test_tarball_url <- NULL @@ -14,7 +16,8 @@ test_that("install_cmdstan() successfully installs cmdstan", { expect_message( expect_output( install_cmdstan(dir = dir, cores = 2, quiet = FALSE, overwrite = TRUE, - release_url = cmdstan_test_tarball_url), + release_url = cmdstan_test_tarball_url, + wsl = os_is_wsl()), "Compiling, linking C++ code", fixed = TRUE ), @@ -25,13 +28,13 @@ test_that("install_cmdstan() successfully installs cmdstan", { test_that("install_cmdstan() errors if installation already exists", { install_dir <- cmdstan_default_install_path() - dir <- file.path(install_dir, "cmdstan-2.23.0") + dir <- file.path(install_dir, paste0(wsl_prefix, "cmdstan-2.23.0")) if (!dir.exists(dir)) { dir.create(dir) } expect_warning( install_cmdstan(dir = install_dir, overwrite = FALSE, - version = "2.23.0"), + version = "2.23.0", wsl = os_is_wsl()), "An installation already exists", fixed = TRUE ) @@ -44,24 +47,25 @@ test_that("install_cmdstan() errors if it times out", { dir <- tempdir(check = TRUE) } ver <- latest_released_version() - dir_exists <- dir.exists(file.path(dir, paste0("cmdstan-",ver))) + dir_exists <- dir.exists(file.path(dir, paste0(wsl_prefix, "cmdstan-",ver))) # with quiet=TRUE expect_warning( expect_message( install_cmdstan(dir = dir, timeout = 1, quiet = TRUE, overwrite = dir_exists, - release_url = cmdstan_test_tarball_url), + release_url = cmdstan_test_tarball_url, wsl = os_is_wsl()), if (dir_exists) "* Removing the existing installation" else "* * Installing CmdStan from https://github.com", fixed = TRUE ), "increasing the value of the 'timeout' argument and running again with 'quiet=FALSE'", fixed = TRUE ) - dir_exists <- dir.exists(file.path(dir, paste0("cmdstan-",ver))) + dir_exists <- dir.exists(file.path(dir, paste0(wsl_prefix,"cmdstan-",ver))) # with quiet=FALSE expect_warning( expect_message( install_cmdstan(dir = dir, timeout = 1, quiet = FALSE, overwrite = dir_exists, - release_url = cmdstan_test_tarball_url), + release_url = cmdstan_test_tarball_url, + wsl = os_is_wsl()), if (dir_exists) "* Removing the existing installation" else "* * Installing CmdStan from https://github.com", fixed = TRUE ), @@ -72,15 +76,16 @@ test_that("install_cmdstan() errors if it times out", { test_that("install_cmdstan() errors if invalid version or URL", { expect_error( - install_cmdstan(version = "2.23.2"), + install_cmdstan(version = "2.23.2", wsl = os_is_wsl()), "Download of CmdStan failed. Please check if the supplied version number is valid." ) expect_error( - install_cmdstan(release_url = "https://github.com/stan-dev/cmdstan/releases/download/v2.23.2/cmdstan-2.23.2.tar.gz"), + install_cmdstan(release_url = "https://github.com/stan-dev/cmdstan/releases/download/v2.23.2/cmdstan-2.23.2.tar.gz", + wsl = os_is_wsl()), "Download of CmdStan failed. Please check if the supplied release URL is valid." ) expect_error( - install_cmdstan(release_url = "https://github.com/stan-dev/cmdstan/releases/tag/v2.24.0"), + install_cmdstan(release_url = "https://github.com/stan-dev/cmdstan/releases/tag/v2.24.0", wsl = os_is_wsl()), "cmdstanr supports installing from .tar.gz archives only" ) }) @@ -95,7 +100,8 @@ test_that("install_cmdstan() works with version and release_url", { expect_message( expect_output( install_cmdstan(dir = dir, overwrite = TRUE, cores = 4, - release_url = "https://github.com/stan-dev/cmdstan/releases/download/v2.26.1/cmdstan-2.26.1.tar.gz"), + release_url = "https://github.com/stan-dev/cmdstan/releases/download/v2.26.1/cmdstan-2.26.1.tar.gz", + wsl = os_is_wsl()), "Compiling, linking C++ code", fixed = TRUE ), @@ -108,7 +114,8 @@ test_that("install_cmdstan() works with version and release_url", { install_cmdstan(dir = dir, overwrite = TRUE, cores = 4, version = "2.27.0", # the URL is intentionally invalid to test that the version has higher priority - release_url = "https://github.com/stan-dev/cmdstan/releases/download/v2.27.3/cmdstan-2.27.3.tar.gz"), + release_url = "https://github.com/stan-dev/cmdstan/releases/download/v2.27.3/cmdstan-2.27.3.tar.gz", + wsl = os_is_wsl()), "Compiling, linking C++ code", fixed = TRUE ), @@ -118,7 +125,7 @@ test_that("install_cmdstan() works with version and release_url", { "version and release_url shouldn't both be specified", fixed = TRUE ) - expect_true(dir.exists(file.path(dir, "cmdstan-2.27.0"))) + expect_true(dir.exists(file.path(dir, paste0(wsl_prefix, "cmdstan-2.27.0")))) set_cmdstan_path(cmdstan_default_path()) }) @@ -148,6 +155,7 @@ test_that("toolchain checks on Unix work", { test_that("toolchain checks on Windows with RTools 3.5 work", { skip_if_not(os_is_windows()) + skip_if(os_is_wsl()) skip_if(R.Version()$major > "3") path_backup <- Sys.getenv("PATH") diff --git a/tests/testthat/test-model-compile.R b/tests/testthat/test-model-compile.R index f38f3b90b..e2bb85283 100644 --- a/tests/testthat/test-model-compile.R +++ b/tests/testthat/test-model-compile.R @@ -73,7 +73,7 @@ test_that("compile() method works with spaces in path", { expect_interactive_message(mod_spaces$compile(), "Compiling Stan program...") file.remove(stan_model_with_spaces) file.remove(exe) - file.remove(dir_with_spaces) + unlink(dir_with_spaces, recursive = TRUE) }) test_that("compile() method overwrites binaries", { @@ -117,7 +117,7 @@ test_that("compilation works with include_paths", { test_that("name in STANCFLAGS is set correctly", { out <- utils::capture.output(mod$compile(quiet = FALSE, force_recompile = TRUE)) - if(os_is_windows()) { + if(os_is_windows() && !os_is_wsl()) { out_no_name <- "bin/stanc.exe --name='bernoulli_model' --o" out_name <- "bin/stanc.exe --name='bernoulli2_model' --o" } else { @@ -428,7 +428,7 @@ test_that("check_syntax() works with pedantic=TRUE", { }) test_that("compiliation errors if folder with the model name exists", { - skip_if(os_is_windows()) + skip_if(os_is_windows() && !os_is_wsl()) model_code <- " parameters { real y; @@ -449,7 +449,7 @@ test_that("compiliation errors if folder with the model name exists", { cmdstan_model(stan_file), "There is a subfolder matching the model name in the same folder as the model! Please remove or rename the subfolder and try again." ) - file.remove(exe) + unlink(exe, recursive = TRUE) }) test_that("cpp_options_to_compile_flags() works", { @@ -473,16 +473,20 @@ test_that("include_paths_stanc3_args() works", { dir.create(path_1) } path_1 <- repair_path(path_1) - expect_equal(include_paths_stanc3_args(path_1), paste0("--include-paths=", path_1)) + path_1_compare <- ifelse(os_is_wsl(), wsl_safe_path(path_1), path_1) + expect_equal( + include_paths_stanc3_args(path_1), + paste0("--include-paths=", path_1_compare)) path_2 <- file.path(tempdir(), "folder2") if (!dir.exists(path_2)) { dir.create(path_2) } path_2 <- repair_path(path_2) + path_2_compare <- ifelse(os_is_wsl(), wsl_safe_path(path_2), path_2) expect_equal( include_paths_stanc3_args(c(path_1, path_2)), c( - paste0("--include-paths=", path_1, ",", path_2) + paste0("--include-paths=", path_1_compare, ",", path_2_compare) ) ) }) @@ -581,7 +585,7 @@ test_that("cmdstan_model errors with no args ", { }) test_that("cmdstan_model works with user_header", { - skip_if(os_is_macos() | os_is_windows()) + skip_if(os_is_macos() | (os_is_windows() && !os_is_wsl())) tmpfile <- tempfile(fileext = ".hpp") hpp <- " diff --git a/tests/testthat/test-model-sample_mpi.R b/tests/testthat/test-model-sample_mpi.R index e0b8ac8a5..6b348f2ed 100644 --- a/tests/testthat/test-model-sample_mpi.R +++ b/tests/testthat/test-model-sample_mpi.R @@ -31,6 +31,15 @@ test_that("sample_mpi() works", { } cpp_options = list(cxx="mpicxx", stan_mpi = TRUE, tbb_cxx_type=tbb_cxx_type) mod_mpi <- cmdstan_model(mpi_file, cpp_options = cpp_options) + + if (os_is_wsl()) { + # Default GHA WSL install runs as root, which MPI discourages + # Specify that this is safe to ignore for this test + Sys.setenv("OMPI_ALLOW_RUN_AS_ROOT"=1) + Sys.setenv("OMPI_ALLOW_RUN_AS_ROOT_CONFIRM"=1) + Sys.setenv("WSLENV"="OMPI_ALLOW_RUN_AS_ROOT/u:OMPI_ALLOW_RUN_AS_ROOT_CONFIRM/u") + } + utils::capture.output( f <- mod_mpi$sample_mpi(chains = 1, mpi_args = list("n" = 1)) ) diff --git a/tests/testthat/test-path.R b/tests/testthat/test-path.R index 190f48d3b..daed18ae3 100644 --- a/tests/testthat/test-path.R +++ b/tests/testthat/test-path.R @@ -90,7 +90,7 @@ test_that("Warning message is thrown if can't detect version number", { }) test_that("cmdstan_ext() works", { - if (os_is_windows()) { + if (os_is_windows() && !os_is_wsl()) { expect_identical(cmdstan_ext(), ".exe") expect_identical(cmdstan_ext("path"), "path.exe") } else {