Merge pull request #677 from andrjohns/wsl-support

Allow executing cmdstan through WSL

.github/workflows/R-CMD-check-wsl.yaml

@@ -0,0 +1,103 @@
+---
+# Github Actions workflow to check CmdStanR
+# yamllint disable rule:line-length
+
+name: Unit tests - WSL Backend
+
+'on':
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  WSL-R-CMD-check:
+    if: "! contains(github.event.head_commit.message, '[ci skip]')"
+    runs-on: windows-latest
+
+    name: windows-latest-WSLv1
+
+    env:
+      R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+      NOT_CRAN: true
+
+    steps:
+      - name: cmdstan env vars
+        run: |
+          echo "CMDSTAN_PATH=${HOME}/.cmdstan" >> $GITHUB_ENV
+        shell: bash
+
+      - uses: n1hility/cancel-previous-runs@v2
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+        if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/master'"
+
+      - uses: actions/checkout@v3
+
+      - uses: r-lib/actions/[email protected]
+        with:
+          r-version: 'release'
+          rtools-version: '42'
+      - uses: r-lib/actions/setup-pandoc@v1
+
+      - name: Query dependencies
+        run: |
+          install.packages('remotes')
+          saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2)
+          writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version")
+        shell: Rscript {0}
+
+      - name: Install dependencies
+        run: |
+          remotes::install_deps(dependencies = TRUE)
+          remotes::install_cran("rcmdcheck")
+          remotes::install_local(path = ".")
+          install.packages("curl")
+        shell: Rscript {0}
+
+      - uses: Vampire/setup-wsl@v1
+        with:
+            distribution: Ubuntu-22.04
+            use-cache: 'true'
+            set-as-default: 'true'
+      - name: Install WSL Dependencies
+        run: |
+          # Bugfix for current gzip (for unpacking apt packages) under WSLv1:
+          # https://github.com/microsoft/WSL/issues/8219#issuecomment-1110508016
+          echo -en '\x10' | sudo dd of=/usr/bin/gzip count=1 bs=1 conv=notrunc seek=$((0x189))
+          sudo apt-get update
+          sudo apt-get install -y build-essential libopenmpi-dev
+        shell: wsl-bash {0}
+
+      - name: Install cmdstan
+        run: |
+          cmdstanr::install_cmdstan(cores = 2, wsl = TRUE, overwrite = TRUE)
+        shell: Rscript {0}
+
+      - name: Session info
+        run: |
+          options(width = 100)
+          pkgs <- installed.packages()[, "Package"]
+          sessioninfo::session_info(pkgs, include_base = TRUE)
+        shell: Rscript {0}
+
+      - name: Check
+        env:
+          _R_CHECK_CRAN_INCOMING_: false
+        run: rcmdcheck::rcmdcheck(args = c("--no-manual", "--as-cran"), error_on = "warning", check_dir = "check")
+        shell: Rscript {0}
+
+      - name: Show testthat output
+        if: always()
+        run: find check -name 'testthat.Rout*' -exec cat '{}' \; || true
+        shell: bash
+
+      - name: Upload check results
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: wsl-backend-results
+          path: check

DESCRIPTION

@@ -13,7 +13,9 @@ Authors@R:
       person(given = "Mike", family = "Lawrence", role = "ctb"),
       person(given = c("William", "Michael"), family = "Landau", role = "ctb",
            email = "[email protected]", comment = c(ORCID = "0000-0003-1878-3253")),
-      person(given = "Jacob", family = "Socolar", role = "ctb"))
+      person(given = "Jacob", family = "Socolar", role = "ctb"),
+      person(given = "Andrew", family = "Johnson", role = "ctb",
+            comment = c(ORCID = "0000-0001-7000-8065 ")))
 Description: A lightweight interface to 'Stan' <https://mc-stan.org>.
     The 'CmdStanR' interface is an alternative to 'RStan' that calls the command
     line interface for compilation and running algorithms instead of interfacing

R/args.R

@@ -138,16 +138,16 @@ CmdStanArgs <- R6::R6Class(
       }
 
       if (!is.null(self$init)) {
-        args$init <- paste0("init=", self$init[idx])
+        args$init <- paste0("init=", wsl_safe_path(self$init[idx]))
       }
 
       if (!is.null(self$data_file)) {
-        args$data <- c("data", paste0("file=", self$data_file))
+        args$data <- c("data", paste0("file=", wsl_safe_path(self$data_file)))
       }
 
-      args$output <- c("output", paste0("file=", output_file))
+      args$output <- c("output", paste0("file=", wsl_safe_path(output_file)))
       if (!is.null(latent_dynamics_file)) {
-        args$output <- c(args$output, paste0("diagnostic_file=", latent_dynamics_file))
+        args$output <- c(args$output, paste0("diagnostic_file=", wsl_safe_path(latent_dynamics_file)))
       }
       if (!is.null(self$refresh)) {
         args$output <- c(args$output, paste0("refresh=", self$refresh))
@@ -158,7 +158,7 @@ CmdStanArgs <- R6::R6Class(
       }
 
       if (!is.null(profile_file)) {
-        args$output <- c(args$output, paste0("profile_file=", profile_file))
+        args$output <- c(args$output, paste0("profile_file=", wsl_safe_path(profile_file)))
       }
       if (!is.null(self$opencl_ids)) {
         args$opencl <- c("opencl", paste0("platform=", self$opencl_ids[1]), paste0("device=", self$opencl_ids[2]))
@@ -167,7 +167,7 @@ CmdStanArgs <- R6::R6Class(
       self$method_args$compose(idx, args)
     },
     command = function() {
-      paste0(if (!os_is_windows()) "./", basename(self$exe_file))
+      paste0(if (!os_is_windows() || os_is_wsl()) "./", basename(self$exe_file))
     }
   )
 )
@@ -1024,6 +1024,10 @@ compose_arg <- function(self, arg_name, cmdstan_arg_name = NULL, idx = NULL) {
   if (is.null(val)) {
     return(NULL)
   }
+
+  if (os_is_wsl() && (arg_name %in% c("metric_file", "fitted_params"))) {
+    val <- sapply(val, wsl_safe_path)
+  }
   if (!is.null(idx) && length(val) >= idx) {
     val <- val[idx]
   }

R/csv.R

@@ -734,6 +734,15 @@ read_csv_metadata <- function(csv_file) {
   if (length(gradients) > 0) {
     csv_file_info$gradients <- gradients
   }
+
+  # Revert any WSL-updated paths before returning the metadata
+  if (os_is_wsl()) {
+    csv_file_info$init <- wsl_safe_path(csv_file_info$init, revert = TRUE)
+    csv_file_info$profile_file <- wsl_safe_path(csv_file_info$profile_file,
+                                                revert = TRUE)
+    csv_file_info$fitted_params <- wsl_safe_path(csv_file_info$fitted_params,
+                                                  revert = TRUE)
+  }
   csv_file_info
 }
 

R/install.R

@@ -57,6 +57,8 @@
 #' @param check_toolchain (logical) Should `install_cmdstan()` attempt to check
 #'   that the required toolchain is installed and properly configured. The
 #'   default is `TRUE`.
+#' @param wsl (logical) Should CmdStan be installed and run through the Windows
+#'  Subsystem for Linux (WSL). The default is `FALSE`.
 #'
 #' @examples
 #' \dontrun{
@@ -81,7 +83,19 @@ install_cmdstan <- function(dir = NULL,
                             version = NULL,
                             release_url = NULL,
                             cpp_options = list(),
-                            check_toolchain = TRUE) {
+                            check_toolchain = TRUE,
+                            wsl = FALSE) {
+  # Use environment variable to record WSL usage throughout install,
+  # post-installation will simply check for 'wsl-' prefix in cmdstan path
+  if (isTRUE(wsl)) {
+    if (!os_is_windows()) {
+      warning("wsl=TRUE is only available on Windows, and will be ignored!",
+              call. = FALSE)
+      wsl <- FALSE
+    } else {
+      Sys.setenv("CMDSTANR_USE_WSL" = 1)
+    }
+  }
   if (check_toolchain) {
     check_cmdstan_toolchain(fix = FALSE, quiet = quiet)
   }
@@ -111,6 +125,7 @@ install_cmdstan <- function(dir = NULL,
     release_url <- paste0("https://github.com/stan-dev/cmdstan/releases/download/v",
                           version, "/cmdstan-", version, cmdstan_arch_suffix(version), ".tar.gz")
   }
+  wsl_prefix <- ifelse(isTRUE(wsl), "wsl-", "")
   if (!is.null(release_url)) {
     if (!endsWith(release_url, ".tar.gz")) {
       stop(release_url, " is not a .tar.gz archive!",
@@ -122,14 +137,14 @@ install_cmdstan <- function(dir = NULL,
     tar_name <- utils::tail(split_url[[1]], n = 1)
     cmdstan_ver <- substr(tar_name, 0, nchar(tar_name) - 7)
     tar_gz_file <- paste0(cmdstan_ver, ".tar.gz")
-    dir_cmdstan <- file.path(dir, cmdstan_ver)
+    dir_cmdstan <- file.path(dir, paste0(wsl_prefix, cmdstan_ver))
     dest_file <- file.path(dir, tar_gz_file)
   } else {
     ver <- latest_released_version()
     message("* Latest CmdStan release is v", ver)
     cmdstan_ver <- paste0("cmdstan-", ver, cmdstan_arch_suffix(ver))
     tar_gz_file <- paste0(cmdstan_ver, ".tar.gz")
-    dir_cmdstan <- file.path(dir, cmdstan_ver)
+    dir_cmdstan <- file.path(dir, paste0(wsl_prefix, cmdstan_ver))
     message("* Installing CmdStan v", ver, " in ", dir_cmdstan)
     message("* Downloading ", tar_gz_file, " from GitHub...")
     download_url <- github_download_url(ver)
@@ -171,7 +186,7 @@ install_cmdstan <- function(dir = NULL,
       append = TRUE
     )
   }
-  if (is_rtools42_toolchain()) {
+  if (is_rtools42_toolchain() && !os_is_wsl()) {
     cmdstan_make_local(
       dir = dir_cmdstan,
       cpp_options = list(
@@ -204,6 +219,9 @@ install_cmdstan <- function(dir = NULL,
       "\nrebuild_cmdstan(cores = ...)"
     )
   }
+  if (isTRUE(wsl)) {
+    Sys.unsetenv("CMDSTANR_USE_WSL")
+  }
 }
 
 
@@ -265,7 +283,9 @@ cmdstan_make_local <- function(dir = cmdstan_path(),
 #'
 check_cmdstan_toolchain <- function(fix = FALSE, quiet = FALSE) {
   if (os_is_windows()) {
-    if (R.version$major >= "4") {
+    if (os_is_wsl()) {
+      check_wsl_toolchain()
+    } else if (R.version$major >= "4") {
       check_rtools4x_windows_toolchain(fix = fix, quiet = quiet)
     } else {
       check_rtools35_windows_toolchain(fix = fix, quiet = quiet)
@@ -379,8 +399,8 @@ build_cmdstan <- function(dir,
       toolchain_PATH_env_var(),
       tbb_path(dir = dir)
     ),
-    processx::run(
-      run_cmd,
+    wsl_compatible_run(
+      command = run_cmd,
       args = c(translation_args, paste0("-j", cores), "build"),
       wd = dir,
       echo_cmd = is_verbose_mode(),
@@ -401,9 +421,9 @@ clean_cmdstan <- function(dir = cmdstan_path(),
       toolchain_PATH_env_var(),
       tbb_path(dir = dir)
     ),
-    processx::run(
-      make_cmd(),
-      args = c("clean-all"),
+    wsl_compatible_run(
+      command = make_cmd(),
+      args = "clean-all",
       wd = dir,
       echo_cmd = is_verbose_mode(),
       echo = !quiet || is_verbose_mode(),
@@ -420,9 +440,10 @@ build_example <- function(dir, cores, quiet, timeout) {
       toolchain_PATH_env_var(),
       tbb_path(dir = dir)
     ),
-    processx::run(
-      make_cmd(),
-      args = c(paste0("-j", cores), cmdstan_ext(file.path("examples", "bernoulli", "bernoulli"))),
+    wsl_compatible_run(
+      command = make_cmd(),
+      args = c(paste0("-j", cores),
+                cmdstan_ext(file.path("examples", "bernoulli", "bernoulli"))),
       wd = dir,
       echo_cmd = is_verbose_mode(),
       echo = !quiet || is_verbose_mode(),
@@ -499,6 +520,41 @@ install_toolchain <- function(quiet = FALSE) {
   invisible(NULL)
 }
 
+check_wsl_toolchain <- function() {
+  wsl_inaccessible <- processx::run(command = "wsl",
+                                     args = "uname",
+                                     error_on_status = FALSE)
+  if (wsl_inaccessible$status) {
+    stop("\n", "A WSL distribution is not installed or is not accessible.",
+         "\n", "Please see the Microsoft documentation for guidance on installing WSL: ",
+         "\n", "https://docs.microsoft.com/en-us/windows/wsl/install",
+         call. = FALSE)
+  }
+
+  make_not_present <- processx::run(command = "wsl",
+                                    args = c("which", "make"),
+                                    error_on_status = FALSE)
+
+  gpp_not_present <- processx::run(command = "wsl",
+                                    args = c("which", "g++"),
+                                    error_on_status = FALSE)
+
+  clangpp_not_present <- processx::run(command = "wsl",
+                                   args = c("which", "clang++"),
+                                   windows_verbatim_args = TRUE,
+                                   error_on_status = FALSE)
+
+  if (make_not_present$status || (gpp_not_present$status
+        && clangpp_not_present$status)) {
+    stop("\n", "Your distribution is missing the needed utilities for compiling C++.",
+         "\n", "Please launch your WSL and install them using the appropriate command:",
+         "\n", "Debian/Ubuntu: sudo apt-get install build-essential",
+         "\n", "Fedora: sudo dnf group install \"C Development Tools and Libraries\"",
+         "\n", "Arch: pacman -Sy base-devel",
+         call. = FALSE)
+  }
+}
+
 check_rtools4x_windows_toolchain <- function(fix = FALSE, quiet = FALSE) {
   rtools_path <- rtools4x_home_path()
   rtools_version <- if (is_rtools42_toolchain()) "Rtools42" else "Rtools40"