Refactor bundlePackages()

NEWS.md

@@ -1,5 +1,10 @@
 # rsconnect 0.8.30 (development version)
 
+* `deployApp()` now uses a stricter policy for determining whether or not
+  a local package can be successfully installed on the deployment server.
+  This means that you're more likely to get a clean failure prior to 
+  deployment (#659).
+
 * The logic used by `deployApp()` for determining whether you publish a 
   new update or update an existing app has been simplified. Now `appName`,
   `account`, and `server` are used to find existing deployments. If none

R/appDependencies.R

@@ -1,35 +1,53 @@
-#' Detect Application Dependencies
+#' Detect application dependencies
+#'
+#' @description
+#' `appDependencies()` recursively detects all R package dependencies for an
+#' application by parsing all `.R` and `.Rmd` files and looking for calls
+#' to `library()`, `require()`, `requireNamespace()`, `::`, and so on.
+#' It then adds all recursive dependencies to create a complete manifest of
+#' package packages need to be installed to run the app.
+#'
+#' # Remote installation
+#'
+#' When deployed, the app must first install all of these packages, and
+#' rsconnect does its best to ensure that all the packages have the same
+#' version as you are running locally. It knows how to install packages from
+#' the following sources:
+#'
+#' * CRAN (`CRAN`) and CRAN-like repositories (`CustomCRANLikeRepository`).
+#' * BioConductor (`Bioconductor`)
+#' * Packages installed from GitHub (`github`), GitLab (`gitlab`), or
+#'   BitBucket (`bitbucket`).
+#'
+#' It does not know how to install packages that you have built and installed
+#' locally so if you attempt to deploy an app that depends on such a package
+#' it will fail. To resolve this issue, you'll need to install from a known
+#' source.
+#'
+#' # Suggested packages
+#'
+#' The `Suggests` field is not included when determining recursive dependencies,
+#' so it's possible that not every package required to run your application will
+#' be detected.
+#'
+#' For example, ggplot2's `geom_hex()` requires the hexbin package to be
+#' installed, but it is only suggested by ggplot2. So if you app uses
+#' `geom_hex()` it will fail, reporting that the hexbin package is not
+#' installed.
+#'
+#' You can overcome this problem with (e.g.) `requireNamespace(hexbin)`.
+#' This will tell rsconnect that your app needs the hexbin package, without
+#' otherwise affecting your code.
 #'
-#' Recursively detect all package dependencies for an application. This function
-#' parses all .R files in the application directory to determine what packages
-#' the application depends on; and for each of those packages what other
-#' packages they depend on.
 #' @inheritParams deployApp
 #' @param appDir Directory containing application. Defaults to current working
 #'   directory.
-#' @return Returns a data frame listing the package
-#'   dependencies detected for the application: \tabular{ll}{ `package`
-#'   \tab Name of package \cr `version` \tab Version of package\cr }
-#' @details Dependencies are determined by parsing application source code and
-#'   looking for calls to `library`, `require`, `::`, and
-#'   `:::`.
-#'
-#'   Recursive dependencies are detected by examining the `Depends`,
-#'   `Imports`, and `LinkingTo` fields of the packages immediately
-#'   dependend on by the application.
-#'
-#' @note Since the `Suggests` field is not included when determining
-#'   recursive dependencies of packages, it's possible that not every package
-#'   required to run your application will be detected.
-#'
-#'   In this case, you can force a package to be included dependency by
-#'   inserting call(s) to `require` within your source directory. This code
-#'   need not actually execute, for example you could create a standalone file
-#'   named `dependencies.R` with the following code: \cr \cr
-#'   `require(xts)` \cr `require(colorspace)` \cr
-#'
-#'   This will force the `xts` and `colorspace` packages to be
-#'   installed along with the rest of your application when it is deployed.
+#' @return A data frame with columns:
+#'   * `package`: package name.
+#'   * `version`: local version.
+#'   * `source`: where the package was installed from.
+#'   * `repository`: for CRAN and CRAN-like repositories, the url to the
+#'      repo.
 #' @examples
 #' \dontrun{
 #'
@@ -50,5 +68,5 @@ appDependencies <- function(appDir = getwd(), appFiles = NULL) {
   on.exit(unlink(bundleDir, recursive = TRUE), add = TRUE)
 
   deps <- snapshotRDependencies(bundleDir)
-  deps[c("Package", "Version", "Source")]
+  deps[c("Package", "Version", "Source", "Repository")]
 }

R/bundle.R

@@ -898,6 +898,13 @@ snapshotRDependencies <- function(appDir, implicit_dependencies = c(), verbose =
     sapply(repos, "[[", 1)
   )
 
+  # get packages records defined in the lockfile
+  records <- utils::tail(df, -1)
+  records[c("Source", "Repository")] <- findPackageRepoAndSource(records, repos)
+  records
+}
+
+findPackageRepoAndSource <- function(records, repos) {
   # read available.packages filters (allow user to override if necessary;
   # this is primarily to allow debugging)
   #
@@ -906,10 +913,7 @@ snapshotRDependencies <- function(appDir, implicit_dependencies = c(), verbose =
   # in use can still be marked as available on CRAN -- for example, currently
   # the package "foreign" requires "R (>= 4.0.0)" but older versions of R
   # can still successfully install older versions from the CRAN archive
-  filters <- getOption(
-    "available_packages_filters",
-    default = c("duplicates")
-  )
+  filters <- getOption("available_packages_filters", default = "duplicates")
 
   # get Bioconductor repos if any
   biocRepos <- repos[grep("BioC", names(repos), perl = TRUE, value = TRUE)]
@@ -933,50 +937,58 @@ snapshotRDependencies <- function(appDir, implicit_dependencies = c(), verbose =
     name = names(named.repos),
     url = as.character(named.repos),
     contrib.url = contrib.url(named.repos, type = "source"),
-    stringsAsFactors = FALSE)
-
-  # get packages records defined in the lockfile
-  records <- utils::tail(df, -1)
+    stringsAsFactors = FALSE
+  )
 
-  # if the package is in a named CRAN-like repository capture it
+  # Sources are created by packrat:
+  # https://github.com/rstudio/packrat/blob/main/R/pkg.R#L328
   tmp <- lapply(seq_len(nrow(records)), function(i) {
-
     pkg <- records[i, "Package"]
     source <- records[i, "Source"]
     repository <- NA
-    # capture Bioconcutor repository
-    if (identical(source, "Bioconductor")) {
-      if (pkg %in% biocPackages) {
-        repository <- biocPackages[pkg, "Repository"]
-      }
-    } else if (isSCMSource(source)) {
-      # leave source+SCM packages alone.
+    if (identical(source, "Bioconductor") && pkg %in% biocPackages) {
+      # installed from known Bioconductor repo
+      repository <- biocPackages[pkg, "Repository"]
+    } else if (source %in% c("github", "bitbucket", "gitlab")) {
+      # leave SCM packages alone.
     } else if (pkg %in% rownames(repo.packages)) {
       # capture CRAN-like repository
 
-      # Find this package in the set of available packages then use its
-      # contrib.url to map back to the configured repositories.
-      package.contrib <- repo.packages[pkg, "Repository"]
-      package.repo.index <- vapply(repo.lookup$contrib.url,
-                                   function(url) grepl(url, package.contrib, fixed = TRUE), logical(1))
-      package.repo <- repo.lookup[package.repo.index, ][1, ]
-      # If the incoming package comes from CRAN, keep the CRAN name in place
-      # even if that means using a different name than the repos list.
-      #
-      # The "cran" source is a well-known location for shinyapps.io.
-      #
-      # shinyapps.io isn't going to use the manifest-provided CRAN URL,
-      # but other consumers (Connect) will.
-      if (tolower(source) != "cran") {
-        source <- package.repo$name
+      # If the name of a source-installed package matches a CRAN package,
+      # packrat automatically fills in the repository information. But that
+      # causes problems if you're working with a developement version that's
+      # newer than CRAN.
+      repo_version <- package_version(repo.packages[pkg, "Version"])
+      local_version <- package_version(records[i, "Version"])
+      if (local_version <= repo_version) {
+        # Find this package in the set of available packages then use its
+        # contrib.url to map back to the configured repositories.
+        package.contrib <- repo.packages[pkg, "Repository"]
+        package.repo.index <- grepl(repo.lookup$contrib.url, package.contrib, fixed = TRUE)
+        package.repo <- repo.lookup[package.repo.index, ][1, ]
+        # If the incoming package comes from CRAN, keep the CRAN name in place
+        # even if that means using a different name than the repos list.
+        #
+        # The "cran" source is a well-known location for shinyapps.io.
+        #
+        # shinyapps.io isn't going to use the manifest-provided CRAN URL,
+        # but other consumers (Connect) will.
+        if (tolower(source) != "cran") {
+          source <- package.repo$name
+        }
+        repository <- package.repo$url
+      } else {
+        # Ahead of repo, so don't know how to install
+        source <- NA
       }
-      repository <- package.repo$url
+    } else {
+      # Don't know how to install
+      source <- NA
     }
     # validatePackageSource will emit a warning for packages with NA repository.
-    data.frame(Source = source, Repository = repository)
+    data.frame(Source = source, Repository = repository, stringsAsFactors = FALSE)
   })
-  records[, c("Source", "Repository")] <- do.call("rbind", tmp)
-  return(records)
+  do.call("rbind", tmp)
 }
 
 addPackratSnapshot <- function(bundleDir, implicit_dependencies = c(), verbose = FALSE) {