From e2dce76626f583d41e905b2864e96f346167fd68 Mon Sep 17 00:00:00 2001 From: walkowif <59475134+walkowif@users.noreply.github.com> Date: Tue, 14 Nov 2023 17:35:43 +0100 Subject: [PATCH] Add support for binary dependencies (#9) --- README.md | 47 ++++++++++++++++++++++++++++++++++++++++++++--- cmd/download.go | 13 +++++++++++-- cmd/parse.go | 3 ++- 3 files changed, 57 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8f21d10..c496ff2 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Real-life example with multiple input packages and repositories. Please see below for [an example](#configuration-file) how to set package and repository lists more easily in a configuration file. ```bash -locksmith --inputPackageList https://raw.githubusercontent.com/insightsengineering/formatters/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/rtables/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/scda/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/scda.2022/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/nestcolor/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/tern/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/rlistings/main/DESCRIPTION --inputRepositoryList BioC=https://bioconductor.org/packages/release/bioc,CRAN=https://cran.rstudio.com/ +locksmith --inputPackageList https://raw.githubusercontent.com/insightsengineering/formatters/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/rtables/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/scda/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/scda.2022/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/nestcolor/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/tern/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/rlistings/main/DESCRIPTION --inputRepositoryList BioC=https://bioconductor.org/packages/release/bioc,CRAN=https://cran.rstudio.com ``` In order to download the packages from GitHub or GitLab repositories, please set the environment variables containing the Personal Access Tokens. @@ -63,8 +63,8 @@ inputPackages: - https://raw.githubusercontent.com/insightsengineering/scda/main/DESCRIPTION - https://raw.githubusercontent.com/insightsengineering/scda.2022/main/DESCRIPTION inputRepositories: - - Bioconductor.BioCsoft=https://bioconductor.org/packages/release/bioc/ - - CRAN=https://cran.rstudio.com/ + - Bioconductor.BioCsoft=https://bioconductor.org/packages/release/bioc + - CRAN=https://cran.rstudio.com ``` The example above shows an alternative way of providing input packages, and input repositories, @@ -72,6 +72,47 @@ as opposed to `inputPackageList` and `inputRepositoryList` CLI flags/YAML keys. Additionally, `inputPackageList`/`inputRepositoryList` CLI flags take precendence over `inputPackages`/`inputRepositories` YAML keys. +## Binary dependencies + +For `locksmith` in order to generate an `renv.lock` with binary R packages, it is necessary to provide URLs to binary repositories in `inputRepositories`/`inputRepositoryList`. + +Examples illustrating the expected format of URLs to repositories with binary packages: + +* Linux: + * `https://packagemanager.posit.co/cran/__linux__//latest` +* Windows: + * `https://cloud.r-project.org/bin/windows/contrib/` + * `https://www.bioconductor.org/packages/release/bioc/bin/windows/contrib/` + * `https://packagemanager.posit.co/cran/latest/bin/windows/contrib/` +* macOS: + * `https://cloud.r-project.org/bin/macosx/contrib/` + * `https://www.bioconductor.org/packages/release/bioc/bin/macosx/big-sur-arm64/contrib/` + * `https://www.bioconductor.org/packages/release/bioc/bin/macosx/big-sur-x86_64/contrib/` + * `https://packagemanager.posit.co/cran/latest/bin/macosx/big-sur-x86_64/contrib/` + * `https://packagemanager.posit.co/cran/latest/bin/macosx/big-sur-arm64/contrib/` + +where `` is e.g. `4.2`, `4.3` etc. + +In all cases the URL points to a directory where the `PACKAGES` file is located. + +As a result, the configuration file could look like this: + +* for macOS: + + ```yaml + inputRepositories: + - CRAN-macOS=https://cloud.r-project.org/bin/macosx/contrib/4.2 + - Bioc-macOS=https://www.bioconductor.org/packages/release/bioc/bin/macosx/big-sur-x86_64/contrib/4.3 + ``` + +* for Windows: + + ```yaml + inputRepositories: + - CRAN-Windows=https://cloud.r-project.org/bin/windows/contrib/4.2 + - Bioc-Windows=https://www.bioconductor.org/packages/release/bioc/bin/windows/contrib/4.3 + ``` + ## Environment variables `locksmith` reads environment variables with `LOCKSMITH_` prefix and tries to match them with CLI flags. diff --git a/cmd/download.go b/cmd/download.go index d30785d..c55c38f 100644 --- a/cmd/download.go +++ b/cmd/download.go @@ -237,11 +237,20 @@ func DownloadPackagesFiles(repositoryList []string, downloadFileFunction func(string, map[string]string) (int, int64, string)) map[string]string { inputPackagesFiles := make(map[string]string) for _, repository := range repositoryList { - statusCode, _, packagesFileContent := downloadFileFunction(repository+"/src/contrib/PACKAGES", map[string]string{}) + var packagesFileURL string + if strings.Contains(repository, "/bin/windows/") || strings.Contains(repository, "/bin/macosx") { + // If we're dealing with a repository with binary Windows or macOS packages, + // we're expecting it to be in a specific format documented in the README. + packagesFileURL = repository + "/PACKAGES" + } else { + packagesFileURL = repository + "/src/contrib/PACKAGES" + } + log.Debug("Downloading ", packagesFileURL) + statusCode, _, packagesFileContent := downloadFileFunction(packagesFileURL, map[string]string{}) if statusCode == 200 { inputPackagesFiles[repository] = packagesFileContent } else { - log.Warn("An error occurred while downloading ", repository+"/src/contrib/PACKAGES") + log.Warn("An error occurred while downloading ", packagesFileURL) } } return inputPackagesFiles diff --git a/cmd/parse.go b/cmd/parse.go index 9eef07b..eda29aa 100644 --- a/cmd/parse.go +++ b/cmd/parse.go @@ -36,6 +36,7 @@ func ParseDescriptionFileList(inputDescriptionFiles []DescriptionFile) []Package func ParsePackagesFiles(repositoryPackageFiles map[string]string) map[string]PackagesFile { packagesFilesMap := make(map[string]PackagesFile) for repository, packagesFile := range repositoryPackageFiles { + log.Debug("Parsing PACKAGES file for ", repository) packagesFilesMap[repository] = ProcessPackagesFile(packagesFile) } return packagesFilesMap @@ -45,7 +46,7 @@ func ParsePackagesFiles(repositoryPackageFiles map[string]string) map[string]Pac // with those fields/properties that are required for further processing. func ProcessPackagesFile(content string) PackagesFile { var allPackages PackagesFile - for _, lineGroup := range strings.Split(content, "\n\n") { + for _, lineGroup := range strings.Split(strings.ReplaceAll(content, "\r\n", "\n"), "\n\n") { if lineGroup == "" { continue }