Skip to content

Commit

Permalink
Update docstrings
Browse files Browse the repository at this point in the history
  • Loading branch information
@walkowif
walkowif committed Oct 25, 2023
1 parent 324aace commit a770d14
Show file tree
Hide file tree
Showing 7 changed files with 109 additions and 72 deletions.
8 changes: 7 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,15 @@ locksmith --logLevel debug --exampleParameter 'exampleValue'
Real-life example with multiple input packages and repositories.

```bash
locksmith --inputPackageList https://raw.githubusercontent.com/insightsengineering/formatters/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/rtables/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/scda/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/scda.2022/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/nestcolor/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/tern/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/rlistings/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/citril/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/scda.test/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/citril.metadata/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/chevron/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/dunlin/main/DESCRIPTION --inputRepositoryList https://bioconductor.org/packages/release/bioc,https://cran.rstudio.com/
locksmith --inputPackageList https://raw.githubusercontent.com/insightsengineering/formatters/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/rtables/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/scda/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/scda.2022/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/nestcolor/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/tern/main/DESCRIPTION,https://raw.githubusercontent.com/insightsengineering/rlistings/main/DESCRIPTION --inputRepositoryList BioC=https://bioconductor.org/packages/release/bioc,CRAN=https://cran.rstudio.com/
```

In order to download the packages from GitHub or GitLab repositories, please set the environment variables containing the Personal Access Tokens.
* For GitHub, set the `LOCKSMITH_GITHUBTOKEN` environment variable.
* For GitLab, set the `LOCKSMITH_GITLABTOKEN` environment variable.

By default `locksmith` will save the resulting output file to `renv.lock`.

## Configuration file

If you'd like to set the above options in a configuration file, by default `locksmith` checks `~/.locksmith`, `~/.locksmith.yaml` and `~/.locksmith.yml` files.
Expand Down
39 changes: 27 additions & 12 deletions cmd/construct.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,10 @@ import (
"strings"
)

func constructOutputPackageList(packages []PackageDescription, packagesFiles map[string]PackagesFile,
// ConstructOutputPackageList generates a list of all packages and their dependencies
// which should be included in the output renv.lock file,
// based on the list of package descriptions, and information contained in the PACKAGES files.
func ConstructOutputPackageList(packages []PackageDescription, packagesFiles map[string]PackagesFile,
repositoryList []string) []PackageDescription {
var outputPackageList []PackageDescription
// Add all input packages to output list, as the packages should be downloaded from git repositories.
Expand All @@ -36,13 +39,13 @@ func constructOutputPackageList(packages []PackageDescription, packagesFiles map
skipDependency := false
if d.DependencyType == "Depends" || d.DependencyType == "Imports" ||
d.DependencyType == "Suggests" || d.DependencyType == "LinkingTo" {
if checkIfSkipDependency("", p.Package, d.DependencyName,
if CheckIfSkipDependency("", p.Package, d.DependencyName,
d.VersionOperator, d.VersionValue, &outputPackageList) {
skipDependency = true
}
if !skipDependency {
log.Info(p.Package, " → ", d.DependencyName)
resolveDependenciesRecursively(
ResolveDependenciesRecursively(
&outputPackageList, d.DependencyName, d.VersionOperator,
d.VersionValue, repositoryList, packagesFiles, 1,
)
Expand All @@ -53,7 +56,11 @@ func constructOutputPackageList(packages []PackageDescription, packagesFiles map
return outputPackageList
}

func resolveDependenciesRecursively(outputList *[]PackageDescription, name string, versionOperator string,
// ResolveDependenciesRecursively checks dependencies of the package, and their required versions.
// Checks if the required version is already included in the output package list
// (later used to generate the renv.lock), or if the dependency should be downloaded from a package repository.
// Repeats the process recursively for all dependencies not yet processed.
func ResolveDependenciesRecursively(outputList *[]PackageDescription, name string, versionOperator string,
versionValue string, repositoryList []string, packagesFiles map[string]PackagesFile, recursionLevel int) {
var indentation string
for i := 0; i < recursionLevel; i++ {
Expand All @@ -67,7 +74,7 @@ func resolveDependenciesRecursively(outputList *[]PackageDescription, name strin
log.Warn(indentation, name, " not found in top repository.")
}
// Check if package in the repository is available in sufficient version.
if !checkIfVersionSufficient(p.Version, versionOperator, versionValue) {
if !CheckIfVersionSufficient(p.Version, versionOperator, versionValue) {
// Try to retrieve the package from the next repository.
log.Warn(
indentation, p.Package, " in repository ", r,
Expand All @@ -87,10 +94,10 @@ func resolveDependenciesRecursively(outputList *[]PackageDescription, name strin
for _, d := range p.Dependencies {
if d.DependencyType == "Depends" || d.DependencyType == "Imports" ||
d.DependencyType == "LinkingTo" {
if !checkIfSkipDependency(indentation, p.Package, d.DependencyName,
if !CheckIfSkipDependency(indentation, p.Package, d.DependencyName,
d.VersionOperator, d.VersionValue, outputList) {
log.Info(indentation, p.Package, " → ", d.DependencyName)
resolveDependenciesRecursively(
ResolveDependenciesRecursively(
outputList, d.DependencyName, d.VersionOperator,
d.VersionValue, repositoryList, packagesFiles, recursionLevel+1,
)
Expand All @@ -112,7 +119,10 @@ func resolveDependenciesRecursively(outputList *[]PackageDescription, name strin
)
}

func checkIfBasePackage(name string) bool {
// CheckIfBasePackage checks whether the package should be treated as a base R package
// (included in every R installation) or if it should be treated as a dependency
// to be downloaded from a package repository.
func CheckIfBasePackage(name string) bool {
var basePackages = []string{
"base", "compiler", "datasets", "graphics", "grDevices", "grid",
"methods", "parallel", "splines", "stats", "stats4", "tcltk", "tools",
Expand All @@ -121,9 +131,12 @@ func checkIfBasePackage(name string) bool {
return stringInSlice(name, basePackages)
}

func checkIfSkipDependency(indentation string, packageName string, dependencyName string,
// CheckIfSkipDependency checks if processing of the package (dependency) should be skipped.
// Dependency should be skipped if it is a base R package, or has already been added to output
// package list (later used to generate the renv.lock).
func CheckIfSkipDependency(indentation string, packageName string, dependencyName string,
versionOperator string, versionValue string, outputList *[]PackageDescription) bool {
if checkIfBasePackage(dependencyName) {
if CheckIfBasePackage(dependencyName) {
log.Trace(indentation, "Skipping package ", dependencyName, " as it is a base R package.")
return true
}
Expand All @@ -133,7 +146,7 @@ func checkIfSkipDependency(indentation string, packageName string, dependencyNam
for i := 0; i < len(*outputList); i++ {
if dependencyName == (*outputList)[i].Package {
// Dependency found on the output list.
if checkIfVersionSufficient((*outputList)[i].Version, versionOperator, versionValue) {
if CheckIfVersionSufficient((*outputList)[i].Version, versionOperator, versionValue) {
var requirementMessage string
if versionOperator != "" && versionValue != "" {
requirementMessage = " according to the requirement " + versionOperator + " " + versionValue
Expand Down Expand Up @@ -172,7 +185,9 @@ func splitVersion(r rune) bool {
return r == '.' || r == '-'
}

func checkIfVersionSufficient(availableVersionValue string, versionOperator string,
// CheckIfVersionSufficient checks if availableVersionValue fulfills the requirement
// expressed by versionOperator ('>=' or '>') and requiredVersionValue.
func CheckIfVersionSufficient(availableVersionValue string, versionOperator string,
requiredVersionValue string) bool {
// Check if there are any version requirements at all.
if versionOperator == "" && requiredVersionValue == "" {
Expand Down
51 changes: 27 additions & 24 deletions cmd/download.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,9 @@ type GitHubObject struct {
Sha string `json:"sha"`
}

// Returns HTTP status code for downloaded file, number of bytes in downloaded content,
// and the downloaded content itself.
func downloadTextFile(url string, parameters map[string]string) (int, int64, string) { // #nosec G402
// DownloadTextFile returns HTTP status code for downloaded file, number of bytes
// in downloaded content, and the downloaded content itself as a string.
func DownloadTextFile(url string, parameters map[string]string) (int, int64, string) { // #nosec G402
tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
client := &http.Client{Transport: tr}
req, err := http.NewRequest("GET", url, nil)
Expand All @@ -69,12 +69,13 @@ func downloadTextFile(url string, parameters map[string]string) (int, int64, str
return -1, 0, ""
}

// Retrieve information about GitLab repository (project path, repository name and commit SHA) from
// projectURL repository GitLab API endpoint.
func getGitLabProjectAndSha(projectURL string, remoteRef string, token map[string]string,
// GetGitLabProjectAndSha retrieves information about GitLab repository
// (project path, repository name and commit SHA)
// from projectURL GitLab API endpoint.
func GetGitLabProjectAndSha(projectURL string, remoteRef string, token map[string]string,
downloadFileFunction func(string, map[string]string) (int, int64, string)) (string, string, string) {
var remoteUsername, remoteRepo, remoteSha string
log.Trace("Downloading project information from ", projectURL)
log.Trace("Downloading data for GitLab project from ", projectURL)
statusCode, _, projectDataResponse := downloadFileFunction(projectURL, token)
if statusCode == 200 {
var projectData GitLabAPIResponse
Expand All @@ -85,7 +86,7 @@ func getGitLabProjectAndSha(projectURL string, remoteRef string, token map[strin
remoteUsername = strings.Join(projectPath[:projectPathLength-1], "/")
remoteRepo = projectPath[projectPathLength-1]
} else {
log.Warn("An error occurred while retrieving project information from ", projectURL)
log.Warn("An error occurred while retrieving project data from ", projectURL)
}
match, errMatch := regexp.MatchString(`v\d+(\.\d+)*`, remoteRef)
var urlPath string
Expand All @@ -104,14 +105,14 @@ func getGitLabProjectAndSha(projectURL string, remoteRef string, token map[strin
checkError(err)
remoteSha = tagOrBranchData.Commit.ID
} else {
log.Warn("An error occurred while retrieving tag information from ", tagOrBranchURL)
log.Warn("An error occurred while retrieving data from ", tagOrBranchURL)
}
checkError(errMatch)
return remoteUsername, remoteRepo, remoteSha
}

// Retrieve SHA of the remoteRef from the 'remoteUsername/remoteRepo' GitHub repository.
func getGitHubSha(remoteUsername string, remoteRepo string, remoteRef string, token map[string]string,
// GetGitHubSha retrieves SHA of the remoteRef from the remoteUsername/remoteRepo GitHub repository.
func GetGitHubSha(remoteUsername string, remoteRepo string, remoteRef string, token map[string]string,
downloadFileFunction func(string, map[string]string) (int, int64, string)) string {
var remoteSha string
log.Trace("Downloading data for GitHub project ", remoteUsername, "/", remoteRepo)
Expand Down Expand Up @@ -139,15 +140,16 @@ func getGitHubSha(remoteUsername string, remoteRepo string, remoteRef string, to
return remoteSha
}

// Get information about packages stored in git repositories.
func processDescriptionURL(descriptionURL string,
// ProcessDescriptionURL gets information about the git repository in which the packages is stored
// based on the provided descriptionURL to the package DESCRIPTION file.
func ProcessDescriptionURL(descriptionURL string,
downloadFileFunction func(string, map[string]string) (int, int64, string),
) (map[string]string, string, string, string, string, string, string, string, string) {
token := make(map[string]string)
var remoteType, remoteRef, remoteHost, remoteUsername, remoteRepo string
var remoteSubdir, remoteSha, packageSource string
if strings.HasPrefix(descriptionURL, "https://raw.githubusercontent.com") {
// Expecting URL in form:
// Expecting GitHub URL in form:
// https://raw.githubusercontent.com/<organization>/<repo-name>/<ref-name>/<optional-subdirectories>/DESCRIPTION
token["Authorization"] = "token " + gitHubToken
remoteType = "github"
Expand All @@ -157,15 +159,15 @@ func processDescriptionURL(descriptionURL string,
remoteUsername = strings.Split(shorterURL, "/")[0]
remoteRepo = strings.Split(shorterURL, "/")[1]
remoteRef = strings.Split(shorterURL, "/")[2]
remoteSha = getGitHubSha(remoteUsername, remoteRepo, remoteRef, token, downloadFileFunction)
remoteSha = GetGitHubSha(remoteUsername, remoteRepo, remoteRef, token, downloadFileFunction)
// Check whether package is stored in a subdirectory of the git repository.
for i, j := range strings.Split(shorterURL, "/") {
if j == "DESCRIPTION" {
remoteSubdir = strings.Join(strings.Split(shorterURL, "/")[3:i], "/")
}
}
} else {
// Expecting URL in form:
// Expecting GitLab URL in form:
// https://example.gitlab.com/api/v4/projects/<project-id>/repository/files/<optional-subdirectories>/DESCRIPTION/raw?ref=<ref-name>
// <optional-subdirectories> contains '/' encoded as '%2F'
re := regexp.MustCompile(`ref=.*$`)
Expand All @@ -182,20 +184,21 @@ func processDescriptionURL(descriptionURL string,
descriptionPath := strings.Split(strings.ReplaceAll(descriptionPath, "%2F", "/"), "/")
remoteSubdir = strings.Join(descriptionPath[:len(descriptionPath)-1], "/")
}
remoteUsername, remoteRepo, remoteSha = getGitLabProjectAndSha(projectURL, remoteRef, token, downloadFileFunction)
remoteUsername, remoteRepo, remoteSha = GetGitLabProjectAndSha(projectURL, remoteRef, token, downloadFileFunction)
}
return token, remoteType, packageSource, remoteHost, remoteUsername, remoteRepo, remoteSubdir, remoteRef, remoteSha
}

// Downloads DESCRIPTION files from the list of supplied URLs.
// Returns a list of structures representing the contents of DESCRIPTION file for the packages,
// and the git repositories storing the packages.
func downloadDescriptionFiles(packageDescriptionList []string,
// DownloadDescriptionFiles downloads DESCRIPTION files from packageDescriptionList.
// It returns a list of structures representing:
// * the contents of DESCRIPTION file for the packages and
// * various information about git repositories storing the packages.
func DownloadDescriptionFiles(packageDescriptionList []string,
downloadFileFunction func(string, map[string]string) (int, int64, string)) []DescriptionFile {
var inputDescriptionFiles []DescriptionFile
for _, packageDescriptionURL := range packageDescriptionList {
token, remoteType, packageSource, remoteHost, remoteUsername, remoteRepo, remoteSubdir, remoteRef, remoteSha :=
processDescriptionURL(packageDescriptionURL, downloadFileFunction)
ProcessDescriptionURL(packageDescriptionURL, downloadFileFunction)
log.Info(
"Downloading ", packageDescriptionURL, "\nremoteType = ", remoteType,
", remoteUsername = ", remoteUsername, ", remoteRepo = ", remoteRepo,
Expand All @@ -221,10 +224,10 @@ func downloadDescriptionFiles(packageDescriptionList []string,
return inputDescriptionFiles
}

// Downloads PACKAGES files from repository URLs specified in the repositoryList.
// DownloadPackagesFiles downloads PACKAGES files from repository URLs specified in the repositoryList.
// Returns a map from repository URL to the string with the contents of PACKAGES file
// for that repository.
func downloadPackagesFiles(repositoryList []string,
func DownloadPackagesFiles(repositoryList []string,
downloadFileFunction func(string, map[string]string) (int, int64, string)) map[string]string {
inputPackagesFiles := make(map[string]string)
for _, repository := range repositoryList {
Expand Down
Loading

0 comments on commit a770d14

Please sign in to comment.