Merge pull request #4 from HDash/fix-github-files

Handle more than 100 results from repo branch query
RajLabMSSM · Dec 16, 2024 · a4e4b26 · a4e4b26
2 parents 1e3594d + e3c326b
commit a4e4b26
Show file tree

Hide file tree

Showing 7 changed files with 78 additions and 49 deletions.
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,7 +1,7 @@
 Package: echogithub
 Type: Package
 Title: echoverse module: Extract data and metadata from GitHub
-Version: 0.99.2
+Version: 0.99.3
 Authors@R: 
     c(person(given = "Brian",
            family = "Schilder",
@@ -17,7 +17,12 @@ Authors@R:
            family = "Raj",
            role = c("aut"),
            email = "towfique.raj@mssm.edu",
-           comment = c(ORCID = "0000-0002-9355-5704"))
+           comment = c(ORCID = "0000-0002-9355-5704")),
+    person(given = "Hiranyamaya",
+           family = "Dash",
+           role = c("ctb"),
+           email = "hdash.work@gmail.com",
+           comment = c(ORCID = "0009-0005-5514-505X"))
            ) 
 Description: echoverse module: Extract data and metadata from GitHub.
 URL: https://github.com/RajLabMSSM/echogithub
@@ -56,7 +61,7 @@ Suggests:
 Remotes:
     github::neurogenomics/cranlogs,
     github::neurogenomics/rworkflows
-RoxygenNote: 7.2.3
+RoxygenNote: 7.3.2
 VignetteBuilder: knitr
 License: GPL-3
 Config/testthat/edition: 3
diff --git a/NEWS.md b/NEWS.md
@@ -1,3 +1,11 @@
+# echogithub 0.99.3
+
+## Bug fixes
+
+* `github_branches`
+    - Handle repos with more than 100 branches.
+
+
 # echogithub 0.99.2
 
 ## New features
@@ -68,4 +76,4 @@
 * Switched to using `gh` instead of `httr` to avoid API limits imposed by GitHub.
     - Kept `httr` as alternative method.
 * `is_url`:
-    - Add `RCurl::url.exists` check.
+    - Add `RCurl::url.exists` check.
diff --git a/R/github_branches.R b/R/github_branches.R
@@ -1,33 +1,33 @@
 #' GitHub branches
-#' 
+#'
 #' List all branches for a given GitHub repository.
-#' @param owner Owner of the GitHub repository. 
+#' @param owner Owner of the GitHub repository.
 #' If \code{NULL}, will automatically try to infer the owner
 #'  name from the \emph{DESCRIPTION file}
 #'  (assuming you're working directory is a local R package repo).
-#' @param repo GitHub repository name. 
+#' @param repo GitHub repository name.
 #' If \code{NULL}, will automatically try to infer the repo name
 #'  name from the \emph{DESCRIPTION file}
 #'  (assuming you're working directory is a local R package repo).
-#' @param branch [Optional] If \code{branch} is supplied 
-#' (as a character vector of one or more branch names), 
+#' @param branch [Optional] If \code{branch} is supplied
+#' (as a character vector of one or more branch names),
 #' will check to see if that branch exists. If it does, only that branch will
 #' be returned. If it doesn't, an error will be thrown.
 #' @param master_or_main  If \code{branch} is supplied and
 #'  is either \code{"master"} or \code{"main"},
-#' automatically interpret "master" and "main" as synonymous and return 
-#' whichever branch exists. 
+#' automatically interpret "master" and "main" as synonymous and return
+#' whichever branch exists.
 #' @param as_datatable Return the results as a \link[data.table]{data.table}
 #' (\code{TRUE}), or as a character vector of branch names
 #'  (default: \code{FALSE}).
 #' @param error Throw an error when no matching branches are fond.
 #' @inheritParams github_files
 #' @inheritParams description_extract
 #' @returns Character vector or \link[data.table]{data.table} of branches.
-#' 
+#'
 #' @export
 #' @importFrom gh gh_token gh
-#' @examples 
+#' @examples
 #' branches <- github_branches(owner="RajLabMSSM", repo="echolocatoR")
 github_branches <- function(owner = NULL,
                             repo = NULL,
@@ -37,36 +37,45 @@ github_branches <- function(owner = NULL,
                             token = gh::gh_token(),
                             desc_file = NULL,
                             error = FALSE,
-                            verbose = TRUE){ 
+                            verbose = TRUE){
     name <- NULL;
-  
+
     out <- infer_owner_repo(owner = owner,
-                            repo = repo, 
-                            desc_file = desc_file, 
+                            repo = repo,
+                            desc_file = desc_file,
                             verbose = verbose)
     owner <- out$owner
-    repo <- out$repo 
+    repo <- out$repo
     #### Search branches ####
     messager("Searching for all branches in:",paste(owner,repo,sep="/"),
              v=verbose)
     endpoint <- paste(
         "https://api.github.com/repos",owner,repo,"branches",
         sep="/"
     )
-    gh_response <- gh::gh(endpoint = endpoint,
-                          .token = token,
-                          per_page = 100) 
-    dt <- gh_to_dt(gh_response)
-    dt <- cbind(owner=owner, repo=repo, dt)
-    #### Filter branches ####
-    if(!is.null(branch)){
-        #### Detect synonymous branches ####
-        if(isTRUE(master_or_main) && 
-           any(c("master","main") %in% branch)){
-               branch <- unique(c("master","main",branch))
-           }
-        dt <- dt[name %in% branch,]
+    page <- 1
+    repeat {
+      # Keep iterating pages until we find the branch or run out of pages
+        gh_response <- gh::gh(endpoint = endpoint,
+                              .token = token,
+                              per_page = 100,
+                              page = page)
+        if(length(gh_response) == 0) break
+        dt <- gh_to_dt(gh_response)
+        dt <- cbind(owner=owner, repo=repo, dt)
+        #### Filter branches ####
+        if(!is.null(branch)){
+            #### Detect synonymous branches ####
+            if(isTRUE(master_or_main) &&
+               any(c("master","main") %in% branch)){
+                   branch <- unique(c("master","main",branch))
+               }
+            dt <- dt[name %in% branch,]
+        }
+        if(nrow(dt)>0) break
+        page <- page + 1
     }
+
     #### Report ####
     if(nrow(dt)>0){
         messager(paste0(
@@ -78,9 +87,9 @@ github_branches <- function(owner = NULL,
         if(isTRUE(error)) {
             stop(stp)
         } else {
-            messager("WARNING:",stp,"Returning NULL.",v=verbose) 
+            messager("WARNING:",stp,"Returning NULL.",v=verbose)
             return(NULL)
-        } 
+        }
     }
     #### Return ####
     if(isTRUE(as_datatable)){

diff --git a/README.md b/README.md
@@ -1,19 +1,20 @@
 <img src='https://github.com/RajLabMSSM/echogithub/raw/master/inst/hex/hex.png' title='Hex sticker for echogithub' height='300'><br>
 [![License:
 GPL-3](https://img.shields.io/badge/license-GPL--3-blue.svg)](https://cran.r-project.org/web/licenses/GPL-3)
-[![](https://img.shields.io/badge/devel%20version-0.99.2-black.svg)](https://github.com/RajLabMSSM/echogithub)
+[![](https://img.shields.io/badge/devel%20version-0.99.3-black.svg)](https://github.com/RajLabMSSM/echogithub)
 [![](https://img.shields.io/github/languages/code-size/RajLabMSSM/echogithub.svg)](https://github.com/RajLabMSSM/echogithub)
 [![](https://img.shields.io/github/last-commit/RajLabMSSM/echogithub.svg)](https://github.com/RajLabMSSM/echogithub/commits/master)
 <br> [![R build
 status](https://github.com/RajLabMSSM/echogithub/workflows/rworkflows/badge.svg)](https://github.com/RajLabMSSM/echogithub/actions)
-[![](https://codecov.io/gh/RajLabMSSM/echogithub/branch/master/graph/badge.svg)](https://codecov.io/gh/RajLabMSSM/echogithub)
+[![](https://codecov.io/gh/RajLabMSSM/echogithub/branch/master/graph/badge.svg)](https://app.codecov.io/gh/RajLabMSSM/echogithub)
 <br>
 <a href='https://app.codecov.io/gh/RajLabMSSM/echogithub/tree/master' target='_blank'><img src='https://codecov.io/gh/RajLabMSSM/echogithub/branch/master/graphs/icicle.svg' title='Codecov icicle graph' width='200' height='50' style='vertical-align: top;'></a>  
 <h4>  
-Authors: <i>Brian Schilder, Jack Humphrey, Towfique Raj</i>  
+Authors: <i>Brian Schilder, Jack Humphrey, Towfique Raj, Hiranyamaya
+Dash</i>  
 </h4>
 <h5>  
-README updated: <i>Mar-10-2023</i>  
+README updated: <i>Dec-03-2024</i>  
 </h5>
 
 ## `echogithub`: Extract data and metadata from GitHub.

diff --git a/man/description_extract.Rd b/man/description_extract.Rd
diff --git a/man/github_branches.Rd b/man/github_branches.Rd
diff --git a/tests/testthat/test-github_files.R b/tests/testthat/test-github_files.R
@@ -1,9 +1,9 @@
-test_that("github_files works", { 
+test_that("github_files works", {
 
-    # files <- github_files(owner = "RajLabMSSM",
-    #                       repo = "Fine_Mapping_Shiny",
-    #                       query = ".md$", 
-    #                       download = TRUE)
-    # testthat::expect_true(methods::is(files, "data.table"))    
-    # testthat::expect_true(nrow(files)>=1)    
+    files <- github_files(owner = "RajLabMSSM",
+                          repo = "Fine_Mapping_Shiny",
+                          query = ".md$",
+                          download = TRUE)
+    testthat::expect_true(methods::is(files, "data.table"))
+    testthat::expect_true(nrow(files)>=1)
 })