Fix #1122

ropensci · Aug 24, 2023 · 1df7c94 · 1df7c94
1 parent c3d1887
commit 1df7c94
Show file tree

Hide file tree

Showing 5 changed files with 167 additions and 105 deletions.
diff --git a/R/class_gcp.R b/R/class_gcp.R
@@ -77,32 +77,26 @@ store_read_object.tar_gcp <- function(store) {
   )
   on.exit(unlink(scratch))
   dir_create(dirname(scratch))
-  old_try_attempts <- getOption("googleAuthR.tryAttempts")
-  on.exit(options(googleAuthR.tryAttempts = old_try_attempts), add = TRUE)
-  option <- store$resources$network$max_tries
-  if_any(is.null(option), NULL, options(googleAuthR.tryAttempts = option))
   gcp_gcs_download(
     key = key,
     bucket = bucket,
     file = scratch,
     version = store_gcp_version(path),
-    verbose = store$resources$gcp$verbose %|||% FALSE
+    verbose = store$resources$gcp$verbose %|||% FALSE,
+    max_tries = store$resources$network$max_tries %|||% 5L
   )
   store_convert_object(store, store_read_path(store, scratch))
 }
 
 #' @export
 store_exist_object.tar_gcp <- function(store, name = NULL) {
   path <- store$file$path
-  old_try_attempts <- getOption("googleAuthR.tryAttempts")
-  on.exit(options(googleAuthR.tryAttempts = old_try_attempts), add = TRUE)
-  option <- store$resources$network$max_tries
-  if_any(is.null(option), NULL, options(googleAuthR.tryAttempts = option))
   gcp_gcs_exists(
     key = store_gcp_key(path),
     bucket = store_gcp_bucket(path),
     version = store_gcp_version(path),
-    verbose = store$resources$gcp$verbose %|||% FALSE
+    verbose = store$resources$gcp$verbose %|||% FALSE,
+    max_tries = store$resources$network$max_tries %|||% 5L
   )
 }
 
@@ -119,16 +113,13 @@ store_delete_object.tar_gcp <- function(store, name = NULL) {
     "from trying to delete it.\nMessage: "
   )
   message <- sprintf(message, name, bucket, key, name)
-  old_try_attempts <- getOption("googleAuthR.tryAttempts")
-  on.exit(options(googleAuthR.tryAttempts = old_try_attempts), add = TRUE)
-  option <- store$resources$network$max_tries
-  if_any(is.null(option), NULL, options(googleAuthR.tryAttempts = option))
   tryCatch(
     gcp_gcs_delete(
       key = key,
       bucket =  bucket,
       version = version,
-      verbose = store$resources$gcp$verbose %|||% FALSE
+      verbose = store$resources$gcp$verbose %|||% FALSE,
+      max_tries = store$resources$network$max_tries %|||% 5L
     ),
     error = function(condition) {
       tar_throw_validate(message, conditionMessage(condition))
@@ -147,10 +138,6 @@ store_upload_object.tar_gcp <- function(store) {
 store_upload_object_gcp <- function(store) {
   key <- store_gcp_key(store$file$path)
   bucket <- store_gcp_bucket(store$file$path)
-  old_try_attempts <- getOption("googleAuthR.tryAttempts")
-  on.exit(options(googleAuthR.tryAttempts = old_try_attempts), add = TRUE)
-  option <- store$resources$network$max_tries
-  if_any(is.null(option), NULL, options(googleAuthR.tryAttempts = option))
   head <- if_any(
     file_exists_stage(store$file),
     gcp_gcs_upload(
@@ -159,7 +146,8 @@ store_upload_object_gcp <- function(store) {
       bucket = bucket,
       metadata = list("targets-hash" = store$file$hash),
       predefined_acl = store$resources$gcp$predefined_acl %|||% "private",
-      verbose = store$resources$gcp$verbose %|||% FALSE
+      verbose = store$resources$gcp$verbose %|||% FALSE,
+      max_tries = store$resources$network$max_tries %|||% 5L
     ),
     tar_throw_file(
       "Cannot upload non-existent gcp staging file ",
@@ -191,15 +179,12 @@ store_has_correct_hash.tar_gcp <- function(store) {
 
 store_gcp_hash <- function(store) {
   path <- store$file$path
-  old_try_attempts <- getOption("googleAuthR.tryAttempts")
-  on.exit(options(googleAuthR.tryAttempts = old_try_attempts), add = TRUE)
-  option <- store$resources$network$max_tries
-  if_any(is.null(option), NULL, options(googleAuthR.tryAttempts = option))
   head <- gcp_gcs_head(
     key = store_gcp_key(path),
     bucket = store_gcp_bucket(path),
     version = store_gcp_version(path),
-    verbose = store$resources$gcp$verbose %|||% FALSE
+    verbose = store$resources$gcp$verbose %|||% FALSE,
+    max_tries = store$resources$network$max_tries %|||% 5L
   )
   head$metadata[["targets-hash"]]
 }

diff --git a/R/class_gcp_file.R b/R/class_gcp_file.R
@@ -52,16 +52,13 @@ store_read_object.tar_gcp_file <- function(store) {
     pattern = basename(store_gcp_key(path))
   )
   dir_create(dirname(scratch))
-  old_try_attempts <- getOption("googleAuthR.tryAttempts")
-  on.exit(options(googleAuthR.tryAttempts = old_try_attempts), add = TRUE)
-  option <- store$resources$network$max_tries
-  if_any(is.null(option), NULL, options(googleAuthR.tryAttempts = option))
   gcp_gcs_download(
     key = key,
     bucket = bucket,
     file = scratch,
     version = store_gcp_version(path),
-    verbose = store$resources$gcp$verbose %|||% FALSE
+    verbose = store$resources$gcp$verbose %|||% FALSE,
+    max_tries = store$resources$network$max_tries %|||% 5L
   )
   stage <- store_gcp_file_stage(path)
   dir_create(dirname(stage))

diff --git a/R/utils_gcp.R b/R/utils_gcp.R
@@ -9,9 +9,13 @@ gcp_gcs_head <- function(
   key,
   bucket = gcp_gcs_bucket(),
   version = NULL,
-  verbose = FALSE
+  verbose = FALSE,
+  max_tries
 ) {
-  gcp_gcs_auth(verbose = verbose)
+  old_try_attempts <- getOption("googleAuthR.tryAttempts")
+  on.exit(options(googleAuthR.tryAttempts = old_try_attempts), add = TRUE)
+  if_any(is.null(max_tries), NULL, options(googleAuthR.tryAttempts = max_tries))
+  gcp_gcs_auth(verbose = verbose, max_tries = max_tries)
   if_any(verbose, identity, suppressMessages) (
     tryCatch(
       googleCloudStorageR::gcs_get_object(
@@ -27,22 +31,27 @@ gcp_gcs_head <- function(
 
 gcp_gcs_exists <- function(
   key,
-  bucket = gcp_gcs_bucket(),
+  bucket = gcp_gcs_bucket(verbose = verbose, max_tries = max_tries),
   version = NULL,
-  verbose = FALSE
+  verbose = FALSE,
+  max_tries
 ) {
   !is.null(
     gcp_gcs_head(
       key = key,
       bucket = bucket,
       version = version,
-      verbose = verbose
+      verbose = verbose,
+      max_tries = max_tries
     )
   )
 }
 
-gcp_gcs_bucket <- function(verbose = FALSE) {
-  gcp_gcs_auth(verbose = verbose)
+gcp_gcs_bucket <- function(verbose = FALSE, max_tries) {
+  old_try_attempts <- getOption("googleAuthR.tryAttempts")
+  on.exit(options(googleAuthR.tryAttempts = old_try_attempts), add = TRUE)
+  if_any(is.null(max_tries), NULL, options(googleAuthR.tryAttempts = max_tries))
+  gcp_gcs_auth(verbose = verbose, max_tries = max_tries)
   if_any(verbose, identity, suppressMessages) (
     googleCloudStorageR::gcs_get_global_bucket()
   )
@@ -51,11 +60,15 @@ gcp_gcs_bucket <- function(verbose = FALSE) {
 gcp_gcs_download <- function(
   file,
   key,
-  bucket = gcp_gcs_bucket(),
+  bucket = gcp_gcs_bucket(verbose = verbose, max_tries = max_tries),
   version = NULL,
-  verbose = FALSE
+  verbose = FALSE,
+  max_tries
 ) {
-  gcp_gcs_auth(verbose = verbose)
+  old_try_attempts <- getOption("googleAuthR.tryAttempts")
+  on.exit(options(googleAuthR.tryAttempts = old_try_attempts), add = TRUE)
+  if_any(is.null(max_tries), NULL, options(googleAuthR.tryAttempts = max_tries))
+  gcp_gcs_auth(verbose = verbose, max_tries = max_tries)
   if_any(verbose, identity, suppressMessages) (
     googleCloudStorageR::gcs_get_object(
       key,
@@ -69,11 +82,15 @@ gcp_gcs_download <- function(
 
 gcp_gcs_delete <- function(
   key,
-  bucket = gcp_gcs_bucket(),
+  bucket = gcp_gcs_bucket(verbose = verbose, max_tries = max_tries),
   version = NULL,
-  verbose = FALSE
+  verbose = FALSE,
+  max_tries
 ) {
-  gcp_gcs_auth(verbose = verbose)
+  old_try_attempts <- getOption("googleAuthR.tryAttempts")
+  on.exit(options(googleAuthR.tryAttempts = old_try_attempts), add = TRUE)
+  if_any(is.null(max_tries), NULL, options(googleAuthR.tryAttempts = max_tries))
+  gcp_gcs_auth(verbose = verbose, max_tries = max_tries)
   if_any(verbose, identity, suppressMessages) (
     tryCatch(
       googleCloudStorageR::gcs_delete_object(
@@ -90,12 +107,16 @@ gcp_gcs_delete <- function(
 gcp_gcs_upload <- function(
   file,
   key,
-  bucket = gcp_gcs_bucket(),
+  bucket = gcp_gcs_bucket(verbose = verbose, max_tries = max_tries),
   metadata = list(),
   predefined_acl = "private",
-  verbose = FALSE
+  verbose = FALSE,
+  max_tries
 ) {
-  gcp_gcs_auth(verbose = verbose)
+  old_try_attempts <- getOption("googleAuthR.tryAttempts")
+  on.exit(options(googleAuthR.tryAttempts = old_try_attempts), add = TRUE)
+  if_any(is.null(max_tries), NULL, options(googleAuthR.tryAttempts = max_tries))
+  gcp_gcs_auth(verbose = verbose, max_tries = max_tries)
   meta <- NULL
   if (length(metadata) > 0) {
     meta <- googleCloudStorageR::gcs_metadata_object(
@@ -114,10 +135,13 @@ gcp_gcs_upload <- function(
   )
 }
 
-gcp_gcs_auth <- function(verbose = FALSE) {
+gcp_gcs_auth <- function(verbose = FALSE, max_tries) {
   if (isTRUE(tar_runtime$gcp_auth)) {
     return()
   }
+  old_try_attempts <- getOption("googleAuthR.tryAttempts")
+  on.exit(options(googleAuthR.tryAttempts = old_try_attempts), add = TRUE)
+  if_any(is.null(max_tries), NULL, options(googleAuthR.tryAttempts = max_tries))
   if_any(verbose, identity, suppressMessages) (
     googleCloudStorageR::gcs_auth(
       token = gargle::token_fetch(

diff --git a/tests/gcp/test-class_gcp_parquet.R b/tests/gcp/test-class_gcp_parquet.R
@@ -5,7 +5,7 @@ tar_test("gcp_parquet format returns data frames", {
   skip_if_not_installed("arrow")
   bucket_name <- random_bucket_name()
   # needs to be a GCP project the tester auth has access to
-  gcp_gcs_auth()
+  gcp_gcs_auth(max_tries = 5)
   project <- Sys.getenv("GCE_DEFAULT_PROJECT_ID")
   googleCloudStorageR::gcs_create_bucket(bucket_name, projectId = project)
   on.exit(gcp_gcs_delete_bucket(bucket_name))