From f2b4858f1d9013e4b05a0e971d2a2851b4b2ac54 Mon Sep 17 00:00:00 2001 From: Sam Albers Date: Tue, 9 Jul 2024 16:13:08 -0700 Subject: [PATCH 01/17] replace with test url --- R/utils.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/utils.R b/R/utils.R index 4a542bdd..fb2fdc1c 100644 --- a/R/utils.R +++ b/R/utils.R @@ -12,12 +12,12 @@ catalogue_base_url <- function() { getOption("bcdata.catalogue_gui_url", - default = "https://catalogue.data.gov.bc.ca/") + default = "https://toyger.data.gov.bc.ca/") } catalogue_base_api_url <- function() { getOption("bcdata.catalogue_api_url", - default = "https://catalogue.data.gov.bc.ca/api/3") + default = "https://toyger.data.gov.bc.ca/api/3") } wfs_base_url <- function(host = bcdc_web_service_host()) { From 01afa76bdfed70b4c1011e95a928a469d5c906f8 Mon Sep 17 00:00:00 2001 From: Andy Teucher Date: Wed, 14 Aug 2024 12:52:05 -0700 Subject: [PATCH 02/17] Extract names from nested lest --- R/bcdc-get-citation.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/bcdc-get-citation.R b/R/bcdc-get-citation.R index b5cf57b0..7ff2682e 100644 --- a/R/bcdc-get-citation.R +++ b/R/bcdc-get-citation.R @@ -93,8 +93,8 @@ bcdc_get_citation.bcdc_record <- function(record) { } clean_org_name <- function(rec) { - name <- jsonlite::fromJSON(rec$contacts)$name + name <- vapply(rec$contacts, function(x) x$name, FUN.VALUE = character(1)) name <- trimws(name) name <- paste0(name, collapse = ", ") - sub(",([^,]*)$", " &\\1", name) + gsub(",([^,]*)$", " &\\1", name) } From 2144c8f53d763fd5f2b3a90dfcabf62782f7ec22 Mon Sep 17 00:00:00 2001 From: Andy Teucher Date: Wed, 14 Aug 2024 12:53:29 -0700 Subject: [PATCH 03/17] Parse details df from nested list --- R/describe-feature.R | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/R/describe-feature.R b/R/describe-feature.R index 78d73134..41a05054 100644 --- a/R/describe-feature.R +++ b/R/describe-feature.R @@ -125,14 +125,14 @@ feature_helper <- function(whse_name){ xml_df } - - obj_desc_join <- function(record) { stopifnot(inherits(record, "bcdc_record")) wfs_resource <- get_wfs_resource_from_record(record) whse_name <- wfs_resource$object_name - wfs_df <- jsonlite::fromJSON(wfs_resource$details) + wfs_df <- purrr::list_rbind( + purrr::map(wfs_resource$details, as.data.frame) + ) dplyr::left_join( feature_helper(whse_name), From eb581107f360fd020e64e15fa57d91a84baeae03 Mon Sep 17 00:00:00 2001 From: Andy Teucher Date: Wed, 14 Aug 2024 12:54:01 -0700 Subject: [PATCH 04/17] Remove sector from facets list, add publish_state --- R/bcdc_search.R | 4 ++-- tests/testthat/test-get_record.R | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/R/bcdc_search.R b/R/bcdc_search.R index 29ab0968..aaee28f2 100644 --- a/R/bcdc_search.R +++ b/R/bcdc_search.R @@ -14,7 +14,7 @@ #' #' @param facet the facet(s) for which to retrieve valid values. Can be one or #' more of: -#' `"license_id", "download_audience", "res_format", "sector", "organization", "groups"` +#' `"license_id", "download_audience", "res_format", "publish_state", "organization", "groups"` #' #' @return A data frame of values for the selected facet #' @export @@ -30,7 +30,7 @@ #' ) #' } bcdc_search_facets <- function(facet = c("license_id", "download_audience", - "res_format", "sector", + "res_format", "publish_state", "organization", "groups")) { if(!has_internet()) stop("No access to internet", call. = FALSE) # nocov diff --git a/tests/testthat/test-get_record.R b/tests/testthat/test-get_record.R index 3c446f0c..fa129f41 100644 --- a/tests/testthat/test-get_record.R +++ b/tests/testthat/test-get_record.R @@ -30,7 +30,7 @@ test_that("bcdc_search_facets works", { skip_if_net_down() ret_names <- c("facet", "count", "display_name", "name") lapply(c("license_id", "download_audience", "res_format", - "sector", "organization"), + "publish_state", "organization"), function(x) expect_named(bcdc_search_facets(x)) ) expect_error(bcdc_search_facets("foo"), "'arg' should be one of") From 61764595424b9165e3b50620b867e951de9660ff Mon Sep 17 00:00:00 2001 From: Andy Teucher Date: Thu, 15 Aug 2024 11:51:32 -0700 Subject: [PATCH 05/17] document --- man/bcdc_search_facets.Rd | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/man/bcdc_search_facets.Rd b/man/bcdc_search_facets.Rd index 12b3e56d..85885809 100644 --- a/man/bcdc_search_facets.Rd +++ b/man/bcdc_search_facets.Rd @@ -5,14 +5,14 @@ \title{Get the valid values for a facet (that you can use in \code{\link[=bcdc_search]{bcdc_search()}})} \usage{ bcdc_search_facets( - facet = c("license_id", "download_audience", "res_format", "sector", "organization", - "groups") + facet = c("license_id", "download_audience", "res_format", "publish_state", + "organization", "groups") ) } \arguments{ \item{facet}{the facet(s) for which to retrieve valid values. Can be one or more of: -\verb{"license_id", "download_audience", "res_format", "sector", "organization", "groups"}} +\verb{"license_id", "download_audience", "res_format", "publish_state", "organization", "groups"}} } \value{ A data frame of values for the selected facet From 5a9f19dfc2c3a4702dd1f0cd1d23d96f5b8a9e5a Mon Sep 17 00:00:00 2001 From: Andy Teucher Date: Thu, 15 Aug 2024 12:17:06 -0700 Subject: [PATCH 06/17] temporarily skip get-data tests due to missing data in test catalogue --- tests/testthat/test-get-data.R | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/testthat/test-get-data.R b/tests/testthat/test-get-data.R index 7a23188d..858826f6 100644 --- a/tests/testthat/test-get-data.R +++ b/tests/testthat/test-get-data.R @@ -10,6 +10,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. +skip("Temporary skip for missing data in test catalogue") + test_that("bcdc_get_data collects an sf object for a valid record and resource id", { skip_if_net_down() skip_on_cran() From 93a40e945f6be7f064be6da1d2cb1e21b3043b2a Mon Sep 17 00:00:00 2001 From: Andy Teucher Date: Fri, 16 Aug 2024 11:34:34 -0700 Subject: [PATCH 07/17] Don't use BCDC_KEY while testing test server --- .github/workflows/R-CMD-check.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 93b42a7f..ee29df24 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -29,7 +29,7 @@ jobs: env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} R_KEEP_PKG_SOURCE: yes - BCDC_KEY: ${{ secrets.BCDC_KEY }} + # BCDC_KEY: ${{ secrets.BCDC_KEY }} BCDC_TEST_RECORD: ${{ secrets.BCDC_TEST_RECORD }} steps: From b1b9c1cdea571aaab5b00995072a424e9e19b88c Mon Sep 17 00:00:00 2001 From: Andy Teucher Date: Fri, 16 Aug 2024 11:42:10 -0700 Subject: [PATCH 08/17] formatting --- R/utils.R | 57 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/R/utils.R b/R/utils.R index fb2fdc1c..e20c4ec6 100644 --- a/R/utils.R +++ b/R/utils.R @@ -148,8 +148,10 @@ bcdc_auth <- function() { ## Check if there is internet ## h/t to https://github.com/ropensci/handlr/blob/pluralize/tests/testthat/helper-handlr.R has_internet <- function() { - z <- try(suppressWarnings(readLines('https://www.google.com', n = 1)), - silent = TRUE) + z <- try( + suppressWarnings(readLines('https://www.google.com', n = 1)), + silent = TRUE + ) !inherits(z, "try-error") } @@ -285,13 +287,15 @@ read_from_url <- function(resource, ...){ fun$package, " package.") handle_excel(tmp, ...) - tryCatch(do.call(fun$fun, list(tmp, ...)), - error = function(e) { - stop("Reading the data set failed with the following error message:\n\n ", e, - "\nThe file can be found here:\n '", - tmp, "'\nif you would like to try to read it manually.\n", - call. = FALSE) - }) +tryCatch( + do.call(fun$fun, list(tmp, ...)), + error = function(e) { + stop("Reading the data set failed with the following error message:\n\n ", e, + "\nThe file can be found here:\n '", + tmp, "'\nif you would like to try to read it manually.\n", + call. = FALSE) + } +) } @@ -305,11 +309,11 @@ resource_to_tibble <- function(x){ package_id = safe_map_chr(x, "package_id"), location = simplify_string(safe_map_chr(x, "resource_storage_location")) ) - - mutate(res_df, - wfs_available = wfs_available(res_df), - bcdata_available = wfs_available | other_format_available(res_df)) -} + + dplyr::mutate(res_df, + wfs_available = wfs_available(res_df), + bcdata_available = wfs_available | other_format_available(res_df)) + } #' @importFrom rlang "%||%" safe_map_chr <- function(x, name) { @@ -329,10 +333,13 @@ pagination_sort_col <- function(cols_df) { if (idcol %in% cols) return(idcol) } #Otherwise use the first column - this is likely pretty fragile - warning("Unable to find a suitable column to sort on for pagination. Using", - " the first column (", cols[1], - "). Please check your data for obvious duplicated or missing rows.", - call. = FALSE) + warning( + "Unable to find a suitable column to sort on for pagination. Using", + " the first column (", + cols[1], + "). Please check your data for obvious duplicated or missing rows.", + call. = FALSE + ) cols[1] } @@ -348,11 +355,15 @@ handle_zip <- function(x) { files <- list_supported_files(dir) # check if it's a shapefile - if (length(files) > 1L) { - stop("More than one supported file in zip file. It has been downloaded and ", - "extracted to '", dir, "', where you can access its contents manually.", - call. = FALSE) - } +if (length(files) > 1L) { + stop( + "More than one supported file in zip file. It has been downloaded and ", + "extracted to '", + dir, + "', where you can access its contents manually.", + call. = FALSE + ) +} files } From b9f1ac61ca4d47af20e6404d246d891becd1245d Mon Sep 17 00:00:00 2001 From: Andy Teucher Date: Fri, 16 Aug 2024 15:14:16 -0700 Subject: [PATCH 09/17] Skip specific tests that fail due to no data --- tests/testthat/test-get-data.R | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/tests/testthat/test-get-data.R b/tests/testthat/test-get-data.R index 858826f6..e1d9ab12 100644 --- a/tests/testthat/test-get-data.R +++ b/tests/testthat/test-get-data.R @@ -10,8 +10,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. -skip("Temporary skip for missing data in test catalogue") - test_that("bcdc_get_data collects an sf object for a valid record and resource id", { skip_if_net_down() skip_on_cran() @@ -43,7 +41,9 @@ test_that("bcdc_get_data works with slug and full url with corresponding resourc }) -test_that("bcdc_get_data works with a non-wms record with only one resource",{ +test_that("bcdc_get_data works with a non-wms record with only one resource", { + skip("Temporary skip for missing data in test catalogue") + skip_if_net_down() skip_on_cran() name <- "ee9d4ee0-6a34-4dff-89e0-9add9a969168" # "criminal-code-traffic-offences" @@ -51,6 +51,8 @@ test_that("bcdc_get_data works with a non-wms record with only one resource",{ }) test_that("bcdc_get_data works when using read_excel arguments", { + skip("Temporary skip for missing data in test catalogue") + skip_if_net_down() skip_on_cran() ret <- bcdc_get_data("2e469ff2-dadb-45ea-af9d-f5683a4b9465", @@ -60,7 +62,9 @@ test_that("bcdc_get_data works when using read_excel arguments", { expect_equal(nrow(ret), 3L, ignore_attr = TRUE) }) -test_that("bcdc_get_data works with an xls when specifying a specific resource",{ +test_that("bcdc_get_data works with an xls when specifying a specific resource", { + skip("Temporary skip for missing data in test catalogue") + skip_if_net_down() skip_on_cran() name <- 'bc-grizzly-bear-habitat-classification-and-rating' @@ -68,6 +72,8 @@ test_that("bcdc_get_data works with an xls when specifying a specific resource", }) test_that("bcdc_get_data will return non-wms resources",{ + skip("Temporary skip for missing data in test catalogue") + skip_if_net_down() skip_on_cran() expect_s3_class(bcdc_get_data(record = '76b1b7a3-2112-4444-857a-afccf7b20da8', @@ -86,6 +92,8 @@ test_that("bcdc_get_data works with a zipped shp file", { }) test_that("unknown single file (shp) inside zip", { + skip("Temporary skip for missing data in test catalogue") + skip_if_net_down() skip_on_cran() expect_s3_class(bcdc_get_data("e31f7488-27fa-4330-ae86-160a0deb8a59"), @@ -101,6 +109,8 @@ test_that("fails when resource doesn't exist", { }) test_that("fails when multiple files in a zip", { + skip("Temporary skip for missing data in test catalogue") + skip_if_net_down() skip_on_cran() expect_error(bcdc_get_data("300c0980-b5e3-4202-b0da-d816f14fadad", @@ -109,6 +119,8 @@ test_that("fails when multiple files in a zip", { }) test_that("fails informatively when can't read a file", { + skip("Temporary skip for missing data in test catalogue") + skip_if_net_down() skip_on_cran() expect_error( @@ -119,7 +131,7 @@ test_that("fails informatively when can't read a file", { "Reading the data set failed with the following error message:") }) -test_that("bcdc_get_data can return the wms resource when it is specified by resource",{ +test_that("bcdc_get_data can return the wms resource when it is specified by resource", { skip_if_net_down() skip_on_cran() expect_s3_class(bcdc_get_data('76b1b7a3-2112-4444-857a-afccf7b20da8', @@ -134,6 +146,8 @@ test_that("a wms record with only one resource works with only the record id",{ }) test_that("bcdc_get_data works with a bcdc_record object", { + skip("Temporary skip for missing data in test catalogue") + skip_if_net_down() skip_on_cran() record <- bcdc_get_record("bc-college-region-boundaries") @@ -173,6 +187,8 @@ test_that("bcdc_get_data fails when >1 resource not specified & noninteractive", }) test_that("bcdc_get_data handles sheet name specification", { + skip("Temporary skip for missing data in test catalogue") + skip_if_net_down() skip_on_cran() expect_message(bcdc_get_data('8620ce82-4943-43c4-9932-40730a0255d6'), 'This .xlsx resource contains the following sheets:') @@ -181,6 +197,8 @@ test_that("bcdc_get_data handles sheet name specification", { }) test_that("bcdc_get_data returns a list object when resource has a json extension", { + skip("Temporary skip for missing data in test catalogue") + skip_if_net_down() skip_on_cran() expect_type(bcdc_get_data("8e24f2bc-ab7a-49df-9418-539387180f33"), "list") From c840ddff64ff842cd8828eb42717375214fd3d89 Mon Sep 17 00:00:00 2001 From: Andy Teucher Date: Fri, 16 Aug 2024 15:18:59 -0700 Subject: [PATCH 10/17] formatting --- tests/testthat/test-get-data.R | 182 ++++++++++++++++++++++++--------- 1 file changed, 132 insertions(+), 50 deletions(-) diff --git a/tests/testthat/test-get-data.R b/tests/testthat/test-get-data.R index e1d9ab12..54f8a7ee 100644 --- a/tests/testthat/test-get-data.R +++ b/tests/testthat/test-get-data.R @@ -13,8 +13,10 @@ test_that("bcdc_get_data collects an sf object for a valid record and resource id", { skip_if_net_down() skip_on_cran() - bc_airports <- bcdc_get_data('76b1b7a3-2112-4444-857a-afccf7b20da8', - resource = "4d0377d9-e8a1-429b-824f-0ce8f363512c") + bc_airports <- bcdc_get_data( + '76b1b7a3-2112-4444-857a-afccf7b20da8', + resource = "4d0377d9-e8a1-429b-824f-0ce8f363512c" + ) expect_s3_class(bc_airports, "sf") expect_equal(attr(bc_airports, "sf_column"), "geometry") }) @@ -23,16 +25,40 @@ test_that("bcdc_get_data collects an sf object for a valid record and resource i test_that("bcdc_get_data works with slug and full url with corresponding resource", { skip_if_net_down() skip_on_cran() - expect_s3_class(ret1 <- bcdc_get_data("https://catalogue.data.gov.bc.ca/dataset/bc-airports", resource = "4d0377d9-e8a1-429b-824f-0ce8f363512c"), - "sf") - expect_s3_class(ret2 <- bcdc_get_data("bc-airports", resource = "4d0377d9-e8a1-429b-824f-0ce8f363512c"), - "sf") - expect_s3_class(ret3 <- bcdc_get_data("https://catalogue.data.gov.bc.ca/dataset/76b1b7a3-2112-4444-857a-afccf7b20da8", resource = "4d0377d9-e8a1-429b-824f-0ce8f363512c"), - "sf") - expect_s3_class(ret4 <- bcdc_get_data("76b1b7a3-2112-4444-857a-afccf7b20da8", resource = "4d0377d9-e8a1-429b-824f-0ce8f363512c"), - "sf") - expect_s3_class(ret5 <- bcdc_get_data("https://catalogue.data.gov.bc.ca/dataset/76b1b7a3-2112-4444-857a-afccf7b20da8/resource/4d0377d9-e8a1-429b-824f-0ce8f363512c"), - "sf") + expect_s3_class( + ret1 <- bcdc_get_data( + "https://catalogue.data.gov.bc.ca/dataset/bc-airports", + resource = "4d0377d9-e8a1-429b-824f-0ce8f363512c" + ), + "sf" + ) + expect_s3_class( + ret2 <- bcdc_get_data( + "bc-airports", + resource = "4d0377d9-e8a1-429b-824f-0ce8f363512c" + ), + "sf" + ) + expect_s3_class( + ret3 <- bcdc_get_data( + "https://catalogue.data.gov.bc.ca/dataset/76b1b7a3-2112-4444-857a-afccf7b20da8", + resource = "4d0377d9-e8a1-429b-824f-0ce8f363512c" + ), + "sf" + ) + expect_s3_class( + ret4 <- bcdc_get_data( + "76b1b7a3-2112-4444-857a-afccf7b20da8", + resource = "4d0377d9-e8a1-429b-824f-0ce8f363512c" + ), + "sf" + ) + expect_s3_class( + ret5 <- bcdc_get_data( + "https://catalogue.data.gov.bc.ca/dataset/76b1b7a3-2112-4444-857a-afccf7b20da8/resource/4d0377d9-e8a1-429b-824f-0ce8f363512c" + ), + "sf" + ) for (x in list(ret2, ret3, ret4, ret5)) { expect_equal(dim(x), dim(ret1)) @@ -55,9 +81,12 @@ test_that("bcdc_get_data works when using read_excel arguments", { skip_if_net_down() skip_on_cran() - ret <- bcdc_get_data("2e469ff2-dadb-45ea-af9d-f5683a4b9465", - resource = "18510a60-de82-440a-b806-06fba70eaf9d", - skip = 4, n_max = 3) + ret <- bcdc_get_data( + "2e469ff2-dadb-45ea-af9d-f5683a4b9465", + resource = "18510a60-de82-440a-b806-06fba70eaf9d", + skip = 4, + n_max = 3 + ) expect_s3_class(ret, "tbl") expect_equal(nrow(ret), 3L, ignore_attr = TRUE) }) @@ -68,7 +97,10 @@ test_that("bcdc_get_data works with an xls when specifying a specific resource", skip_if_net_down() skip_on_cran() name <- 'bc-grizzly-bear-habitat-classification-and-rating' - expect_s3_class(bcdc_get_data(name, resource = '7b09f82f-e7d0-44bf-9310-b94039b323a8'), "tbl") + expect_s3_class( + bcdc_get_data(name, resource = '7b09f82f-e7d0-44bf-9310-b94039b323a8'), + "tbl" + ) }) test_that("bcdc_get_data will return non-wms resources",{ @@ -76,19 +108,33 @@ test_that("bcdc_get_data will return non-wms resources",{ skip_if_net_down() skip_on_cran() - expect_s3_class(bcdc_get_data(record = '76b1b7a3-2112-4444-857a-afccf7b20da8', - resource = 'fcccba36-b528-4731-8978-940b3cc04f69'), "tbl") - - expect_s3_class(bcdc_get_data(record = 'fa542137-a976-49a6-856d-f1201adb2243', - resource = 'dc1098a7-a4b8-49a3-adee-9badd4429279'), "tbl") + expect_s3_class( + bcdc_get_data( + record = '76b1b7a3-2112-4444-857a-afccf7b20da8', + resource = 'fcccba36-b528-4731-8978-940b3cc04f69' + ), + "tbl" + ) + + expect_s3_class( + bcdc_get_data( + record = 'fa542137-a976-49a6-856d-f1201adb2243', + resource = 'dc1098a7-a4b8-49a3-adee-9badd4429279' + ), + "tbl" + ) }) test_that("bcdc_get_data works with a zipped shp file", { skip_if_net_down() skip_on_cran() - expect_s3_class(bcdc_get_data(record = '481d6d4d-a536-4df9-9e9c-7473cd2ed89e', - resource = '41c9bff0-4e25-49fc-a3e2-2a2e426ac71d'), - "sf") + expect_s3_class( + bcdc_get_data( + record = '481d6d4d-a536-4df9-9e9c-7473cd2ed89e', + resource = '41c9bff0-4e25-49fc-a3e2-2a2e426ac71d' + ), + "sf" + ) }) test_that("unknown single file (shp) inside zip", { @@ -96,16 +142,19 @@ test_that("unknown single file (shp) inside zip", { skip_if_net_down() skip_on_cran() - expect_s3_class(bcdc_get_data("e31f7488-27fa-4330-ae86-160a0deb8a59"), - "sf") + expect_s3_class( + bcdc_get_data("e31f7488-27fa-4330-ae86-160a0deb8a59"), + "sf" + ) }) test_that("fails when resource doesn't exist", { skip_if_net_down() skip_on_cran() - expect_error(bcdc_get_data("300c0980-b5e3-4202-b0da-d816f14fadad", - resource = "not-a-real-resource"), - "The specified resource does not exist in this record") + expect_error(bcdc_get_data( + "300c0980-b5e3-4202-b0da-d816f14fadad", + resource = "not-a-real-resource" + ), "The specified resource does not exist in this record") }) test_that("fails when multiple files in a zip", { @@ -113,9 +162,10 @@ test_that("fails when multiple files in a zip", { skip_if_net_down() skip_on_cran() - expect_error(bcdc_get_data("300c0980-b5e3-4202-b0da-d816f14fadad", - resource = "c212a8a7-c625-4464-b9c8-4527c843f52f"), - "More than one supported file in zip file") + expect_error(bcdc_get_data( + "300c0980-b5e3-4202-b0da-d816f14fadad", + resource = "c212a8a7-c625-4464-b9c8-4527c843f52f" + ), "More than one supported file in zip file") }) test_that("fails informatively when can't read a file", { @@ -125,8 +175,10 @@ test_that("fails informatively when can't read a file", { skip_on_cran() expect_error( suppressWarnings( - bcdc_get_data(record = '523dce9d-b464-44a5-b733-2022e94546c3', - resource = '4cc98644-f6eb-410b-9df0-f9b2beac9717') + bcdc_get_data( + record = '523dce9d-b464-44a5-b733-2022e94546c3', + resource = '4cc98644-f6eb-410b-9df0-f9b2beac9717' + ) ), "Reading the data set failed with the following error message:") }) @@ -134,15 +186,23 @@ test_that("fails informatively when can't read a file", { test_that("bcdc_get_data can return the wms resource when it is specified by resource", { skip_if_net_down() skip_on_cran() - expect_s3_class(bcdc_get_data('76b1b7a3-2112-4444-857a-afccf7b20da8', - resource = "4d0377d9-e8a1-429b-824f-0ce8f363512c"), "sf") + expect_s3_class( + bcdc_get_data( + '76b1b7a3-2112-4444-857a-afccf7b20da8', + resource = "4d0377d9-e8a1-429b-824f-0ce8f363512c" + ), + "sf" + ) }) test_that("a wms record with only one resource works with only the record id",{ skip_if_net_down() skip_on_cran() - expect_s3_class(bcdc_get_data("bc-college-region-boundaries"), "sf") + expect_s3_class( + bcdc_get_data("bc-college-region-boundaries"), + "sf" + ) }) test_that("bcdc_get_data works with a bcdc_record object", { @@ -154,36 +214,46 @@ test_that("bcdc_get_data works with a bcdc_record object", { expect_s3_class(bcdc_get_data(record), "sf") record <- bcdc_get_record('fa542137-a976-49a6-856d-f1201adb2243') - expect_s3_class(bcdc_get_data(record, - resource = 'dc1098a7-a4b8-49a3-adee-9badd4429279'), - "tbl") + expect_s3_class( + bcdc_get_data(record, resource = 'dc1098a7-a4b8-49a3-adee-9badd4429279'), + "tbl" + ) }) test_that("bcdc_get_data fails with invalid input", { skip_if_net_down() skip_on_cran() - expect_error(bcdc_get_data(35L), - "No bcdc_get_data method for an object of class integer") + expect_error( + bcdc_get_data(35L), + "No bcdc_get_data method for an object of class integer" + ) }) test_that("bcdc_get_data works with BCGW name", { skip_if_net_down() skip_on_cran() - expect_s3_class(bcdc_get_data("WHSE_IMAGERY_AND_BASE_MAPS.GSR_AIRPORTS_SVW"), "bcdc_sf") + expect_s3_class( + bcdc_get_data("WHSE_IMAGERY_AND_BASE_MAPS.GSR_AIRPORTS_SVW"), + "bcdc_sf" + ) }) test_that("bcdc_get_data fails when no downloadable resources", { skip_if_net_down() skip_on_cran() - expect_error(bcdc_get_data("4e237966-3db8-4e28-8e59-296bf0b8d8e4"), - "There are no resources that bcdata can download from this record") + expect_error( + bcdc_get_data("4e237966-3db8-4e28-8e59-296bf0b8d8e4"), + "There are no resources that bcdata can download from this record" + ) }) test_that("bcdc_get_data fails when >1 resource not specified & noninteractive", { skip_if_net_down() skip_on_cran() - expect_error(bcdc_get_data("21c72822-2502-4431-b9a2-92fc9401ef12"), - "The record you are trying to access appears to have more than one resource.") + expect_error( + bcdc_get_data("21c72822-2502-4431-b9a2-92fc9401ef12"), + "The record you are trying to access appears to have more than one resource." + ) }) test_that("bcdc_get_data handles sheet name specification", { @@ -191,9 +261,21 @@ test_that("bcdc_get_data handles sheet name specification", { skip_if_net_down() skip_on_cran() - expect_message(bcdc_get_data('8620ce82-4943-43c4-9932-40730a0255d6'), 'This .xlsx resource contains the following sheets:') - expect_error(bcdc_get_data('8620ce82-4943-43c4-9932-40730a0255d6', sheet = "foo"), "Error: Sheet 'foo' not found") - expect_s3_class(bcdc_get_data('8620ce82-4943-43c4-9932-40730a0255d6', sheet = "Multi Unit Homes"), "data.frame") + expect_message( + bcdc_get_data('8620ce82-4943-43c4-9932-40730a0255d6'), + 'This .xlsx resource contains the following sheets:' + ) + expect_error( + bcdc_get_data('8620ce82-4943-43c4-9932-40730a0255d6', sheet = "foo"), + "Error: Sheet 'foo' not found" + ) + expect_s3_class( + bcdc_get_data( + '8620ce82-4943-43c4-9932-40730a0255d6', + sheet = "Multi Unit Homes" + ), + "data.frame" + ) }) test_that("bcdc_get_data returns a list object when resource has a json extension", { From 8e4f9d47161ff3f75ad490981ce3e4d419fcecdb Mon Sep 17 00:00:00 2001 From: Andy Teucher Date: Fri, 16 Aug 2024 15:34:55 -0700 Subject: [PATCH 11/17] Use catalogue_base_url() in tests instead of hardcoded url --- tests/testthat/test-bcdc-get-citation.R | 2 +- tests/testthat/test-get-data.R | 6 +- tests/testthat/test-get_record.R | 64 +++++++++++++++------ tests/testthat/test-query-geodata-collect.R | 19 ++++-- 4 files changed, 64 insertions(+), 27 deletions(-) diff --git a/tests/testthat/test-bcdc-get-citation.R b/tests/testthat/test-bcdc-get-citation.R index 3bb684b3..b6caa089 100644 --- a/tests/testthat/test-bcdc-get-citation.R +++ b/tests/testthat/test-bcdc-get-citation.R @@ -16,6 +16,6 @@ test_that("bcdc_get_citation take a character and returns a bibentry",{ rec <- bcdc_get_record(point_record) expect_s3_class(bcdc_get_citation(rec), c("citation", "bibentry")) expect_s3_class(bcdc_get_citation(point_record), c("citation", "bibentry")) - test_url <- paste0("https://catalogue.data.gov.bc.ca/dataset/", point_record) + test_url <- glue::glue("{catalogue_base_url()}/dataset/{point_record}") expect_s3_class(bcdc_get_citation(test_url), c("citation", "bibentry")) }) diff --git a/tests/testthat/test-get-data.R b/tests/testthat/test-get-data.R index 54f8a7ee..20b0bdac 100644 --- a/tests/testthat/test-get-data.R +++ b/tests/testthat/test-get-data.R @@ -27,7 +27,7 @@ test_that("bcdc_get_data works with slug and full url with corresponding resourc skip_on_cran() expect_s3_class( ret1 <- bcdc_get_data( - "https://catalogue.data.gov.bc.ca/dataset/bc-airports", + glue::glue("{catalogue_base_url()}/dataset/bc-airports"), resource = "4d0377d9-e8a1-429b-824f-0ce8f363512c" ), "sf" @@ -41,7 +41,7 @@ test_that("bcdc_get_data works with slug and full url with corresponding resourc ) expect_s3_class( ret3 <- bcdc_get_data( - "https://catalogue.data.gov.bc.ca/dataset/76b1b7a3-2112-4444-857a-afccf7b20da8", + glue::glue("{catalogue_base_url()}/dataset/76b1b7a3-2112-4444-857a-afccf7b20da8"), resource = "4d0377d9-e8a1-429b-824f-0ce8f363512c" ), "sf" @@ -55,7 +55,7 @@ test_that("bcdc_get_data works with slug and full url with corresponding resourc ) expect_s3_class( ret5 <- bcdc_get_data( - "https://catalogue.data.gov.bc.ca/dataset/76b1b7a3-2112-4444-857a-afccf7b20da8/resource/4d0377d9-e8a1-429b-824f-0ce8f363512c" + glue::glue("{catalogue_base_url()}/dataset/76b1b7a3-2112-4444-857a-afccf7b20da8/resource/4d0377d9-e8a1-429b-824f-0ce8f363512c") ), "sf" ) diff --git a/tests/testthat/test-get_record.R b/tests/testthat/test-get_record.R index fa129f41..7c94fe5d 100644 --- a/tests/testthat/test-get_record.R +++ b/tests/testthat/test-get_record.R @@ -13,14 +13,26 @@ test_that("bcdc_get_record works with slug and full url", { skip_on_cran() skip_if_net_down() - expect_s3_class(ret1 <- bcdc_get_record("https://catalogue.data.gov.bc.ca/dataset/bc-airports"), - "bcdc_record") - expect_s3_class(ret2 <- bcdc_get_record("bc-airports"), - "bcdc_record") - expect_s3_class(ret3 <- bcdc_get_record("https://catalogue.data.gov.bc.ca/dataset/76b1b7a3-2112-4444-857a-afccf7b20da8"), - "bcdc_record") - expect_s3_class(ret4 <- bcdc_get_record("76b1b7a3-2112-4444-857a-afccf7b20da8"), - "bcdc_record") + expect_s3_class( + ret1 <- bcdc_get_record( + glue::glue("{catalogue_base_url()}/dataset/bc-airports") + ), + "bcdc_record" + ) + expect_s3_class( + ret2 <- bcdc_get_record("bc-airports"), + "bcdc_record" + ) + expect_s3_class( + ret3 <- bcdc_get_record( + glue::glue("{catalogue_base_url()}/dataset/76b1b7a3-2112-4444-857a-afccf7b20da8") + ), + "bcdc_record" + ) + expect_s3_class( + ret4 <- bcdc_get_record("76b1b7a3-2112-4444-857a-afccf7b20da8"), + "bcdc_record" + ) expect_equal(ret1$title, "BC Airports") lapply(list(ret2, ret3, ret4), expect_equal, ret1) }) @@ -29,9 +41,15 @@ test_that("bcdc_search_facets works", { skip_on_cran() skip_if_net_down() ret_names <- c("facet", "count", "display_name", "name") - lapply(c("license_id", "download_audience", "res_format", - "publish_state", "organization"), - function(x) expect_named(bcdc_search_facets(x)) + lapply( + c( + "license_id", + "download_audience", + "res_format", + "publish_state", + "organization" + ), + function(x) expect_named(bcdc_search_facets(x)) ) expect_error(bcdc_search_facets("foo"), "'arg' should be one of") }) @@ -83,8 +101,10 @@ test_that("bcdc_search works", { expect_s3_class(bcdc_search("forest"), "bcdc_recordlist") expect_s3_class(bcdc_search("regional district", res_format = "fgdb"), "bcdc_recordlist") - expect_error(bcdc_search(organization = "foo"), - "foo is not a valid value for organization") + expect_error( + bcdc_search(organization = "foo"), + "foo is not a valid value for organization" + ) }) test_that("a record with bcgeographicwarehouse AND wms is return by bcdc_get_record",{ @@ -122,8 +142,10 @@ test_that("a data frame with 8 columns of expected types is returned by bcdc_tid expect_type(d$bcdata_available, "logical") expect_equal(d, bcdc_tidy_resources('76b1b7a3-2112-4444-857a-afccf7b20da8')) expect_error(bcdc_tidy_resources(list()), "No bcdc_tidy_resources method for an object of class") - expect_error(bcdc_tidy_resources("WHSE_IMAGERY_AND_BASE_MAPS.GSR_AIRPORTS_SVW"), - "No bcdc_tidy_resources method for a BCGW object name") + expect_error( + bcdc_tidy_resources("WHSE_IMAGERY_AND_BASE_MAPS.GSR_AIRPORTS_SVW"), + "No bcdc_tidy_resources method for a BCGW object name" + ) }) test_that("bcdc_get_record works with/without authentication", { @@ -135,16 +157,20 @@ test_that("bcdc_get_record works with/without authentication", { on.exit(Sys.setenv(BCDC_KEY = key_val)) # record NOT requiring auth - expect_message(res <- bcdc_get_record(point_record), - "Authorizing with your stored API key") + expect_message( + res <- bcdc_get_record(point_record), + "Authorizing with your stored API key" + ) expect_s3_class(res, "bcdc_record") # record requiring auth auth_record_id <- Sys.getenv("BCDC_TEST_RECORD") skip_if_not(nzchar(key_val)) - expect_message(res <- bcdc_get_record(auth_record_id), - "Authorizing with your stored API key") + expect_message( + res <- bcdc_get_record(auth_record_id), + "Authorizing with your stored API key" + ) expect_s3_class(res, "bcdc_record") Sys.unsetenv("BCDC_KEY") diff --git a/tests/testthat/test-query-geodata-collect.R b/tests/testthat/test-query-geodata-collect.R index 131ddc61..8c94e4c8 100644 --- a/tests/testthat/test-query-geodata-collect.R +++ b/tests/testthat/test-query-geodata-collect.R @@ -31,7 +31,9 @@ test_that("bcdc_query_geodata succeeds with a records over 10000 rows", { skip_if_net_down() skip("Skipping the BEC test, though available for testing") expect_s3_class( - collect(bcdc_query_geodata("terrestrial-protected-areas-representation-by-biogeoclimatic-unit")), + collect( + bcdc_query_geodata("terrestrial-protected-areas-representation-by-biogeoclimatic-unit") + ), "bcdc_sf" ) }) @@ -41,7 +43,10 @@ test_that("bcdc_query_geodata works with slug and full url using collect", { skip_on_cran() skip_if_net_down() expect_s3_class( - ret1 <- bcdc_query_geodata("https://catalogue.data.gov.bc.ca/dataset/bc-airports") %>% collect(), + ret1 <- bcdc_query_geodata( + glue::glue("{catalogue_base_url()}/dataset/bc-airports") + ) %>% + collect(), "sf" ) expect_s3_class( @@ -49,7 +54,9 @@ test_that("bcdc_query_geodata works with slug and full url using collect", { "sf" ) expect_s3_class( - ret3 <- bcdc_query_geodata("https://catalogue.data.gov.bc.ca/dataset/76b1b7a3-2112-4444-857a-afccf7b20da8") + ret3 <- bcdc_query_geodata( + glue::glue("{catalogue_base_url()}/dataset/76b1b7a3-2112-4444-857a-afccf7b20da8") + ) %>% collect(), "sf" ) @@ -58,7 +65,11 @@ test_that("bcdc_query_geodata works with slug and full url using collect", { "sf" ) expect_s3_class( - ret5 <- bcdc_query_geodata("https://catalogue.data.gov.bc.ca/dataset/76b1b7a3-2112-4444-857a-afccf7b20da8/resource/4d0377d9-e8a1-429b-824f-0ce8f363512c") + ret5 <- bcdc_query_geodata( + glue::glue( + "{catalogue_base_url()}/dataset/76b1b7a3-2112-4444-857a-afccf7b20da8/resource/4d0377d9-e8a1-429b-824f-0ce8f363512c" + ) + ) %>% collect(), "sf" ) From d2b04040953fef76513fd11a9846deaa167fc8df Mon Sep 17 00:00:00 2001 From: Andy Teucher Date: Wed, 4 Dec 2024 16:50:46 -0800 Subject: [PATCH 12/17] Unskip tests --- tests/testthat/test-get-data.R | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/tests/testthat/test-get-data.R b/tests/testthat/test-get-data.R index 20b0bdac..16b2799b 100644 --- a/tests/testthat/test-get-data.R +++ b/tests/testthat/test-get-data.R @@ -68,8 +68,6 @@ test_that("bcdc_get_data works with slug and full url with corresponding resourc test_that("bcdc_get_data works with a non-wms record with only one resource", { - skip("Temporary skip for missing data in test catalogue") - skip_if_net_down() skip_on_cran() name <- "ee9d4ee0-6a34-4dff-89e0-9add9a969168" # "criminal-code-traffic-offences" @@ -77,8 +75,6 @@ test_that("bcdc_get_data works with a non-wms record with only one resource", { }) test_that("bcdc_get_data works when using read_excel arguments", { - skip("Temporary skip for missing data in test catalogue") - skip_if_net_down() skip_on_cran() ret <- bcdc_get_data( @@ -92,8 +88,6 @@ test_that("bcdc_get_data works when using read_excel arguments", { }) test_that("bcdc_get_data works with an xls when specifying a specific resource", { - skip("Temporary skip for missing data in test catalogue") - skip_if_net_down() skip_on_cran() name <- 'bc-grizzly-bear-habitat-classification-and-rating' @@ -104,8 +98,6 @@ test_that("bcdc_get_data works with an xls when specifying a specific resource", }) test_that("bcdc_get_data will return non-wms resources",{ - skip("Temporary skip for missing data in test catalogue") - skip_if_net_down() skip_on_cran() expect_s3_class( @@ -138,8 +130,6 @@ test_that("bcdc_get_data works with a zipped shp file", { }) test_that("unknown single file (shp) inside zip", { - skip("Temporary skip for missing data in test catalogue") - skip_if_net_down() skip_on_cran() expect_s3_class( @@ -158,8 +148,6 @@ test_that("fails when resource doesn't exist", { }) test_that("fails when multiple files in a zip", { - skip("Temporary skip for missing data in test catalogue") - skip_if_net_down() skip_on_cran() expect_error(bcdc_get_data( @@ -169,8 +157,6 @@ test_that("fails when multiple files in a zip", { }) test_that("fails informatively when can't read a file", { - skip("Temporary skip for missing data in test catalogue") - skip_if_net_down() skip_on_cran() expect_error( @@ -206,8 +192,6 @@ test_that("a wms record with only one resource works with only the record id",{ }) test_that("bcdc_get_data works with a bcdc_record object", { - skip("Temporary skip for missing data in test catalogue") - skip_if_net_down() skip_on_cran() record <- bcdc_get_record("bc-college-region-boundaries") @@ -257,8 +241,6 @@ test_that("bcdc_get_data fails when >1 resource not specified & noninteractive", }) test_that("bcdc_get_data handles sheet name specification", { - skip("Temporary skip for missing data in test catalogue") - skip_if_net_down() skip_on_cran() expect_message( @@ -279,8 +261,6 @@ test_that("bcdc_get_data handles sheet name specification", { }) test_that("bcdc_get_data returns a list object when resource has a json extension", { - skip("Temporary skip for missing data in test catalogue") - skip_if_net_down() skip_on_cran() expect_type(bcdc_get_data("8e24f2bc-ab7a-49df-9418-539387180f33"), "list") From 36f4694adfaefcc4b2857b39efad9c25136673a8 Mon Sep 17 00:00:00 2001 From: Andy Teucher Date: Wed, 4 Dec 2024 16:50:54 -0800 Subject: [PATCH 13/17] Document --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 7f694918..61192763 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -62,7 +62,7 @@ VignetteBuilder: knitr Encoding: UTF-8 Roxygen: list(markdown = TRUE) -RoxygenNote: 7.3.1.9000 +RoxygenNote: 7.3.2 Collate: 'bcdata-package.R' 'bcdc-get-citation.R' From adf5ab279080ada41669d60218f56a4296cdac08 Mon Sep 17 00:00:00 2001 From: Andy Teucher Date: Wed, 4 Dec 2024 17:36:09 -0800 Subject: [PATCH 14/17] Update NEWS --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index 2c07e0ff..584dfa05 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,6 +5,7 @@ `local()` for local functions more restrictive; updated tests and examples (#341). * Deprecate the `bcdata.single_download_limit` option, as it was mostly redundant with `bcdata.chunk_limit`, and should always be set by the server. Please set the page size limit for paginated requests via the `bcdata.chunk_limit` option (#332) +* Updated internals to adapt to changes in B.C. Data Catalogue (#342m, #343) # bcdata 0.4.1 From 890d5dcde0588cad595ebffb4125f409515e271a Mon Sep 17 00:00:00 2001 From: Andy Teucher Date: Wed, 11 Dec 2024 14:22:38 -0800 Subject: [PATCH 15/17] restore catalogue base url --- R/utils.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/utils.R b/R/utils.R index e20c4ec6..e8098881 100644 --- a/R/utils.R +++ b/R/utils.R @@ -12,12 +12,12 @@ catalogue_base_url <- function() { getOption("bcdata.catalogue_gui_url", - default = "https://toyger.data.gov.bc.ca/") + default = "https://catalogue.data.gov.bc.ca/") } catalogue_base_api_url <- function() { getOption("bcdata.catalogue_api_url", - default = "https://toyger.data.gov.bc.ca/api/3") + default = "https://catalogue.data.gov.bc.ca/api/3") } wfs_base_url <- function(host = bcdc_web_service_host()) { From f830c7330862a6aadcd05df0203390f6f88cc8e5 Mon Sep 17 00:00:00 2001 From: Andy Teucher Date: Wed, 11 Dec 2024 14:28:50 -0800 Subject: [PATCH 16/17] enable testing with BCDC_KEY --- .github/workflows/R-CMD-check.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 50b90cb5..588a51a2 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -29,7 +29,7 @@ jobs: env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} R_KEEP_PKG_SOURCE: yes - # BCDC_KEY: ${{ secrets.BCDC_KEY }} + BCDC_KEY: ${{ secrets.BCDC_KEY }} BCDC_TEST_RECORD: ${{ secrets.BCDC_TEST_RECORD }} steps: From 08611542bd68164ed729edc8a4aa167c6af4e754 Mon Sep 17 00:00:00 2001 From: Andy Teucher Date: Wed, 11 Dec 2024 14:31:01 -0800 Subject: [PATCH 17/17] Actually old key won't work --- .github/workflows/R-CMD-check.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 588a51a2..50b90cb5 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -29,7 +29,7 @@ jobs: env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} R_KEEP_PKG_SOURCE: yes - BCDC_KEY: ${{ secrets.BCDC_KEY }} + # BCDC_KEY: ${{ secrets.BCDC_KEY }} BCDC_TEST_RECORD: ${{ secrets.BCDC_TEST_RECORD }} steps: