Skip to content

Commit

Permalink
Add timestamp
Browse files Browse the repository at this point in the history
  • Loading branch information
asadow committed Mar 18, 2024
1 parent 8a7e6f8 commit 2f37404
Show file tree
Hide file tree
Showing 6 changed files with 33 additions and 29 deletions.
3 changes: 3 additions & 0 deletions R/data.R → R/pull.R
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@ mm_data <- function(endpoint, ..., allfields = TRUE) {

successes <- responses |> httr2::resps_successes()
pages <- successes |> purrr::map(mm_resp_extract)
timestamps <- successes |>
purrr::map(\(x) x |> purrr::pluck("headers", "Date") |> parse_header_date())
pages <- purrr::map2(pages, timestamps, \(x, y) x |> dplyr::mutate(ts = !!y))

no_data_i <- pages |> purrr::map_lgl(is.null) |> which()

Expand Down
5 changes: 5 additions & 0 deletions R/utils-parse.R
Original file line number Diff line number Diff line change
Expand Up @@ -86,3 +86,8 @@ extract_schema <- function(parsed) {
tidyr::unnest(type) |>
as.data.frame()
}

parse_header_date <- function(resp) {
tz <- stringr::str_extract(resp, "\\b\\w+$")
as.POSIXct(resp, format = "%a, %d %b %Y %H:%M:%S", tz = tz)
}
2 changes: 1 addition & 1 deletion inst/httptest2/redact.R
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
headers <- c("Cache-Control", "Server", "X-Powered-By",
"Access-Control-Allow-Origin", "WWW-Authenticate", "X-Powered-By",
"X-Robots-Tag", "Date", "Content-Length", "Set-Cookie",
"X-Robots-Tag", "Content-Length", "Set-Cookie",
"Strict-Transport-Security")

function(response) {
Expand Down
5 changes: 3 additions & 2 deletions tests/testthat/mm_data/status-b4ea5e.R
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,9 @@ structure(list(method = "GET", url = "status?ALLFIELDS=1", status_code = 200L,
headers = structure(list(`Cache-Control` = "REDACTED", `Content-Type` = "application/hal+json",
Server = "REDACTED", `X-Powered-By` = "REDACTED", `Access-Control-Allow-Origin` = "REDACTED",
`WWW-Authenticate` = "REDACTED", `X-Powered-By` = "REDACTED",
`X-Robots-Tag` = "REDACTED", Date = "REDACTED", `Content-Length` = "REDACTED",
`Set-Cookie` = "REDACTED", `Strict-Transport-Security` = "REDACTED"), class = "httr2_headers"),
`X-Robots-Tag` = "REDACTED", Date = "Mon, 18 Mar 2024 18:25:54 GMT",
`Content-Length` = "REDACTED", `Set-Cookie` = "REDACTED",
`Strict-Transport-Security` = "REDACTED"), class = "httr2_headers"),
body = as.raw(c(0x7b, 0x22, 0x5f, 0x65, 0x6d, 0x62, 0x65,
0x64, 0x64, 0x65, 0x64, 0x22, 0x3a, 0x7b, 0x22, 0x53, 0x74,
0x61, 0x74, 0x75, 0x73, 0x20, 0x43, 0x6f, 0x64, 0x65, 0x22,
Expand Down
45 changes: 20 additions & 25 deletions tests/testthat/statuses/statuses-b4ea5e.R
Original file line number Diff line number Diff line change
@@ -1,25 +1,20 @@
structure(list(
method = "GET", url = "statuses?ALLFIELDS=1",
status_code = 404L, headers = structure(list(
`Cache-Control` = "REDACTED",
`Content-Type` = "application/problem+json", Server = "REDACTED",
`X-Powered-By` = "REDACTED", `Access-Control-Allow-Origin` = "REDACTED",
`WWW-Authenticate` = "REDACTED", `X-Powered-By` = "REDACTED",
`X-Robots-Tag` = "REDACTED", Date = "REDACTED",
`Content-Length` = "REDACTED",
`Set-Cookie` = "REDACTED", `Strict-Transport-Security` = "REDACTED"
), class = "httr2_headers"),
body = as.raw(c(
0x7b, 0x22, 0x74, 0x79, 0x70, 0x65, 0x22,
0x3a, 0x22, 0x61, 0x62, 0x6f, 0x75, 0x74, 0x3a, 0x62, 0x6c,
0x61, 0x6e, 0x6b, 0x22, 0x2c, 0x22, 0x74, 0x69, 0x74, 0x6c,
0x65, 0x22, 0x3a, 0x22, 0x4e, 0x6f, 0x74, 0x20, 0x46, 0x6f,
0x75, 0x6e, 0x64, 0x22, 0x2c, 0x22, 0x73, 0x74, 0x61, 0x74,
0x75, 0x73, 0x22, 0x3a, 0x34, 0x30, 0x34, 0x2c, 0x22, 0x64,
0x65, 0x74, 0x61, 0x69, 0x6c, 0x22, 0x3a, 0x22, 0x54, 0x68,
0x69, 0x73, 0x20, 0x69, 0x73, 0x20, 0x6e, 0x6f, 0x74, 0x20,
0x61, 0x20, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x20, 0x77, 0x65,
0x62, 0x20, 0x41, 0x50, 0x49, 0x20, 0x65, 0x6e, 0x64, 0x70,
0x6f, 0x69, 0x6e, 0x74, 0x2e, 0x22, 0x7d, 0x20, 0x0d, 0x0a
)), cache = rlang::caller_env()
), class = "httr2_response")
structure(list(method = "GET", url = "statuses?ALLFIELDS=1",
status_code = 404L, headers = structure(list(`Cache-Control` = "REDACTED",
`Content-Type` = "application/problem+json", Server = "REDACTED",
`X-Powered-By` = "REDACTED", `Access-Control-Allow-Origin` = "REDACTED",
`WWW-Authenticate` = "REDACTED", `X-Powered-By` = "REDACTED",
`X-Robots-Tag` = "REDACTED", Date = "Mon, 18 Mar 2024 18:25:57 GMT",
`Content-Length` = "REDACTED", `Set-Cookie` = "REDACTED",
`Strict-Transport-Security` = "REDACTED"), class = "httr2_headers"),
body = as.raw(c(0x7b, 0x22, 0x74, 0x79, 0x70, 0x65, 0x22,
0x3a, 0x22, 0x61, 0x62, 0x6f, 0x75, 0x74, 0x3a, 0x62, 0x6c,
0x61, 0x6e, 0x6b, 0x22, 0x2c, 0x22, 0x74, 0x69, 0x74, 0x6c,
0x65, 0x22, 0x3a, 0x22, 0x4e, 0x6f, 0x74, 0x20, 0x46, 0x6f,
0x75, 0x6e, 0x64, 0x22, 0x2c, 0x22, 0x73, 0x74, 0x61, 0x74,
0x75, 0x73, 0x22, 0x3a, 0x34, 0x30, 0x34, 0x2c, 0x22, 0x64,
0x65, 0x74, 0x61, 0x69, 0x6c, 0x22, 0x3a, 0x22, 0x54, 0x68,
0x69, 0x73, 0x20, 0x69, 0x73, 0x20, 0x6e, 0x6f, 0x74, 0x20,
0x61, 0x20, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x20, 0x77, 0x65,
0x62, 0x20, 0x41, 0x50, 0x49, 0x20, 0x65, 0x6e, 0x64, 0x70,
0x6f, 0x69, 0x6e, 0x74, 0x2e, 0x22, 0x7d)), request = "REDACTED",
cache = new.env(parent = emptyenv())), class = "httr2_response")
2 changes: 1 addition & 1 deletion tests/testthat/test-get.R
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ with_mock_dir("mm_data", {
got <- mm_data("status")

expect_s3_class(got, "tbl_df")
expect_named(got, c("ampc_required", "description", "status"))
expect_named(got, c("ampc_required", "description", "status", "ts"))
})
})

Expand Down

0 comments on commit 2f37404

Please sign in to comment.