From 016f57579229aebd44254234f75f63aeba3c4fc7 Mon Sep 17 00:00:00 2001 From: pachamaltese Date: Thu, 14 Mar 2019 13:13:52 -0300 Subject: [PATCH 01/23] halfway svg render, still downloads instead of showing --- R/content-types.R | 2 +- R/images.R | 10 +++++++--- R/parse-block.R | 4 +++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/R/content-types.R b/R/content-types.R index a3730d0dc..f60c0022f 100644 --- a/R/content-types.R +++ b/R/content-types.R @@ -64,7 +64,7 @@ getCharacterSet <- function(contentType){ as.character( ifelse( charsetStart > -1, - substr(contentType, charsetStart, nchar(contentType)), + substr(contentType, charsetStart, nchar(contentType)), default ) ) diff --git a/R/images.R b/R/images.R index 423e3a58a..db198440b 100644 --- a/R/images.R +++ b/R/images.R @@ -1,6 +1,6 @@ -#' @param imageFun The function to call to setup the image device (e.g. `png`) -#' @param args A list of supplemental arguments to be passed into jpeg() -#' @importFrom grDevices dev.off jpeg png +#' @param imageFun The function to call to setup the image device (`png`, `jpeg` or `svg`) +#' @param args A list of supplemental arguments to be passed into png(), jpeg() or svg() +#' @importFrom grDevices dev.off jpeg png svg #' @noRd render_image <- function(imageFun, contentType, args=NULL){ list( @@ -31,3 +31,7 @@ render_jpeg <- function(args){ render_png <- function(args){ render_image(png, "image/png", args) } + +render_svg <- function(args){ + render_image(svg, "image/svg", args) +} diff --git a/R/parse-block.R b/R/parse-block.R index 4a6c339ad..1a49530b3 100644 --- a/R/parse-block.R +++ b/R/parse-block.R @@ -149,7 +149,7 @@ parseBlock <- function(lineNum, file){ } - imageMat <- stringi::stri_match(line, regex="^#['\\*]\\s*@(jpeg|png)([\\s\\(].*)?\\s*$") + imageMat <- stringi::stri_match(line, regex="^#['\\*]\\s*@(jpeg|png|svg)([\\s\\(].*)?\\s*$") if (!is.na(imageMat[1,1])){ if (!is.null(image)){ # Must have already assigned. @@ -262,6 +262,8 @@ evaluateBlock <- function(srcref, file, expr, envir, addEndpoint, addFilter, mou ep$registerHooks(render_png(imageArgs)) } else if (block$image == "jpeg"){ ep$registerHooks(render_jpeg(imageArgs)) + } else if (block$image == "svg"){ + ep$registerHooks(render_svg(imageArgs)) } else { stop("Image format not found: ", block$image) } From f3a4ac736056bc05c4166533d51bb845d361601d Mon Sep 17 00:00:00 2001 From: pachamaltese Date: Thu, 14 Mar 2019 16:33:50 -0300 Subject: [PATCH 02/23] well rendered svg in firefox --- R/images.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/images.R b/R/images.R index db198440b..42dafdfae 100644 --- a/R/images.R +++ b/R/images.R @@ -33,5 +33,5 @@ render_png <- function(args){ } render_svg <- function(args){ - render_image(svg, "image/svg", args) + render_image(svg, "image/svg+xml", args) } From 70e5974dec27a47cec030604caac19803999be09 Mon Sep 17 00:00:00 2001 From: Pachamaltese Date: Tue, 25 Feb 2020 15:39:04 -0300 Subject: [PATCH 03/23] test for svg --- tests/testthat/test-image.R | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/testthat/test-image.R b/tests/testthat/test-image.R index 832fdbfb7..0d590753c 100644 --- a/tests/testthat/test-image.R +++ b/tests/testthat/test-image.R @@ -26,6 +26,10 @@ test_that("Images are properly rendered", { expect_equal(resp$headers$`Content-type`, "image/jpeg") expect_gt(length(resp$body), 100) # This changes based on R ver/OS, may not be useful. expect_lt(length(resp$body), fullsizeJPEG) # Should be smaller than the full one + + resp <- r$serve(make_req("GET", "/svg"), PlumberResponse$new()) + expect_equal(resp$status, 200) + expect_equal(resp$headers$`Content-type`, "image/svg+xml") }) test_that("render_image arguments supplement", { From 428b9713bb72b0d9ae4d3223ae8537f4e0bebbc7 Mon Sep 17 00:00:00 2001 From: Pachamaltese Date: Tue, 25 Feb 2020 16:23:44 -0300 Subject: [PATCH 04/23] revert test --- tests/testthat/test-image.R | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/testthat/test-image.R b/tests/testthat/test-image.R index 0d590753c..832fdbfb7 100644 --- a/tests/testthat/test-image.R +++ b/tests/testthat/test-image.R @@ -26,10 +26,6 @@ test_that("Images are properly rendered", { expect_equal(resp$headers$`Content-type`, "image/jpeg") expect_gt(length(resp$body), 100) # This changes based on R ver/OS, may not be useful. expect_lt(length(resp$body), fullsizeJPEG) # Should be smaller than the full one - - resp <- r$serve(make_req("GET", "/svg"), PlumberResponse$new()) - expect_equal(resp$status, 200) - expect_equal(resp$headers$`Content-type`, "image/svg+xml") }) test_that("render_image arguments supplement", { From d56ebd1e0a1c9d2ff216dac9ea89c3899df3ea31 Mon Sep 17 00:00:00 2001 From: Pachamaltese Date: Tue, 25 Feb 2020 16:49:59 -0300 Subject: [PATCH 05/23] working svg test --- tests/testthat/files/image.R | 5 +++++ tests/testthat/test-image.R | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/tests/testthat/files/image.R b/tests/testthat/files/image.R index bf0f8ff14..706171cfd 100644 --- a/tests/testthat/files/image.R +++ b/tests/testthat/files/image.R @@ -10,6 +10,11 @@ function() { plot(1:10) } +#* @svg +#* @get /svg +function() { + plot(1:10) +} #' @png (width = 150, height=150) #' @get /littlepng diff --git a/tests/testthat/test-image.R b/tests/testthat/test-image.R index 832fdbfb7..9104ecc6c 100644 --- a/tests/testthat/test-image.R +++ b/tests/testthat/test-image.R @@ -26,6 +26,10 @@ test_that("Images are properly rendered", { expect_equal(resp$headers$`Content-type`, "image/jpeg") expect_gt(length(resp$body), 100) # This changes based on R ver/OS, may not be useful. expect_lt(length(resp$body), fullsizeJPEG) # Should be smaller than the full one + + resp <- r$serve(make_req("GET", "/svg"), PlumberResponse$new()) + expect_equal(resp$status, 200) + expect_equal(resp$headers$`Content-type`, "image/svg+xml") # without +xml doesn't work in firefox }) test_that("render_image arguments supplement", { From 563f2c6010be1ecbf1b265535a81c4dfad4819f0 Mon Sep 17 00:00:00 2001 From: Mauricio Vargas Date: Sat, 15 Jan 2022 19:23:02 -0500 Subject: [PATCH 06/23] update feather serializers + add parquet serializer --- DESCRIPTION | 3 ++- NAMESPACE | 2 ++ R/content-types.R | 16 +--------------- R/parse-body.R | 20 +++++++++++++++----- R/serializer.R | 24 ++++++++++++++++++++---- man/parsers.Rd | 7 ++++++- man/serializers.Rd | 7 ++++++- 7 files changed, 52 insertions(+), 27 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 38915ecc0..71ee7d57d 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -10,6 +10,7 @@ Authors@R: c( person("Bruno", "Tremblay", role = "ctb", email = "cran@neoxone.com"), person("Frans", "van Dunné", role = "ctb", email = "frans@ixpantia.com"), person("Sebastiaan", "Vandewoude", role="ctb", email = "sebastiaanvandewoude@gmail.com"), + person("Mauricio", "Vargas Sepulveda", role="ctb", email = "mavargas11@uc.cl"), person(family = "RStudio", role = c("cph", "fnd"))) License: MIT + file LICENSE BugReports: https://github.com/rstudio/plumber/issues @@ -44,7 +45,7 @@ Suggests: later, readr, yaml, - feather, + arrow, future, rstudioapi, spelling, diff --git a/NAMESPACE b/NAMESPACE index d12ec0541..fbef9ad95 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -33,6 +33,7 @@ export(parser_json) export(parser_multi) export(parser_none) export(parser_octet) +export(parser_parquet) export(parser_rds) export(parser_read_file) export(parser_text) @@ -83,6 +84,7 @@ export(serializer_html) export(serializer_htmlwidget) export(serializer_jpeg) export(serializer_json) +export(serializer_parquet) export(serializer_pdf) export(serializer_png) export(serializer_print) diff --git a/R/content-types.R b/R/content-types.R index 12db5796f..006995c51 100644 --- a/R/content-types.R +++ b/R/content-types.R @@ -43,6 +43,7 @@ knownContentTypes <- c( xlam = "application/vnd.ms-excel.addin.macroEnabled.12", xlsb = "application/vnd.ms-excel.sheet.binary.macroEnabled.12", feather = "application/feather", + parquet = "application/parquet", rds = "application/rds", tsv = "application/tab-separated-values", csv = "application/csv", @@ -74,20 +75,6 @@ cleanup_content_type <- function(type) { if (stri_detect_fixed(type, ";")) { type <- stri_split_fixed(type, ";")[[1]][1] } -<<<<<<< HEAD - charsetStart <- attr( - gregexpr(".*charset=(.*)", contentType, perl = T)[[1]], - "capture.start" - ) - charsetStart <- as.integer(charsetStart) - as.character( - ifelse( - charsetStart > -1, - substr(contentType, charsetStart, nchar(contentType)), - default - ) - ) -======= type } @@ -110,5 +97,4 @@ get_fileext <- function(type) { get_character_set <- function(content_type = NULL) { if (is.null(content_type)) return("UTF-8") stri_match_first_regex(paste(content_type,"; charset=UTF-8"), "charset=([^;\\s]*)")[,2] ->>>>>>> 9e7265d791003ce43ffbb36f65b3543242b65743 } diff --git a/R/parse-body.R b/R/parse-body.R index d734badc6..3134589c1 100644 --- a/R/parse-body.R +++ b/R/parse-body.R @@ -480,18 +480,27 @@ parser_rds <- function(...) { }) } -#' @describeIn parsers feather parser. See [feather::read_feather()] for more details. +#' @describeIn parsers feather parser. See [arrow::read_feather()] for more details. #' @export parser_feather <- function(...) { parser_read_file(function(tmpfile) { - if (!requireNamespace("feather", quietly = TRUE)) { - stop("`feather` must be installed for `parser_feather` to work") + if (!requireNamespace("arrow", quietly = TRUE)) { + stop("`arrow` must be installed for `parser_feather` to work") } - feather::read_feather(tmpfile, ...) + arrow::read_feather(tmpfile, ...) }) } - +#' @describeIn parsers parquet parser. See [arrow::read_parquet()] for more details. +#' @export +parser_parquet <- function(...) { + parser_read_file(function(tmpfile) { + if (!requireNamespace("arrow", quietly = TRUE)) { + stop("`arrow` must be installed for `parser_parquet` to work") + } + arrow::read_parquet(tmpfile, ...) + }) +} #' @describeIn parsers Octet stream parser. Returns the raw content. #' @export @@ -569,6 +578,7 @@ register_parsers_onLoad <- function() { register_parser("form", parser_form, fixed = "application/x-www-form-urlencoded") register_parser("rds", parser_rds, fixed = "application/rds") register_parser("feather", parser_feather, fixed = "application/feather") + register_parser("parquet", parser_parquet, fixed = "application/parquet") register_parser("text", parser_text, fixed = "text/plain", regex = "^text/") register_parser("tsv", parser_tsv, fixed = c("application/tab-separated-values", "text/tab-separated-values")) # yaml types: https://stackoverflow.com/a/38000954/591574 diff --git a/R/serializer.R b/R/serializer.R index 4a6b81b8d..eb638d527 100644 --- a/R/serializer.R +++ b/R/serializer.R @@ -263,17 +263,32 @@ serializer_rds <- function(version = "2", ascii = FALSE, ..., type = "applicatio }) } -#' @describeIn serializers feather serializer. See also: [feather::write_feather()] +#' @describeIn serializers feather serializer. See also: [arrow::write_feather()] #' @export serializer_feather <- function(type = "application/feather") { - if (!requireNamespace("feather", quietly = TRUE)) { - stop("`feather` must be installed for `serializer_feather` to work") + if (!requireNamespace("arrow", quietly = TRUE)) { + stop("`arrow` must be installed for `serializer_feather` to work") } serializer_write_file( fileext = ".feather", type = type, write_fn = function(val, tmpfile) { - feather::write_feather(val, tmpfile) + arrow::write_feather(val, tmpfile) + } + ) +} + +#' @describeIn serializers parquet serializer. See also: [arrow::write_parquet()] +#' @export +serializer_parquet <- function(type = "application/parquet") { + if (!requireNamespace("arrow", quietly = TRUE)) { + stop("`arrow` must be installed for `serializer_parquet` to work") + } + serializer_write_file( + fileext = ".parquet", + type = type, + write_fn = function(val, tmpfile) { + arrow::write_parquet(val, tmpfile) } ) } @@ -614,6 +629,7 @@ add_serializers_onLoad <- function() { register_serializer("csv", serializer_csv) register_serializer("tsv", serializer_tsv) register_serializer("feather", serializer_feather) + register_serializer("parquet", serializer_parquet) register_serializer("yaml", serializer_yaml) register_serializer("geojson", serializer_geojson) diff --git a/man/parsers.Rd b/man/parsers.Rd index d4be32d6f..64c7c81fa 100644 --- a/man/parsers.Rd +++ b/man/parsers.Rd @@ -11,6 +11,7 @@ \alias{parser_read_file} \alias{parser_rds} \alias{parser_feather} +\alias{parser_parquet} \alias{parser_octet} \alias{parser_multi} \alias{parser_none} @@ -36,6 +37,8 @@ parser_rds(...) parser_feather(...) +parser_parquet(...) + parser_octet() parser_multi() @@ -86,7 +89,9 @@ This parser should be used when reading from a file is required. \item \code{parser_rds}: RDS parser. See \code{\link[=readRDS]{readRDS()}} for more details. -\item \code{parser_feather}: feather parser. See \code{\link[feather:read_feather]{feather::read_feather()}} for more details. +\item \code{parser_feather}: feather parser. See \code{\link[arrow:read_feather]{arrow::read_feather()}} for more details. + +\item \code{parser_parquet}: parquet parser. See \code{\link[arrow:read_parquet]{arrow::read_parquet()}} for more details. \item \code{parser_octet}: Octet stream parser. Returns the raw content. diff --git a/man/serializers.Rd b/man/serializers.Rd index eee549680..39b51cd21 100644 --- a/man/serializers.Rd +++ b/man/serializers.Rd @@ -11,6 +11,7 @@ \alias{serializer_geojson} \alias{serializer_rds} \alias{serializer_feather} +\alias{serializer_parquet} \alias{serializer_yaml} \alias{serializer_text} \alias{serializer_format} @@ -47,6 +48,8 @@ serializer_rds(version = "2", ascii = FALSE, ..., type = "application/rds") serializer_feather(type = "application/feather") +serializer_parquet(type = "application/parquet") + serializer_yaml(..., type = "text/x-yaml; charset=UTF-8") serializer_text( @@ -135,7 +138,9 @@ more details on Plumber serializers and how to customize their behavior. \item \code{serializer_rds}: RDS serializer. See also: \code{\link[base:serialize]{base::serialize()}} -\item \code{serializer_feather}: feather serializer. See also: \code{\link[feather:read_feather]{feather::write_feather()}} +\item \code{serializer_feather}: feather serializer. See also: \code{\link[arrow:write_feather]{arrow::write_feather()}} + +\item \code{serializer_parquet}: parquet serializer. See also: \code{\link[arrow:write_parquet]{arrow::write_parquet()}} \item \code{serializer_yaml}: YAML serializer. See also: \code{\link[yaml:as.yaml]{yaml::as.yaml()}} From 42d1ed67484536f318f97b19d33f6a500bda0c42 Mon Sep 17 00:00:00 2001 From: Mauricio Vargas Date: Sat, 15 Jan 2022 19:25:15 -0500 Subject: [PATCH 07/23] update NEWS --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index f86cd9ec8..a67918c56 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,6 +5,8 @@ ## New features * Introduces new GeoJSON serializer and parser. GeoJSON objects are parsed into `sf` objects and `sf` or `sfc` objects will be serialized into GeoJSON. (@josiahparry, #830) +* Updates feather serializer to use the arrow package +* Adds parquet serializer by using the arrow package ## Bug fixes From d6d58659d1b9e26bb7d0a3cf421d72dfa5d72172 Mon Sep 17 00:00:00 2001 From: Mauricio Vargas Date: Sat, 15 Jan 2022 19:29:43 -0500 Subject: [PATCH 08/23] update tests for feather + add tests for parquet --- tests/testthat/test-parse-body.R | 24 ++++++++++++-- tests/testthat/test-serializer-feather.R | 6 ++-- tests/testthat/test-serializer-parquet.R | 40 ++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 5 deletions(-) create mode 100644 tests/testthat/test-serializer-parquet.R diff --git a/tests/testthat/test-parse-body.R b/tests/testthat/test-parse-body.R index d1f89434f..6c8825f6f 100644 --- a/tests/testthat/test-parse-body.R +++ b/tests/testthat/test-parse-body.R @@ -90,7 +90,7 @@ test_that("Test tsv parser", { }) test_that("Test feather parser", { - skip_if_not_installed("feather") + skip_if_not_installed("arrow") tmp <- tempfile() on.exit({ @@ -98,7 +98,7 @@ test_that("Test feather parser", { }, add = TRUE) r_object <- iris - feather::write_feather(r_object, tmp) + arrow::write_feather(r_object, tmp) val <- readBin(tmp, "raw", 10000) parsed <- parse_body(val, "application/feather", make_parser("feather")) @@ -109,6 +109,26 @@ test_that("Test feather parser", { expect_equal(parsed, r_object) }) +test_that("Test parquet parser", { + skip_if_not_installed("arrow") + + tmp <- tempfile() + on.exit({ + file.remove(tmp) + }, add = TRUE) + + r_object <- iris + arrow::write_parquet(r_object, tmp) + val <- readBin(tmp, "raw", 10000) + + parsed <- parse_body(val, "application/parquet", make_parser("parquet")) + # convert from parquet tibble to data.frame + parsed <- as.data.frame(parsed, stringsAsFactors = FALSE) + attr(parsed, "spec") <- NULL + + expect_equal(parsed, r_object) +}) + test_that("Test geojson parser", { skip_if_not_installed("geojsonsf") skip_if_not_installed("sf") diff --git a/tests/testthat/test-serializer-feather.R b/tests/testthat/test-serializer-feather.R index c191997da..7a2a3529e 100644 --- a/tests/testthat/test-serializer-feather.R +++ b/tests/testthat/test-serializer-feather.R @@ -1,7 +1,7 @@ context("feather serializer") test_that("feather serializes properly", { - skip_if_not_installed("feather") + skip_if_not_installed("arrow") d <- data.frame(a=1, b=2, c="hi") val <- serializer_feather()(d, data.frame(), PlumberResponse$new(), stop) @@ -18,7 +18,7 @@ test_that("feather serializes properly", { }) test_that("Errors call error handler", { - skip_if_not_installed("feather") + skip_if_not_installed("arrow") errors <- 0 errHandler <- function(req, res, err){ @@ -31,7 +31,7 @@ test_that("Errors call error handler", { }) test_that("Errors are rendered correctly with debug TRUE", { - skip_if_not_installed("feather") + skip_if_not_installed("arrow") pr <- pr() %>% pr_get("/", function() stop("myerror"), serializer = serializer_feather()) %>% pr_set_debug(TRUE) capture.output(res <- pr$serve(make_req(pr = pr), PlumberResponse$new("csv"))) diff --git a/tests/testthat/test-serializer-parquet.R b/tests/testthat/test-serializer-parquet.R new file mode 100644 index 000000000..0f831c707 --- /dev/null +++ b/tests/testthat/test-serializer-parquet.R @@ -0,0 +1,40 @@ +context("parquet serializer") + +test_that("parquet serializes properly", { + skip_if_not_installed("arrow") + + d <- data.frame(a=1, b=2, c="hi") + val <- serializer_parquet()(d, data.frame(), PlumberResponse$new(), stop) + expect_equal(val$status, 200L) + expect_equal(val$headers$`Content-Type`, "application/parquet") + + # can test by doing a full round trip if we believe the parser works via `test-parse-body.R` + parsed <- parse_body(val$body, "application/parquet", make_parser("parquet")) + # convert from parquet tibble to data.frame + parsed <- as.data.frame(parsed, stringsAsFactors = FALSE) + attr(parsed, "spec") <- NULL + + expect_equal(parsed, d) +}) + +test_that("Errors call error handler", { + skip_if_not_installed("arrow") + + errors <- 0 + errHandler <- function(req, res, err){ + errors <<- errors + 1 + } + + expect_equal(errors, 0) + serializer_parquet()(parse(text="hi"), data.frame(), PlumberResponse$new("csv"), errorHandler = errHandler) + expect_equal(errors, 1) +}) + +test_that("Errors are rendered correctly with debug TRUE", { + skip_if_not_installed("arrow") + + pr <- pr() %>% pr_get("/", function() stop("myerror"), serializer = serializer_parquet()) %>% pr_set_debug(TRUE) + capture.output(res <- pr$serve(make_req(pr = pr), PlumberResponse$new("csv"))) + + expect_match(res$body, "Error in (function () : myerror", fixed = TRUE) +}) From ef4a2e57eadd35bac15ec719a7dab540afb3a1d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pach=C3=A1?= Date: Tue, 18 Jan 2022 14:57:11 -0500 Subject: [PATCH 09/23] Update R/content-types.R Co-authored-by: Neal Richardson --- R/content-types.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/content-types.R b/R/content-types.R index 006995c51..a73eadb37 100644 --- a/R/content-types.R +++ b/R/content-types.R @@ -42,7 +42,7 @@ knownContentTypes <- c( dotx = "application/vnd.openxmlformats-officedocument.wordprocessingml.template", xlam = "application/vnd.ms-excel.addin.macroEnabled.12", xlsb = "application/vnd.ms-excel.sheet.binary.macroEnabled.12", - feather = "application/feather", + feather = "application/vnd.apache.arrow.file", parquet = "application/parquet", rds = "application/rds", tsv = "application/tab-separated-values", From 421f2bb86a1fabca760a180a27115d73488ed0e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pach=C3=A1?= Date: Tue, 18 Jan 2022 14:57:57 -0500 Subject: [PATCH 10/23] Update R/serializer.R Co-authored-by: Barret Schloerke --- R/serializer.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/serializer.R b/R/serializer.R index eb638d527..50a232412 100644 --- a/R/serializer.R +++ b/R/serializer.R @@ -280,7 +280,7 @@ serializer_feather <- function(type = "application/feather") { #' @describeIn serializers parquet serializer. See also: [arrow::write_parquet()] #' @export -serializer_parquet <- function(type = "application/parquet") { +serializer_parquet <- function(type = "application/vnd.apache.parquet") { if (!requireNamespace("arrow", quietly = TRUE)) { stop("`arrow` must be installed for `serializer_parquet` to work") } From f5bc085dd7c699034862c624bbbed28dab3a92c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pach=C3=A1?= Date: Tue, 18 Jan 2022 14:58:07 -0500 Subject: [PATCH 11/23] Update R/content-types.R Co-authored-by: Barret Schloerke --- R/content-types.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/content-types.R b/R/content-types.R index a73eadb37..91d1fc7a6 100644 --- a/R/content-types.R +++ b/R/content-types.R @@ -43,7 +43,7 @@ knownContentTypes <- c( xlam = "application/vnd.ms-excel.addin.macroEnabled.12", xlsb = "application/vnd.ms-excel.sheet.binary.macroEnabled.12", feather = "application/vnd.apache.arrow.file", - parquet = "application/parquet", + parquet = "application/vnd.apache.parquet", rds = "application/rds", tsv = "application/tab-separated-values", csv = "application/csv", From 20f27d7ec9e30a8425abf8e7840a241359b81761 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pach=C3=A1?= Date: Tue, 18 Jan 2022 14:58:26 -0500 Subject: [PATCH 12/23] Update R/parse-body.R Co-authored-by: Barret Schloerke --- R/parse-body.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/parse-body.R b/R/parse-body.R index 3134589c1..cb01bbdd5 100644 --- a/R/parse-body.R +++ b/R/parse-body.R @@ -577,7 +577,7 @@ register_parsers_onLoad <- function() { register_parser("octet", parser_octet, fixed = "application/octet-stream") register_parser("form", parser_form, fixed = "application/x-www-form-urlencoded") register_parser("rds", parser_rds, fixed = "application/rds") - register_parser("feather", parser_feather, fixed = "application/feather") + register_parser("feather", parser_feather, fixed = c("application/vnd.apache.arrow.file", "application/feather")) register_parser("parquet", parser_parquet, fixed = "application/parquet") register_parser("text", parser_text, fixed = "text/plain", regex = "^text/") register_parser("tsv", parser_tsv, fixed = c("application/tab-separated-values", "text/tab-separated-values")) From 5a418f909af91e57b0642d81c0517a55cff5aab6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pach=C3=A1?= Date: Tue, 18 Jan 2022 14:58:35 -0500 Subject: [PATCH 13/23] Update tests/testthat/test-serializer-parquet.R Co-authored-by: Barret Schloerke --- tests/testthat/test-serializer-parquet.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testthat/test-serializer-parquet.R b/tests/testthat/test-serializer-parquet.R index 0f831c707..b26764489 100644 --- a/tests/testthat/test-serializer-parquet.R +++ b/tests/testthat/test-serializer-parquet.R @@ -9,7 +9,7 @@ test_that("parquet serializes properly", { expect_equal(val$headers$`Content-Type`, "application/parquet") # can test by doing a full round trip if we believe the parser works via `test-parse-body.R` - parsed <- parse_body(val$body, "application/parquet", make_parser("parquet")) + parsed <- parse_body(val$body, "application/vnd.apache.parquet", make_parser("parquet")) # convert from parquet tibble to data.frame parsed <- as.data.frame(parsed, stringsAsFactors = FALSE) attr(parsed, "spec") <- NULL From 28c0fb2d2d8125e2874f0c8b4820873e0e4f07d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pach=C3=A1?= Date: Tue, 18 Jan 2022 14:59:24 -0500 Subject: [PATCH 14/23] Update tests/testthat/test-parse-body.R Co-authored-by: Barret Schloerke --- tests/testthat/test-parse-body.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testthat/test-parse-body.R b/tests/testthat/test-parse-body.R index 6c8825f6f..5f2aa555d 100644 --- a/tests/testthat/test-parse-body.R +++ b/tests/testthat/test-parse-body.R @@ -101,7 +101,7 @@ test_that("Test feather parser", { arrow::write_feather(r_object, tmp) val <- readBin(tmp, "raw", 10000) - parsed <- parse_body(val, "application/feather", make_parser("feather")) + parsed <- parse_body(val, "application/vnd.apache.arrow.file", make_parser("feather")) # convert from feather tibble to data.frame parsed <- as.data.frame(parsed, stringsAsFactors = FALSE) attr(parsed, "spec") <- NULL From c8c1f52ab47fcb87bd1bdbb51ff700675cc84b2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pach=C3=A1?= Date: Tue, 18 Jan 2022 15:11:30 -0500 Subject: [PATCH 15/23] Update R/serializer.R Co-authored-by: Barret Schloerke --- R/serializer.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/serializer.R b/R/serializer.R index 50a232412..76ef759c8 100644 --- a/R/serializer.R +++ b/R/serializer.R @@ -265,7 +265,7 @@ serializer_rds <- function(version = "2", ascii = FALSE, ..., type = "applicatio #' @describeIn serializers feather serializer. See also: [arrow::write_feather()] #' @export -serializer_feather <- function(type = "application/feather") { +serializer_feather <- function(type = "application/vnd.apache.arrow.file") { if (!requireNamespace("arrow", quietly = TRUE)) { stop("`arrow` must be installed for `serializer_feather` to work") } From e849328559bff63ff16d796549ebdc3c9a6600b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pach=C3=A1?= Date: Tue, 18 Jan 2022 15:21:18 -0500 Subject: [PATCH 16/23] Update R/parse-body.R Co-authored-by: Barret Schloerke --- R/parse-body.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/parse-body.R b/R/parse-body.R index cb01bbdd5..6ca067d86 100644 --- a/R/parse-body.R +++ b/R/parse-body.R @@ -578,7 +578,7 @@ register_parsers_onLoad <- function() { register_parser("form", parser_form, fixed = "application/x-www-form-urlencoded") register_parser("rds", parser_rds, fixed = "application/rds") register_parser("feather", parser_feather, fixed = c("application/vnd.apache.arrow.file", "application/feather")) - register_parser("parquet", parser_parquet, fixed = "application/parquet") + register_parser("parquet", parser_parquet, fixed = "application/vnd.apache.parquet") register_parser("text", parser_text, fixed = "text/plain", regex = "^text/") register_parser("tsv", parser_tsv, fixed = c("application/tab-separated-values", "text/tab-separated-values")) # yaml types: https://stackoverflow.com/a/38000954/591574 From 8fc7422e10fd6cf34c182587b79da67e255208e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pach=C3=A1?= Date: Tue, 18 Jan 2022 15:21:49 -0500 Subject: [PATCH 17/23] Update DESCRIPTION Co-authored-by: Barret Schloerke --- DESCRIPTION | 1 - 1 file changed, 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 71ee7d57d..ee7f6d37a 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -10,7 +10,6 @@ Authors@R: c( person("Bruno", "Tremblay", role = "ctb", email = "cran@neoxone.com"), person("Frans", "van Dunné", role = "ctb", email = "frans@ixpantia.com"), person("Sebastiaan", "Vandewoude", role="ctb", email = "sebastiaanvandewoude@gmail.com"), - person("Mauricio", "Vargas Sepulveda", role="ctb", email = "mavargas11@uc.cl"), person(family = "RStudio", role = c("cph", "fnd"))) License: MIT + file LICENSE BugReports: https://github.com/rstudio/plumber/issues From 44ebd93cf8529390de14dba3c76f4aca45e82f68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pach=C3=A1?= Date: Tue, 18 Jan 2022 15:22:07 -0500 Subject: [PATCH 18/23] Update NEWS.md Co-authored-by: Barret Schloerke --- NEWS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index a67918c56..417d09e2b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,8 +5,8 @@ ## New features * Introduces new GeoJSON serializer and parser. GeoJSON objects are parsed into `sf` objects and `sf` or `sfc` objects will be serialized into GeoJSON. (@josiahparry, #830) -* Updates feather serializer to use the arrow package -* Adds parquet serializer by using the arrow package +* Update feather serializer to use the arrow package. The new default feather MIME type is `application/vnd.apache.arrow.file`. (@pachadotdev #849) +* Add parquet serializer and parser by using the arrow package (@pachadotdev #849) ## Bug fixes From 872fa4cde0fd0e1d9c8832f3396f8525200505dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pach=C3=A1?= Date: Tue, 18 Jan 2022 15:24:05 -0500 Subject: [PATCH 19/23] Update tests/testthat/test-serializer-parquet.R Co-authored-by: Barret Schloerke --- tests/testthat/test-serializer-parquet.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testthat/test-serializer-parquet.R b/tests/testthat/test-serializer-parquet.R index b26764489..f14864de9 100644 --- a/tests/testthat/test-serializer-parquet.R +++ b/tests/testthat/test-serializer-parquet.R @@ -6,7 +6,7 @@ test_that("parquet serializes properly", { d <- data.frame(a=1, b=2, c="hi") val <- serializer_parquet()(d, data.frame(), PlumberResponse$new(), stop) expect_equal(val$status, 200L) - expect_equal(val$headers$`Content-Type`, "application/parquet") + expect_equal(val$headers$`Content-Type`, "application/vnd.apache.parquet") # can test by doing a full round trip if we believe the parser works via `test-parse-body.R` parsed <- parse_body(val$body, "application/vnd.apache.parquet", make_parser("parquet")) From 59e81a9306e33894ae17c9e7da3426c84cc5db4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pach=C3=A1?= Date: Tue, 18 Jan 2022 15:24:16 -0500 Subject: [PATCH 20/23] Update tests/testthat/test-parse-body.R Co-authored-by: Barret Schloerke --- tests/testthat/test-parse-body.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testthat/test-parse-body.R b/tests/testthat/test-parse-body.R index 5f2aa555d..ec2b37703 100644 --- a/tests/testthat/test-parse-body.R +++ b/tests/testthat/test-parse-body.R @@ -121,7 +121,7 @@ test_that("Test parquet parser", { arrow::write_parquet(r_object, tmp) val <- readBin(tmp, "raw", 10000) - parsed <- parse_body(val, "application/parquet", make_parser("parquet")) + parsed <- parse_body(val, "application/vnd.apache.parquet", make_parser("parquet")) # convert from parquet tibble to data.frame parsed <- as.data.frame(parsed, stringsAsFactors = FALSE) attr(parsed, "spec") <- NULL From 32f6aab3bdc145e5472d70fa7a63ce851d68e9bf Mon Sep 17 00:00:00 2001 From: Mauricio Vargas Date: Tue, 18 Jan 2022 15:25:34 -0500 Subject: [PATCH 21/23] adding @schloerke suggestions --- tests/testthat/test-serializer-parquet.R | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/tests/testthat/test-serializer-parquet.R b/tests/testthat/test-serializer-parquet.R index f14864de9..2e12444c4 100644 --- a/tests/testthat/test-serializer-parquet.R +++ b/tests/testthat/test-serializer-parquet.R @@ -1,30 +1,9 @@ -context("parquet serializer") - -test_that("parquet serializes properly", { - skip_if_not_installed("arrow") - - d <- data.frame(a=1, b=2, c="hi") - val <- serializer_parquet()(d, data.frame(), PlumberResponse$new(), stop) - expect_equal(val$status, 200L) - expect_equal(val$headers$`Content-Type`, "application/vnd.apache.parquet") - - # can test by doing a full round trip if we believe the parser works via `test-parse-body.R` - parsed <- parse_body(val$body, "application/vnd.apache.parquet", make_parser("parquet")) - # convert from parquet tibble to data.frame - parsed <- as.data.frame(parsed, stringsAsFactors = FALSE) - attr(parsed, "spec") <- NULL - - expect_equal(parsed, d) -}) - test_that("Errors call error handler", { skip_if_not_installed("arrow") - errors <- 0 errHandler <- function(req, res, err){ errors <<- errors + 1 } - expect_equal(errors, 0) serializer_parquet()(parse(text="hi"), data.frame(), PlumberResponse$new("csv"), errorHandler = errHandler) expect_equal(errors, 1) @@ -32,9 +11,7 @@ test_that("Errors call error handler", { test_that("Errors are rendered correctly with debug TRUE", { skip_if_not_installed("arrow") - pr <- pr() %>% pr_get("/", function() stop("myerror"), serializer = serializer_parquet()) %>% pr_set_debug(TRUE) capture.output(res <- pr$serve(make_req(pr = pr), PlumberResponse$new("csv"))) - expect_match(res$body, "Error in (function () : myerror", fixed = TRUE) }) From 0dfea4ebe5cb7cc9ef9228f5053d444a5fdde017 Mon Sep 17 00:00:00 2001 From: Mauricio Vargas Date: Tue, 18 Jan 2022 15:45:14 -0500 Subject: [PATCH 22/23] update tests + vignette --- man/serializers.Rd | 4 ++-- tests/testthat/test-serializer-feather.R | 4 ++-- tests/testthat/test-serializer-parquet.R | 2 ++ vignettes/rendering-output.Rmd | 3 ++- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/man/serializers.Rd b/man/serializers.Rd index 39b51cd21..4f13eca49 100644 --- a/man/serializers.Rd +++ b/man/serializers.Rd @@ -46,9 +46,9 @@ serializer_geojson(..., type = "application/geo+json") serializer_rds(version = "2", ascii = FALSE, ..., type = "application/rds") -serializer_feather(type = "application/feather") +serializer_feather(type = "application/vnd.apache.arrow.file") -serializer_parquet(type = "application/parquet") +serializer_parquet(type = "application/vnd.apache.parquet") serializer_yaml(..., type = "text/x-yaml; charset=UTF-8") diff --git a/tests/testthat/test-serializer-feather.R b/tests/testthat/test-serializer-feather.R index 7a2a3529e..cfa4bc591 100644 --- a/tests/testthat/test-serializer-feather.R +++ b/tests/testthat/test-serializer-feather.R @@ -6,10 +6,10 @@ test_that("feather serializes properly", { d <- data.frame(a=1, b=2, c="hi") val <- serializer_feather()(d, data.frame(), PlumberResponse$new(), stop) expect_equal(val$status, 200L) - expect_equal(val$headers$`Content-Type`, "application/feather") + expect_equal(val$headers$`Content-Type`, "application/vnd.apache.arrow.file") # can test by doing a full round trip if we believe the parser works via `test-parse-body.R` - parsed <- parse_body(val$body, "application/feather", make_parser("feather")) + parsed <- parse_body(val$body, "application/vnd.apache.arrow.file", make_parser("feather")) # convert from feather tibble to data.frame parsed <- as.data.frame(parsed, stringsAsFactors = FALSE) attr(parsed, "spec") <- NULL diff --git a/tests/testthat/test-serializer-parquet.R b/tests/testthat/test-serializer-parquet.R index 2e12444c4..533f2b50e 100644 --- a/tests/testthat/test-serializer-parquet.R +++ b/tests/testthat/test-serializer-parquet.R @@ -1,3 +1,5 @@ +context("parquet serializer") + test_that("Errors call error handler", { skip_if_not_installed("arrow") errors <- 0 diff --git a/vignettes/rendering-output.Rmd b/vignettes/rendering-output.Rmd index bda83f923..2ef0f9d37 100644 --- a/vignettes/rendering-output.Rmd +++ b/vignettes/rendering-output.Rmd @@ -59,7 +59,8 @@ Annotation | Content Type | Description/References `@serializer rds` | `application/rds` | Object processed with `base::serialize()` `@serializer csv` | `text/csv` | Object processed with `readr::format_csv()` `@serializer tsv` | `text/tab-separated-values` | Object processed with `readr::format_tsv()` -`@serializer feather` | `application/feather` | Object processed with `feather::write_feather()` +`@serializer feather` | `application/vnd.apache.arrow.file` | Object processed with `arrow::write_feather()` +`@serializer parquet` | `application/parquet` | Object processed with `arrow::write_parquet()` `@serializer yaml` | `text/x-yaml` | Object processed with `yaml::as_yaml()` `@serializer htmlwidget` | `text/html; charset=utf-8` | `htmlwidgets::saveWidget()` `@serializer text` | `text/plain` | Text output processed by `as.character()` From afb58c55a97002e8d6c72fbeb6d83684269cc2c8 Mon Sep 17 00:00:00 2001 From: Mauricio Vargas Date: Tue, 18 Jan 2022 15:47:29 -0500 Subject: [PATCH 23/23] remove parquet tests (same as feather) --- tests/testthat/test-serializer-parquet.R | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 tests/testthat/test-serializer-parquet.R diff --git a/tests/testthat/test-serializer-parquet.R b/tests/testthat/test-serializer-parquet.R deleted file mode 100644 index 533f2b50e..000000000 --- a/tests/testthat/test-serializer-parquet.R +++ /dev/null @@ -1,19 +0,0 @@ -context("parquet serializer") - -test_that("Errors call error handler", { - skip_if_not_installed("arrow") - errors <- 0 - errHandler <- function(req, res, err){ - errors <<- errors + 1 - } - expect_equal(errors, 0) - serializer_parquet()(parse(text="hi"), data.frame(), PlumberResponse$new("csv"), errorHandler = errHandler) - expect_equal(errors, 1) -}) - -test_that("Errors are rendered correctly with debug TRUE", { - skip_if_not_installed("arrow") - pr <- pr() %>% pr_get("/", function() stop("myerror"), serializer = serializer_parquet()) %>% pr_set_debug(TRUE) - capture.output(res <- pr$serve(make_req(pr = pr), PlumberResponse$new("csv"))) - expect_match(res$body, "Error in (function () : myerror", fixed = TRUE) -})