From a5c94b65366fe38ad33b809f56e057ee2066225f Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Fri, 31 Jul 2020 17:17:59 -0400 Subject: [PATCH 1/5] Add feather serializer and parser --- DESCRIPTION | 3 ++- R/content-types.R | 1 + R/parse-body.R | 12 +++++++++ R/serializer.R | 20 +++++++++++++++ tests/testthat/test-parse-body.R | 20 +++++++++++++++ tests/testthat/test-serializer-feather.R | 31 ++++++++++++++++++++++++ 6 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 tests/testthat/test-serializer-feather.R diff --git a/DESCRIPTION b/DESCRIPTION index 2b06dd8e6..c08e378ef 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -41,7 +41,8 @@ Suggests: analogsea (>= 0.7.0), later, readr, - yaml + yaml, + feather Remotes: rstudio/swagger RoxygenNote: 7.1.1 diff --git a/R/content-types.R b/R/content-types.R index dc8c73e1c..881ddaf27 100644 --- a/R/content-types.R +++ b/R/content-types.R @@ -42,6 +42,7 @@ knownContentTypes <- list( dotx='application/vnd.openxmlformats-officedocument.wordprocessingml.template', xlam='application/vnd.ms-excel.addin.macroEnabled.12', xlsb='application/vnd.ms-excel.sheet.binary.macroEnabled.12', + feather='application/feather', rds='application/rds', tsv="text/tab-separated-values", csv="text/csv") diff --git a/R/parse-body.R b/R/parse-body.R index 925e7a433..1e3a2c8e8 100644 --- a/R/parse-body.R +++ b/R/parse-body.R @@ -428,6 +428,18 @@ parser_rds <- function(...) { }) } +#' @describeIn parsers feather parser. See [feather::read_feather()] for more details. +#' @export +parser_feather <- function(...) { + parser_read_file(function(tmpfile) { + if (!requireNamespace("feather", quietly = TRUE)) { + stop("`feather` must be installed for `parser_feather` to work") + } + feather::read_feather(tmpfile, ...) + }) +} + + #' @describeIn parsers Octet stream parser. Will add a filename attribute if the filename exists #' @export diff --git a/R/serializer.R b/R/serializer.R index 60a7af6a9..cfc5eea6a 100644 --- a/R/serializer.R +++ b/R/serializer.R @@ -209,6 +209,26 @@ serializer_rds <- function(version = "2", ascii = FALSE, ...) { }) } +#' @describeIn serializers feather serializer. See [feather::write_feather] for more details. +#' @export +serializer_feather <- function() { + if (!requireNamespace("feather", quietly = TRUE)) { + stop("`feather` must be installed for `serializer_feather` to work") + } + serializer_content_type("application/feather; charset=UTF-8", function(val) { + tmpfile <- tempfile(fileext = ".feather") + on.exit({ + if (file.exists(tmpfile)) { + unlink(tmpfile) + } + }, add = TRUE) + + feather::write_feather(val, tmpfile) + readBin(tmpfile, what = "raw", n = file.info(tmpfile)$size) + }) +} + + #' @describeIn serializers YAML serializer. See [yaml::as.yaml()] for more details. #' @export serializer_yaml <- function(...) { diff --git a/tests/testthat/test-parse-body.R b/tests/testthat/test-parse-body.R index 0b7cb6621..fd71b2e67 100644 --- a/tests/testthat/test-parse-body.R +++ b/tests/testthat/test-parse-body.R @@ -84,6 +84,26 @@ test_that("Test tsv parser", { expect_equal(parsed, r_object) }) +test_that("Test feather parser", { + skip_if_not_installed("feather") + + tmp <- tempfile() + on.exit({ + file.remove(tmp) + }, add = TRUE) + + r_object <- iris + feather::write_feather(r_object, tmp) + val <- readBin(tmp, "raw", 10000) + + parsed <- parse_body(val, "application/feather", make_parser("feather")) + # convert from feather tibble to data.frame + parsed <- as.data.frame(parsed, stringsAsFactors = FALSE) + attr(parsed, "spec") <- NULL + + expect_equal(parsed, r_object) +}) + test_that("Test multipart parser", { # also tests rds and the octet -> content type conversion diff --git a/tests/testthat/test-serializer-feather.R b/tests/testthat/test-serializer-feather.R new file mode 100644 index 000000000..2c404d89b --- /dev/null +++ b/tests/testthat/test-serializer-feather.R @@ -0,0 +1,31 @@ +context("feather serializer") + +test_that("feather serializes properly", { + skip_if_not_installed("feather") + + d <- data.frame(a=1, b=2, c="hi") + val <- serializer_feather()(d, data.frame(), PlumberResponse$new(), stop) + expect_equal(val$status, 200L) + expect_equal(val$headers$`Content-Type`, "application/feather; charset=UTF-8") + + # can test by doing a full round trip if we believe the parser works via `test-parse-body.R` + parsed <- parse_body(val$body, "application/feather", make_parser("feather")) + # convert from feather tibble to data.frame + parsed <- as.data.frame(parsed, stringsAsFactors = FALSE) + attr(parsed, "spec") <- NULL + + expect_equal(parsed, d) +}) + +test_that("Errors call error handler", { + skip_if_not_installed("feather") + + errors <- 0 + errHandler <- function(req, res, err){ + errors <<- errors + 1 + } + + expect_equal(errors, 0) + serializer_feather()(parse(text="hi"), data.frame(), PlumberResponse$new("csv"), errorHandler = errHandler) + expect_equal(errors, 1) +}) From 979ff12d21ef4b238edd260ebaa9f8c44cf14867 Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Fri, 31 Jul 2020 17:19:04 -0400 Subject: [PATCH 2/5] Register feather parser and serializer --- R/parse-body.R | 1 + R/serializer.R | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/R/parse-body.R b/R/parse-body.R index 1e3a2c8e8..316cfdc2b 100644 --- a/R/parse-body.R +++ b/R/parse-body.R @@ -495,6 +495,7 @@ register_parsers_onLoad <- function() { register_parser("octet", parser_octet, fixed = "application/octet-stream") register_parser("query", parser_query, fixed = "application/x-www-form-urlencoded") register_parser("rds", parser_rds, fixed = "application/rds") + register_parser("feather", parser_feather, fixed = "application/feather") register_parser("text", parser_text, fixed = "text/plain", regex = "^text/") register_parser("tsv", parser_tsv, fixed = c("application/tab-separated-values", "text/tab-separated-values")) register_parser("yaml", parser_yaml, fixed = c("application/yaml", "application/x-yaml", "text/yaml", "text/x-yaml")) diff --git a/R/serializer.R b/R/serializer.R index cfc5eea6a..85d668c89 100644 --- a/R/serializer.R +++ b/R/serializer.R @@ -332,11 +332,12 @@ add_serializers_onLoad <- function() { addSerializer("json", serializer_json) addSerializer("unboxedJSON", serializer_unboxed_json) addSerializer("rds", serializer_rds) + addSerializer("feather", serializer_feather) addSerializer("xml", serializer_xml) addSerializer("yaml", serializer_yaml) addSerializer("text", serializer_text) - addSerializer("format", serializer_format) - addSerializer("print", serializer_print) - addSerializer("cat", serializer_cat) + addSerializer("format", serializer_format) + addSerializer("print", serializer_print) + addSerializer("cat", serializer_cat) addSerializer("htmlwidget", serializer_htmlwidget) } From 2da1bd3ae0d2fcd6a5a6b4d8d013d6a9c21a2e76 Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Fri, 31 Jul 2020 17:19:17 -0400 Subject: [PATCH 3/5] safe guard tsv/csv tests --- tests/testthat/test-parse-body.R | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/testthat/test-parse-body.R b/tests/testthat/test-parse-body.R index fd71b2e67..d403a6d9f 100644 --- a/tests/testthat/test-parse-body.R +++ b/tests/testthat/test-parse-body.R @@ -49,6 +49,8 @@ test_that("Test yaml parser", { }) test_that("Test csv parser", { + skip_if_not_installed("readr") + tmp <- tempfile() on.exit({ file.remove(tmp) @@ -67,6 +69,8 @@ test_that("Test csv parser", { }) test_that("Test tsv parser", { + skip_if_not_installed("readr") + tmp <- tempfile() on.exit({ file.remove(tmp) From 7448574528e4cf2edae96478afaab749df993f33 Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Fri, 31 Jul 2020 17:19:46 -0400 Subject: [PATCH 4/5] Docs --- NAMESPACE | 2 ++ R/parse-body.R | 10 +++++----- man/parsers.Rd | 15 ++++++++++----- man/serializers.Rd | 5 +++++ 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index 181546553..4ea5037ac 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -19,6 +19,7 @@ export(include_md) export(include_rmd) export(options_plumber) export(parser_csv) +export(parser_feather) export(parser_json) export(parser_multi) export(parser_none) @@ -62,6 +63,7 @@ export(registered_uis) export(serializer_cat) export(serializer_content_type) export(serializer_csv) +export(serializer_feather) export(serializer_format) export(serializer_headers) export(serializer_html) diff --git a/R/parse-body.R b/R/parse-body.R index 316cfdc2b..357fe4269 100644 --- a/R/parse-body.R +++ b/R/parse-body.R @@ -336,7 +336,7 @@ parser_query <- function() { } -#' @describeIn parsers JSON parser +#' @describeIn parsers JSON parser. See [jsonlite::parse_json()] for more details. (Defaults to using `simplifyVectors = TRUE`) #' @export parser_json <- function(...) { parser_text(function(txt_value) { @@ -359,7 +359,7 @@ parser_text <- function(parse_fn = identity) { } -#' @describeIn parsers YAML parser +#' @describeIn parsers YAML parser. See [yaml::yaml.load()] for more details. #' @export parser_yaml <- function(...) { parser_text(function(val) { @@ -370,7 +370,7 @@ parser_yaml <- function(...) { }) } -#' @describeIn parsers CSV parser +#' @describeIn parsers CSV parser. See [readr::read_csv()] for more details. #' @export parser_csv <- function(...) { parse_fn <- function(raw_val) { @@ -385,7 +385,7 @@ parser_csv <- function(...) { } -#' @describeIn parsers TSV parser +#' @describeIn parsers TSV parser. See [readr::read_tsv()] for more details. #' @export parser_tsv <- function(...) { parse_fn <- function(raw_val) { @@ -419,7 +419,7 @@ parser_read_file <- function(read_fn = readLines) { } -#' @describeIn parsers RDS parser +#' @describeIn parsers RDS parser. See [readRDS()] for more details. #' @export parser_rds <- function(...) { parser_read_file(function(tmpfile) { diff --git a/man/parsers.Rd b/man/parsers.Rd index 3b0980ef1..22f4969dd 100644 --- a/man/parsers.Rd +++ b/man/parsers.Rd @@ -9,6 +9,7 @@ \alias{parser_tsv} \alias{parser_read_file} \alias{parser_rds} +\alias{parser_feather} \alias{parser_octet} \alias{parser_multi} \alias{parser_none} @@ -30,6 +31,8 @@ parser_read_file(read_fn = readLines) parser_rds(...) +parser_feather(...) + parser_octet() parser_multi() @@ -64,20 +67,22 @@ See \code{\link[=registered_parsers]{registered_parsers()}} for a list of regist \itemize{ \item \code{parser_query}: Query string parser -\item \code{parser_json}: JSON parser +\item \code{parser_json}: JSON parser. See \code{\link[jsonlite:read_json]{jsonlite::parse_json()}} for more details. (Defaults to using \code{simplifyVectors = TRUE}) \item \code{parser_text}: Helper parser to parse plain text -\item \code{parser_yaml}: YAML parser +\item \code{parser_yaml}: YAML parser. See \code{\link[yaml:yaml.load]{yaml::yaml.load()}} for more details. -\item \code{parser_csv}: CSV parser +\item \code{parser_csv}: CSV parser. See \code{\link[readr:read_delim]{readr::read_csv()}} for more details. -\item \code{parser_tsv}: TSV parser +\item \code{parser_tsv}: TSV parser. See \code{\link[readr:read_delim]{readr::read_tsv()}} for more details. \item \code{parser_read_file}: Helper parser that writes the binary post body to a file and reads it back again using \code{read_fn}. This parser should be used when reading from a file is required. -\item \code{parser_rds}: RDS parser +\item \code{parser_rds}: RDS parser. See \code{\link[=readRDS]{readRDS()}} for more details. + +\item \code{parser_feather}: feather parser. See \code{\link[feather:read_feather]{feather::read_feather()}} for more details. \item \code{parser_octet}: Octet stream parser. Will add a filename attribute if the filename exists diff --git a/man/serializers.Rd b/man/serializers.Rd index 36cd36ea1..a29d40ec3 100644 --- a/man/serializers.Rd +++ b/man/serializers.Rd @@ -8,6 +8,7 @@ \alias{serializer_json} \alias{serializer_unboxed_json} \alias{serializer_rds} +\alias{serializer_feather} \alias{serializer_yaml} \alias{serializer_text} \alias{serializer_format} @@ -30,6 +31,8 @@ serializer_unboxed_json(auto_unbox = TRUE, ...) serializer_rds(version = "2", ascii = FALSE, ...) +serializer_feather() + serializer_yaml(...) serializer_text(..., serialize_fn = as.character) @@ -84,6 +87,8 @@ more details on Plumber serializers and how to customize their behavior. \item \code{serializer_rds}: RDS serializer. See \code{\link[=serialize]{serialize()}} for more details. +\item \code{serializer_feather}: feather serializer. See \link[feather:read_feather]{feather::write_feather} for more details. + \item \code{serializer_yaml}: YAML serializer. See \code{\link[yaml:as.yaml]{yaml::as.yaml()}} for more details. \item \code{serializer_text}: Text serializer. See \code{\link[=as.character]{as.character()}} for more details. From 2f4ce5c9b532ec15880b3454559c037fa2a28e40 Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Fri, 31 Jul 2020 17:23:43 -0400 Subject: [PATCH 5/5] Update NEWS.md --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index 4a0ca73c0..b89cf5be0 100644 --- a/NEWS.md +++ b/NEWS.md @@ -55,6 +55,8 @@ both UIs integration are available from https://github.com/meztez/rapidoc/ and h ### Minor new features and improvements +* Added `serializer_feather()` and `parser_feather()` (#626) + * When `plumb()`ing a file, arguments supplied to parsers and serializers may be values defined earlier in the file. (@meztez, #620) * Updated Docker files. New Docker repo is now [`rstudio/plumber`](https://hub.docker.com/r/rstudio/plumber/tags). Updates heavily inspired from @mskyttner (#459). (#589)