diff --git a/.Rbuildignore b/.Rbuildignore index 2c46fa8a..b91aead1 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -12,3 +12,6 @@ ^appveyor\.yml$ ^\.lintr$ ^data-raw$ +^doc$ +^Meta$ +^docker-compose.yml$ diff --git a/.gitignore b/.gitignore index d1e57080..a9778d08 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,6 @@ inst/doc inst/jrxml/*.jasper vignettes/*.pdf -docs/ \ No newline at end of file +docs/ +/doc/ +/Meta/ diff --git a/DESCRIPTION b/DESCRIPTION index 45cebe67..51503e32 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: rapbase Type: Package Title: Base Functions and Resources for Rapporteket -Version: 1.23.1.9000 +Version: 1.23.1.9001 Authors@R: c( person(given = "Are", family = "Edvardsen", @@ -46,7 +46,7 @@ Imports: sship (>= 0.9.0), utils, yaml -RoxygenNote: 7.2.1 +RoxygenNote: 7.2.3 URL: https://github.com/Rapporteket/rapbase BugReports: https://github.com/Rapporteket/rapbase/issues Suggests: @@ -55,3 +55,4 @@ Suggests: rvest, testthat, withr +VignetteBuilder: knitr diff --git a/NEWS.md b/NEWS.md index adcc922c..d41a845a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,12 @@ +# rapbase 1.24.0 + +In summary, registries at Rapporteket may now run as standalone container apps. Thus, shiny-server is no longer a requirement for app deployment. Below is a summary of what has been done. + +* Extended handling of user attributes when behind an app proxy (spring boot/shinyproxy) +* Added a database backend for staging data +* Per app (registry) encryption of staging data (regardless of file or database backend) +* Added a vignette with a short description of staging data server side set-up + # rapbase 1.23.1 * Fixed Fixed bug in log sanitizer function ([#130](https://github.com/Rapporteket/rapbase/pull/130)) diff --git a/R/moduleExport.R b/R/moduleExport.R index ef2a8afa..31af0314 100644 --- a/R/moduleExport.R +++ b/R/moduleExport.R @@ -237,7 +237,8 @@ exportDb <- function(registryName, compress = FALSE, session) { conf <- rapbase::getConfig()[[registryName]] cmd <- paste0( "mysqldump ", - "--no-tablespaces --single-transaction --add-drop-database " + "--no-tablespaces --single-transaction --add-drop-database ", + "--column-statistics=0 " ) cmd <- paste0( cmd, "-B -u ", conf$user, " -p", conf$pass, " -h ", conf$host, diff --git a/R/moduleNavbarWidget.R b/R/moduleNavbarWidget.R index abc7cc8d..a3426526 100644 --- a/R/moduleNavbarWidget.R +++ b/R/moduleNavbarWidget.R @@ -6,7 +6,12 @@ #' #' These modules take use of the shiny session object to obtain data for the #' widget. Hence, a Rapporteket like context will be needed for these modules to -#' function properly. +#' function properly. For deployment of (shiny) application as containers make +#' sure to migrate to \code{navbarWidgetServer2()}. In addition to serving the +#' user information widget, this function provides a list of reactive user +#' attributes. Hence, when using \code{navbarWidgetServer2()} the source of +#' (static) user attributes is no longer the shiny session object but rather the +#' list object (of reactive user attributes) returned by this function. #' #' @param id Character string providing module namespace #' @param addUserInfo Logical defining if an "about" hyperlink is to be added @@ -22,11 +27,12 @@ #' called from outside the registry environment \code{caller} must be set to #' the actual name of the R package. #' -#' @return Shiny objects, mostly. \code{navbarWidgetServer()} invisibly returns -#' a list of reactive values representing user metadata and privileges. See -#' \code{\link{userAttribute}} for further details on these values. +#' @return Shiny objects, mostly. \code{navbarWidgetServer2()} invisibly returns +#' a list of reactive values representing user metadata and privileges. See +#' \code{\link{userAttribute}} for further details on these values. #' @name navbarWidget -#' @aliases navbarWidgetInput navbarWidgetServer2 navbarWidgetApp +#' @aliases navbarWidgetInput navbarWidgetServer navbarWidgetServer2 +#' navbarWidgetApp #' @examples #' ## client user interface function #' ui <- shiny::tagList( diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..fcb3d50e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,39 @@ +version: '3.3' + +services: + db: + image: mysql:5.7 + command: --innodb-log-file-size=500M --innodb_strict_mode=0 + restart: "no" + environment: + MYSQL_ROOT_PASSWORD: root + + dev: + depends_on: + - db + image: rapporteket/dev:nightly + volumes: + - .:/home/rstudio/rapbase/ + - ~/.ssh:/home/rstudio/.ssh + - ~/.gitconfig:/home/rstudio/.gitconfig + ports: + - "8787:8787" + - "3838:3838" + dns: + - 8.8.8.8 + restart: "no" + environment: + PASSWORD: password + DB_HOST: db + DB_USER: root + DB_PASS: root + + adminer: + depends_on: + - db + image: adminer + restart: "no" + environment: + ADMINER_PLUGINS: frames + ports: + - 8888:8080 diff --git a/man/navbarWidget.Rd b/man/navbarWidget.Rd index 5d5163ec..1611499b 100644 --- a/man/navbarWidget.Rd +++ b/man/navbarWidget.Rd @@ -3,9 +3,9 @@ \name{navbarWidget} \alias{navbarWidget} \alias{navbarWidgetInput} +\alias{navbarWidgetServer} \alias{navbarWidgetServer2} \alias{navbarWidgetApp} -\alias{navbarWidgetServer} \title{Shiny modules providing GUI and server logic for user info widget} \usage{ navbarWidgetInput(id, addUserInfo = TRUE, selectOrganization = FALSE) @@ -40,9 +40,9 @@ called from outside the registry environment \code{caller} must be set to the actual name of the R package.} } \value{ -Shiny objects, mostly. \code{navbarWidgetServer()} invisibly returns -a list of reactive values representing user metadata and privileges. See -\code{\link{userAttribute}} for further details on these values. +Shiny objects, mostly. \code{navbarWidgetServer2()} invisibly returns + a list of reactive values representing user metadata and privileges. See + \code{\link{userAttribute}} for further details on these values. } \description{ Shiny modules for making a user information widget in registry shiny apps at @@ -52,7 +52,12 @@ number of code lines for each registry. \details{ These modules take use of the shiny session object to obtain data for the widget. Hence, a Rapporteket like context will be needed for these modules to -function properly. +function properly. For deployment of (shiny) application as containers make +sure to migrate to \code{navbarWidgetServer2()}. In addition to serving the +user information widget, this function provides a list of reactive user +attributes. Hence, when using \code{navbarWidgetServer2()} the source of +(static) user attributes is no longer the shiny session object but rather the +list object (of reactive user attributes) returned by this function. } \examples{ ## client user interface function diff --git a/vignettes/.gitignore b/vignettes/.gitignore new file mode 100644 index 00000000..097b2416 --- /dev/null +++ b/vignettes/.gitignore @@ -0,0 +1,2 @@ +*.html +*.R diff --git a/vignettes/stagingData.Rmd b/vignettes/stagingData.Rmd new file mode 100644 index 00000000..655dd453 --- /dev/null +++ b/vignettes/stagingData.Rmd @@ -0,0 +1,74 @@ +--- +title: "Staging data: principles and set-up" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{Staging data: principles and set-up} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + +```{r, include = FALSE} +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>" +) +``` + +## Introduction +Registry applications at Rapporteket normally obtains data by opening connections to a database containing raw registry data. Before figures, tables and reports can be made, data usually have to be filtered, combined and analyzed. Depending on the amount of data and the complexity of the analysis this process can be time-consuming. Quite often, such pre-processing can be generalized and does not have to be run each and every time a user interacts with a registry application. Staging data allow for such pre-processed data to be stored for quick and easy retrieval and therefore reduce the time the registry applications need to spend on processing data. + +This document aim to describe how staging of data works and how to enable it at Rapporteket. + +## Backend +Staging data may be stored as binary files or as blobs in a database. In any case, creating and retrieving staging data follow the same scheme regardless of how it is stored. Selecting a backend is configurable and will apply globally, _i.e._ for all registry applications at Rapporteket. + +### File +To store staging data as binary files make sure to set the property __target__ to the value __file__ in _$R_RAP_CONFIG_PATH/rapbaseConfig.yml_: + +```yaml +... + # Staging data + staging: + target: file + key: staging +... +``` + +When staging data is stored as files the property __key__ is not in use and may take any value. For a given registry _MyRegistry_ staging of the data set _MyData_ will then be stored with the file path _R_RAP_CONFIG_PATH/stagingData/MyRegistry/MyData_. + +### Database +To store staging data as blobs in a database make sure to set the property __target__ to the value __db__ in _$R_RAP_CONFIG_PATH/rapbaseConfig.yml_: + +```yaml +... + # Staging data + staging: + target: db + key: stagingDb +... +``` + +When staging data is stored in a database __key__ must match the corresponding entry in _$R_RAP_CONFIG_PATH/dbConfig.yml_ that provides the database connection credentials: + +```yaml +... +stagingDb: + host: database_server_hostname_or_ip + name: database_name + user: some_username + pass: password_for_some_username + disp: optional_information +... +``` + +The database server must exist and accept connections as defined in the above configuration. If the database itself does not exist when an registry application request staging of data, it will be created. + +## Encryption +Prior to storage the R data structure to become staging data is serialized to binary, encrypted and compressed. When reading data already staged this process is reversed. Encryption is based on the AES256 algorithm and the key applied is generated from the password of the database credentials as defined in _\$R_RAP_CONFIG_PATH/dbConfig.yml_ for the corresponding registry. Hence, there will be a registry specific encryption for staged data meaning that from a common place of storage, staged data will not be accessible across registries. Protection as provided by the encryption solely rest on the accessibility of _\$R_RAP_CONFIG_PATH/dbConfig.yml_ that holds the information needed to decrypt staging data. + + +## Caveats +If the key used for encryption is lost staging data will no longer be accessible. If the key cannot be restored all staging data encrypted with this key will also be forever lost. Hence, if the purpose is to remove staging data, destroying the key will be a very potent method to ensure proper deletion, but please note that making staged data inaccessible may lead to errors in registry applications that uses staged data. For any number of reasons database credentials for registries at Rapporteket may also at some point be altered having the unintentional effect of making staged data inaccessible. + +## Implementation +For developers of registry applications at Rapporteket that wants to implement staging data, please refer to the staging data function documentation: `help("rapbase::stagingData")`.