Switch to gypsum backend for saving and loading datasets.

.Rbuildignore

@@ -0,0 +1,3 @@
+^scripts$
+^README.md$
+^\.github$

.gitignore

@@ -1,2 +1,3 @@
 *.rds
 *.html
+*.swp

DESCRIPTION

@@ -1,7 +1,7 @@
 Package: celldex
-Title: Reference Index for Cell Types 
-Version: 1.13.0
-Date: 2023-05-24
+Title: Index of Reference Cell Type Datasets
+Version: 1.13.1
+Date: 2024-02-27
 Authors@R: c(person("Dvir", "Aran", email="dvir.aran@ucsf.edu", role=c("aut")),
     person("Aaron", "Lun", email="infinite.monkeys.with.keyboards@gmail.com", role=c("aut", "cre", "cph")),
     person("Daniel", "Bunis", role="aut"),
@@ -15,18 +15,29 @@ License: GPL-3
 Depends: SummarizedExperiment
 Imports: 
     utils,
+    methods,
+    Matrix, 
     ExperimentHub,
     AnnotationHub,
     AnnotationDbi,
     S4Vectors,
     DelayedArray,
-    DelayedMatrixStats
+    DelayedMatrixStats,
+    gypsum,
+    alabaster.base,
+    alabaster.matrix,
+    alabaster.se,
+    DBI,
+    RSQLite,
+    jsonlite
 Suggests:
     testthat,
     knitr,
     rmarkdown,
     BiocStyle,
     DT,
+    jsonvalidate,
+    BiocManager,
     ensembldb
 biocViews: 
     ExperimentHub,
@@ -38,4 +49,4 @@ VignetteBuilder: knitr
 Encoding: UTF-8
 URL: https://github.com/LTLA/celldex
 BugReports: https://support.bioconductor.org/
-RoxygenNote: 7.1.0
+RoxygenNote: 7.3.1

NAMESPACE

@@ -7,13 +7,54 @@ export(ImmGenData)
 export(MonacoImmuneData)
 export(MouseRNAseqData)
 export(NovershternHematopoieticData)
+export(defineTextQuery)
+export(fetchLatestVersion)
+export(fetchMetadata)
+export(fetchReference)
+export(listReferences)
+export(listVersions)
+export(saveReference)
+export(searchReferences)
+export(surveyReferences)
+import(methods)
+importClassesFrom(Matrix,dgCMatrix)
+importClassesFrom(Matrix,lgCMatrix)
+importClassesFrom(alabaster.matrix,ReloadedArray)
 importFrom(AnnotationDbi,mapIds)
 importFrom(AnnotationHub,AnnotationHub)
+importFrom(DBI,dbConnect)
+importFrom(DBI,dbDisconnect)
+importFrom(DBI,dbGetQuery)
 importFrom(DelayedArray,DelayedArray)
+importFrom(DelayedArray,is_sparse)
+importFrom(DelayedArray,type)
 importFrom(DelayedMatrixStats,colAnyNAs)
 importFrom(DelayedMatrixStats,rowAnyNAs)
 importFrom(ExperimentHub,ExperimentHub)
+importFrom(RSQLite,SQLite)
 importFrom(S4Vectors,DataFrame)
+importFrom(SummarizedExperiment,"assay<-")
 importFrom(SummarizedExperiment,SummarizedExperiment)
+importFrom(SummarizedExperiment,assay)
+importFrom(SummarizedExperiment,assayNames)
+importFrom(SummarizedExperiment,colData)
 importFrom(SummarizedExperiment,rowData)
+importFrom(alabaster.base,altReadObject)
+importFrom(alabaster.base,altReadObjectFunction)
+importFrom(alabaster.base,readObject)
+importFrom(alabaster.base,readObjectFile)
+importFrom(alabaster.base,saveObject)
+importFrom(gypsum,cacheDirectory)
+importFrom(gypsum,defineTextQuery)
+importFrom(gypsum,fetchLatest)
+importFrom(gypsum,fetchMetadataDatabase)
+importFrom(gypsum,fetchMetadataSchema)
+importFrom(gypsum,listAssets)
+importFrom(gypsum,saveFile)
+importFrom(gypsum,saveVersion)
+importFrom(gypsum,searchMetadataTextFilter)
+importFrom(gypsum,validateMetadata)
+importFrom(jsonlite,fromJSON)
+importFrom(jsonlite,toJSON)
 importFrom(utils,read.delim)
+importMethodsFrom(alabaster.se,saveObject)

R/BlueprintEncodeData.R

@@ -49,17 +49,19 @@
 #' ref.se <- BlueprintEncodeData(rm.NA = "rows")
 #'
 #' @export
-BlueprintEncodeData <- function(rm.NA = c("rows","cols","both","none"), 
-    ensembl=FALSE, cell.ont=c("all", "nonna", "none"))
-{
+BlueprintEncodeData <- function(rm.NA = c("rows","cols","both","none"), ensembl=FALSE, cell.ont=c("all", "nonna", "none"), legacy=FALSE) {
     rm.NA <- match.arg(rm.NA)
-    se <- .create_se("blueprint_encode", 
-        version = list(logcounts="1.0.0", coldata="1.2.0"),
-        assays="logcounts", rm.NA = rm.NA,
-        has.rowdata = FALSE, has.coldata = TRUE)
+    cell.ont <- match.arg(cell.ont)
 
-    se <- .convert_to_ensembl(se, "Hs", ensembl)
-    se <- .add_ontology(se, "blueprint_encode", match.arg(cell.ont))
+    if (!legacy && rm.NA == "rows" && cell.ont == "all") {
+        se <- fetchReference("blueprint_encode", "2024-02-26", realize.assays=TRUE)
+    } else {
+        se <- .create_se("blueprint_encode", 
+            version = list(logcounts="1.0.0", coldata="1.2.0"),
+            assays="logcounts", rm.NA = rm.NA,
+            has.rowdata = FALSE, has.coldata = TRUE)
+        se <- .add_ontology(se, "blueprint_encode", cell.ont)
+    }
 
-    se
+    .convert_to_ensembl(se, "Hs", ensembl)
 }

R/DatabaseImmuneExpressionData.R

@@ -58,14 +58,18 @@
 #' ref.se <- DatabaseImmuneCellExpressionData()
 #' 
 #' @export
-DatabaseImmuneCellExpressionData <- function(ensembl=FALSE, cell.ont=c("all", "nonna", "none")) {
-    version <- "1.0.0"
-    se <- .create_se("dice", version, 
-        assays="logcounts", rm.NA = "none",
-        has.rowdata = FALSE, has.coldata = TRUE)
+DatabaseImmuneCellExpressionData <- function(ensembl=FALSE, cell.ont=c("all", "nonna", "none"), legacy=FALSE) {
+    cell.ont <- match.arg(cell.ont)
 
-    se <- .convert_to_ensembl(se, "Hs", ensembl)
-    se <- .add_ontology(se, "dice", match.arg(cell.ont))
+    if (!legacy && cell.ont == "all") {
+        se <- fetchReference("dice", "2024-02-26", realize.assays=TRUE)
+    } else {
+        version <- "1.0.0"
+        se <- .create_se("dice", version, 
+            assays="logcounts", rm.NA = "none",
+            has.rowdata = FALSE, has.coldata = TRUE)
+        se <- .add_ontology(se, "dice", match.arg(cell.ont))
+    }
 
-    se
+    .convert_to_ensembl(se, "Hs", ensembl)
 }

R/HumanPrimaryCellAtlasData.R

@@ -10,6 +10,8 @@
 #' If \code{"nonna"}, all samples without a valid term are discarded;
 #' if \code{"all"}, all samples are returned with (possibly \code{NA}) terms;
 #' if \code{"none"}, terms are not added.
+#' @param legacy Logical scalar indicating whether to pull data from ExperimentHub.
+#' By default, we use data from the gypsum backend.
 #'
 #' @details
 #' This function provides normalized expression values for 713 microarray samples from
@@ -43,14 +45,18 @@
 #'
 #' @export
 #' @importFrom SummarizedExperiment rowData
-HumanPrimaryCellAtlasData <- function(ensembl=FALSE, cell.ont=c("all", "nonna", "none")) {
-    version <- "1.0.0"
-    se <- .create_se("hpca", version,
-        assays="logcounts", rm.NA = "none",
-        has.rowdata = FALSE, has.coldata = TRUE)
+HumanPrimaryCellAtlasData <- function(ensembl=FALSE, cell.ont=c("all", "nonna", "none"), legacy=FALSE) {
+    cell.ont <- match.arg(cell.ont)
 
-    se <- .convert_to_ensembl(se, "Hs", ensembl)
-    se <- .add_ontology(se, "hpca", match.arg(cell.ont))
+    if (!legacy && cell.ont == "all") {
+        se <- fetchReference("hpca", "2024-02-26", realize.assays=TRUE)
+    } else {
+        version <- "1.0.0"
+        se <- .create_se("hpca", version,
+            assays="logcounts", rm.NA = "none",
+            has.rowdata = FALSE, has.coldata = TRUE)
+        se <- .add_ontology(se, "hpca", cell.ont)
+    }
 
-    se
+    .convert_to_ensembl(se, "Hs", ensembl)
 }

R/ImmGenData.R

@@ -38,13 +38,19 @@
 #' ref.se <- ImmGenData()
 #' 
 #' @export
-ImmGenData <- function(ensembl=FALSE, cell.ont=c("all", "nonna", "none")) {
-    se <- .create_se("immgen", version = "1.0.0",
-        assays="logcounts", rm.NA = "none",
-        has.rowdata = FALSE, has.coldata = TRUE)
+ImmGenData <- function(ensembl=FALSE, cell.ont=c("all", "nonna", "none"), legacy=FALSE) {
+    cell.ont <- match.arg(cell.ont)
+
+    if (!legacy && cell.ont == "all") {
+        se <- fetchReference("immgen", "2024-02-26", realize.assays=TRUE)
+    } else {
+        se <- .create_se("immgen", version = "1.0.0",
+            assays="logcounts", rm.NA = "none",
+            has.rowdata = FALSE, has.coldata = TRUE)
+        se <- .add_ontology(se, "immgen", cell.ont)
+    }
 
     se <- .convert_to_ensembl(se, "Mm", ensembl)
-    se <- .add_ontology(se, "immgen", match.arg(cell.ont))
 
     se
 }

R/MonacoImmuneData.R

@@ -72,14 +72,18 @@
 #' ref.se <- MonacoImmuneData()
 #' 
 #' @export
-MonacoImmuneData <- function(ensembl=FALSE, cell.ont=c("all", "nonna", "none")) {
-    version <- "1.0.0"
-    se <- .create_se("monaco_immune", version, 
-        assays="logcounts", rm.NA = "none",
-        has.rowdata = FALSE, has.coldata = TRUE)
+MonacoImmuneData <- function(ensembl=FALSE, cell.ont=c("all", "nonna", "none"), legacy=FALSE) {
+    cell.ont <- match.arg(cell.ont)
 
-    se <- .convert_to_ensembl(se, "Hs", ensembl)
-    se <- .add_ontology(se, "monaco", match.arg(cell.ont))
+    if (!legacy && cell.ont == "all") {
+        se <- fetchReference("monaco_immune", "2024-02-26", realize.assays=TRUE)
+    } else {
+        version <- "1.0.0"
+        se <- .create_se("monaco_immune", version, 
+            assays="logcounts", rm.NA = "none",
+            has.rowdata = FALSE, has.coldata = TRUE)
+        se <- .add_ontology(se, "monaco", cell.ont)
+    }
 
-    se
+    .convert_to_ensembl(se, "Hs", ensembl)
 }

R/MouseRNAseqData.R

@@ -52,14 +52,18 @@
 #' ref.se <- MouseRNAseqData()
 #' 
 #' @export
-MouseRNAseqData <- function(ensembl=FALSE, cell.ont=c("all", "nonna", "none")) {
-    version <- "1.0.0"
-    se <- .create_se("mouse.rnaseq", version, 
-        assays="logcounts", rm.NA = "none",
-        has.rowdata = FALSE, has.coldata = TRUE)
+MouseRNAseqData <- function(ensembl=FALSE, cell.ont=c("all", "nonna", "none"), legacy=FALSE) {
+    cell.ont <- match.arg(cell.ont)
 
-    se <- .convert_to_ensembl(se, "Mm", ensembl)
-    se <- .add_ontology(se, "mouse_rnaseq", match.arg(cell.ont))
+    if (!legacy && cell.ont == "all") {
+        se <- fetchReference("mouse_rnaseq", "2024-02-26", realize.assays=TRUE)
+    } else {
+        version <- "1.0.0"
+        se <- .create_se("mouse.rnaseq", version, 
+            assays="logcounts", rm.NA = "none",
+            has.rowdata = FALSE, has.coldata = TRUE)
+        se <- .add_ontology(se, "mouse_rnaseq", cell.ont)
+    }
 
-    se
+    .convert_to_ensembl(se, "Mm", ensembl)
 }

R/NovershternHematopoieticData.R

@@ -87,14 +87,18 @@
 #' ref.se <- NovershternHematopoieticData()
 #' 
 #' @export
-NovershternHematopoieticData <- function(ensembl=FALSE, cell.ont=c("all", "nonna", "none")) {
-    se <- .create_se("dmap", 
-        version = c(logcounts="1.0.0", coldata="1.2.0"), 
-        assays="logcounts", rm.NA = "none",
-        has.rowdata = FALSE, has.coldata = TRUE)
+NovershternHematopoieticData <- function(ensembl=FALSE, cell.ont=c("all", "nonna", "none"), legacy=FALSE) {
+    cell.ont <- match.arg(cell.ont)
 
-    se <- .convert_to_ensembl(se, "Hs", ensembl)
-    se <- .add_ontology(se, "novershtern", match.arg(cell.ont))
+    if (!legacy && cell.ont == "all") {
+        se <- fetchReference("novershtern_hematopoietic", "2024-02-26", realize.assays=TRUE)
+    } else {
+        se <- .create_se("dmap", 
+            version = c(logcounts="1.0.0", coldata="1.2.0"), 
+            assays="logcounts", rm.NA = "none",
+            has.rowdata = FALSE, has.coldata = TRUE)
+        se <- .add_ontology(se, "novershtern", cell.ont)
+    }
 
-    se
+    .convert_to_ensembl(se, "Hs", ensembl)
 }