diff --git a/DESCRIPTION b/DESCRIPTION index 18bee4ec..51b2bb08 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: camtraptor Title: Read, Explore and Visualize Camera Trap Data Packages -Version: 0.13.0 +Version: 0.14.0 Authors@R: c( person(given = "Damiano", family = "Oldoni", diff --git a/R/read_camtrap_dp.R b/R/read_camtrap_dp.R index f990b890..e9b7100c 100644 --- a/R/read_camtrap_dp.R +++ b/R/read_camtrap_dp.R @@ -16,10 +16,12 @@ #' media file to speed up reading larger Camtrap DP packages. #' @param path Path to the directory containing the datapackage. Use `file` #' with path or URL to a `datapackage.json` file instead. -#' @return A list containing three (tibble) data.frames: 1. `observations` 2. -#' `deployments` 3. `media` +#' @return A list containing three (tibble) data.frames: +#' 1. `observations` +#' 2. `deployments` +#' 3. `media` #' -#' and a list with metadata: `datapackage` +#' and a list with metadata: `datapackage`. #' #' @export #' @@ -33,6 +35,19 @@ #' #' # Read Camtrap DP package and ignore media file #' muskrat_coypu <- read_camtrap_dp(camtrap_dp_file, media = FALSE) +#' +#' # If parsing issues while reading deployments, observations or media arise, +#' use readr::problems() +#' camtrap_dp_file_with_issues <- system.file( +#' "extdata", +#' "mica_parsing_issues", +#' "datapackage_for_parsing_issues.json", +#' package = "camtraptor" +#' ) +#' muskrat_coypu_with_issues <- read_camtrap_dp(camtrap_dp_file_with_issues, media = TRUE) +#' readr::problems(muskrat_coypu_with_issues$deployments) +#' readr::problems(muskrat_coypu_with_issues$observations) +#' readr::problems(muskrat_coypu_with_issues$media) #' } read_camtrap_dp <- function(file = NULL, media = TRUE, @@ -65,7 +80,23 @@ read_camtrap_dp <- function(file = NULL, # read files package <- frictionless::read_package(file) deployments <- frictionless::read_resource(package, "deployments") + issues_deployments <- readr::problems(deployments) + if (nrow(issues_deployments) > 0) { + warning( + glue("One or more parsing issues occurred while reading deployments. ", + "On how to use readr::problems() with datapackages, ", + "see examples in documentation of function read_camtrap_dp." + )) + } observations <- frictionless::read_resource(package, "observations") + issues_observations <- readr::problems(observations) + if (nrow(issues_observations) > 0) { + warning( + glue("One or more parsing issues occurred while reading observations. ", + "On how to use readr::problems() with datapackages, ", + "see examples in documentation of function read_camtrap_dp." + )) + } # get taxonomic info taxon_infos <- get_species(list( @@ -82,25 +113,34 @@ read_camtrap_dp <- function(file = NULL, by = c("taxonID", "scientificName")) observations <- observations %>% dplyr::relocate(dplyr::one_of(cols_taxon_infos), .after = .data$cameraSetup) + # Inherit parsing issues from reading + attr(observations, which = "problems") <- issues_observations } if (media == TRUE) { media <- frictionless::read_resource(package, "media") + issues_media <- readr::problems(media) + if (nrow(issues_media) > 0) { + warning(glue("One or more parsing issues occurred while reading media. ", + "On how to use readr::problems() with datapackages, ", + "see examples in documentation of function read_camtrap_dp.") + ) + } } # return list if (is.data.frame(media)) { list( "datapackage" = package, - "deployments" = dplyr::tibble(deployments), - "media" = dplyr::tibble(media), - "observations" = dplyr::tibble(observations) + "deployments" = deployments, + "media" = media, + "observations" = observations ) } else { list( "datapackage" = package, - "deployments" = dplyr::tibble(deployments), + "deployments" = deployments, "media" = NULL, - "observations" = dplyr::tibble(observations) + "observations" = observations ) } } diff --git a/inst/extdata/mica_parsing_issues/datapackage_for_parsing_issues.json b/inst/extdata/mica_parsing_issues/datapackage_for_parsing_issues.json new file mode 100644 index 00000000..d99b022d --- /dev/null +++ b/inst/extdata/mica_parsing_issues/datapackage_for_parsing_issues.json @@ -0,0 +1,309 @@ +{ + "name": "mica-muskrat-and-coypu-20210707160815", + "id": "https://doi.org/10.5281/zenodo.4893244", + "profile": "https://raw.githubusercontent.com/tdwg/camtrap-dp/0.1.6/camtrap-dp-profile.json", + "created": "2021-07-07T16:08:15Z", + "sources": [ + { + "title": "Agouti", + "path": "https://www.agouti.eu", + "email": "agouti@wur.nl" + } + ], + "contributors": [ + { + "title": "Abel De Boer", + "email": "adeboer@wetterskipfryslan.nl", + "role": "contributor" + }, + { + "title": "Axel Neukermans", + "email": "axel.neukermans@inbo.be", + "role": "author", + "organization": "Research Institute for Nature and Forest (INBO)" + }, + { + "title": "Björn Matthies", + "email": "bjoern.matthies@lwk-niedersachsen.de", + "role": "contributor" + }, + { + "title": "Brecht Neukermans", + "email": "brecht.neukermans@inbo.be", + "role": "author", + "organization": "Research Institute for Nature and Forest (INBO)" + }, + { + "title": "Claudia Maistrelli", + "email": "claudia.maistrelli@tiho-hannover.de", + "role": "contributor" + }, + { + "title": "Danny Van der beeck", + "email": "daniel.vanderbeeck@gmail.com", + "role": "contributor" + }, + { + "title": "Dan Slootmaekers", + "email": "d.slootmaekers@vmm.be", + "role": "contributor" + }, + { + "title": "Dennis Donckers", + "email": "dennis.donckers2@telenet.be", + "role": "contributor" + }, + { + "title": "Emma Cartuyvels", + "email": "emma.cartuyvels@inbo.be", + "role": "author", + "organization": "Research Institute for Nature and Forest (INBO)" + }, + { + "title": "Frank Huysentruyt", + "email": "frank.huysentruyt@inbo.be", + "role": "contributor", + "organization": "Research Institute for Nature and Forest (INBO)" + }, + { + "title": "Friederike Gethöffer", + "email": "friederike.gethoeffer@tiho-hannover.de", + "role": "contributor" + }, + { + "title": "Heiko Fritz", + "email": "foersterheiko@gmx.de", + "role": "contributor" + }, + { + "title": "Jan Lodewijkx", + "email": "j.lodewijkx@vmm.be", + "role": "contributor" + }, + { + "title": "Jim Casaer", + "email": "jim.casaer@inbo.be", + "role": "maintainer", + "organization": "Research Institute for Nature and Forest (INBO)" + }, + { + "title": "Kurt Schamp", + "email": "kurt.schamp@inbo.be", + "role": "contributor" + }, + { + "title": "Lilja Fromme", + "email": "lilja.fromme@tiho-hannover.de", + "role": "contributor" + }, + { + "title": "Lydia Liebgott", + "email": "lydia.liebgott@tiho-hannover.de", + "role": "contributor" + }, + { + "title": "Peter Desmet", + "email": "peter.desmet@inbo.be", + "path": "https://orcid.org/0000-0002-8442-8025", + "role": "maintainer", + "organization": "Research Institute for Nature and Forest (INBO)" + }, + { + "title": "Tim Adriaens", + "email": "tim.adriaens@inbo.be", + "role": "contributor", + "organization": "Research Institute for Nature and Forest (INBO)" + }, + { + "title": "Yasmine Verzelen", + "email": "yasmine.verzelen@inbo.be", + "role": "contributor", + "organization": "Research Institute for Nature and Forest (INBO)" + }, + { + "title": "Yorick Liefting", + "email": "yorick.liefting@wur.nl", + "role": "maintainer", + "organization": "Wageningen University" + } + ], + "organizations": [ + { + "title": "Research Institute for Nature and Forest (INBO)", + "path": "https://inbo.be" + } + ], + "project": { + "title": "MICA - Muskrat and Coypu", + "description": "This project is part of the LIFE project MICA, in which innovative techniques are tested for a more efficient control of muskrat and coypu populations, both invasive species. Camera traps were located in areas where the presence of muskrat and/or coypu was suspected.", + "samplingDesign": "targeted", + "captureMethod": [ + "motion detection", + "time lapse" + ], + "animalTypes": [ + "unmarked" + ], + "classificationLevel": "sequence", + "sequenceInterval": 120, + "_id": "86cabc14-d475-4439-98a7-e7b590bed60e" + }, + "spatial": { + "type": "Feature", + "bbox": [ + 3.51755, + 50.69905, + 7.0243, + 53.27052 + ], + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 3.51755, + 50.69905 + ], + [ + 7.0243, + 50.69905 + ], + [ + 7.0243, + 53.27052 + ], + [ + 3.51755, + 53.27052 + ], + [ + 3.51755, + 50.69905 + ] + ] + ] + } + }, + "temporal": { + "start": "2019-10-09", + "end": "2021-03-28" + }, + "taxonomic": [ + { + "taxonID": "DGP6", + "taxonIDReference": "https://www.catalogueoflife.org", + "scientificName": "Anas platyrhynchos", + "vernacularNames": { + "en": "mallard", + "nl": "wilde eend" + } + }, + { + "taxonID": "DGPL", + "taxonIDReference": "https://www.catalogueoflife.org", + "scientificName": "Anas strepera", + "vernacularNames": { + "en": "gadwall", + "nl": "krakeend" + } + }, + { + "taxonID": "32FH", + "taxonIDReference": "https://www.catalogueoflife.org", + "scientificName": "Ardea", + "vernacularNames": { + "en": "great herons", + "nl": "reigers" + } + }, + { + "taxonID": "GCHS", + "taxonIDReference": "https://www.catalogueoflife.org", + "scientificName": "Ardea cinerea", + "vernacularNames": { + "en": "grey heron", + "nl": "blauwe reiger" + } + }, + { + "taxonID": "RQPW", + "taxonIDReference": "https://www.catalogueoflife.org", + "scientificName": "Castor fiber", + "vernacularNames": { + "en": "Eurasian beaver", + "nl": "bever" + } + }, + { + "taxonID": "6MB3T", + "taxonIDReference": "https://www.catalogueoflife.org", + "scientificName": "Homo sapiens", + "vernacularNames": { + "en": "human", + "nl": "mens" + } + }, + { + "taxonID": "3Y9VW", + "taxonIDReference": "https://www.catalogueoflife.org", + "scientificName": "Martes foina", + "vernacularNames": { + "en": "beech marten", + "nl": "steenmarter" + } + }, + { + "taxonID": "44QYC", + "taxonIDReference": "https://www.catalogueoflife.org", + "scientificName": "Mustela putorius", + "vernacularNames": { + "en": "European polecat", + "nl": "bunzing" + } + }, + { + "taxonID": "5BSG3", + "taxonIDReference": "https://www.catalogueoflife.org", + "scientificName": "Vulpes vulpes", + "vernacularNames": { + "en": "red fox", + "nl": "vos" + } + } + ], + "platform": { + "title": "Agouti", + "path": "https://agouti.eu", + "packageID": "3bdce07e-f0b1-4566-948a-e1f0a0d0a214" + }, + "resources": [ + { + "name": "deployments", + "path": "deployments_parsing_issues.csv", + "profile": "tabular-data-resource", + "format": "csv", + "mediatype": "text/csv", + "encoding": "utf-8", + "schema": "https://raw.githubusercontent.com/tdwg/camtrap-dp/0.1.6/deployments-table-schema.json" + }, + { + "name": "media", + "path": "media_parsing_issues.csv", + "profile": "tabular-data-resource", + "format": "csv", + "mediatype": "text/csv", + "encoding": "utf-8", + "schema": "https://raw.githubusercontent.com/tdwg/camtrap-dp/0.1.6/media-table-schema.json" + }, + { + "name": "observations", + "path": "observations_parsing_issues.csv", + "profile": "tabular-data-resource", + "format": "csv", + "mediatype": "text/csv", + "encoding": "utf-8", + "schema": "https://raw.githubusercontent.com/tdwg/camtrap-dp/0.1.6/observations-table-schema.json" + } + ] +} diff --git a/inst/extdata/mica_parsing_issues/deployments_parsing_issues.csv b/inst/extdata/mica_parsing_issues/deployments_parsing_issues.csv new file mode 100644 index 00000000..bf967eaa --- /dev/null +++ b/inst/extdata/mica_parsing_issues/deployments_parsing_issues.csv @@ -0,0 +1,5 @@ +deploymentID,locationID,locationName,longitude,latitude,coordinateUncertainty,start,end,setupBy,cameraID,cameraModel,cameraInterval,cameraHeight,cameraTilt,cameraHeading,timestampIssues,baitUse,session,array,featureType,habitat,tags,comments,_id +29b7d356-4bb4-4ec4-b792-2af5cc32efa8,2df5259b-b4b4-4f43-8cf7-effcced06d6f,B_DL_val 5_beek kleine vijver,5.655,51.181,,2020-07-29TT05:29:41Z,2020-08-08T04:20:40Z,Danny Van der beeck,,,0,0.7,,,,,,,,,boven de stroom,van 29/07/2020 tot 08/08/2020 120 foto's, +577b543a-2cf1-4b23-b6d2-cda7e2eac372,ff1535c0-6b5d-44be-b3ef-c4d4204dad74,B_DL_val 3_dikke boom,5.659,51.184,,2020-06-1921:00:00Z,2020-06-28T23:33:22Z,Danny Van der beeck,,,0,0.8,,,,,,,,,linkeroever,van 19/06/2020 tot 29/06/2020 63 foto's, +62c200a9-0e03-4495-bcd8-032944f6f5a1,ce943ced-1bcf-4140-9a2e-e8ee5e8c10e6,B_DM_val 4_'t WAD,4.013,50.699,,2021-03-27T20:38:18Z,2021-04-18T21:25:00Z,Davy,,,0,1,,,,,,,,,,, +7ca633fa-64f8-4cfc-a628-6b0c419056d7,3232bcfd-5dfa-496e-b7ab-14593bb1b7f1,Mica Viane,3.898,50.742,,2019-10-09T11:18:07Z,2019-10-23T10:00:16Z,Axel Neukermans,,,0,2,,,,,,,,,boven de stroom,CAM_244 HC600 Boven de Stroom, diff --git a/inst/extdata/mica_parsing_issues/media_parsing_issues.csv b/inst/extdata/mica_parsing_issues/media_parsing_issues.csv new file mode 100644 index 00000000..43e63172 --- /dev/null +++ b/inst/extdata/mica_parsing_issues/media_parsing_issues.csv @@ -0,0 +1,5 @@ +mediaID,deploymentID,sequenceID,captureMethod,timestamp,filePath,fileName,fileMediatype,exifData,favourite,comments,_id +3e65dfaa-b239-4834-b809-2bc1b06e26cc,29b7d356-4bb4-4ec4-b792-2af5cc32efa8,8f5ffbf2-52c4-4dd2-b502-d93a8aa64640,motion detection,2020-07-29T05:29:41Z,https://multimedia.agouti.eu/assets/3e65dfaa-b239-4834-b809-2bc1b06e26cc/file,20200815213846-RCNX0061.JPG,image/jpeg,,,, +dbfba6c1-ec77-4048-9b3b-aac3ace43c24,29b7d356-4bb4-4ec4-b792-2af5cc32efa8,8f5ffbf2-52c4-4dd2-b502-d93a8aa64640,motion detection,2020-07-29TTT05:29:42Z,https://multimedia.agouti.eu/assets/dbfba6c1-ec77-4048-9b3b-aac3ace43c24/file,20200815213846-RCNX0062.JPG,image/jpeg,,,, +006d72c5-dcbc-4e7f-ac29-e6ca1644693f,29b7d356-4bb4-4ec4-b792-2af5cc32efa8,8f5ffbf2-52c4-4dd2-b502-d93a8aa64640,motion detection,2020-07-29T05:29:43Z,https://multimedia.agouti.eu/assets/006d72c5-dcbc-4e7f-ac29-e6ca1644693f/file,20200815213846-RCNX0063.JPG,image/jpeg,,,, +d7fde0a6-8b6f-41a5-be4a-ef3950aef757,29b7d356-4bb4-4ec4-b792-2af5cc32efa8,8f5ffbf2-52c4-4dd2-b502-d93a8aa64640,motion detection,2020-07-29T05:29:43Z,https://multimedia.agouti.eu/assets/d7fde0a6-8b6f-41a5-be4a-ef3950aef757/file,20200815213846-RCNX0064.JPG,image/jpeg,,,, diff --git a/inst/extdata/mica_parsing_issues/observations_parsing_issues.csv b/inst/extdata/mica_parsing_issues/observations_parsing_issues.csv new file mode 100644 index 00000000..c9d209f5 --- /dev/null +++ b/inst/extdata/mica_parsing_issues/observations_parsing_issues.csv @@ -0,0 +1,5 @@ +observationID,deploymentID,sequenceID,mediaID,timestamp,observationType,cameraSetup,taxonID,scientificName,count,countNew,lifeStage,sex,behaviour,individualID,classificationMethod,classifiedBy,classificationTimestamp,classificationConfidence,comments,_id +ef2f7140-ae97-4b44-8309-ab1ffbc02879,29b7d356-4bb4-4ec4-b792-2af5cc32efa8,8f5ffbf2-52c4-4dd2-b502-d93a8aa64640,,2020-07-29TT05:29:41Z,unknown,FALSE,,,,,,,,,human,,,,, +68686a75-ad44-4676-b45e-2b85f60e4d11,29b7d356-4bb4-4ec4-b792-2af5cc32efa8,1d98da96-c3ce-4479-9b97-8883cd33724f,,2020-07-2905:38:55Z,blank,FALSE,,,,,,,,,human,,,,, +3d065f23-426a-449b-8693-f3c6b2ac9c7a,29b7d356-4bb4-4ec4-b792-2af5cc32efa8,89b807ca-697f-4452-8349-b235622c94fa,,2020-07-29T05:46:48Z,animal,FALSE,DGPL,Anas strepera,4,,subadult,unknown,,,human,Danny Van der beeck,2020-08-17T06:57:28Z,,, +1fa8b00a-8f0e-485e-b98e-eb9bb1ee5a9f,29b7d356-4bb4-4ec4-b792-2af5cc32efa8,710eac2a-e261-478d-84e8-de99f418138c,,2020-07-30T04:29:31Z,animal,FALSE,DGPL,Anas strepera,1,,adult,female,,,human,Danny Van der beeck,2020-08-17T06:57:50Z,,, diff --git a/man/read_camtrap_dp.Rd b/man/read_camtrap_dp.Rd index 1e1b309f..53224695 100644 --- a/man/read_camtrap_dp.Rd +++ b/man/read_camtrap_dp.Rd @@ -16,10 +16,14 @@ media file to speed up reading larger Camtrap DP packages.} with path or URL to a \code{datapackage.json} file instead.} } \value{ -A list containing three (tibble) data.frames: 1. \code{observations} 2. -\code{deployments} 3. \code{media} +A list containing three (tibble) data.frames: +\enumerate{ +\item \code{observations} +\item \code{deployments} +\item \code{media} +} -and a list with metadata: \code{datapackage} +and a list with metadata: \code{datapackage}. } \description{ This function reads camera trap data formatted following the \href{https://github.com/tdwg/camtrap-dp}{Camera Trap Data Package (Camtrap DP)} format. The @@ -41,5 +45,18 @@ muskrat_coypu <- read_camtrap_dp(camtrap_dp_file) # Read Camtrap DP package and ignore media file muskrat_coypu <- read_camtrap_dp(camtrap_dp_file, media = FALSE) + +# If parsing issues while reading deployments, observations or media arise, +use readr::problems() +camtrap_dp_file_with_issues <- system.file( + "extdata", + "mica_parsing_issues", + "datapackage_for_parsing_issues.json", + package = "camtraptor" +) +muskrat_coypu_with_issues <- read_camtrap_dp(camtrap_dp_file_with_issues, media = TRUE) +readr::problems(muskrat_coypu_with_issues$deployments) +readr::problems(muskrat_coypu_with_issues$observations) +readr::problems(muskrat_coypu_with_issues$media) } } diff --git a/tests/testthat/test-read_camtrap_dp.R b/tests/testthat/test-read_camtrap_dp.R index 2dd0feeb..288610ea 100644 --- a/tests/testthat/test-read_camtrap_dp.R +++ b/tests/testthat/test-read_camtrap_dp.R @@ -3,6 +3,24 @@ test_that("file is checked properly", { expect_error(read_camtrap_dp(1)) }) +testthat::test_that("test warnings", { + local_edition(2) + camtrap_dp_file_with_issues <- system.file("extdata", "mica_parsing_issues", "datapackage_for_parsing_issues.json", package = "camtraptor") + # deployments + expect_warning(camtraptor::read_camtrap_dp( + file = camtrap_dp_file_with_issues), + "One or more parsing issues occurred while reading deployments.") + # observations + expect_warning(camtraptor::read_camtrap_dp( + file = camtrap_dp_file_with_issues), + "One or more parsing issues occurred while reading observations.") + # media + expect_warning(camtraptor::read_camtrap_dp( + file = camtrap_dp_file_with_issues), + "One or more parsing issues occurred while reading media.") +}) + + test_that("media is checked properly", { dp_path <- system.file("extdata", "mica", "datapackage.json", package = "camtraptor")