diff --git a/data-serving/data-service/src/controllers/case.ts b/data-serving/data-service/src/controllers/case.ts index 50b6ec89c..cc4124c90 100644 --- a/data-serving/data-service/src/controllers/case.ts +++ b/data-serving/data-service/src/controllers/case.ts @@ -124,10 +124,11 @@ export class CasesController { return; } - // don't export any note + // don't export notes or sourceEntryIds c.forEach((aCase: LeanDocument) => { delete aCase.restrictedNotes; delete aCase.notes; + delete aCase.caseReference.sourceEntryId; }); res.json(await Promise.all(c.map((aCase) => dtoFromCase(aCase)))); @@ -232,6 +233,8 @@ export class CasesController { doc = await cursor.next(); while (doc != null) { delete doc.restrictedNotes; + delete doc.notes; + delete doc.caseReference.sourceEntryId; const caseDTO = await dtoFromCase(doc); const parsedCase = parseDownloadedCase(caseDTO); const stringifiedCase = stringify([parsedCase], { @@ -254,6 +257,7 @@ export class CasesController { while (doc != null) { delete doc.restrictedNotes; delete doc.notes; + delete doc.caseReference.sourceEntryId; const normalizedDoc = await denormalizeFields(doc); if (!doc.hasOwnProperty('SGTF')) { normalizedDoc.SGTF = 'NA'; diff --git a/data-serving/data-service/test/controllers/case.test.ts b/data-serving/data-service/test/controllers/case.test.ts index e4d8d865d..4a18c9436 100644 --- a/data-serving/data-service/test/controllers/case.test.ts +++ b/data-serving/data-service/test/controllers/case.test.ts @@ -122,6 +122,13 @@ describe('GET', () => { const res = await request(app).get(`/api/cases/${c._id}`).expect(200); expect(res.body[0].notes).toBeUndefined(); }); + it('should not show the sourceEntryId for a case', async () => { + const c = new Case(minimalCase); + c.caseReference.sourceEntryId = 'Sourcey McSourceFace'; + await c.save(); + const res = await request(app).get(`/api/cases/${c._id}`).expect(200); + expect(res.body[0].caseReference.sourceEntryId).toBeUndefined(); + }); it('should convert age bucket to age range', async () => { const c = new Case(minimalCase); const bucket = await AgeBucket.findOne({}); diff --git a/data-serving/scripts/export-data/data_dictionary.txt b/data-serving/scripts/export-data/data_dictionary.txt index 83f5d04d3..c3ad62535 100644 --- a/data-serving/scripts/export-data/data_dictionary.txt +++ b/data-serving/scripts/export-data/data_dictionary.txt @@ -24,29 +24,24 @@ METADATA Additional sources (URLs) for this case - 3. caseReference.sourceEntryId - - Unique source entry ID. This is only present in sources that - offer this information. - - 4. caseReference.sourceId * + 3. caseReference.sourceId * Unique source ID for this case. Each case is ingested from a specific source URL, which has an unique ID. This is stable for a particular source. - 5. caseReference.sourceUrl * + 4. caseReference.sourceUrl * Data URL from which this case was ingested. - 6. caseReference.uploadIds [] * + 5. caseReference.uploadIds [] * Subsequent uploads following the initial upload of a case can change the data of a case (only in sources that provide an unique ID in caseReference.sourceEntryId). This field records the unique upload IDs that updated this case. - 7. caseReference.verificationStatus * + 6. caseReference.verificationStatus * Case verification status Values: VERIFIED | UNVERIFIED | EXCLUDED @@ -63,27 +58,27 @@ DEMOGRAPHICS Generally, we prefer to ingest demographic information over location, if they are not available in the same dataset. - 8. demographics.ageRange.end + 7. demographics.ageRange.end Upper age range of individual (0 - 120) - 9. demographics.ageRange.start + 8. demographics.ageRange.start Lower age range of individual (0 - 120) -10. demographics.ethnicity +9. demographics.ethnicity Ethnicity of individual -11. demographics.gender +10. demographics.gender Gender of individual (Male | Female | Non-binary/Third gender | Other) -12. demographics.nationalities [] +11. demographics.nationalities [] All the nationalities of the individual -13. demographics.occupation +12. demographics.occupation Occupation of the individual @@ -91,136 +86,136 @@ EVENTS All .date values are dates in YYYY-MM-DD format. -14. events.confirmed.date * -15. events.confirmed.value +13. events.confirmed.date * +14. events.confirmed.value Confirmed date. If value is present, indicates method of confirmation. -16. events.firstClinicalConsultation.date +15. events.firstClinicalConsultation.date First clinical consultation date -17. events.hospitalAdmission.date -18. events.hospitalAdmission.value +16. events.hospitalAdmission.date +17. events.hospitalAdmission.value Hospital admission date, value (Yes | No) -19. events.icuAdmission.date -20. events.icuAdmission.value +18. events.icuAdmission.date +19. events.icuAdmission.value Intensive Care Unit admission date, value (Yes | No) -21. events.onsetSymptoms.date +20. events.onsetSymptoms.date Date of onset of symptoms -22. events.outcome.date -23. events.outcome.value +21. events.outcome.date +22. events.outcome.value Outcome date, values are Death | Recovered | hospitalAdmission | icuAdmission | Unknown -24. events.selfIsolation.date +23. events.selfIsolation.date Date that individual started self-isolating LOCATION -25. location.administrativeAreaLevel1 +24. location.administrativeAreaLevel1 Admin1 level location of individual (usually state or province) -26. location.administrativeAreaLevel2 +25. location.administrativeAreaLevel2 Admin2 level location of individual (usually district) -27. location.administrativeAreaLevel3 +26. location.administrativeAreaLevel3 Admin3 level location of individual (usually city) -28. location.country * +27. location.country * Country that case was reported in. -29. location.geoResolution * +28. location.geoResolution * Geo-resolution of location (how coarse the location is) Country | Admin1 | Admin2 | Admin3 | Point -30. location.geometry.latitude * +29. location.geometry.latitude * Geolocated latitude (-90 to 90) Positive values are North, negative values are South -31. location.geometry.longitude * +30. location.geometry.longitude * Geolocated longitude (-180 to 180) Positive values are East, negative values are West -32. location.name +31. location.name Full name of location (example: Lyon, Auvergne-Rhône-Alpes, France) -33. location.place +32. location.place Name of the place this location refers to (example: Boston Children's Hospital) PATHOGENS -34. pathogens [] +33. pathogens [] Pathogens other than SARS-CoV-2 PRE-EXISTING CONDITIONS -35. preexistingConditions.hasPreexistingConditions +34. preexistingConditions.hasPreexistingConditions Whether the patient has pre-existing conditions Boolean: True | False -36. preexistingConditions.values [] +35. preexistingConditions.values [] List of pre-existing conditions REVISION METADATA -37. revisionMetadata.creationMetadata.date +36. revisionMetadata.creationMetadata.date Date this case was first created -38. revisionMetadata.creationMetadata.notes +37. revisionMetadata.creationMetadata.notes Notes added by the curator for this case -39. revisionMetadata.editMetadata.date +38. revisionMetadata.editMetadata.date Date this case was last edited -40. revisionMetadata.editMetadata.notes +39. revisionMetadata.editMetadata.notes Notes added by the curator for last edit -41. revisionMetadata.revisionNumber +40. revisionMetadata.revisionNumber Revision number of the case (positive integer) SGTF -42. SGTF +41. SGTF S-Gene Target failure (0 = no deletion, 1 = deletion (S-)) SYMPTOMS -43. symptoms.status +42. symptoms.status Symptom status (Asymptomatic | Symptomatic | Presymptomatic | null) -44. symptoms.values [] +43. symptoms.values [] List of symptoms @@ -228,96 +223,96 @@ TRANSMISSION How this case got infected and by who if known -45. transmission.linkedCaseIds [] +44. transmission.linkedCaseIds [] UUID of a related case in the system -46. transmission.places [] +45. transmission.places [] Places where transmission occurred -47. transmission.routes [] +46. transmission.routes [] Routes of transmission TRAVEL HISTORY -48. travelHistory.travel.dateRange.end -49. travelHistory.travel.dateRange.start +47. travelHistory.travel.dateRange.end +48. travelHistory.travel.dateRange.start Start and end dates for travel history -50. travelHistory.travel.location.administrativeAreaLevel1 [] -51. travelHistory.travel.location.administrativeAreaLevel2 [] -52. travelHistory.travel.location.administrativeAreaLevel3 [] -53. travelHistory.travel.location.country [] -54. travelHistory.travel.location.geoResolution [] +49. travelHistory.travel.location.administrativeAreaLevel1 [] +50. travelHistory.travel.location.administrativeAreaLevel2 [] +51. travelHistory.travel.location.administrativeAreaLevel3 [] +52. travelHistory.travel.location.country [] +53. travelHistory.travel.location.geoResolution [] These have the same meaning as in LOCATION, except that these pertain to travel history of the individual. Unlike the fields in location, the fields here are all comma-separated arrays, with each item corresponding to a travel location in the last 30 days. -55. travelHistory.travel.location.geometry.coordinates [] +54. travelHistory.travel.location.geometry.coordinates [] Comma-separated tuples of latitude and longitude. If the individual visited latitude m1 and longitude n1 this would be represented as "(m1, n1)". If there was another travel coordinate (m2, n2), then this would be represented as "(m1, n1),(m2, n2)" -56. travelHistory.travel.location.name [] -57. travelHistory.travel.location.place [] +55. travelHistory.travel.location.name [] +56. travelHistory.travel.location.place [] Same as LOCATION, except these are arrays -58. travelHistory.travel.methods [] +57. travelHistory.travel.methods [] Corresponding travel methods (such as air, ship, rail ...) -59. travelHistory.travel.purpose [] +58. travelHistory.travel.purpose [] Purpose of travel -60. travelHistory.traveledPrior30Days +59. travelHistory.traveledPrior30Days Whether the patient has travelled in the past 30 days Boolean: True | False VACCINES -61. vaccines.0.batch +60. vaccines.0.batch First vaccine batch -62. vaccines.0.date +61. vaccines.0.date Date of first vaccine -63. vaccines.0.name +62. vaccines.0.name Name of first vaccine -64. vaccines.0.sideEffects [] +63. vaccines.0.sideEffects [] List of side-effects experienced after vaccine -65. vaccines.1.batch -66. vaccines.1.date -67. vaccines.1.name -68. vaccines.1.sideEffects -69. vaccines.2.batch -70. vaccines.2.date -71. vaccines.2.name -72. vaccines.2.sideEffects -73. vaccines.3.batch -74. vaccines.3.date -75. vaccines.3.name -76. vaccines.3.sideEffects +64. vaccines.1.batch +65. vaccines.1.date +66. vaccines.1.name +67. vaccines.1.sideEffects +68. vaccines.2.batch +69. vaccines.2.date +70. vaccines.2.name +71. vaccines.2.sideEffects +72. vaccines.3.batch +73. vaccines.3.date +74. vaccines.3.name +75. vaccines.3.sideEffects Same as before, for subsequent vaccines taken by the same individual VARIANT OF CONCERN -77. variantOfConcern +76. variantOfConcern Variant of concern that was detected. This uses the Pango lineage. diff --git a/data-serving/scripts/export-data/fields.txt b/data-serving/scripts/export-data/fields.txt index 9dd1a942a..bfb139c90 100644 --- a/data-serving/scripts/export-data/fields.txt +++ b/data-serving/scripts/export-data/fields.txt @@ -1,6 +1,5 @@ _id caseReference.additionalSources -caseReference.sourceEntryId caseReference.sourceId caseReference.sourceUrl caseReference.uploadIds diff --git a/data-serving/scripts/export-data/setup_country_export.sh b/data-serving/scripts/export-data/setup_country_export.sh index 582e574c0..18c75211c 100755 --- a/data-serving/scripts/export-data/setup_country_export.sh +++ b/data-serving/scripts/export-data/setup_country_export.sh @@ -11,7 +11,7 @@ require_env "${BUCKET:-}" "Specify S3 bucket to output files in BUCKET" echo "Setting up country export job definitions for environment {ENV}..." -CASECOUNT_URL=${CASECOUNT_URL:-https://covid-19-aggregates-eu.s3.amazonaws.com/country/latest.json} +CASECOUNT_URL=${CASECOUNT_URL:-https://covid-19-aggregates.s3.amazonaws.com/country/latest.json} # mongoexport rate in cases/s # actual rate is higher, but this allows some wiggle room # in calculation of Batch job timeouts diff --git a/data-serving/scripts/export-data/test_transform_mongoexport_bucketed_ages.csv b/data-serving/scripts/export-data/test_transform_mongoexport_bucketed_ages.csv index f2f830a14..72794f6e6 100644 --- a/data-serving/scripts/export-data/test_transform_mongoexport_bucketed_ages.csv +++ b/data-serving/scripts/export-data/test_transform_mongoexport_bucketed_ages.csv @@ -1,3 +1,3 @@ -_id,caseReference.additionalSources,caseReference.sourceEntryId,caseReference.sourceId,caseReference.sourceUrl,caseReference.uploadIds,caseReference.verificationStatus,demographics.ageBuckets,demographics.ethnicity,demographics.gender,demographics.nationalities,demographics.occupation,events,location.administrativeAreaLevel1,location.administrativeAreaLevel2,location.administrativeAreaLevel3,location.country,location.geoResolution,location.geometry.latitude,location.geometry.longitude,location.name,location.place,location.query,pathogens,preexistingConditions.hasPreexistingConditions,preexistingConditions.values,revisionMetadata.creationMetadata.curator,revisionMetadata.creationMetadata.date,revisionMetadata.creationMetadata.notes,revisionMetadata.editMetadata.curator,revisionMetadata.editMetadata.date,revisionMetadata.editMetadata.notes,revisionMetadata.revisionNumber,SGTF,symptoms.status,symptoms.values,transmission.linkedCaseIds,transmission.places,transmission.routes,travelHistory.travel.dateRange.end,travelHistory.travel.dateRange.start,travelHistory.travel.location.name,travelHistory.travel.methods,travelHistory.travel.purpose,travelHistory.traveledPrior30Days,vaccines.0.name,vaccines.0.batch,vaccines.0.date,vaccines.0.sideEffects,vaccines.1.name,vaccines.1.batch,vaccines.1.date,vaccines.1.sideEffects,vaccines.2.name,vaccines.2.batch,vaccines.2.date,vaccines.2.sideEffects,vaccines.3.name,vaccines.3.batch,vaccines.3.date,vaccines.3.sideEffects -ObjectId(6817283abaa89324a90109aa),[],,787123878aa90909811aaff1,http://foo/bar.csv,"[""bb12399abbb19230900aa123""]",UNVERIFIED,"[""001"", ""002""]",,Male,[],,"[{""name"":""confirmed"",""dateRange"":{""start"":{""$date"":""2021-10-01T00:00:00.000Z""},""end"":{""$date"":""2021-01-01T00:00:00.000Z""}}}]",,,,Antarctica,Country,-79.402,0.323,Antarctica,,,[],,,ingestion@example.com,2021-01-02T13:42:34.991Z,,,,,0,,,,,,,,,,,true,,,,,,,,,,,,,,,,, -ObjectId(798989a98998acc98989a1bb),[],,787123878aa90909811aaff1,http://foo/bar.csv,"[""bb12399abbb19230900aa123""]",UNVERIFIED,"[""003""]",,Female,[],,"[{""name"":""confirmed"",""dateRange"":{""start"":{""$date"":""2021-01-05T00:00:00.000Z""},""end"":{""$date"":""2021-01-05T00:00:00.000Z""}}}]",,,,Antarctica,Country,-79.402,0.323,Antarctica,,,[],,,ingestion@example.com,2021-01-02T13:42:34.991Z,,,,,0,,,,,,,,,,,true,,,,,,,,,,,,,,,,, +_id,caseReference.additionalSources,caseReference.sourceId,caseReference.sourceUrl,caseReference.uploadIds,caseReference.verificationStatus,demographics.ageBuckets,demographics.ethnicity,demographics.gender,demographics.nationalities,demographics.occupation,events,location.administrativeAreaLevel1,location.administrativeAreaLevel2,location.administrativeAreaLevel3,location.country,location.geoResolution,location.geometry.latitude,location.geometry.longitude,location.name,location.place,location.query,pathogens,preexistingConditions.hasPreexistingConditions,preexistingConditions.values,revisionMetadata.creationMetadata.curator,revisionMetadata.creationMetadata.date,revisionMetadata.creationMetadata.notes,revisionMetadata.editMetadata.curator,revisionMetadata.editMetadata.date,revisionMetadata.editMetadata.notes,revisionMetadata.revisionNumber,SGTF,symptoms.status,symptoms.values,transmission.linkedCaseIds,transmission.places,transmission.routes,travelHistory.travel.dateRange.end,travelHistory.travel.dateRange.start,travelHistory.travel.location.name,travelHistory.travel.methods,travelHistory.travel.purpose,travelHistory.traveledPrior30Days,vaccines.0.name,vaccines.0.batch,vaccines.0.date,vaccines.0.sideEffects,vaccines.1.name,vaccines.1.batch,vaccines.1.date,vaccines.1.sideEffects,vaccines.2.name,vaccines.2.batch,vaccines.2.date,vaccines.2.sideEffects,vaccines.3.name,vaccines.3.batch,vaccines.3.date,vaccines.3.sideEffects +ObjectId(6817283abaa89324a90109aa),[],787123878aa90909811aaff1,http://foo/bar.csv,"[""bb12399abbb19230900aa123""]",UNVERIFIED,"[""001"", ""002""]",,Male,[],,"[{""name"":""confirmed"",""dateRange"":{""start"":{""$date"":""2021-10-01T00:00:00.000Z""},""end"":{""$date"":""2021-01-01T00:00:00.000Z""}}}]",,,,Antarctica,Country,-79.402,0.323,Antarctica,,,[],,,ingestion@example.com,2021-01-02T13:42:34.991Z,,,,,0,,,,,,,,,,,true,,,,,,,,,,,,,,,,, +ObjectId(798989a98998acc98989a1bb),[],787123878aa90909811aaff1,http://foo/bar.csv,"[""bb12399abbb19230900aa123""]",UNVERIFIED,"[""003""]",,Female,[],,"[{""name"":""confirmed"",""dateRange"":{""start"":{""$date"":""2021-01-05T00:00:00.000Z""},""end"":{""$date"":""2021-01-05T00:00:00.000Z""}}}]",,,,Antarctica,Country,-79.402,0.323,Antarctica,,,[],,,ingestion@example.com,2021-01-02T13:42:34.991Z,,,,,0,,,,,,,,,,,true,,,,,,,,,,,,,,,,, diff --git a/data-serving/scripts/export-data/test_transform_mongoexport_bucketed_ages_expected.csv b/data-serving/scripts/export-data/test_transform_mongoexport_bucketed_ages_expected.csv index 45aecb5b0..e7cce1b8c 100644 --- a/data-serving/scripts/export-data/test_transform_mongoexport_bucketed_ages_expected.csv +++ b/data-serving/scripts/export-data/test_transform_mongoexport_bucketed_ages_expected.csv @@ -1,3 +1,3 @@ -_id,caseReference.additionalSources,caseReference.sourceEntryId,caseReference.sourceId,caseReference.sourceUrl,caseReference.uploadIds,caseReference.verificationStatus,demographics.ageRange.end,demographics.ageRange.start,demographics.ethnicity,demographics.gender,demographics.nationalities,demographics.occupation,events.confirmed.date,events.confirmed.value,events.firstClinicalConsultation.date,events.hospitalAdmission.date,events.hospitalAdmission.value,events.icuAdmission.date,events.icuAdmission.value,events.onsetSymptoms.date,events.outcome.date,events.outcome.value,events.selfIsolation.date,location.administrativeAreaLevel1,location.administrativeAreaLevel2,location.administrativeAreaLevel3,location.country,location.geometry.latitude,location.geometry.longitude,location.geoResolution,location.name,location.place,pathogens,preexistingConditions.hasPreexistingConditions,preexistingConditions.values,revisionMetadata.creationMetadata.date,revisionMetadata.creationMetadata.notes,revisionMetadata.editMetadata.date,revisionMetadata.editMetadata.notes,revisionMetadata.revisionNumber,SGTF,symptoms.status,symptoms.values,transmission.linkedCaseIds,transmission.places,transmission.routes,travelHistory.travel.dateRange.end,travelHistory.travel.dateRange.start,travelHistory.travel.location.administrativeAreaLevel1,travelHistory.travel.location.administrativeAreaLevel2,travelHistory.travel.location.administrativeAreaLevel3,travelHistory.travel.location.country,travelHistory.travel.location.geometry.coordinates,travelHistory.travel.location.geoResolution,travelHistory.travel.location.name,travelHistory.travel.location.place,travelHistory.travel.methods,travelHistory.travel.purpose,travelHistory.traveledPrior30Days,vaccines.0.batch,vaccines.0.date,vaccines.0.name,vaccines.0.sideEffects,vaccines.1.batch,vaccines.1.date,vaccines.1.name,vaccines.1.sideEffects,vaccines.2.batch,vaccines.2.date,vaccines.2.name,vaccines.2.sideEffects,vaccines.3.batch,vaccines.3.date,vaccines.3.name,vaccines.3.sideEffects,variantOfConcern -ObjectId(6817283abaa89324a90109aa),,,787123878aa90909811aaff1,http://foo/bar.csv,bb12399abbb19230900aa123,UNVERIFIED,5,0,,Male,,,2021-01-01,,,,,,,,,,,,,,Antarctica,-79.402,0.323,Country,Antarctica,,,,,2021-01-02T13:42:34.991Z,,,,0,NA,,,,,,,,,,,,,,,,,true,,,,,,,,,,,,,,,,,, -ObjectId(798989a98998acc98989a1bb),,,787123878aa90909811aaff1,http://foo/bar.csv,bb12399abbb19230900aa123,UNVERIFIED,9,6,,Female,,,2021-01-05,,,,,,,,,,,,,,Antarctica,-79.402,0.323,Country,Antarctica,,,,,2021-01-02T13:42:34.991Z,,,,0,NA,,,,,,,,,,,,,,,,,true,,,,,,,,,,,,,,,,,, +_id,caseReference.additionalSources,caseReference.sourceId,caseReference.sourceUrl,caseReference.uploadIds,caseReference.verificationStatus,demographics.ageRange.end,demographics.ageRange.start,demographics.ethnicity,demographics.gender,demographics.nationalities,demographics.occupation,events.confirmed.date,events.confirmed.value,events.firstClinicalConsultation.date,events.hospitalAdmission.date,events.hospitalAdmission.value,events.icuAdmission.date,events.icuAdmission.value,events.onsetSymptoms.date,events.outcome.date,events.outcome.value,events.selfIsolation.date,location.administrativeAreaLevel1,location.administrativeAreaLevel2,location.administrativeAreaLevel3,location.country,location.geometry.latitude,location.geometry.longitude,location.geoResolution,location.name,location.place,pathogens,preexistingConditions.hasPreexistingConditions,preexistingConditions.values,revisionMetadata.creationMetadata.date,revisionMetadata.creationMetadata.notes,revisionMetadata.editMetadata.date,revisionMetadata.editMetadata.notes,revisionMetadata.revisionNumber,SGTF,symptoms.status,symptoms.values,transmission.linkedCaseIds,transmission.places,transmission.routes,travelHistory.travel.dateRange.end,travelHistory.travel.dateRange.start,travelHistory.travel.location.administrativeAreaLevel1,travelHistory.travel.location.administrativeAreaLevel2,travelHistory.travel.location.administrativeAreaLevel3,travelHistory.travel.location.country,travelHistory.travel.location.geometry.coordinates,travelHistory.travel.location.geoResolution,travelHistory.travel.location.name,travelHistory.travel.location.place,travelHistory.travel.methods,travelHistory.travel.purpose,travelHistory.traveledPrior30Days,vaccines.0.batch,vaccines.0.date,vaccines.0.name,vaccines.0.sideEffects,vaccines.1.batch,vaccines.1.date,vaccines.1.name,vaccines.1.sideEffects,vaccines.2.batch,vaccines.2.date,vaccines.2.name,vaccines.2.sideEffects,vaccines.3.batch,vaccines.3.date,vaccines.3.name,vaccines.3.sideEffects,variantOfConcern +ObjectId(6817283abaa89324a90109aa),,787123878aa90909811aaff1,http://foo/bar.csv,bb12399abbb19230900aa123,UNVERIFIED,5,0,,Male,,,2021-01-01,,,,,,,,,,,,,,Antarctica,-79.402,0.323,Country,Antarctica,,,,,2021-01-02T13:42:34.991Z,,,,0,NA,,,,,,,,,,,,,,,,,true,,,,,,,,,,,,,,,,,, +ObjectId(798989a98998acc98989a1bb),,787123878aa90909811aaff1,http://foo/bar.csv,bb12399abbb19230900aa123,UNVERIFIED,9,6,,Female,,,2021-01-05,,,,,,,,,,,,,,Antarctica,-79.402,0.323,Country,Antarctica,,,,,2021-01-02T13:42:34.991Z,,,,0,NA,,,,,,,,,,,,,,,,,true,,,,,,,,,,,,,,,,,, diff --git a/data-serving/scripts/export-data/test_transform_mongoexport_expected.csv b/data-serving/scripts/export-data/test_transform_mongoexport_expected.csv index cf96dba24..ede14630c 100644 --- a/data-serving/scripts/export-data/test_transform_mongoexport_expected.csv +++ b/data-serving/scripts/export-data/test_transform_mongoexport_expected.csv @@ -1,3 +1,3 @@ -_id,caseReference.additionalSources,caseReference.sourceEntryId,caseReference.sourceId,caseReference.sourceUrl,caseReference.uploadIds,caseReference.verificationStatus,demographics.ageRange.end,demographics.ageRange.start,demographics.ethnicity,demographics.gender,demographics.nationalities,demographics.occupation,events.confirmed.date,events.confirmed.value,events.firstClinicalConsultation.date,events.hospitalAdmission.date,events.hospitalAdmission.value,events.icuAdmission.date,events.icuAdmission.value,events.onsetSymptoms.date,events.outcome.date,events.outcome.value,events.selfIsolation.date,location.administrativeAreaLevel1,location.administrativeAreaLevel2,location.administrativeAreaLevel3,location.country,location.geometry.latitude,location.geometry.longitude,location.geoResolution,location.name,location.place,pathogens,preexistingConditions.hasPreexistingConditions,preexistingConditions.values,revisionMetadata.creationMetadata.date,revisionMetadata.creationMetadata.notes,revisionMetadata.editMetadata.date,revisionMetadata.editMetadata.notes,revisionMetadata.revisionNumber,SGTF,symptoms.status,symptoms.values,transmission.linkedCaseIds,transmission.places,transmission.routes,travelHistory.travel.dateRange.end,travelHistory.travel.dateRange.start,travelHistory.travel.location.administrativeAreaLevel1,travelHistory.travel.location.administrativeAreaLevel2,travelHistory.travel.location.administrativeAreaLevel3,travelHistory.travel.location.country,travelHistory.travel.location.geometry.coordinates,travelHistory.travel.location.geoResolution,travelHistory.travel.location.name,travelHistory.travel.location.place,travelHistory.travel.methods,travelHistory.travel.purpose,travelHistory.traveledPrior30Days,vaccines.0.batch,vaccines.0.date,vaccines.0.name,vaccines.0.sideEffects,vaccines.1.batch,vaccines.1.date,vaccines.1.name,vaccines.1.sideEffects,vaccines.2.batch,vaccines.2.date,vaccines.2.name,vaccines.2.sideEffects,vaccines.3.batch,vaccines.3.date,vaccines.3.name,vaccines.3.sideEffects,variantOfConcern -ObjectId(6817283abaa89324a90109aa),,,787123878aa90909811aaff1,http://foo/bar.csv,bb12399abbb19230900aa123,UNVERIFIED,69,60,,Male,,,2021-01-01,,,,,,,,,,,,,,Antarctica,-79.402,0.323,Country,Antarctica,,,[],,ingestion@example.com,2021-01-02T13:42:34.991Z,,,,0,,,,,,,,,,,,,,,,,true,,,,,,,,,,,,,,,,,, -ObjectId(798989a98998acc98989a1bb),,,787123878aa90909811aaff1,http://foo/bar.csv,bb12399abbb19230900aa123,UNVERIFIED,29,20,,Female,,,2021-01-05,,,,,,,,,,,,,,Antarctica,-79.402,0.323,Country,Antarctica,,,[],,ingestion@example.com,2021-01-02T13:42:34.991Z,,,,0,,,,,,,,,,,,,,,,,true,,,,,,,,,,,,,,,,,, +_id,caseReference.additionalSources,caseReference.sourceId,caseReference.sourceUrl,caseReference.uploadIds,caseReference.verificationStatus,demographics.ageRange.end,demographics.ageRange.start,demographics.ethnicity,demographics.gender,demographics.nationalities,demographics.occupation,events.confirmed.date,events.confirmed.value,events.firstClinicalConsultation.date,events.hospitalAdmission.date,events.hospitalAdmission.value,events.icuAdmission.date,events.icuAdmission.value,events.onsetSymptoms.date,events.outcome.date,events.outcome.value,events.selfIsolation.date,location.administrativeAreaLevel1,location.administrativeAreaLevel2,location.administrativeAreaLevel3,location.country,location.geometry.latitude,location.geometry.longitude,location.geoResolution,location.name,location.place,pathogens,preexistingConditions.hasPreexistingConditions,preexistingConditions.values,revisionMetadata.creationMetadata.date,revisionMetadata.creationMetadata.notes,revisionMetadata.editMetadata.date,revisionMetadata.editMetadata.notes,revisionMetadata.revisionNumber,SGTF,symptoms.status,symptoms.values,transmission.linkedCaseIds,transmission.places,transmission.routes,travelHistory.travel.dateRange.end,travelHistory.travel.dateRange.start,travelHistory.travel.location.administrativeAreaLevel1,travelHistory.travel.location.administrativeAreaLevel2,travelHistory.travel.location.administrativeAreaLevel3,travelHistory.travel.location.country,travelHistory.travel.location.geometry.coordinates,travelHistory.travel.location.geoResolution,travelHistory.travel.location.name,travelHistory.travel.location.place,travelHistory.travel.methods,travelHistory.travel.purpose,travelHistory.traveledPrior30Days,vaccines.0.batch,vaccines.0.date,vaccines.0.name,vaccines.0.sideEffects,vaccines.1.batch,vaccines.1.date,vaccines.1.name,vaccines.1.sideEffects,vaccines.2.batch,vaccines.2.date,vaccines.2.name,vaccines.2.sideEffects,vaccines.3.batch,vaccines.3.date,vaccines.3.name,vaccines.3.sideEffects,variantOfConcern +ObjectId(6817283abaa89324a90109aa),,787123878aa90909811aaff1,http://foo/bar.csv,bb12399abbb19230900aa123,UNVERIFIED,69,60,,Male,,,2021-01-01,,,,,,,,,,,,,,Antarctica,-79.402,0.323,Country,Antarctica,,,[],,ingestion@example.com,2021-01-02T13:42:34.991Z,,,,0,,,,,,,,,,,,,,,,,true,,,,,,,,,,,,,,,,,, +ObjectId(798989a98998acc98989a1bb),,787123878aa90909811aaff1,http://foo/bar.csv,bb12399abbb19230900aa123,UNVERIFIED,29,20,,Female,,,2021-01-05,,,,,,,,,,,,,,Antarctica,-79.402,0.323,Country,Antarctica,,,[],,ingestion@example.com,2021-01-02T13:42:34.991Z,,,,0,,,,,,,,,,,,,,,,,true,,,,,,,,,,,,,,,,,, diff --git a/data-serving/scripts/export-data/test_transform_mongoexport_expected.json b/data-serving/scripts/export-data/test_transform_mongoexport_expected.json index b04e3e19a..a5eac0f99 100644 --- a/data-serving/scripts/export-data/test_transform_mongoexport_expected.json +++ b/data-serving/scripts/export-data/test_transform_mongoexport_expected.json @@ -1,4 +1,4 @@ [ - {"SGTF": "0", "_id": "ObjectId(6817283abaa89324a90109aa)", "caseReference.additionalSources": null, "caseReference.sourceEntryId": "", "caseReference.sourceId": "787123878aa90909811aaff1", "caseReference.sourceUrl": "http://foo/bar.csv", "caseReference.uploadIds": "bb12399abbb19230900aa123", "caseReference.verificationStatus": "UNVERIFIED", "demographics.ageRange.end": "69", "demographics.ageRange.start": "60", "demographics.ethnicity": "", "demographics.gender": "Male", "demographics.nationalities": null, "demographics.occupation": "", "events.confirmed.date": "2021-01-01", "events.confirmed.value": null, "events.firstClinicalConsultation.date": "", "events.hospitalAdmission.date": "", "events.hospitalAdmission.value": "", "events.icuAdmission.date": "", "events.icuAdmission.value": "", "events.onsetSymptoms.date": "", "events.outcome.date": "", "events.outcome.value": "", "events.selfIsolation.date": "", "location.administrativeAreaLevel1": "", "location.administrativeAreaLevel2": "", "location.administrativeAreaLevel3": "", "location.country": "Antarctica", "location.geoResolution": "Country", "location.geometry.latitude": "-79.402", "location.geometry.longitude": "0.323", "location.name": "Antarctica", "location.place": "", "pathogens": "", "preexistingConditions.hasPreexistingConditions": "[]", "preexistingConditions.values": "", "revisionMetadata.creationMetadata.date": "ingestion@example.com", "revisionMetadata.creationMetadata.notes": "2021-01-02T13:42:34.991Z", "revisionMetadata.editMetadata.date": "", "revisionMetadata.editMetadata.notes": "", "revisionMetadata.revisionNumber": "", "symptoms.status": "", "symptoms.values": "", "transmission.linkedCaseIds": "", "transmission.places": "", "transmission.routes": "", "travelHistory.travel.dateRange.end": "", "travelHistory.travel.dateRange.start": "", "travelHistory.travel.location.administrativeAreaLevel1": "", "travelHistory.travel.location.administrativeAreaLevel2": "", "travelHistory.travel.location.administrativeAreaLevel3": "", "travelHistory.travel.location.country": "", "travelHistory.travel.location.geoResolution": "", "travelHistory.travel.location.geometry.coordinates": "", "travelHistory.travel.location.name": "", "travelHistory.travel.location.place": "", "travelHistory.travel.methods": "", "travelHistory.travel.purpose": "true", "travelHistory.traveledPrior30Days": "", "vaccines.0.batch": "", "vaccines.0.date": "", "vaccines.0.name": "", "vaccines.0.sideEffects": "", "vaccines.1.batch": "", "vaccines.1.date": "", "vaccines.1.name": "", "vaccines.1.sideEffects": "", "vaccines.2.batch": "", "vaccines.2.date": "", "vaccines.2.name": "", "vaccines.2.sideEffects": "", "vaccines.3.batch": "", "vaccines.3.date": "", "vaccines.3.name": "", "vaccines.3.sideEffects": null, "variantOfConcern": ""} -, {"SGTF": "0", "_id": "ObjectId(798989a98998acc98989a1bb)", "caseReference.additionalSources": null, "caseReference.sourceEntryId": "", "caseReference.sourceId": "787123878aa90909811aaff1", "caseReference.sourceUrl": "http://foo/bar.csv", "caseReference.uploadIds": "bb12399abbb19230900aa123", "caseReference.verificationStatus": "UNVERIFIED", "demographics.ageRange.end": "29", "demographics.ageRange.start": "20", "demographics.ethnicity": "", "demographics.gender": "Female", "demographics.nationalities": null, "demographics.occupation": "", "events.confirmed.date": "2021-01-05", "events.confirmed.value": null, "events.firstClinicalConsultation.date": "", "events.hospitalAdmission.date": "", "events.hospitalAdmission.value": "", "events.icuAdmission.date": "", "events.icuAdmission.value": "", "events.onsetSymptoms.date": "", "events.outcome.date": "", "events.outcome.value": "", "events.selfIsolation.date": "", "location.administrativeAreaLevel1": "", "location.administrativeAreaLevel2": "", "location.administrativeAreaLevel3": "", "location.country": "Antarctica", "location.geoResolution": "Country", "location.geometry.latitude": "-79.402", "location.geometry.longitude": "0.323", "location.name": "Antarctica", "location.place": "", "pathogens": "", "preexistingConditions.hasPreexistingConditions": "[]", "preexistingConditions.values": "", "revisionMetadata.creationMetadata.date": "ingestion@example.com", "revisionMetadata.creationMetadata.notes": "2021-01-02T13:42:34.991Z", "revisionMetadata.editMetadata.date": "", "revisionMetadata.editMetadata.notes": "", "revisionMetadata.revisionNumber": "", "symptoms.status": "", "symptoms.values": "", "transmission.linkedCaseIds": "", "transmission.places": "", "transmission.routes": "", "travelHistory.travel.dateRange.end": "", "travelHistory.travel.dateRange.start": "", "travelHistory.travel.location.administrativeAreaLevel1": "", "travelHistory.travel.location.administrativeAreaLevel2": "", "travelHistory.travel.location.administrativeAreaLevel3": "", "travelHistory.travel.location.country": "", "travelHistory.travel.location.geoResolution": "", "travelHistory.travel.location.geometry.coordinates": "", "travelHistory.travel.location.name": "", "travelHistory.travel.location.place": "", "travelHistory.travel.methods": "", "travelHistory.travel.purpose": "true", "travelHistory.traveledPrior30Days": "", "vaccines.0.batch": "", "vaccines.0.date": "", "vaccines.0.name": "", "vaccines.0.sideEffects": "", "vaccines.1.batch": "", "vaccines.1.date": "", "vaccines.1.name": "", "vaccines.1.sideEffects": "", "vaccines.2.batch": "", "vaccines.2.date": "", "vaccines.2.name": "", "vaccines.2.sideEffects": "", "vaccines.3.batch": "", "vaccines.3.date": "", "vaccines.3.name": "", "vaccines.3.sideEffects": null, "variantOfConcern": ""} + {"SGTF": "0", "_id": "ObjectId(6817283abaa89324a90109aa)", "caseReference.additionalSources": null, "caseReference.sourceId": "787123878aa90909811aaff1", "caseReference.sourceUrl": "http://foo/bar.csv", "caseReference.uploadIds": "bb12399abbb19230900aa123", "caseReference.verificationStatus": "UNVERIFIED", "demographics.ageRange.end": "69", "demographics.ageRange.start": "60", "demographics.ethnicity": "", "demographics.gender": "Male", "demographics.nationalities": null, "demographics.occupation": "", "events.confirmed.date": "2021-01-01", "events.confirmed.value": null, "events.firstClinicalConsultation.date": "", "events.hospitalAdmission.date": "", "events.hospitalAdmission.value": "", "events.icuAdmission.date": "", "events.icuAdmission.value": "", "events.onsetSymptoms.date": "", "events.outcome.date": "", "events.outcome.value": "", "events.selfIsolation.date": "", "location.administrativeAreaLevel1": "", "location.administrativeAreaLevel2": "", "location.administrativeAreaLevel3": "", "location.country": "Antarctica", "location.geoResolution": "Country", "location.geometry.latitude": "-79.402", "location.geometry.longitude": "0.323", "location.name": "Antarctica", "location.place": "", "pathogens": "", "preexistingConditions.hasPreexistingConditions": "[]", "preexistingConditions.values": "", "revisionMetadata.creationMetadata.date": "ingestion@example.com", "revisionMetadata.creationMetadata.notes": "2021-01-02T13:42:34.991Z", "revisionMetadata.editMetadata.date": "", "revisionMetadata.editMetadata.notes": "", "revisionMetadata.revisionNumber": "", "symptoms.status": "", "symptoms.values": "", "transmission.linkedCaseIds": "", "transmission.places": "", "transmission.routes": "", "travelHistory.travel.dateRange.end": "", "travelHistory.travel.dateRange.start": "", "travelHistory.travel.location.administrativeAreaLevel1": "", "travelHistory.travel.location.administrativeAreaLevel2": "", "travelHistory.travel.location.administrativeAreaLevel3": "", "travelHistory.travel.location.country": "", "travelHistory.travel.location.geoResolution": "", "travelHistory.travel.location.geometry.coordinates": "", "travelHistory.travel.location.name": "", "travelHistory.travel.location.place": "", "travelHistory.travel.methods": "", "travelHistory.travel.purpose": "true", "travelHistory.traveledPrior30Days": "", "vaccines.0.batch": "", "vaccines.0.date": "", "vaccines.0.name": "", "vaccines.0.sideEffects": "", "vaccines.1.batch": "", "vaccines.1.date": "", "vaccines.1.name": "", "vaccines.1.sideEffects": "", "vaccines.2.batch": "", "vaccines.2.date": "", "vaccines.2.name": "", "vaccines.2.sideEffects": "", "vaccines.3.batch": "", "vaccines.3.date": "", "vaccines.3.name": "", "vaccines.3.sideEffects": null, "variantOfConcern": ""} +, {"SGTF": "0", "_id": "ObjectId(798989a98998acc98989a1bb)", "caseReference.additionalSources": null, "caseReference.sourceId": "787123878aa90909811aaff1", "caseReference.sourceUrl": "http://foo/bar.csv", "caseReference.uploadIds": "bb12399abbb19230900aa123", "caseReference.verificationStatus": "UNVERIFIED", "demographics.ageRange.end": "29", "demographics.ageRange.start": "20", "demographics.ethnicity": "", "demographics.gender": "Female", "demographics.nationalities": null, "demographics.occupation": "", "events.confirmed.date": "2021-01-05", "events.confirmed.value": null, "events.firstClinicalConsultation.date": "", "events.hospitalAdmission.date": "", "events.hospitalAdmission.value": "", "events.icuAdmission.date": "", "events.icuAdmission.value": "", "events.onsetSymptoms.date": "", "events.outcome.date": "", "events.outcome.value": "", "events.selfIsolation.date": "", "location.administrativeAreaLevel1": "", "location.administrativeAreaLevel2": "", "location.administrativeAreaLevel3": "", "location.country": "Antarctica", "location.geoResolution": "Country", "location.geometry.latitude": "-79.402", "location.geometry.longitude": "0.323", "location.name": "Antarctica", "location.place": "", "pathogens": "", "preexistingConditions.hasPreexistingConditions": "[]", "preexistingConditions.values": "", "revisionMetadata.creationMetadata.date": "ingestion@example.com", "revisionMetadata.creationMetadata.notes": "2021-01-02T13:42:34.991Z", "revisionMetadata.editMetadata.date": "", "revisionMetadata.editMetadata.notes": "", "revisionMetadata.revisionNumber": "", "symptoms.status": "", "symptoms.values": "", "transmission.linkedCaseIds": "", "transmission.places": "", "transmission.routes": "", "travelHistory.travel.dateRange.end": "", "travelHistory.travel.dateRange.start": "", "travelHistory.travel.location.administrativeAreaLevel1": "", "travelHistory.travel.location.administrativeAreaLevel2": "", "travelHistory.travel.location.administrativeAreaLevel3": "", "travelHistory.travel.location.country": "", "travelHistory.travel.location.geoResolution": "", "travelHistory.travel.location.geometry.coordinates": "", "travelHistory.travel.location.name": "", "travelHistory.travel.location.place": "", "travelHistory.travel.methods": "", "travelHistory.travel.purpose": "true", "travelHistory.traveledPrior30Days": "", "vaccines.0.batch": "", "vaccines.0.date": "", "vaccines.0.name": "", "vaccines.0.sideEffects": "", "vaccines.1.batch": "", "vaccines.1.date": "", "vaccines.1.name": "", "vaccines.1.sideEffects": "", "vaccines.2.batch": "", "vaccines.2.date": "", "vaccines.2.name": "", "vaccines.2.sideEffects": "", "vaccines.3.batch": "", "vaccines.3.date": "", "vaccines.3.name": "", "vaccines.3.sideEffects": null, "variantOfConcern": ""} ] diff --git a/data-serving/scripts/export-data/test_transform_mongoexport_expected.tsv b/data-serving/scripts/export-data/test_transform_mongoexport_expected.tsv index fd990e3f7..fddfd0e4e 100644 --- a/data-serving/scripts/export-data/test_transform_mongoexport_expected.tsv +++ b/data-serving/scripts/export-data/test_transform_mongoexport_expected.tsv @@ -1,3 +1,3 @@ -_id caseReference.additionalSources caseReference.sourceEntryId caseReference.sourceId caseReference.sourceUrl caseReference.uploadIds caseReference.verificationStatus demographics.ageRange.end demographics.ageRange.start demographics.ethnicity demographics.gender demographics.nationalities demographics.occupation events.confirmed.date events.confirmed.value events.firstClinicalConsultation.date events.hospitalAdmission.date events.hospitalAdmission.value events.icuAdmission.date events.icuAdmission.value events.onsetSymptoms.date events.outcome.date events.outcome.value events.selfIsolation.date location.administrativeAreaLevel1 location.administrativeAreaLevel2 location.administrativeAreaLevel3 location.country location.geometry.latitude location.geometry.longitude location.geoResolution location.name location.place pathogens preexistingConditions.hasPreexistingConditions preexistingConditions.values revisionMetadata.creationMetadata.date revisionMetadata.creationMetadata.notes revisionMetadata.editMetadata.date revisionMetadata.editMetadata.notes revisionMetadata.revisionNumber SGTF symptoms.status symptoms.values transmission.linkedCaseIds transmission.places transmission.routes travelHistory.travel.dateRange.end travelHistory.travel.dateRange.start travelHistory.travel.location.administrativeAreaLevel1 travelHistory.travel.location.administrativeAreaLevel2 travelHistory.travel.location.administrativeAreaLevel3 travelHistory.travel.location.country travelHistory.travel.location.geometry.coordinates travelHistory.travel.location.geoResolution travelHistory.travel.location.name travelHistory.travel.location.place travelHistory.travel.methods travelHistory.travel.purpose travelHistory.traveledPrior30Days vaccines.0.batch vaccines.0.date vaccines.0.name vaccines.0.sideEffects vaccines.1.batch vaccines.1.date vaccines.1.name vaccines.1.sideEffects vaccines.2.batch vaccines.2.date vaccines.2.name vaccines.2.sideEffects vaccines.3.batch vaccines.3.date vaccines.3.name vaccines.3.sideEffects variantOfConcern -ObjectId(6817283abaa89324a90109aa) 787123878aa90909811aaff1 http://foo/bar.csv bb12399abbb19230900aa123 UNVERIFIED 69 60 Male 2021-01-01 Antarctica -79.402 0.323 Country Antarctica [] ingestion@example.com 2021-01-02T13:42:34.991Z 0 true -ObjectId(798989a98998acc98989a1bb) 787123878aa90909811aaff1 http://foo/bar.csv bb12399abbb19230900aa123 UNVERIFIED 29 20 Female 2021-01-05 Antarctica -79.402 0.323 Country Antarctica [] ingestion@example.com 2021-01-02T13:42:34.991Z 0 true +_id caseReference.additionalSources caseReference.sourceId caseReference.sourceUrl caseReference.uploadIds caseReference.verificationStatus demographics.ageRange.end demographics.ageRange.start demographics.ethnicity demographics.gender demographics.nationalities demographics.occupation events.confirmed.date events.confirmed.value events.firstClinicalConsultation.date events.hospitalAdmission.date events.hospitalAdmission.value events.icuAdmission.date events.icuAdmission.value events.onsetSymptoms.date events.outcome.date events.outcome.value events.selfIsolation.date location.administrativeAreaLevel1 location.administrativeAreaLevel2 location.administrativeAreaLevel3 location.country location.geometry.latitude location.geometry.longitude location.geoResolution location.name location.place pathogens preexistingConditions.hasPreexistingConditions preexistingConditions.values revisionMetadata.creationMetadata.date revisionMetadata.creationMetadata.notes revisionMetadata.editMetadata.date revisionMetadata.editMetadata.notes revisionMetadata.revisionNumber SGTF symptoms.status symptoms.values transmission.linkedCaseIds transmission.places transmission.routes travelHistory.travel.dateRange.end travelHistory.travel.dateRange.start travelHistory.travel.location.administrativeAreaLevel1 travelHistory.travel.location.administrativeAreaLevel2 travelHistory.travel.location.administrativeAreaLevel3 travelHistory.travel.location.country travelHistory.travel.location.geometry.coordinates travelHistory.travel.location.geoResolution travelHistory.travel.location.name travelHistory.travel.location.place travelHistory.travel.methods travelHistory.travel.purpose travelHistory.traveledPrior30Days vaccines.0.batch vaccines.0.date vaccines.0.name vaccines.0.sideEffects vaccines.1.batch vaccines.1.date vaccines.1.name vaccines.1.sideEffects vaccines.2.batch vaccines.2.date vaccines.2.name vaccines.2.sideEffects vaccines.3.batch vaccines.3.date vaccines.3.name vaccines.3.sideEffects variantOfConcern +ObjectId(6817283abaa89324a90109aa) 787123878aa90909811aaff1 http://foo/bar.csv bb12399abbb19230900aa123 UNVERIFIED 69 60 Male 2021-01-01 Antarctica -79.402 0.323 Country Antarctica [] ingestion@example.com 2021-01-02T13:42:34.991Z 0 true +ObjectId(798989a98998acc98989a1bb) 787123878aa90909811aaff1 http://foo/bar.csv bb12399abbb19230900aa123 UNVERIFIED 29 20 Female 2021-01-05 Antarctica -79.402 0.323 Country Antarctica [] ingestion@example.com 2021-01-02T13:42:34.991Z 0 true diff --git a/data-serving/scripts/export-data/test_transform_mongoexport_header.csv b/data-serving/scripts/export-data/test_transform_mongoexport_header.csv index facdb2b40..360d6a183 100644 --- a/data-serving/scripts/export-data/test_transform_mongoexport_header.csv +++ b/data-serving/scripts/export-data/test_transform_mongoexport_header.csv @@ -1 +1 @@ -_id,caseReference.additionalSources,caseReference.sourceEntryId,caseReference.sourceId,caseReference.sourceUrl,caseReference.uploadIds,caseReference.verificationStatus,demographics.ageRange.end,demographics.ageRange.start,demographics.ethnicity,demographics.gender,demographics.nationalities,demographics.occupation,events,genomeSequences,location.administrativeAreaLevel1,location.administrativeAreaLevel2,location.administrativeAreaLevel3,location.country,location.geoResolution,location.geometry.latitude,location.geometry.longitude,location.name,location.place,location.query,pathogens,preexistingConditions.hasPreexistingConditions,preexistingConditions.values,revisionMetadata.creationMetadata.curator,revisionMetadata.creationMetadata.date,revisionMetadata.creationMetadata.notes,revisionMetadata.editMetadata.curator,revisionMetadata.editMetadata.date,revisionMetadata.editMetadata.notes,revisionMetadata.revisionNumber,SGTF,symptoms.status,symptoms.values,transmission.linkedCaseIds,transmission.places,transmission.routes,travelHistory.travel.dateRange.end,travelHistory.travel.dateRange.start,travelHistory.travel.location.name,travelHistory.travel.methods,travelHistory.travel.purpose,travelHistory.traveledPrior30Days,vaccines.0.name,vaccines.0.batch,vaccines.0.date,vaccines.0.sideEffects,vaccines.1.name,vaccines.1.batch,vaccines.1.date,vaccines.1.sideEffects,vaccines.2.name,vaccines.2.batch,vaccines.2.date,vaccines.2.sideEffects,vaccines.3.name,vaccines.3.batch,vaccines.3.date,vaccines.3.sideEffects +_id,caseReference.additionalSources,caseReference.sourceId,caseReference.sourceUrl,caseReference.uploadIds,caseReference.verificationStatus,demographics.ageRange.end,demographics.ageRange.start,demographics.ethnicity,demographics.gender,demographics.nationalities,demographics.occupation,events,genomeSequences,location.administrativeAreaLevel1,location.administrativeAreaLevel2,location.administrativeAreaLevel3,location.country,location.geoResolution,location.geometry.latitude,location.geometry.longitude,location.name,location.place,location.query,pathogens,preexistingConditions.hasPreexistingConditions,preexistingConditions.values,revisionMetadata.creationMetadata.curator,revisionMetadata.creationMetadata.date,revisionMetadata.creationMetadata.notes,revisionMetadata.editMetadata.curator,revisionMetadata.editMetadata.date,revisionMetadata.editMetadata.notes,revisionMetadata.revisionNumber,SGTF,symptoms.status,symptoms.values,transmission.linkedCaseIds,transmission.places,transmission.routes,travelHistory.travel.dateRange.end,travelHistory.travel.dateRange.start,travelHistory.travel.location.name,travelHistory.travel.methods,travelHistory.travel.purpose,travelHistory.traveledPrior30Days,vaccines.0.name,vaccines.0.batch,vaccines.0.date,vaccines.0.sideEffects,vaccines.1.name,vaccines.1.batch,vaccines.1.date,vaccines.1.sideEffects,vaccines.2.name,vaccines.2.batch,vaccines.2.date,vaccines.2.sideEffects,vaccines.3.name,vaccines.3.batch,vaccines.3.date,vaccines.3.sideEffects diff --git a/data-serving/scripts/export-data/transform.py b/data-serving/scripts/export-data/transform.py index a5d18743a..c636dff40 100644 --- a/data-serving/scripts/export-data/transform.py +++ b/data-serving/scripts/export-data/transform.py @@ -39,6 +39,7 @@ "events", "notes", "travelHistory.travel", + "caseReference.sourceEntryId" ] __TRAVEL = [ diff --git a/docs/data_landscape.md b/docs/data_landscape.md index 07b6d82b3..ca64903c3 100644 --- a/docs/data_landscape.md +++ b/docs/data_landscape.md @@ -23,8 +23,8 @@ Various buckets (data containers) are used for both temporary and long-term stor Aggregated data from the line list used by the map visualisation. -* covid-19-aggregates-eu -* covid-19-aggregates-dev-eu +* covid-19-aggregates +* covid-19-aggregates-dev ### Export diff --git a/ingestion/monitoring/completeness.py b/ingestion/monitoring/completeness.py index c9f951870..ae018c669 100644 --- a/ingestion/monitoring/completeness.py +++ b/ingestion/monitoring/completeness.py @@ -114,8 +114,8 @@ def setup_logger(): setup_logger() endpoint_url = os.getenv("ENDPOINT_URL") objects = data_files( - os.getenv("COUNTRY_EXPORT_BUCKET", "covid-19-country-export-eu"), + os.getenv("COUNTRY_EXPORT_BUCKET", "covid-19-country-export"), endpoint_url=endpoint_url, ) data = completeness_s3_many(objects, endpoint_url) - upload(data, os.getenv("METRICS_BUCKET", "covid-19-aggregates-eu"), endpoint_url) + upload(data, os.getenv("METRICS_BUCKET", "covid-19-aggregates"), endpoint_url) diff --git a/ingestion/monitoring/daily_metrics.py b/ingestion/monitoring/daily_metrics.py index 0854c64c9..a4d64b512 100644 --- a/ingestion/monitoring/daily_metrics.py +++ b/ingestion/monitoring/daily_metrics.py @@ -11,7 +11,7 @@ import boto3 -BUCKET = "covid-19-aggregates-eu" +BUCKET = "covid-19-aggregates" WEBHOOK_URL = os.environ.get("SLACK_WEBHOOK_METRICS_URL", None) logger = logging.getLogger(__name__) diff --git a/ingestion/monitoring/freshness.py b/ingestion/monitoring/freshness.py index bb2b8d7ca..3e629d799 100644 --- a/ingestion/monitoring/freshness.py +++ b/ingestion/monitoring/freshness.py @@ -91,7 +91,7 @@ def setup_logger(): setup_logger() if not (api_key := os.getenv("GDH_API_KEY")): raise ValueError("Set GDH_API_KEY to your Global.health API key") - bucket = os.getenv("BUCKET", "covid-19-aggregates-eu") + bucket = os.getenv("BUCKET", "covid-19-aggregates") s3_endpoint = os.getenv("S3_ENDPOINT") instance = os.getenv("GDH_URL", DEFAULT_INSTANCE) if sources := fetch_sources(api_key, instance): diff --git a/verification/curator-service/ui/src/components/ViewCase.test.tsx b/verification/curator-service/ui/src/components/ViewCase.test.tsx index 1cd169592..80a02a01c 100644 --- a/verification/curator-service/ui/src/components/ViewCase.test.tsx +++ b/verification/curator-service/ui/src/components/ViewCase.test.tsx @@ -52,7 +52,6 @@ it('loads and displays case', async () => { 'href', 'https://news.org/an-article', ); - expect(getByText('entryId')).toBeInTheDocument(); expect(getByText('abc123')).toBeInTheDocument(); expect(getByText('2020-01-20')).toBeInTheDocument(); expect(getByText('xyz789')).toBeInTheDocument(); diff --git a/verification/curator-service/ui/src/components/ViewCase.tsx b/verification/curator-service/ui/src/components/ViewCase.tsx index e36a73aae..c3a509c27 100644 --- a/verification/curator-service/ui/src/components/ViewCase.tsx +++ b/verification/curator-service/ui/src/components/ViewCase.tsx @@ -313,13 +313,6 @@ function CaseDetails(props: CaseDetailsProps): JSX.Element { isLink /> - - -