1921 delete and reimport sources without stable ids

data-serving/data-service/api/openapi.yaml

@@ -350,6 +350,72 @@ paths:
                     $ref: '#/components/responses/422'
                 '500':
                     $ref: '#/components/responses/500'
+    /cases/markPendingRemoval:
+        post:
+            summary: Mark cases from the identified source as awaiting deletion.
+            operationId: markPendingRemoval
+            tags: [Ingestion]
+            parameters:
+                - in: query
+                  name: sourceId
+                  description: The source identifier for affected cases
+                  required: true
+                  schema:
+                        type: string
+                        pattern: '^[a-f\d]{24}$'
+                - in: query
+                  name: email
+                  description: The curator email address
+                  required: true
+                  schema:
+                        type: string
+            responses:
+                '204':
+                    $ref: '#/components/responses/204'
+                '500':
+                    $ref: '#/components/responses/500'
+    /cases/removePendingCases:
+        post:
+            summary: Remove any cases for the identified source marked as awaiting deletion.
+            operationId: removePendingCases
+            tags: [Ingestion]
+            parameters:
+                - name: sourceId
+                  in: query
+                  description: The source identifier for affected cases
+                  required: true
+                  schema:
+                        type: string
+                        pattern: '^[a-f\d]{24}$'
+            responses:
+                '204':
+                    $ref: '#/components/responses/204'
+                '500':
+                    $ref: '#/components/responses/500'
+    /cases/clearPendingRemovalStatus:
+        post:
+            summary: Clear the 'awaiting deletion' mark from any cases from the identified source.
+            operationId: clearPendingRemovalStatus
+            tags: [Ingestion]
+            parameters:
+                - in: query
+                  name: sourceId
+                  description: The source identifier for affected cases
+                  required: true
+                  schema:
+                        type: string
+                        pattern: '^[a-f\d]{24}$'
+                - in: query
+                  name: email
+                  description: The curator email address
+                  required: true
+                  schema:
+                        type: string
+            responses:
+                '204':
+                    $ref: '#/components/responses/204'
+                '500':
+                    $ref: '#/components/responses/500'
     /cases/symptoms:
         get:
             summary: Lists most frequently used symptoms
@@ -1084,6 +1150,10 @@ components:
                         $ref: '#/components/schemas/BatchUpsertCaseResponse'
         '200':
             description: OK
+        '201':
+            description: Created
+        '204':
+            description: No content
         '400':
             description: Malformed request
         '403':

data-serving/data-service/package.json

@@ -24,9 +24,9 @@
   "author": "",
   "license": "MIT",
   "config": {
-     "mongodbMemoryServer": {
-        "version": "latest"
-      }
+    "mongodbMemoryServer": {
+      "version": "latest"
+    }
   },
   "devDependencies": {
     "@shelf/jest-mongodb": "^1.2.3",

data-serving/data-service/src/controllers/case.ts

@@ -854,6 +854,35 @@ export class CasesController {
         res.status(200).json({ cases: caseIds }).end();
     };
 
+    /** Indicate that each case for the given source is pending deletion. */
+    markPendingRemoval = async (req: Request, res: Response): Promise<void> => {
+        try {
+            req.body.cases.forEach((c: CaseDocument) => {
+                c.pendingRemoval = true;
+                c.save();
+            });
+            res.status(204).end();
+        } catch (err) {
+            res.status(500).json(err).end();
+        }
+    };
+
+    /** Unset the pending deletion flag for cases from this source. */
+    clearPendingRemovalStatus = async (
+        req: Request,
+        res: Response,
+    ): Promise<void> => {
+        try {
+            req.body.cases.forEach((c: CaseDocument) => {
+                c.pendingRemoval = false;
+                c.save();
+            });
+            res.status(204).end();
+        } catch (err) {
+            res.status(500).json(err).end();
+        }
+    };
+
     private excludeRestrictedSourcesFromCaseAggregation(casesQuery: any[]) {
         return _.concat(casesQuery, [
             {

data-serving/data-service/src/controllers/preprocessor.ts

@@ -8,6 +8,7 @@ import {
 import { CaseRevision } from '../model/case-revision';
 import { DocumentQuery } from 'mongoose';
 import _ from 'lodash';
+import { nextTick } from 'process';
 
 // TODO: Type this as RevisionMetadataDocument.
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -206,6 +207,39 @@ export const findCasesToUpdate = async (
     next();
 };
 
+export const findCasesForSource = async (
+    request: Request,
+    response: Response,
+    next: NextFunction,
+): Promise<void> => {
+    // find the appropriate cases
+    const cases = await Case.find({
+        'caseReference.sourceId': request.query.sourceId?.toString(),
+    });
+    request.body.cases = cases;
+    // and store the curator email
+    request.body.curator = {
+        email: request.query.email,
+    };
+
+    next();
+};
+
+export const findPendingCaseIdsForRemovalFromSource = async (
+    request: Request,
+    response: Response,
+    next: NextFunction,
+): Promise<void> => {
+    const caseIds = await Case.find({
+        'caseReference.sourceId': {
+            $eq: request.query.sourceId?.toString(),
+        },
+        pendingRemoval: true,
+    });
+    request.body.caseIds = caseIds.map((c: CaseDocument) => c._id);
+    next();
+};
+
 export const setBatchUpdateRevisionMetadata = async (
     request: Request,
     response: Response,

data-serving/data-service/src/index.ts

@@ -9,7 +9,9 @@ import {
     createBatchUpdateCaseRevisions,
     createBatchUpsertCaseRevisions,
     createCaseRevision,
+    findCasesForSource,
     findCasesToUpdate,
+    findPendingCaseIdsForRemovalFromSource,
     setBatchUpdateRevisionMetadata,
     setBatchUpsertFields,
     setRevisionMetadata,
@@ -133,6 +135,28 @@ apiRouter.post(
     createBatchUpdateCaseRevisions,
     caseController.batchUpdate,
 );
+apiRouter.post(
+    '/cases/markPendingRemoval',
+    findCasesForSource,
+    setBatchUpdateRevisionMetadata,
+    createBatchUpdateCaseRevisions,
+    caseController.markPendingRemoval,
+);
+
+apiRouter.post(
+    '/cases/removePendingCases',
+    findPendingCaseIdsForRemovalFromSource,
+    createBatchDeleteCaseRevisions,
+    caseController.batchDel,
+);
+
+apiRouter.post(
+    '/cases/clearPendingRemovalStatus',
+    findCasesForSource,
+    setBatchUpdateRevisionMetadata,
+    createBatchUpdateCaseRevisions,
+    caseController.clearPendingRemovalStatus,
+);
 apiRouter.put(
     '/cases/:id([a-z0-9]{24})',
     setRevisionMetadata,

data-serving/data-service/src/model/case.ts

@@ -50,6 +50,7 @@ export const caseSchema = new mongoose.Schema(
         revisionMetadata: revisionMetadataSchema,
         notes: String,
         pathogens: [pathogenSchema],
+        pendingRemoval: Boolean,
         preexistingConditions: preexistingConditionsSchema,
         symptoms: symptomsSchema,
         transmission: transmissionSchema,
@@ -120,6 +121,7 @@ export type CaseDocument = mongoose.Document & {
     revisionMetadata: RevisionMetadataDocument;
     notes: string;
     pathogens: [PathogenDocument];
+    pendingRemoval: boolean;
     preexistingConditions: PreexistingConditionsDocument;
     symptoms: SymptomsDocument;
     transmission: TransmissionDocument;

data-serving/data-service/test/controllers/case.test.ts

@@ -1686,3 +1686,114 @@ describe('DELETE', () => {
         expect(await CaseRevision.collection.countDocuments()).toEqual(0);
     });
 });
+
+describe('pending removal markers', async () => {
+    it('marks cases from a given source ID for deletion', async () => {
+        const c1 = new Case(minimalCase);
+        const sourceId1 = '5ea86423bae6982635d2e1f8';
+        const entryId1 = 'def456';
+        c1.set('caseReference.sourceId', sourceId1);
+        c1.set('caseReference.sourceEntryId', entryId1);
+        await c1.save();
+
+        const c2 = new Case(minimalCase);
+        const sourceId2 = '5ea86423bae6982635d2e1f9';
+        const entryId2 = 'abc123';
+        c2.set('caseReference.sourceId', sourceId2);
+        c2.set('caseReference.sourceEntryId', entryId2);
+        await c2.save();
+
+        await request(app)
+            .post(
+                `/api/cases/markPendingRemoval?sourceId=${sourceId1}&email=curator%40example.com`,
+            )
+            .expect(204);
+
+        const pendingCases = await Case.find({
+            pendingRemoval: true,
+        });
+        expect(pendingCases.length).toEqual(1);
+        expect(pendingCases[0].caseReference.sourceEntryId).toEqual(entryId1);
+    });
+
+    it('removes deletion-pending markers from cases with a given source ID', async () => {
+        const c1 = new Case(minimalCase);
+        const sourceId1 = '5ea86423bae6982635d2e1f8';
+        const entryId1 = 'def456';
+        c1.set('caseReference.sourceId', sourceId1);
+        c1.set('caseReference.sourceEntryId', entryId1);
+        c1.pendingRemoval = true;
+        await c1.save();
+
+        const c2 = new Case(minimalCase);
+        const sourceId2 = '5ea86423bae6982635d2e1f9';
+        const entryId2 = 'abc123';
+        c2.set('caseReference.sourceId', sourceId2);
+        c2.set('caseReference.sourceEntryId', entryId2);
+        c2.pendingRemoval = true;
+        await c2.save();
+
+        await request(app)
+            .post(
+                `/api/cases/clearPendingRemovalStatus?sourceId=${sourceId1}&email=curator%40example.com`,
+            )
+            .expect(204);
+
+        const pendingCases = await Case.find({
+            pendingRemoval: true,
+        });
+        expect(pendingCases.length).toEqual(1);
+        expect(pendingCases[0].caseReference.sourceEntryId).toEqual(entryId2);
+    });
+
+    it('deletes pending cases with a given source ID', async () => {
+        const c1 = new Case(minimalCase);
+        const sourceId1 = '5ea86423bae6982635d2e1f8';
+        const entryId1 = 'def456';
+        c1.set('caseReference.sourceId', sourceId1);
+        c1.set('caseReference.sourceEntryId', entryId1);
+        c1.pendingRemoval = true;
+        await c1.save();
+
+        const c2 = new Case(minimalCase);
+        const sourceId2 = '5ea86423bae6982635d2e1f9';
+        const entryId2 = 'abc123';
+        c2.set('caseReference.sourceId', sourceId2);
+        c2.set('caseReference.sourceEntryId', entryId2);
+        c2.pendingRemoval = true;
+        await c2.save();
+
+        await request(app)
+            .post(`/api/cases/removePendingCases?sourceId=${sourceId1}`)
+            .expect(204);
+
+        const allCases = await Case.find({});
+        expect(allCases.length).toEqual(1);
+        expect(allCases[0].caseReference.sourceEntryId).toEqual(entryId2);
+    });
+
+    it('deletes only the pending cases from the source', async () => {
+        const c1 = new Case(minimalCase);
+        const sourceId1 = '5ea86423bae6982635d2e1f8';
+        const entryId1 = 'def456';
+        c1.set('caseReference.sourceId', sourceId1);
+        c1.set('caseReference.sourceEntryId', entryId1);
+        c1.pendingRemoval = true;
+        await c1.save();
+
+        const c2 = new Case(minimalCase);
+        const entryId2 = 'abc123';
+        c2.set('caseReference.sourceId', sourceId1);
+        c2.set('caseReference.sourceEntryId', entryId2);
+        c2.pendingRemoval = false;
+        await c2.save();
+
+        await request(app)
+            .post(`/api/cases/removePendingCases?sourceId=${sourceId1}`)
+            .expect(204);
+
+        const allCases = await Case.find({});
+        expect(allCases.length).toEqual(1);
+        expect(allCases[0].caseReference.sourceEntryId).toEqual(entryId2);
+    });
+});