[new script] [events] Recode data value boolean to optionset

src/data/ProgramsD2Repository.ts

@@ -8,6 +8,7 @@
 import { promiseMap, runMetadata } from "./dhis2-utils";
 import { D2ProgramRules } from "./d2-program-rules/D2ProgramRules";
 import { D2Tracker } from "./D2Tracker";
+import { Program, ProgramType } from "domain/entities/Program";
 
 type MetadataRes = { date: string } & { [k: string]: Array<{ id: string }> };
 
@@ -18,6 +19,39 @@
         this.d2Tracker = new D2Tracker(this.api);
     }
 
+    async get(options: { ids?: Id[]; programTypes?: ProgramType[] }): Async<Program[]> {
+        const { programs } = await this.api.metadata
+            .get({
+                programs: {
+                    fields: {
+                        id: true,
+                        name: true,
+                        programType: true,
+                        programStages: {
+                            id: true,
+                            name: true,
+                            programStageDataElements: {
+                                dataElement: {
+                                    id: true,
+                                    name: true,
+                                    code: true,
+                                    valueType: true,
+                                    optionSet: { id: true, name: true },
+                                },
+                            },
+                        },
+                    },
+                    filter: {
+                        ...(options.ids ? { id: { in: options.ids } } : {}),
+                        ...(options.programTypes ? { programType: { in: options.programTypes } } : {}),
+                    },
+                },
+            })
+            .getData();
+
+        return programs;
+    }
+
     async export(options: { ids: Id[]; orgUnitIds: Id[] | undefined }): Async<ProgramExport> {
         const { ids: programIds, orgUnitIds } = options;
         const metadata = await this.getMetadata(programIds);

src/domain/entities/Program.ts

@@ -0,0 +1,29 @@
+import { Maybe } from "utils/ts-utils";
+import { Id, NamedRef } from "./Base";
+
+export type ProgramType = "WITH_REGISTRATION" | "WITHOUT_REGISTRATION";
+
+export interface Program {
+    id: Id;
+    name: string;
+    programType: ProgramType;
+    programStages: ProgramStage[];
+}
+
+type ProgramStage = {
+    id: Id;
+    programStageDataElements: ProgramStageDataElement[];
+};
+
+type ProgramStageDataElement = {
+    dataElement: DataElement;
+    displayInReports: boolean;
+};
+
+type DataElement = {
+    id: Id;
+    name: string;
+    code: string;
+    valueType: string;
+    optionSet: Maybe<NamedRef>;
+};

src/domain/entities/ProgramEvent.ts

@@ -21,7 +21,7 @@ export interface ProgramEventToSave {
     program: Ref;
     orgUnit: Ref;
     programStage: Ref;
-    dataValues: EventDataValue[];
+    dataValues: EventDataValueToSave[];
     trackedEntityInstanceId?: Id;
     status: EventStatus;
     date: Timestamp;
@@ -43,6 +43,11 @@ export interface EventDataValue {
     lastUpdated: Timestamp;
 }
 
+export interface EventDataValueToSave {
+    dataElementId: Id;
+    value: string;
+}
+
 export class DuplicatedProgramEvents {
     constructor(
         private options: { ignoreDataElementsIds: Maybe<Id[]>; checkDataElementsIds?: Maybe<Id[]> }

src/domain/repositories/ProgramsRepository.ts

@@ -1,9 +1,11 @@
 import { Async } from "domain/entities/Async";
 import { Id } from "domain/entities/Base";
 import { Timestamp } from "domain/entities/Date";
+import { Program, ProgramType } from "domain/entities/Program";
 import { ProgramExport } from "domain/entities/ProgramExport";
 
 export interface ProgramsRepository {
+    get(options: { ids?: Id[]; programTypes?: ProgramType[] }): Async<Program[]>;
     export(options: { ids: Id[] }): Async<ProgramExport>;
     import(programExport: ProgramExport): Async<void>;
     runRules(options: RunRulesOptions): Async<void>;

src/domain/usecases/ProcessEventsOutsideEnrollmentOrgUnitUseCase.ts

@@ -0,0 +1,188 @@
+import _ from "lodash";
+import { promiseMap } from "data/dhis2-utils";
+import { NamedRef } from "domain/entities/Base";
+import { D2Api } from "types/d2-api";
+import logger from "utils/log";
+import { D2TrackerTrackedEntity } from "@eyeseetea/d2-api/api/trackerTrackedEntities";
+import { D2TrackerEnrollment } from "@eyeseetea/d2-api/api/trackerEnrollments";
+import { D2TrackerEvent } from "@eyeseetea/d2-api/api/trackerEvents";
+import { ProgramsRepository } from "domain/repositories/ProgramsRepository";
+
+export class DetectExternalOrgUnitUseCase {
+    pageSize = 1000;
+
+    constructor(private api: D2Api, private programRepository: ProgramsRepository) {}
+
+    async execute(options: { post: boolean }) {
+        const programs = await this.getPrograms();
+
+        await promiseMap(programs, async program => {
+            await this.fixEventsInProgram({ program: program, post: options.post });
+        });
+    }
+
+    private async getPrograms() {
+        logger.info(`Get tracker programs`);
+        const programs = await this.programRepository.get({ programTypes: ["WITH_REGISTRATION"] });
+        logger.info(`Total tracker programs: ${programs.length}`);
+        return programs;
+    }
+
+    async fixEventsInProgram(options: { program: NamedRef; post: boolean }) {
+        const pageCount = await this.getPageCount(options);
+
+        await promiseMap(_.range(1, pageCount + 1), async page => {
+            await this.fixEventsForPage({ ...options, page: page, pageCount: pageCount });
+        });
+    }
+
+    private async getPageCount(options: { program: NamedRef; post: boolean }) {
+        const response = await this.api.tracker.trackedEntities
+            .get({
+                ...params,
+                page: 1,
+                pageSize: 0,
+                totalPages: true,
+                program: options.program.id,
+            })
+            .getData();
+
+        return Math.ceil((response.total || 0) / this.pageSize);
+    }
+
+    async fixEventsForPage(options: { program: NamedRef; page: number; pageCount: number; post: boolean }) {
+        const trackedEntities = await this.getTrackedEntities(options);
+        const mismatchRecords = this.getMismatchRecords(trackedEntities);
+        const report = this.buildReport(mismatchRecords);
+        logger.info(`Events outside its enrollment orgUnit: ${mismatchRecords.length}`);
+        console.log(report);
+
+        if (_(mismatchRecords).isEmpty()) {
+            logger.debug(`No events outside its enrollment orgUnit`);
+        } else if (!options.post) {
+            logger.info(`Add --post to update events (${mismatchRecords.length})`);
+        } else {
+            await this.fixMismatchEvents(mismatchRecords);
+        }
+    }
+
+    private async fixMismatchEvents(mismatchRecords: MismatchRecord[]) {
+        const eventIds = mismatchRecords.map(obj => obj.event.event);
+        const mismatchRecordsByEventId = _.keyBy(mismatchRecords, obj => obj.event.event);
+
+        logger.info(`Get events to update: ${eventIds.join(",")}`);
+        const { instances: events } = await this.api.tracker.events
+            .get({ fields: { $all: true }, event: eventIds.join(";") })
+            .getData();
+
+        const fixedEvents = events.map((event): typeof event => {
+            const obj = mismatchRecordsByEventId[event.event];
+            if (!obj) throw new Error(`Event not found: ${event.event}`);
+            return { ...event, orgUnit: obj.enrollment.orgUnit };
+        });
+
+        await this.saveEvents(fixedEvents);
+    }
+
+    private async saveEvents(fixedEvents: D2TrackerEvent[]) {
+        logger.info(`Post events: ${fixedEvents.length}`);
+
+        const response = await this.api.tracker
+            .post(
+                {
+                    async: false,
+                    skipPatternValidation: true,
+                    skipSideEffects: true,
+                    skipRuleEngine: true,
+                    importMode: "COMMIT",
+                },
+                { events: fixedEvents }
+            )
+            .getData();
+
+        logger.info(`Post result: ${JSON.stringify(response.stats)}`);
+    }
+
+    private buildReport(mismatchRecords: MismatchRecord[]): string {
+        return mismatchRecords
+            .map(obj => {
+                const { trackedEntity: tei, enrollment: enr, event } = obj;
+
+                const msg = [
+                    `trackedEntity: id=${tei.trackedEntity} orgUnit="${enr.orgUnitName}" [${enr.orgUnit}]`,
+                    `event: id=${event.event} orgUnit="${event.orgUnitName}" [${event.orgUnit}]`,
+                ];
+
+                return msg.join(" - ");
+            })
+            .join("\n");
+    }
+
+    private getMismatchRecords(trackedEntities: D2TrackerTrackedEntity[]): MismatchRecord[] {
+        return _(trackedEntities)
+            .flatMap(trackedEntity => {
+                return _(trackedEntity.enrollments)
+                    .flatMap(enrollment => {
+                        return enrollment.events.map(event => {
+                            if (event.orgUnit !== enrollment.orgUnit) {
+                                return {
+                                    trackedEntity: trackedEntity,
+                                    enrollment: enrollment,
+                                    event: event,
+                                };
+                            }
+                        });
+                    })
+                    .compact()
+                    .value();
+            })
+            .value();
+    }
+
+    private async getTrackedEntities(options: {
+        program: NamedRef;
+        page: number;
+        post: boolean;
+        pageCount: number;
+    }): Promise<D2TrackerTrackedEntity[]> {
+        logger.info(`Get events: page ${options.page} of ${options.pageCount}`);
+
+        const response = await this.api.tracker.trackedEntities
+            .get({
+                ...params,
+                page: options.page,
+                pageSize: this.pageSize,
+                program: options.program.id,
+            })
+            .getData();
+
+        logger.info(`Tracked entities: ${response.instances.length}`);
+
+        return response.instances;
+    }
+}
+
+type MismatchRecord = {
+    trackedEntity: D2TrackerTrackedEntity;
+    enrollment: D2TrackerEnrollment;
+    event: D2TrackerEvent;
+};
+
+const params = {
+    ouMode: "ALL",
+    fields: {
+        trackedEntity: true,
+        orgUnit: true,
+        orgUnitName: true,
+        enrollments: {
+            enrollment: true,
+            orgUnit: true,
+            orgUnitName: true,
+            events: {
+                event: true,
+                orgUnit: true,
+                orgUnitName: true,
+            },
+        },
+    },
+} as const;