Release 2022-34 (#4102)

.eslintrc

@@ -41,7 +41,6 @@
         "packages/indicators/**",
         "packages/lesmis-server/**",
         "packages/meditrak-app-server/**",
-        "packages/pdf-export-server/**",
         "packages/psss-server/**",
         "packages/report-server/**",
         "packages/server-boilerplate/**",

codeship-steps.yml

@@ -23,8 +23,6 @@
               command: './packages/devops/scripts/ci/testBackend.sh central-server'
             - name: Test entity-server
               command: './packages/devops/scripts/ci/testBackend.sh entity-server'
-            - name: Test pdf-export-server
-              command: './packages/devops/scripts/ci/testBackend.sh pdf-export-server'
             - name: Test lesmis-server
               command: './packages/devops/scripts/ci/testBackend.sh lesmis-server'
             - name: Test meditrak-app-server

packages/api-client/src/TupaiaApiClient.ts

@@ -5,19 +5,17 @@
  */
 
 import { AuthHandler } from './types';
-import { ApiConnection, AuthApi, EntityApi, CentralApi, PDFExportApi } from './connections';
+import { ApiConnection, AuthApi, EntityApi, CentralApi } from './connections';
 import { PRODUCTION_BASE_URLS, ServiceBaseUrlSet } from './constants';
 
 export class TupaiaApiClient {
   public readonly entity: EntityApi;
   public readonly central: CentralApi;
   public readonly auth: AuthApi;
-  public readonly pdfExport: PDFExportApi;
 
   public constructor(authHandler: AuthHandler, baseUrls: ServiceBaseUrlSet = PRODUCTION_BASE_URLS) {
     this.auth = new AuthApi(new ApiConnection(authHandler, baseUrls.auth));
     this.entity = new EntityApi(new ApiConnection(authHandler, baseUrls.entity));
     this.central = new CentralApi(new ApiConnection(authHandler, baseUrls.central));
-    this.pdfExport = new PDFExportApi(new ApiConnection(authHandler, baseUrls.pdfExport));
   }
 }

packages/api-client/src/connections/index.ts

@@ -8,4 +8,3 @@ export { ApiConnection } from './ApiConnection';
 export { AuthApi } from './AuthApi';
 export { EntityApi } from './EntityApi';
 export { CentralApi } from './CentralApi';
-export { PDFExportApi } from './PDFExportApi';

packages/api-client/src/constants.ts

@@ -5,7 +5,7 @@
 
 export const DATA_TIME_FORMAT = 'YYYY-MM-DD HH:mm:ss';
 
-type ServiceName = 'auth' | 'entity' | 'pdfExport' | 'central' | 'report';
+type ServiceName = 'auth' | 'entity' | 'central' | 'report';
 export type ServiceBaseUrlSet = Record<ServiceName, string>;
 
 const productionSubdomains = [
@@ -18,7 +18,6 @@ const productionSubdomains = [
   'meditrak-api',
   'psss',
   'psss-api',
-  'pdf-export-api',
   'report-api',
   'entity',
   'entity-api',
@@ -38,11 +37,6 @@ const SERVICES = {
     version: 'v1',
     localPort: '8050',
   },
-  pdfExport: {
-    subdomain: 'pdf-export-api',
-    version: 'v1',
-    localPort: '8010',
-  },
   central: {
     subdomain: 'api',
     version: 'v2',
@@ -61,7 +55,6 @@ export const LOCALHOST_BASE_URLS: ServiceBaseUrlSet = {
   auth: getLocalUrl('auth'),
   entity: getLocalUrl('entity'),
   central: getLocalUrl('central'),
-  pdfExport: getLocalUrl('pdfExport'),
   report: getLocalUrl('report'),
 };
 
@@ -75,15 +68,13 @@ export const DEV_BASE_URLS: ServiceBaseUrlSet = {
   auth: getServiceUrl('auth', 'dev'),
   entity: getServiceUrl('entity', 'dev'),
   central: getServiceUrl('central', 'dev'),
-  pdfExport: getServiceUrl('pdfExport', 'dev'),
   report: getServiceUrl('report', 'dev'),
 };
 
 export const PRODUCTION_BASE_URLS: ServiceBaseUrlSet = {
   auth: getServiceUrl('auth'),
   entity: getServiceUrl('entity'),
   central: getServiceUrl('central'),
-  pdfExport: getServiceUrl('pdfExport'),
   report: getServiceUrl('report'),
 };
 
@@ -121,18 +112,16 @@ const getDefaultBaseUrls = (hostname: string): ServiceBaseUrlSet => {
     auth: getServiceUrlForSubdomain('auth', subdomain),
     entity: getServiceUrlForSubdomain('entity', subdomain),
     central: getServiceUrlForSubdomain('central', subdomain),
-    pdfExport: getServiceUrlForSubdomain('pdfExport', subdomain),
     report: getServiceUrlForSubdomain('report', subdomain),
   };
 };
 
 export const getBaseUrlsForHost = (hostname: string): ServiceBaseUrlSet => {
-  const { auth, entity, central, report, pdfExport } = getDefaultBaseUrls(hostname);
+  const { auth, entity, central, report } = getDefaultBaseUrls(hostname);
   return {
     auth: process.env.AUTH_API_URL || auth,
     entity: process.env.ENTITY_API_URL || entity,
     central: process.env.CENTRAL_API_URL || central,
-    pdfExport: process.env.PDF_EXPORT_API_URL || pdfExport,
     report: process.env.REPORT_API_URL || report,
   };
 };

packages/central-server/package.json

@@ -22,7 +22,8 @@
     "start-dev": "yarn package:start:backend-start-dev 9999",
     "start-verbose": "LOG_LEVEL=debug yarn start-dev",
     "test": "yarn workspace @tupaia/database check-test-database-exists && DB_NAME=tupaia_test NODE_ENV=test mocha",
-    "test:coverage": "cross-env NODE_ENV=test nyc mocha"
+    "test:coverage": "cross-env NODE_ENV=test nyc mocha",
+    "create-meditrak-sync-view": "node dist/createMeditrakSyncView.js"
   },
   "dependencies": {
     "@babel/polyfill": "^7.0.0",

packages/central-server/src/apiV2/changesMetadata.js

@@ -0,0 +1,30 @@
+/**
+ * Tupaia
+ * Copyright (c) 2017 - 2022 Beyond Essential Systems Pty Ltd
+ */
+
+import { respond } from '@tupaia/utils';
+import { buildPermissionsBasedMeditrakSyncQuery } from './utilities';
+import { allowNoPermissions } from '../permissions';
+
+/**
+ * Permissions based sync metadata
+ * {
+ *   changeCount: number of changes since last sync
+ *   countries: countries included in the sync
+ *   permissionGroups: permissions groups included in the sync
+ * }
+ * Responds to GET requests to the /changes/metadata endpoint
+ */
+export async function changesMetadata(req, res) {
+  const { models } = req;
+
+  await req.assertPermissions(allowNoPermissions);
+
+  const { query, countries, permissionGroups } = await buildPermissionsBasedMeditrakSyncQuery(req, {
+    select: 'count(*)',
+  });
+  const queryResult = await query.executeOnDatabase(models.database);
+  const changeCount = parseInt(queryResult[0].count);
+  respond(res, { changeCount, countries, permissionGroups });
+}

packages/central-server/src/apiV2/countChanges/LegacyCountChangesHandler.js

@@ -5,7 +5,7 @@
 
 import { getKeysSortedByValues, respond, UnauthenticatedError } from '@tupaia/utils';
 import { getUniversalTypes } from '../../database/utilities';
-import { fetchRequestingMeditrakDevice, getChangesFilter } from '../utilities';
+import { fetchRequestingMeditrakDevice, buildMeditrakSyncQuery } from '../utilities';
 
 const MAX_FAILS_BEFORE_LOG_OUT = 2;
 const MAX_FAILS_BEFORE_TYPE_EXCLUSION = 5;
@@ -135,11 +135,16 @@ export class LegacyCountChangesHandler {
     const { models, query } = this.req;
 
     const universalTypes = getUniversalTypes(models);
-    const filter = await getChangesFilter({
-      ...this.req,
-      query: { ...query, recordTypes: universalTypes.join(',') },
-    });
-    const changesCount = await models.meditrakSyncQueue.count(filter);
+
+    const { query: dbQuery } = await buildMeditrakSyncQuery(
+      {
+        ...this.req,
+        query: { ...query, recordTypes: universalTypes.join(',') },
+      },
+      { select: 'count(*)' },
+    );
+    const result = await dbQuery.executeOnDatabase(models.database);
+    const changesCount = parseInt(result[0].count);
 
     return changesCount > 0;
   }
@@ -163,8 +168,9 @@ export class LegacyCountChangesHandler {
 
   async handle() {
     await this.handleSetUp();
-    const filter = await getChangesFilter(this.req);
-    const changeCount = await this.req.models.meditrakSyncQueue.count(filter);
+    const { query } = await buildMeditrakSyncQuery(this.req, { select: 'count(*)' });
+    const queryResult = await query.executeOnDatabase(this.req.models.database);
+    const changeCount = parseInt(queryResult[0].count);
     respond(this.res, { changeCount });
   }
 }

packages/central-server/src/apiV2/countChanges/countChanges.js

@@ -4,15 +4,15 @@
  */
 
 import { respond, DatabaseError, UnauthenticatedError } from '@tupaia/utils';
-import { getChangesFilter } from '../utilities';
+import { buildMeditrakSyncQuery } from '../utilities';
 import { LegacyCountChangesHandler } from './LegacyCountChangesHandler';
 import { allowNoPermissions } from '../../permissions';
 
 const handleNonLegacyRequest = async (req, res) => {
   const { models } = req;
-
-  const filter = await getChangesFilter(req);
-  const changeCount = await models.meditrakSyncQueue.count(filter);
+  const { query } = await buildMeditrakSyncQuery(req, { select: 'count(*)' });
+  const queryResult = await query.executeOnDatabase(models.database);
+  const changeCount = parseInt(queryResult[0].count);
   respond(res, { changeCount });
 };
 

packages/central-server/src/apiV2/getChanges.js

@@ -3,17 +3,24 @@
  * Copyright (c) 2017 Beyond Essential Systems Pty Ltd
  */
 
+import keyBy from 'lodash.keyby';
+import groupBy from 'lodash.groupby';
 import { respond, DatabaseError } from '@tupaia/utils';
 import { TYPES } from '@tupaia/database';
-import { getChangesFilter, getColumnsForMeditrakApp } from './utilities';
+import {
+  supportsPermissionsBasedSync,
+  buildMeditrakSyncQuery,
+  buildPermissionsBasedMeditrakSyncQuery,
+  getColumnsForMeditrakApp,
+} from './utilities';
 import { allowNoPermissions } from '../permissions';
 
 const MAX_CHANGES_RETURNED = 100;
 
 /**
  * Gets the record ready to sync down to a sync client, transforming any properties as required
  */
-async function getRecordForSync(record) {
+function getRecordForSync(record) {
   const recordWithoutNulls = {};
   // Remove null entries to a) save bandwidth and b) remain consistent with previous mongo based db
   // which simply had no key for undefined properties, whereas postgres uses null
@@ -34,45 +41,67 @@ async function getRecordForSync(record) {
  */
 export async function getChanges(req, res) {
   const { database, models } = req;
-  const { limit = MAX_CHANGES_RETURNED, offset = 0 } = req.query;
+  const { limit = MAX_CHANGES_RETURNED, offset = 0, appVersion } = req.query;
 
   await req.assertPermissions(allowNoPermissions);
 
   try {
-    const filter = await getChangesFilter(req);
-    const changes = await database.find(TYPES.MEDITRAK_SYNC_QUEUE, filter, {
-      sort: ['change_time'],
+    const queryBuilder = supportsPermissionsBasedSync(appVersion)
+      ? buildPermissionsBasedMeditrakSyncQuery
+      : buildMeditrakSyncQuery;
+    const { query } = await queryBuilder(req, {
+      select: (await models.meditrakSyncQueue.fetchFieldNames()).join(', '),
+      sort: 'change_time ASC',
       limit,
       offset,
     });
-    const changesToSend = await Promise.all(
-      changes.map(async change => {
-        const {
-          type: action,
-          record_type: recordType,
-          record_id: recordId,
-          change_time: timestamp,
-        } = change;
-        const columns = await getColumnsForMeditrakApp(models.getModelForDatabaseType(recordType));
-        const changeObject = { action, recordType, timestamp };
-        if (action === 'delete') {
-          changeObject.record = { id: recordId };
-          if (recordType === TYPES.GEOGRAPHICAL_AREA) {
-            // TODO LEGACY Deal with this bug on app end for v3 api
-            changeObject.recordType = 'area';
-          }
+    const changes = await query.executeOnDatabase(database);
+    const changesByRecordType = groupBy(changes, 'record_type');
+    const recordTypesToSync = Object.keys(changesByRecordType);
+    const columnNamesByRecordType = Object.fromEntries(
+      await Promise.all(
+        recordTypesToSync.map(async recordType => [
+          recordType,
+          await getColumnsForMeditrakApp(models.getModelForDatabaseType(recordType)),
+        ]),
+      ),
+    );
+    const changeRecords = (
+      await Promise.all(
+        Object.entries(changesByRecordType).map(async ([recordType, changesForType]) => {
+          const changeIds = changesForType.map(change => change.record_id);
+          const columns = columnNamesByRecordType[recordType];
+          return database.find(recordType, { id: changeIds }, { lean: true, columns });
+        }),
+      )
+    ).flat();
+    const changeRecordsById = keyBy(changeRecords, 'id');
+
+    const changesToSend = changes.map(change => {
+      const {
+        type: action,
+        record_type: recordType,
+        record_id: recordId,
+        change_time: timestamp,
+      } = change;
+      const changeObject = { action, recordType, timestamp };
+      if (action === 'delete') {
+        changeObject.record = { id: recordId };
+        if (recordType === TYPES.GEOGRAPHICAL_AREA) {
+          // TODO LEGACY Deal with this bug on app end for v3 api
+          changeObject.recordType = 'area';
+        }
+      } else {
+        const record = changeRecordsById[recordId];
+        if (!record) {
+          const errorMessage = `Couldn't find record type ${recordType} with id ${recordId}`;
+          changeObject.error = { error: errorMessage };
         } else {
-          const record = await database.findById(recordType, recordId, { lean: true, columns });
-          if (!record) {
-            const errorMessage = `Couldn't find record type ${recordType} with id ${recordId}`;
-            changeObject.error = { error: errorMessage };
-          } else {
-            changeObject.record = await getRecordForSync(record);
-          }
+          changeObject.record = getRecordForSync(record);
         }
-        return changeObject;
-      }),
-    );
+      }
+      return changeObject;
+    });
     respond(res, changesToSend);
     return;
   } catch (error) {

packages/central-server/src/apiV2/index.js

@@ -18,6 +18,7 @@ import { useRouteHandler } from './RouteHandler';
 import { exportRoutes } from './export';
 import { importRoutes } from './import';
 import { authenticate } from './authenticate';
+import { changesMetadata } from './changesMetadata';
 import { countChanges } from './countChanges';
 import { getChanges } from './getChanges';
 import { BESAdminCreateHandler } from './CreateHandler';
@@ -142,6 +143,7 @@ apiV2.use('/import', importRoutes);
  * GET routes
  */
 apiV2.get('/changes/count', catchAsyncErrors(countChanges));
+apiV2.get('/changes/metadata', catchAsyncErrors(changesMetadata));
 apiV2.get('/changes', catchAsyncErrors(getChanges));
 apiV2.get('/socialFeed', catchAsyncErrors(getSocialFeed));
 apiV2.get('/me', useRouteHandler(GETUserForMe));