Merge pull request #4074 from beyondessential/release-2022-32

Release 2022-32

.eslintrc

@@ -41,6 +41,7 @@
         "packages/indicators/**",
         "packages/lesmis-server/**",
         "packages/meditrak-app-server/**",
+        "packages/pdf-export-server/**",
         "packages/psss-server/**",
         "packages/report-server/**",
         "packages/server-boilerplate/**",

codeship-steps.yml

@@ -23,6 +23,8 @@
               command: './packages/devops/scripts/ci/testBackend.sh central-server'
             - name: Test entity-server
               command: './packages/devops/scripts/ci/testBackend.sh entity-server'
+            - name: Test pdf-export-server
+              command: './packages/devops/scripts/ci/testBackend.sh pdf-export-server'
             - name: Test lesmis-server
               command: './packages/devops/scripts/ci/testBackend.sh lesmis-server'
             - name: Test meditrak-app-server

package.json

@@ -51,9 +51,11 @@
   "devDependencies": {
     "@babel/cli": "^7.11.6",
     "@babel/node": "^7.10.5",
+    "@babel/plugin-transform-runtime": "^7.18.10",
     "@beyondessential/eslint-config-jest": "^1.0.0",
     "@beyondessential/eslint-config-js": "^1.1.1",
     "@beyondessential/eslint-config-ts": "^1.3.2",
+    "babel-eslint": "^10.1.0",
     "babel-jest": "^27.0.6",
     "concurrently": "^5.2.0",
     "cypress-dotenv": "^1.2.2",

packages/admin-panel-server/package.json

@@ -19,7 +19,6 @@
     "build-dev": "npm run build",
     "lint": "yarn package:lint:ts",
     "lint:fix": "yarn lint --fix",
-    "prestart": "npm run build",
     "start": "node dist",
     "start-dev": "yarn package:start:backend-start-dev 9994 -ts",
     "test": "yarn package:test",

packages/admin-panel/src/pages/resources/DataSourcesPage.js

@@ -26,7 +26,7 @@ const DataSourceConfigView = row => {
     .map(([key, value]) => (
       <React.Fragment key={key}>
         <dt style={localStyles.config.dt}>{key}:</dt>
-        <dd>{value.toString()}</dd>
+        <dd>{value ? value.toString() : '""'}</dd>
       </React.Fragment>
     ));
 
@@ -77,9 +77,11 @@ const DATA_ELEMENT_FIELDS = [
       default: '{}',
       getJsonFieldSchema: () => [
         {
-          label: 'Regional Server (Choose "No" if stored on country specific server)',
-          fieldName: 'isDataRegional',
-          type: 'boolean',
+          label: 'DHIS Server',
+          fieldName: 'dhisInstanceCode',
+          optionsEndpoint: 'dhisInstances',
+          optionLabelKey: 'dhisInstances.code',
+          optionValueKey: 'dhisInstances.code',
         },
         {
           label: 'Data element code',
@@ -112,9 +114,11 @@ const DATA_GROUP_FIELDS = [
       default: '{}',
       getJsonFieldSchema: () => [
         {
-          label: 'Regional Server (Choose "No" if stored on country specific server)',
-          fieldName: 'isDataRegional',
-          type: 'boolean',
+          label: 'DHIS Server',
+          fieldName: 'dhisInstanceCode',
+          optionsEndpoint: 'dhisInstances',
+          optionLabelKey: 'dhisInstances.code',
+          optionValueKey: 'dhisInstances.code',
         },
       ],
     },

packages/admin-panel/src/pages/resources/SurveysPage.js

@@ -94,8 +94,8 @@ const SURVEY_COLUMNS = [
           editConfig: {
             options: SERVICE_TYPES,
             setFieldsOnChange: (newValue, currentRecord) => {
-              const { isDataRegional = true } = currentRecord['data_group.config'];
-              const config = newValue === 'dhis' ? { isDataRegional } : {};
+              const { dhisInstanceCode = 'regional' } = currentRecord['data_group.config'];
+              const config = newValue === 'dhis' ? { dhisInstanceCode } : {};
               return { 'data_group.config': config };
             },
           },
@@ -109,10 +109,11 @@ const SURVEY_COLUMNS = [
               recordData['data_group.service_type'] === 'dhis'
                 ? [
                     {
-                      label:
-                        'Stored On Regional Server (Choose "No" if stored on country specific server)',
-                      fieldName: 'isDataRegional',
-                      type: 'boolean',
+                      label: 'DHIS Server',
+                      fieldName: 'dhisInstanceCode',
+                      optionsEndpoint: 'dhisInstances',
+                      optionLabelKey: 'dhisInstances.code',
+                      optionValueKey: 'dhisInstances.code',
                     },
                   ]
                 : [],

packages/api-client/src/TupaiaApiClient.ts

@@ -5,17 +5,19 @@
  */
 
 import { AuthHandler } from './types';
-import { ApiConnection, AuthApi, EntityApi, CentralApi } from './connections';
+import { ApiConnection, AuthApi, EntityApi, CentralApi, PDFExportApi } from './connections';
 import { PRODUCTION_BASE_URLS, ServiceBaseUrlSet } from './constants';
 
 export class TupaiaApiClient {
   public readonly entity: EntityApi;
   public readonly central: CentralApi;
   public readonly auth: AuthApi;
+  public readonly pdfExport: PDFExportApi;
 
   public constructor(authHandler: AuthHandler, baseUrls: ServiceBaseUrlSet = PRODUCTION_BASE_URLS) {
     this.auth = new AuthApi(new ApiConnection(authHandler, baseUrls.auth));
     this.entity = new EntityApi(new ApiConnection(authHandler, baseUrls.entity));
     this.central = new CentralApi(new ApiConnection(authHandler, baseUrls.central));
+    this.pdfExport = new PDFExportApi(new ApiConnection(authHandler, baseUrls.pdfExport));
   }
 }

packages/api-client/src/connections/PDFExportApi.ts

@@ -0,0 +1,20 @@
+/*
+ * Tupaia
+ * Copyright (c) 2017 - 2022 Beyond Essential Systems Pty Ltd
+ *
+ */
+
+import { BaseApi } from './BaseApi';
+
+export class PDFExportApi extends BaseApi {
+  public async getPDF(pdfPageUrl: string) {
+    const { data: bufferObject } = await this.connection.post(
+      `pdf`,
+      {},
+      {
+        pdfPageUrl,
+      },
+    );
+    return { data: Buffer.from(bufferObject) };
+  }
+}

packages/api-client/src/connections/index.ts

@@ -8,3 +8,4 @@ export { ApiConnection } from './ApiConnection';
 export { AuthApi } from './AuthApi';
 export { EntityApi } from './EntityApi';
 export { CentralApi } from './CentralApi';
+export { PDFExportApi } from './PDFExportApi';

packages/api-client/src/constants.ts

@@ -5,7 +5,7 @@
 
 export const DATA_TIME_FORMAT = 'YYYY-MM-DD HH:mm:ss';
 
-type ServiceName = 'auth' | 'entity' | 'central' | 'report';
+type ServiceName = 'auth' | 'entity' | 'pdfExport' | 'central' | 'report';
 export type ServiceBaseUrlSet = Record<ServiceName, string>;
 
 const productionSubdomains = [
@@ -18,6 +18,7 @@ const productionSubdomains = [
   'meditrak-api',
   'psss',
   'psss-api',
+  'pdf-export-api',
   'report-api',
   'entity',
   'entity-api',
@@ -37,6 +38,11 @@ const SERVICES = {
     version: 'v1',
     localPort: '8050',
   },
+  pdfExport: {
+    subdomain: 'pdf-export-api',
+    version: 'v1',
+    localPort: '8010',
+  },
   central: {
     subdomain: 'api',
     version: 'v2',
@@ -55,6 +61,7 @@ export const LOCALHOST_BASE_URLS: ServiceBaseUrlSet = {
   auth: getLocalUrl('auth'),
   entity: getLocalUrl('entity'),
   central: getLocalUrl('central'),
+  pdfExport: getLocalUrl('pdfExport'),
   report: getLocalUrl('report'),
 };
 
@@ -68,13 +75,15 @@ export const DEV_BASE_URLS: ServiceBaseUrlSet = {
   auth: getServiceUrl('auth', 'dev'),
   entity: getServiceUrl('entity', 'dev'),
   central: getServiceUrl('central', 'dev'),
+  pdfExport: getServiceUrl('pdfExport', 'dev'),
   report: getServiceUrl('report', 'dev'),
 };
 
 export const PRODUCTION_BASE_URLS: ServiceBaseUrlSet = {
   auth: getServiceUrl('auth'),
   entity: getServiceUrl('entity'),
   central: getServiceUrl('central'),
+  pdfExport: getServiceUrl('pdfExport'),
   report: getServiceUrl('report'),
 };
 
@@ -112,16 +121,18 @@ const getDefaultBaseUrls = (hostname: string): ServiceBaseUrlSet => {
     auth: getServiceUrlForSubdomain('auth', subdomain),
     entity: getServiceUrlForSubdomain('entity', subdomain),
     central: getServiceUrlForSubdomain('central', subdomain),
+    pdfExport: getServiceUrlForSubdomain('pdfExport', subdomain),
     report: getServiceUrlForSubdomain('report', subdomain),
   };
 };
 
 export const getBaseUrlsForHost = (hostname: string): ServiceBaseUrlSet => {
-  const { auth, entity, central, report } = getDefaultBaseUrls(hostname);
+  const { auth, entity, central, report, pdfExport } = getDefaultBaseUrls(hostname);
   return {
     auth: process.env.AUTH_API_URL || auth,
     entity: process.env.ENTITY_API_URL || entity,
     central: process.env.CENTRAL_API_URL || central,
+    pdfExport: process.env.PDF_EXPORT_API_URL || pdfExport,
     report: process.env.REPORT_API_URL || report,
   };
 };

packages/central-server/package.json

@@ -18,7 +18,6 @@
     "build-dev": "npm run build",
     "lint": "yarn package:lint:js",
     "lint:fix": "yarn lint --fix",
-    "prestart": "npm run build",
     "start": "node dist",
     "start-dev": "yarn package:start:backend-start-dev 9999",
     "start-verbose": "LOG_LEVEL=debug yarn start-dev",

packages/central-server/src/apiV2/import/importEntities/getOrCreateParentEntity.js

@@ -64,7 +64,7 @@ export async function getOrCreateParentEntity(
     };
     district = await transactingModels.geographicalArea.findOrCreate(districtObject);
     const defaultMetadata = {
-      dhis: { isDataRegional: true },
+      dhis: { dhisInstanceCode: 'regional' },
       openStreetMaps: { id: districtOsmId },
     };
     const districtEntityMetadata = await getEntityMetadata(
@@ -96,7 +96,7 @@ export async function getOrCreateParentEntity(
       level_name: 'Sub District',
     };
     const defaultMetadata = {
-      dhis: { isDataRegional: true },
+      dhis: { dhisInstanceCode: 'regional' },
       openStreetMaps: { id: subDistrictOsmId },
     };
     const subDistrictEntityMetadata = await getEntityMetadata(

packages/central-server/src/apiV2/import/importEntities/updateCountryEntities.js

@@ -84,7 +84,7 @@ export async function updateCountryEntities(
     type: transactingModels.entity.types.WORLD,
   });
   const defaultMetadata = {
-    dhis: { isDataRegional: true },
+    dhis: { dhisInstanceCode: 'regional' },
   };
   const countryEntityMetadata = await getEntityMetadata(
     transactingModels,

packages/central-server/src/apiV2/index.js

@@ -209,6 +209,7 @@ apiV2.get('/clinics/:recordId?', useRouteHandler(GETClinics));
 apiV2.get('/facilities/:recordId?', useRouteHandler(GETClinics));
 apiV2.get('/geographicalAreas/:recordId?', useRouteHandler(GETGeographicalAreas));
 apiV2.get('/reports/:recordId?', useRouteHandler(GETReports));
+apiV2.get('/dhisInstances/:recordId?', useRouteHandler(BESAdminGETHandler));
 
 /**
  * POST routes

packages/central-server/src/database/models/DataElement.js

@@ -28,7 +28,7 @@ export class DataElementType extends CommonDataElementType {
       },
       {
         service_type: 'dhis',
-        config: { isDataRegional: this.config.isDataRegional },
+        config: { dhisInstanceCode: this.config.dhisInstanceCode },
       },
     );
     await this.attachDataElement(dataElementId);

packages/central-server/src/database/models/DataGroup.js

@@ -25,7 +25,7 @@ export class DataGroupType extends CommonDataGroupType {
       },
       {
         service_type: 'dhis',
-        config: { isDataRegional: this.config.isDataRegional },
+        config: { dhisInstanceCode: this.config.dhisInstanceCode },
       },
     );
     await this.attachDataElement(dataElementId);

packages/central-server/src/database/utilities/assertCanAddDataElementInGroup.js

@@ -3,11 +3,16 @@
  * Copyright (c) 2017 - 2020 Beyond Essential Systems Pty Ltd
  */
 
-const areBothDefinedAndDifferent = (oldValue, newValue) =>
-  oldValue !== undefined && newValue !== undefined && oldValue !== newValue;
+const areBothDefinedAndDifferent = (a, b) => a !== undefined && b !== undefined && a !== b;
 
-const constructErrorMessage = ({ property, value, dataElementCode, otherGroupCode }) =>
-  `Cannot set ${property} to '${value}': data element '${dataElementCode}' is included in data group '${otherGroupCode}', which uses another ${property}`;
+const constructErrorMessage = ({
+  property,
+  newValue,
+  dataElementCode,
+  otherGroupCode,
+  otherValue,
+}) =>
+  `Cannot set ${property} to '${newValue}': data element '${dataElementCode}' is included in data group '${otherGroupCode}', which has ${property}: '${otherValue}'. These must match.`;
 
 export const assertCanAddDataElementInGroup = async (
   models,
@@ -23,30 +28,32 @@ export const assertCanAddDataElementInGroup = async (
   });
   const otherDataGroups = dataGroups.filter(({ code }) => code !== dataGroupCode);
   otherDataGroups.forEach(otherDataGroup => {
-    const { service_type: serviceType, config } = otherDataGroup;
+    const { service_type: otherServiceType, config: otherConfig } = otherDataGroup;
 
     // Tupaia data elements can be in either dhis or tupaia data groups
     if (
       dataElement.service_type !== 'tupaia' &&
-      areBothDefinedAndDifferent(serviceType, newServiceType)
+      areBothDefinedAndDifferent(otherServiceType, newServiceType)
     ) {
       throw new Error(
         constructErrorMessage({
           property: 'service type',
-          value: newServiceType,
+          newValue: newServiceType,
           dataElementCode,
           otherGroupCode: otherDataGroup.code,
+          otherValue: otherServiceType,
         }),
       );
     }
 
-    if (areBothDefinedAndDifferent(config.isDataRegional, newConfig.isDataRegional)) {
+    if (otherConfig.dhisInstanceCode !== newConfig.dhisInstanceCode) {
       throw new Error(
         constructErrorMessage({
           property: 'DHIS server',
-          value: `${config.isDataRegional ? '' : 'non '}regional`,
+          newValue: newConfig.dhisInstanceCode,
           dataElementCode,
           otherGroupCode: otherDataGroup.code,
+          otherValue: otherConfig.dhisInstanceCode,
         }),
       );
     }

packages/central-server/src/dhis/DhisChangeDetailGenerator.js

@@ -69,7 +69,10 @@ export class DhisChangeDetailGenerator extends ChangeDetailGenerator {
       },
     );
 
-    return reduceToDictionary(surveyData, 'id', ({ config }) => config.isDataRegional);
+    return reduceToDictionary(surveyData, 'id', ({ config }) => {
+      // TODO: migrate dhis sync to use new dhisInstanceCode instead of legacy config
+      return config.dhisInstanceCode === 'regional';
+    });
   };
 
   generateDetails = async updateChanges => {

packages/central-server/src/dhis/getDhisApiInstance.js

@@ -4,12 +4,12 @@
  */
 
 import { DhisApi } from '@tupaia/dhis-api';
-import { getDhisConfig } from '@tupaia/utils';
+import { legacy_getDhisConfig } from '@tupaia/utils';
 
 const instances = {};
 
 export const getDhisApiInstance = options => {
-  const { serverName, serverUrl, serverReadOnly } = getDhisConfig(options);
+  const { serverName, serverUrl, serverReadOnly } = legacy_getDhisConfig(options);
   if (!instances[serverName]) {
     instances[serverName] = new DhisApi(serverName, serverUrl, serverReadOnly);
   }

packages/central-server/src/dhis/pushers/Pusher.js

@@ -48,6 +48,8 @@ export class Pusher {
       await this.logResults(results); // await to avoid db lock between delete/update on event push
       return results.wasSuccessful;
     } catch (error) {
+      winston.info('Error while attempting to push to DHIS');
+      winston.info(error.stack);
       await this.logResults({ errors: [error.message] });
       return false;
     }
@@ -102,9 +104,6 @@ export class Pusher {
    * @returns {Promise<>}
    */
   async logResults({ counts, errors = [], data, reference, references = [] }) {
-    if (errors.length > 0) {
-      winston.warn(errors);
-    }
     const errorString = errors.reduce((allErrors, error) => `${allErrors}\n${error}`, '');
     const logRecord = {
       record_type: this.recordType,