RN-646 & RN-629 & RN-630: Added data-table-server (#4152)

.eslintrc

@@ -37,6 +37,7 @@
         "packages/api-client/**",
         "packages/data-api/**",
         "packages/data-lake-api/**",
+        "packages/data-table-server/**",
         "packages/entity-server/**",
         "packages/indicators/**",
         "packages/lesmis-server/**",

codeship-steps.yml

@@ -21,6 +21,8 @@
               command: './packages/devops/scripts/ci/testBackend.sh admin-panel-server'
             - name: Test central-server
               command: './packages/devops/scripts/ci/testBackend.sh central-server'
+            - name: Test data-table-server
+              command: './packages/devops/scripts/ci/testBackend.sh data-table-server'
             - name: Test entity-server
               command: './packages/devops/scripts/ci/testBackend.sh entity-server'
             - name: Test lesmis-server

packages/data-table-server/.env.example

@@ -0,0 +1,22 @@
+AGGREGATION_URL_PREFIX=
+DB_NAME=
+DB_PASSWORD=
+DB_URL=
+DB_PORT=
+DB_USER=
+DHIS_CLIENT_ID=
+DHIS_CLIENT_SECRET=
+DHIS_PASSWORD=
+DHIS_USERNAME=
+JWT_SECRET=
+PORT=
+API_CLIENT_SALT=
+ENTITY_API_URL=
+DATA_LAKE_DB_NAME=
+DATA_LAKE_DB_PASSWORD=
+DATA_LAKE_DB_URL=
+DATA_LAKE_DB_USER=
+SUPERSET_API_USERNAME=
+SUPERSET_API_PASSWORD=
+PALAU_DHIS_CLIENT_SECRET=
+PALAU_DHIS_PASSWORD=

packages/data-table-server/README.md

@@ -0,0 +1,3 @@
+# @tupaia/data-table-server
+
+Microservice for querying Tupaia Data Tables

packages/data-table-server/examples.http

@@ -0,0 +1,30 @@
+@hostname = localhost
+@port = 8010
+@host = {{hostname}}:{{port}}
+@version = v1
+
+@contentType = application/json
+
+# In order to setup authorization, please set 'email' and 'password' in your restClient environement variables 
+# see: https://marketplace.visualstudio.com/items?itemName=humao.rest-client#environment-variables
+@authorization  = Basic {{email}}:{{password}}
+
+
+### /test
+GET http://{{host}}/{{version}}/test HTTP/2.0
+content-type: {{contentType}}
+Authorization: {{authorization}}
+
+
+### Fetch data from analytics data-table
+POST http://{{host}}/{{version}}/dataTable/analytics/fetchData HTTP/1.1
+content-type: {{contentType}}
+Authorization: {{authorization}}
+
+{
+    "dataElementCodes": ["PSSS_AFR_Cases"],
+    "organisationUnitCodes": ["TO"],
+    "hierarchy": "psss",
+    "startDate": "2020-01-01",
+    "endDate" : "2020-12-31"
+}

packages/data-table-server/jest.config.ts

@@ -0,0 +1,12 @@
+/**
+ * Tupaia
+ * Copyright (c) 2017 - 2022 Beyond Essential Systems Pty Ltd
+ */
+
+import baseConfig from '../../jest.config-ts.json';
+
+module.exports = async () => ({
+  ...baseConfig,
+  rootDir: '.',
+  setupFilesAfterEnv: ['../../jest.setup.js'],
+});

packages/data-table-server/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@tupaia/data-table-server",
+  "version": "1.0.0",
+  "private": true,
+  "description": "Microservice for querying Tupaia Data Tables",
+  "homepage": "https://github.com/beyondessential/tupaia",
+  "bugs": {
+    "url": "https://github.com/beyondessential/tupaia/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/beyondessential/tupaia"
+  },
+  "author": "Beyond Essential Systems <admin@tupaia.org> (https://beyondessential.com.au)",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "build": "rm -rf dist && npm run --prefix ../../ package:build:ts",
+    "build-dev": "npm run build",
+    "lint": "yarn package:lint:ts",
+    "lint:fix": "yarn lint --fix",
+    "start": "node dist",
+    "start-dev": "LOG_LEVEL=debug yarn package:start:backend-start-dev 9998 -ts",
+    "start-verbose": "LOG_LEVEL=debug yarn start-dev",
+    "test": "yarn package:test"
+  },
+  "dependencies": {
+    "@tupaia/access-policy": "3.0.0",
+    "@tupaia/aggregator": "1.0.0",
+    "@tupaia/api-client": "3.1.0",
+    "@tupaia/data-broker": "1.0.0",
+    "@tupaia/database": "1.0.0",
+    "@tupaia/server-boilerplate": "1.0.0",
+    "@tupaia/utils": "1.0.0",
+    "dotenv": "^8.2.0",
+    "express": "^4.16.2",
+    "winston": "^3.2.1"
+  }
+}

packages/data-table-server/src/@types/express/index.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Tupaia
+ * Copyright (c) 2017 - 2022 Beyond Essential Systems Pty Ltd
+ */
+
+import { AccessPolicy } from '@tupaia/access-policy';
+import { TupaiaApiClient } from '@tupaia/api-client';
+import { DataTableServerModelRegistry } from '../../types';
+
+declare global {
+  namespace Express {
+    export interface Request {
+      accessPolicy: AccessPolicy;
+      models: DataTableServerModelRegistry;
+      ctx: { services: TupaiaApiClient };
+    }
+  }
+}

packages/data-table-server/src/__tests__/dataTableService/createDataTableService.test.ts

@@ -0,0 +1,56 @@
+/**
+ * Tupaia
+ * Copyright (c) 2017 - 2022 Beyond Essential Systems Pty Ltd
+ */
+
+import { TupaiaApiClient } from '@tupaia/api-client';
+import { DataTableType as DataTableTypeClass } from '@tupaia/database';
+import { createDataTableService } from '../../dataTableService';
+import { AnalyticsDataTableService } from '../../dataTableService/internal/AnalyticsDataTableService';
+import { DataTableType } from '../../models';
+
+describe('createDataTableService', () => {
+  describe('error cases', () => {
+    it('throws an error for an unknown data-table type', () => {
+      const dataTableWithUnknownType = new DataTableTypeClass(
+        {},
+        { type: 'unknown' },
+      ) as DataTableType;
+
+      const createUnknownTypeDataTableService = () =>
+        createDataTableService(dataTableWithUnknownType, {} as TupaiaApiClient);
+
+      expect(createUnknownTypeDataTableService).toThrow(
+        'Cannot build data table for type: unknown',
+      );
+    });
+
+    it('throws an error for an unknown internal data-table', () => {
+      const unknownInternalDataTable = new DataTableTypeClass(
+        {},
+        { type: 'internal', code: 'unknown' },
+      ) as DataTableType;
+
+      const createUnknownInternalDataTableService = () =>
+        createDataTableService(unknownInternalDataTable, {} as TupaiaApiClient);
+
+      expect(createUnknownInternalDataTableService).toThrow(
+        'No internal data-table defined for unknown',
+      );
+    });
+  });
+
+  it('can create an internal data-table service', () => {
+    const analyticsDataTable = new DataTableTypeClass(
+      {},
+      { type: 'internal', code: 'analytics' },
+    ) as DataTableType;
+
+    const analyticsDataTableService = createDataTableService(
+      analyticsDataTable,
+      {} as TupaiaApiClient,
+    );
+
+    expect(analyticsDataTableService instanceof AnalyticsDataTableService).toBe(true);
+  });
+});

packages/data-table-server/src/__tests__/dataTableService/internal/AnalyticsDataTableService.test.ts

@@ -0,0 +1,172 @@
+/**
+ * Tupaia
+ * Copyright (c) 2017 - 2022 Beyond Essential Systems Pty Ltd
+ */
+
+import { TupaiaApiClient } from '@tupaia/api-client';
+import { DataTableType as DataTableTypeClass } from '@tupaia/database';
+import { createDataTableService } from '../../../dataTableService';
+import { DataTableType } from '../../../models';
+
+const TEST_ANALYTICS = [
+  { period: '2020-01-01', organisationUnit: 'TO', dataElement: 'PSSS_AFR_Cases', value: 7 },
+  { period: '2020-01-08', organisationUnit: 'TO', dataElement: 'PSSS_AFR_Cases', value: 12 },
+  { period: '2020-01-15', organisationUnit: 'PG', dataElement: 'PSSS_AFR_Cases', value: 8 },
+  { period: '2020-01-01', organisationUnit: 'PG', dataElement: 'PSSS_ILI_Cases', value: 7 },
+  { period: '2020-01-08', organisationUnit: 'PG', dataElement: 'PSSS_ILI_Cases', value: 12 },
+  { period: '2020-01-15', organisationUnit: 'TO', dataElement: 'PSSS_ILI_Cases', value: 8 },
+];
+
+const fetchFakeAnalytics = (
+  dataElementCodes: string[],
+  {
+    organisationUnitCodes,
+    startDate = '2020-01-01',
+    endDate = '2020-12-31',
+  }: { organisationUnitCodes: string[]; startDate?: string; endDate?: string },
+) => {
+  return {
+    results: TEST_ANALYTICS.filter(
+      analytic =>
+        dataElementCodes.includes(analytic.dataElement) &&
+        organisationUnitCodes.includes(analytic.organisationUnit) &&
+        analytic.period >= startDate &&
+        analytic.period <= endDate,
+    ),
+  };
+};
+
+jest.mock('@tupaia/aggregator', () => ({
+  Aggregator: jest.fn().mockImplementation(() => ({
+    fetchAnalytics: fetchFakeAnalytics,
+  })),
+}));
+
+jest.mock('@tupaia/data-broker', () => ({
+  DataBroker: jest.fn().mockImplementation(() => ({})),
+}));
+
+const analyticsDataTable = new DataTableTypeClass(
+  {},
+  { type: 'internal', code: 'analytics' },
+) as DataTableType;
+
+describe('AnalyticsDataTableService', () => {
+  describe('parameter validation', () => {
+    const testData: [string, unknown, string][] = [
+      [
+        'missing organisationUnitCodes',
+        {
+          hierarchy: 'psss',
+          dataElementCodes: ['PSSS_AFR_Cases'],
+        },
+        'organisationUnitCodes is a required field',
+      ],
+      [
+        'missing hierarchy',
+        {
+          organisationUnitCodes: ['TO'],
+          dataElementCodes: ['PSSS_AFR_Cases'],
+        },
+        'hierarchy is a required field',
+      ],
+      [
+        'missing dataElementCodes',
+        {
+          organisationUnitCodes: ['TO'],
+          hierarchy: 'psss',
+        },
+        'dataElementCodes is a required field',
+      ],
+      [
+        'startDate wrong format',
+        {
+          organisationUnitCodes: ['TO'],
+          hierarchy: 'psss',
+          dataElementCodes: ['PSSS_AFR_Cases'],
+          startDate: 'cat',
+        },
+        'startDate should be in ISO 8601 format',
+      ],
+      [
+        'endDate wrong format',
+        {
+          organisationUnitCodes: ['TO'],
+          hierarchy: 'psss',
+          dataElementCodes: ['PSSS_AFR_Cases'],
+          endDate: 'dog',
+        },
+        'endDate should be in ISO 8601 format',
+      ],
+      [
+        'aggregations wrong format',
+        {
+          organisationUnitCodes: ['TO'],
+          hierarchy: 'psss',
+          dataElementCodes: ['PSSS_AFR_Cases'],
+          aggregations: ['RAW'],
+        },
+        'aggregations[0] must be a `object` type',
+      ],
+    ];
+
+    it.each(testData)('%s', (_, parameters: unknown, expectedError: string) => {
+      const analyticsDataTableService = createDataTableService(
+        analyticsDataTable,
+        {} as TupaiaApiClient,
+      );
+
+      expect(() => analyticsDataTableService.fetchData(parameters)).toThrow(expectedError);
+    });
+  });
+
+  it('can fetch data from Aggregator.fetchAnalytics()', async () => {
+    const analyticsDataTableService = createDataTableService(
+      analyticsDataTable,
+      {} as TupaiaApiClient,
+    );
+
+    const dataElementCodes = ['PSSS_AFR_Cases'];
+    const organisationUnitCodes = ['TO'];
+
+    const analytics = await analyticsDataTableService.fetchData({
+      hierarchy: 'psss',
+      organisationUnitCodes,
+      dataElementCodes,
+    });
+
+    const { results: expectedAnalytics } = fetchFakeAnalytics(dataElementCodes, {
+      organisationUnitCodes,
+    });
+
+    expect(analytics).toEqual(expectedAnalytics);
+  });
+
+  it('passes all parameters to Aggregator.fetchAnalytics()', async () => {
+    const analyticsDataTableService = createDataTableService(
+      analyticsDataTable,
+      {} as TupaiaApiClient,
+    );
+
+    const dataElementCodes = ['PSSS_AFR_Cases', 'PSSS_ILI_Cases'];
+    const organisationUnitCodes = ['PG'];
+    const startDate = '2020-01-05';
+    const endDate = '2020-01-10';
+
+    const analytics = await analyticsDataTableService.fetchData({
+      hierarchy: 'psss',
+      organisationUnitCodes,
+      dataElementCodes,
+      startDate,
+      endDate,
+    });
+
+    const { results: expectedAnalytics } = fetchFakeAnalytics(dataElementCodes, {
+      organisationUnitCodes,
+      startDate,
+      endDate,
+    });
+
+    expect(analytics).toEqual(expectedAnalytics);
+  });
+});