Release 2024-15

.gitattributes

@@ -1 +1,5 @@
+*.md linguist-documentation
 *.pbxproj -text
+packages/types/src/schemas/schemas.ts linguist-generated=true
+packages/types/src/types/models.ts linguist-generated=true
+yarn.lock linguist-generated=true

.github/CONTRIBUTING.md

@@ -101,6 +101,49 @@ your changes and provide feedback. If your pull request has been approved by a r
 will then go through a round of internal testing. Once your changes have passed testing, they
 can be merged into main and be included in an upcoming Tupaia release.
 
+#### Pull request title rules
+
+We use a squash-and-merge strategy when merging PRs into dev. This means the commit messages in dev will match the PR titles. We like to keep standardised commit messages as an easy way to track release changes and for record keeping purposes.
+
+The title of pull requests must be in [Conventional Commit](https://beyond-essential.slab.com/posts/pr-titles-conventional-commits-avgsj3xb) format. This is used to generate
+changelogs, and to provide a consistent format for commits landing in the main branch, as pull
+requests are merged in “squash” mode.
+
+```plain
+type: <description>
+type(scope): <description>
+```
+
+When a Linear card is applicable, the Linear card number should be included:
+
+```plain
+type: TEAM-123: <description>
+type(scope): TEAM-123: <description>
+```
+
+The following types are conventional:
+
+- `ci` for changes to the CI/CD workflows
+- `db` for changes to the database schema, migrations, etc
+- `deps` for changes to dependencies or dependency upgrades
+- `doc` for documentation changes
+- `env` for changes to the environment variables
+- `feat` for new features
+- `fix` for bug fixes
+- `fmt` for automatic formatting changes
+- `merge` for merging between branches (generally between `master` and `dev`)
+- `refactor` for code refactoring
+- `repo` for changes to the repository structure (e.g. `.gitignore`, `.editorconfig`, etc)
+- `revert` for reverting a previous commit
+- `style` for stylistic changes that do not affect the meaning of the code
+- `test` for adding missing tests or correcting existing tests
+- `tweak` for minor changes that do not fit into any other category
+
+When merging, additional change lines may be added to the squashed commit message to provide further
+context to be pulled into changelogs.
+
+Using Conventional Commit format for actual commit messages within pull requests is not required.
+
 ### Note on Forking
 
 While Tupaia is open-source, there is always the possibility for forking the repository, which

.github/workflows/check-pr-title.yaml

@@ -0,0 +1,51 @@
+name: Check PR title
+
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - synchronize
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    env:
+      PR_TITLE: ${{ github.event.pull_request.title }}
+    steps:
+      - id: release
+        name: Check for release PR format
+        if: startsWith(github.event.pull_request.title, 'Release')
+        shell: bash
+        run: |
+          if ! grep -qP '^Release [0-9]{4}-[0-9]{2}$' <<< "$PR_TITLE"; then
+            echo "::warning::Release PR title should be: 'Release <year>-<ISO week number>'"
+            exit 1
+          fi
+
+      - name: Check for Conventional Commit format
+        shell: bash
+        if: steps.release.conclusion == 'skipped'
+        run: |
+          if ! grep -qP '^\w+(\(\w+\))?:\s' <<< "$PR_TITLE"; then
+            echo "::warning::PR title should be in Conventional Commit style, e.g. 'feat: ...'"
+            exit 1
+          fi
+
+      - name: Check for conventional type allow-list
+        if: steps.release.conclusion == 'skipped'
+        shell: bash
+        run: |
+          if ! grep -qP '^(ci|db|deps|doc|env|feat|fix|fmt|merge|refactor|repo|revert|style|test|tweak)(\(\w+\))?:\s' <<< "$PR_TITLE"; then
+            echo "::warning::PR title Conventional Type is not on the list; refer to CONTRIBUTING.md"
+            exit 1
+          fi
+
+      - name: Check for Linear card number for feature branches
+        if: steps.release.conclusion == 'skipped' && startsWith(github.event.pull_request.title, 'feat')
+        shell: bash
+        run: |
+          if ! grep -qP '^\w+(\(\w+\))?:\s[A-Z]+-[0-9]+(:\s+\w+)?' <<< "$PR_TITLE"; then
+            echo "::warning::PR title should start with ticket number, e.g. 'feat(scope): ABC-123: ...'"
+            exit 1
+          fi

.github/workflows/ci.yaml

@@ -95,6 +95,10 @@ jobs:
       API_CLIENT_PASSWORD: api-client
       API_CLIENT_SALT: abc132
       JWT_SECRET: abc132
+      TUPAIA_ADMIN_EMAIL_ADDRESS: test@tupaia.org
+      TUPAIA_FRONT_END_URL: dev.tupaia.org
+      LESMIS_FRONT_END_URL: dev-lesmis.tupaia.org
+      DATATRAK_FRONT_END_URL: dev-datatrak.tupaia.org
 
       # test database
       DB_PG_USER: postgres

.gitignore

@@ -1,5 +1,6 @@
 *.env*
 !*.env.example*
+!*.env.*.example*
 !*.env.encrypted*
 .DS_Store
 **/node_modules

env/aggregation.env.example

@@ -0,0 +1 @@
+AGGREGATION_URL_PREFIX=

env/api-client.env.example

@@ -0,0 +1,7 @@
+AUTH_API_URL=
+DATA_TABLE_API_URL=
+CENTRAL_API_URL=
+ENTITY_API_URL=
+REPORT_API_URL=
+WEB_CONFIG_API_URL=
+API_CLIENT_SALT=

env/aws.env.example

@@ -0,0 +1,3 @@
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+AWS_REGION=

env/data-lake.env.example

@@ -0,0 +1,4 @@
+DATA_LAKE_DB_NAME=
+DATA_LAKE_DB_PASSWORD=
+DATA_LAKE_DB_URL=
+DATA_LAKE_DB_USER=

packages/auth/.env.example → env/db.env.example

@@ -1,4 +1,3 @@
-DB_DISABLE_SSL=
 DB_NAME=
 DB_PASSWORD=
 DB_URL=

env/dhis.env.example

@@ -0,0 +1,8 @@
+PALAU_DHIS_CLIENT_SECRET=
+PALAU_DHIS_PASSWORD=
+PALAU_DHIS_API_URL=
+DHIS_CLIENT_ID=
+DHIS_CLIENT_SECRET=
+DHIS_PASSWORD=
+DHIS_USERNAME=
+IS_PRODUCTION_ENVIRONMENT=

env/external-db-connections.env.example

@@ -0,0 +1,15 @@
+EXT_DB_niwa_PORT=
+EXT_DB_niwa_DATABASE=
+EXT_DB_niwa_USER=
+EXT_DB_niwa_PASSWORD=
+EXT_DB_niwa_HOST=
+EXT_DB_data_lake_HOST=
+EXT_DB_data_lake_PORT=
+EXT_DB_data_lake_DATABASE=
+EXT_DB_data_lake_USER=
+EXT_DB_data_lake_PASSWORD=
+EXT_DB_msupply_scheduled_HOST=
+EXT_DB_msupply_scheduled_PORT=
+EXT_DB_msupply_scheduled_DATABASE=
+EXT_DB_msupply_scheduled_USER=
+EXT_DB_msupply_scheduled_PASSWORD=

env/mail.env.example

@@ -0,0 +1,5 @@
+SITE_EMAIL_ADDRESS=
+SMTP_HOST=
+SMTP_PASSWORD=
+SMTP_USER=
+DEV_EMAIL_ADDRESS=

env/pg.env.example

@@ -0,0 +1,4 @@
+DB_PG_USER=
+DB_PG_PASSWORD= 
+# used in CI/CD, and for running analytics refresh via CLI
+DB_ENABLE_SSL=

env/servers.env.example

@@ -0,0 +1,5 @@
+
+JWT_SECRET=
+SESSION_COOKIE_SECRET=
+
+

env/superset.env.example

@@ -0,0 +1,3 @@
+SUPERSET_API_PROXY_URL=
+SUPERSET_API_USERNAME=
+SUPERSET_API_PASSWORD=

env/weatherbit.env.example

@@ -0,0 +1 @@
+WEATHERBIT_API_KEY=

jest.setup.js

@@ -1,6 +1,6 @@
 /**
  * Tupaia
- * Copyright (c) 2017 - 2020 Beyond Essential Systems Pty Ltd
+ * Copyright (c) 2017 - 2024 Beyond Essential Systems Pty Ltd
  */
 
 /* eslint-env jest */

package.json

@@ -72,7 +72,7 @@
     "babel-preset-react-app": "^10.0.0",
     "concurrently": "^5.2.0",
     "cypress-dotenv": "^1.2.2",
-    "dotenv": "^16.0.3",
+    "dotenv": "16.4.5",
     "eslint": "^7.9.0",
     "eslint-import-resolver-babel-module": "^5.3.1",
     "eslint-plugin-cypress": "^2.11.1",
@@ -87,7 +87,7 @@
     "ts-jest": "^29.1.2",
     "ts-node": "^10.7.0",
     "typescript": "^5.2.2",
-    "vite": "^4.4.8",
+    "vite": "^4.5.3",
     "vite-plugin-compression": "^0.5.1",
     "vite-plugin-ejs": "^1.6.4",
     "yargs": "15.4.1"

packages/admin-panel-server/.env.example

@@ -1,16 +1,4 @@
 PORT=
 
 API_CLIENT_NAME=
-API_CLIENT_PASSWORD=
-API_CLIENT_SALT=
-
-DB_URL=
-DB_NAME=
-DB_PASSWORD=
-DB_USER=
-
-SESSION_COOKIE_SECRET=
-
-CENTRAL_API_URL=
-ENTITY_API_URL=
-REPORT_API_URL=
+API_CLIENT_PASSWORD= 

packages/admin-panel-server/package.json

@@ -30,6 +30,7 @@
     "@tupaia/api-client": "workspace:*",
     "@tupaia/database": "workspace:*",
     "@tupaia/server-boilerplate": "workspace:*",
+    "@tupaia/server-utils": "workspace:*",
     "@tupaia/types": "workspace:*",
     "@tupaia/utils": "workspace:*",
     "api-error-handler": "^1.0.0",
@@ -38,11 +39,11 @@
     "case": "^1.6.3",
     "client-sessions": "^0.8.0",
     "cors": "^2.8.5",
-    "dotenv": "^8.2.0",
-    "express": "^4.16.2",
+    "express": "^4.19.2",
     "lodash": "^4.17.4",
     "multer": "^1.4.3",
-    "winston": "^3.3.3"
+    "winston": "^3.3.3",
+    "xlsx": "^0.10.9"
   },
   "devDependencies": {
     "@types/multer": "^1.4.7"

packages/admin-panel-server/src/app/createApp.ts

@@ -4,7 +4,7 @@
  */
 import { TupaiaDatabase } from '@tupaia/database';
 import { OrchestratorApiBuilder, forwardRequest, handleWith } from '@tupaia/server-boilerplate';
-
+import { getEnvVarOrDefault } from '@tupaia/utils';
 import { AdminPanelSessionModel } from '../models';
 import { hasTupaiaAdminPanelAccess } from '../utils';
 import { upload } from '../middleware';
@@ -42,18 +42,17 @@ import {
   FetchTransformSchemasRoute,
   FetchDataTableBuiltInParamsRequest,
   FetchDataTableBuiltInParamsRoute,
+  ExportEntityHierarchiesRequest,
+  ExportEntityHierarchiesRoute,
 } from '../routes';
 import { authHandlerProvider } from '../auth';
 
-const {
-  CENTRAL_API_URL = 'http://localhost:8090/v2',
-  ENTITY_API_URL = 'http://localhost:8050/v1',
-} = process.env;
-
 /**
  * Set up express server with middleware,
  */
 export async function createApp() {
+  const CENTRAL_API_URL = getEnvVarOrDefault('CENTRAL_API_URL', 'http://localhost:8090/v2');
+  const ENTITY_API_URL = getEnvVarOrDefault('ENTITY_API_URL', 'http://localhost:8050/v1');
   const forwardToEntityApi = forwardRequest(ENTITY_API_URL);
   const builder = new OrchestratorApiBuilder(new TupaiaDatabase(), 'admin-panel')
     .attachApiClientToContext(authHandlerProvider)
@@ -105,6 +104,10 @@ export async function createApp() {
       'export/mapOverlayVisualisation/:mapOverlayVisualisationId',
       handleWith(ExportMapOverlayVisualisationRoute),
     )
+    .get<ExportEntityHierarchiesRequest>(
+      'export/hierarchies',
+      handleWith(ExportEntityHierarchiesRoute),
+    )
     .post<ExportDashboardVisualisationRequest>(
       'export/dashboardVisualisation',
       handleWith(ExportDashboardVisualisationRoute),

packages/admin-panel-server/src/constants.ts

@@ -6,4 +6,3 @@
 
 export const BES_ADMIN_PERMISSION_GROUP = 'BES Admin';
 export const TUPAIA_ADMIN_PANEL_PERMISSION_GROUP = 'Tupaia Admin Panel';
-export const VIZ_BUILDER_USER_PERMISSION_GROUP = 'Viz Builder User';

packages/admin-panel-server/src/index.ts

@@ -4,14 +4,21 @@
  */
 
 import http from 'http';
-import * as dotenv from 'dotenv';
+import path from 'path';
 
 import winston from 'winston';
 import { configureWinston } from '@tupaia/server-boilerplate';
+import { configureDotEnv } from '@tupaia/server-utils';
 import { createApp } from './app';
 
 configureWinston();
-dotenv.config(); // Load the environment variables into process.env
+
+configureDotEnv([
+  path.resolve(__dirname, '../../../env/servers.env'),
+  path.resolve(__dirname, '../../../env/db.env'),
+  path.resolve(__dirname, '../../../env/api-client.env'),
+  path.resolve(__dirname, '../.env'),
+]);
 
 (async () => {
   /**

packages/admin-panel-server/src/routes/ExportEntityHierarchiesRoute.ts

@@ -0,0 +1,56 @@
+/**
+ * Tupaia
+ * Copyright (c) 2017 - 2023 Beyond Essential Systems Pty Ltd
+ */
+
+import { Request } from 'express';
+import xlsx from 'xlsx';
+import { Route } from '@tupaia/server-boilerplate';
+
+export type ExportEntityHierarchiesRequest = Request<
+  { hierarchies: string },
+  { contents: Buffer; filePath: string; type: string },
+  Record<string, never>,
+  Record<string, any>
+>;
+
+export class ExportEntityHierarchiesRoute extends Route<ExportEntityHierarchiesRequest> {
+  protected readonly type = 'download';
+
+  public async buildResponse() {
+    const { hierarchies } = this.req.query;
+    const { entity: entityApi } = this.req.ctx.services;
+
+    const hierarchiesArray = Array.isArray(hierarchies) ? hierarchies : [hierarchies];
+
+    const workbook = xlsx.utils.book_new();
+
+    for (const hierarchy of hierarchiesArray) {
+      const descendants = await entityApi.getDescendantsOfEntity(
+        hierarchy,
+        hierarchy,
+        {
+          fields: ['name', 'code', 'parent_code'],
+        },
+        false,
+        false,
+      );
+
+      const projectEntity = await entityApi.getEntity(hierarchy, hierarchy, {
+        fields: ['name'],
+      });
+
+      const sheetName = projectEntity?.name || hierarchy;
+      const sheet = xlsx.utils.json_to_sheet(descendants);
+      xlsx.utils.book_append_sheet(workbook, sheet, sheetName);
+    }
+
+    const buffer = xlsx.write(workbook, { type: 'buffer', bookType: 'xlsx' });
+
+    return {
+      contents: buffer,
+      filePath: `entity_hierarchies_export_${Date.now()}.xlsx`,
+      type: '.xlsx',
+    };
+  }
+}

packages/admin-panel-server/src/routes/index.ts

@@ -12,3 +12,4 @@ export * from './FetchReportPreviewDataRoute';
 export * from './FetchReportSchemasRoute';
 export * from './UploadTestDataRoute';
 export * from './UserRoute';
+export * from './ExportEntityHierarchiesRoute';