From 310b1714c05a3772b0d65b8478efe8ced71301c9 Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Thu, 28 Mar 2024 14:41:01 +1300 Subject: [PATCH 01/25] =?UTF-8?q?no-issue:=20Don=E2=80=99t=20render=20sche?= =?UTF-8?q?ma=20diffs=20by=20default=20(#5542)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * don’t render `schemas.ts` diffs by default * exclude Markdown from repo language stats * mark `models.ts` as generated --- .gitattributes | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitattributes b/.gitattributes index d42ff18354..506bce338b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,4 @@ +*.md linguist-documentation *.pbxproj -text +packages/types/src/schemas/schemas.ts linguist-generated=true +packages/types/src/types/models.ts linguist-generated=true From 7cbc74221ed03fc078a4b1705b7dac0274c1a7c4 Mon Sep 17 00:00:00 2001 From: Rohan Port <59544282+rohan-bes@users.noreply.github.com> Date: Thu, 28 Mar 2024 14:17:40 +1100 Subject: [PATCH 02/25] no-issue: Add 'output' and fix up config for conditions in viz-builder matrix schema (#5539) --- .../src/VizBuilderApp/constants.js | 11 +- .../src/VizBuilderApp/utils/findVizType.js | 13 +- packages/types/src/schemas/schemas.ts | 899 +++++++++++++++--- packages/types/src/types/index.ts | 1 + .../models-extra/dashboard-item/index.ts | 2 + .../models-extra/dashboard-item/matricies.ts | 30 +- .../types/src/types/models-extra/index.ts | 1 + .../src/components/Matrix/MatrixLegend.tsx | 5 +- 8 files changed, 826 insertions(+), 136 deletions(-) diff --git a/packages/admin-panel/src/VizBuilderApp/constants.js b/packages/admin-panel/src/VizBuilderApp/constants.js index f57e1381a7..f40817d4a9 100644 --- a/packages/admin-panel/src/VizBuilderApp/constants.js +++ b/packages/admin-panel/src/VizBuilderApp/constants.js @@ -12,7 +12,7 @@ import { GaugeChartConfigSchema, IconMapOverlayConfigSchema, LineChartConfigSchema, - MatrixConfigSchema, + MatrixVizBuilderConfigSchema, MultiPhotographViewConfigSchema, MultiSingleValueViewConfigSchema, MultiValueRowViewConfigSchema, @@ -167,9 +167,14 @@ export const DASHBOARD_ITEM_VIZ_TYPES = { // Matrix MATRIX: { name: 'Matrix', - schema: MatrixConfigSchema, + schema: MatrixVizBuilderConfigSchema, + vizMatchesType: viz => viz.type === 'matrix', initialConfig: { type: 'matrix', + output: { + type: 'matrix', + rowField: '', + }, }, }, @@ -185,6 +190,7 @@ export const MAP_OVERLAY_VIZ_TYPES = { ICON: { name: 'Icon', schema: IconMapOverlayConfigSchema, + vizMatchesType: viz => viz.displayType === 'icon', initialConfig: { displayType: 'icon', icon: 'pin', @@ -214,6 +220,7 @@ export const MAP_OVERLAY_VIZ_TYPES = { SPECTRUM: { name: 'Spectrum', schema: SpectrumMapOverlayConfigSchema, + vizMatchesType: viz => viz.displayType === 'shaded-spectrum', initialConfig: { displayType: 'shaded-spectrum', scaleColorScheme: 'default', diff --git a/packages/admin-panel/src/VizBuilderApp/utils/findVizType.js b/packages/admin-panel/src/VizBuilderApp/utils/findVizType.js index 98f2a870a3..b9cb06ce60 100644 --- a/packages/admin-panel/src/VizBuilderApp/utils/findVizType.js +++ b/packages/admin-panel/src/VizBuilderApp/utils/findVizType.js @@ -4,9 +4,16 @@ */ export const findVizType = (viz, vizTypes) => { - // Check that all fields in the viz types initial config are match those in the viz - const vizTypeKeyAndValue = Object.entries(vizTypes).find(([, { initialConfig }]) => - Object.entries(initialConfig).every(([key, value]) => viz.presentation[key] === value), + // Check that all fields in the viz types initial config match those in the viz + const vizTypeKeyAndValue = Object.entries(vizTypes).find( + ([, { vizMatchesType, initialConfig }]) => { + if (vizMatchesType) { + return vizMatchesType(viz.presentation); + } + + // Default to checking if it matches the viz initial config + return Object.entries(initialConfig).every(([key, value]) => viz.presentation[key] === value); + }, ); return vizTypeKeyAndValue ? vizTypeKeyAndValue[0] : 'OTHER'; // Default to 'OTHER' if viz doesn't match any diff --git a/packages/types/src/schemas/schemas.ts b/packages/types/src/schemas/schemas.ts index aa92d14c1e..62c7ab582c 100644 --- a/packages/types/src/schemas/schemas.ts +++ b/packages/types/src/schemas/schemas.ts @@ -1199,14 +1199,7 @@ export const MatrixConfigSchema = { ] } }, - "additionalProperties": false, - "required": [ - "<", - "<=", - "=", - ">", - ">=" - ] + "additionalProperties": false }, { "type": [ @@ -1343,14 +1336,688 @@ export const MatrixConfigSchema = { ] } }, - "additionalProperties": false, - "required": [ - "<", - "<=", - "=", - ">", - ">=" + "additionalProperties": false + }, + { + "type": [ + "string", + "number" ] + } + ] + }, + "legendLabel": { + "type": "string" + } + }, + "required": [ + "condition", + "key" + ] + } + }, + "showRawValue": { + "default": false, + "type": "boolean" + }, + "showNestedRows": { + "default": false, + "type": "boolean" + }, + "applyLocation": { + "description": "Specify if you want to limit where to apply the conditional presentation", + "type": "object", + "properties": { + "columnIndexes": { + "type": "array", + "items": { + "type": "number" + } + } + }, + "additionalProperties": false, + "required": [ + "columnIndexes" + ] + } + }, + "additionalProperties": false + }, + { + "additionalProperties": false, + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "range" + ] + }, + "showRawValue": { + "type": "boolean" + } + }, + "required": [ + "type" + ] + } + ] + }, + "valueType": { + "description": "Specify the valueType for formatting of the value in the matrix", + "enum": [ + "boolean", + "color", + "currency", + "fraction", + "fractionAndPercentage", + "number", + "oneDecimalPlace", + "percentage", + "text", + "view" + ], + "type": "string" + }, + "placeholder": { + "description": "A url to an image to be used when a matrix is collapsed.", + "type": "string" + } + }, + "required": [ + "name", + "type" + ] +} + +export const MatrixVizBuilderConfigSchema = { + "additionalProperties": false, + "type": "object", + "properties": { + "name": { + "description": "The title of the viz", + "type": "string" + }, + "description": { + "description": "A short description that appears above a viz", + "type": "string" + }, + "periodGranularity": { + "description": "Granularity of dates in the viz. Controls the date picker and x axis granularity", + "enum": [ + "day", + "month", + "one_day_at_a_time", + "one_month_at_a_time", + "one_quarter_at_a_time", + "one_week_at_a_time", + "one_year_at_a_time", + "quarter", + "week", + "year" + ], + "type": "string" + }, + "defaultTimePeriod": { + "description": "Initial date range for this viz.\nEither a single offset, or an ISO string / offset for start/end date\neg.\n// Single offset\n\"defaultTimePeriod\": {\n \"unit\": \"week\",\n \"offset\": 7\n}\n\n// Explicit start/end dates\n\"defaultTimePeriod\": {\n \"start\": \"2022-10-01\",\n \"end\": \"2023-06-30\"\n}\n\n// Start/end date offsets\n\"defaultTimePeriod\": {\n \"start\": {\n \"unit\": \"week\",\n \"offset\": -52\n },\n \"end\": {\n \"unit\": \"week\",\n \"offset\": 3\n }\n}", + "anyOf": [ + { + "type": "object", + "properties": { + "unit": { + "description": "Time unit to offset by", + "enum": [ + "day", + "month", + "quarter", + "week", + "year" + ], + "type": "string" + }, + "offset": { + "description": "Offset distance (can be negative to offset to an earlier date)", + "type": "number" + }, + "modifier": { + "description": "Used to modify the offset by either moving the date to the start/end of the modifier unit", + "enum": [ + "end_of", + "start_of" + ], + "type": "string" + }, + "modifierUnit": { + "description": "Time unit to modify the offset by", + "enum": [ + "day", + "month", + "quarter", + "week", + "year" + ], + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "offset", + "unit" + ] + }, + { + "type": "object", + "properties": { + "start": { + "description": "Either an ISO Date string, or an offset object", + "anyOf": [ + { + "type": "object", + "properties": { + "unit": { + "description": "Time unit to offset by", + "enum": [ + "day", + "month", + "quarter", + "week", + "year" + ], + "type": "string" + }, + "offset": { + "description": "Offset distance (can be negative to offset to an earlier date)", + "type": "number" + }, + "modifier": { + "description": "Used to modify the offset by either moving the date to the start/end of the modifier unit", + "enum": [ + "end_of", + "start_of" + ], + "type": "string" + }, + "modifierUnit": { + "description": "Time unit to modify the offset by", + "enum": [ + "day", + "month", + "quarter", + "week", + "year" + ], + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "offset", + "unit" + ] + }, + { + "type": "string" + } + ] + }, + "end": { + "description": "Either an ISO Date string, or an offset object", + "anyOf": [ + { + "type": "object", + "properties": { + "unit": { + "description": "Time unit to offset by", + "enum": [ + "day", + "month", + "quarter", + "week", + "year" + ], + "type": "string" + }, + "offset": { + "description": "Offset distance (can be negative to offset to an earlier date)", + "type": "number" + }, + "modifier": { + "description": "Used to modify the offset by either moving the date to the start/end of the modifier unit", + "enum": [ + "end_of", + "start_of" + ], + "type": "string" + }, + "modifierUnit": { + "description": "Time unit to modify the offset by", + "enum": [ + "day", + "month", + "quarter", + "week", + "year" + ], + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "offset", + "unit" + ] + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + } + ] + }, + "datePickerLimits": { + "description": "Maximum date ranges that the date picker can be used to choose from", + "type": "object", + "properties": { + "start": { + "type": "object", + "properties": { + "unit": { + "description": "Time unit to offset by", + "enum": [ + "day", + "month", + "quarter", + "week", + "year" + ], + "type": "string" + }, + "offset": { + "description": "Offset distance (can be negative to offset to an earlier date)", + "type": "number" + }, + "modifier": { + "description": "Used to modify the offset by either moving the date to the start/end of the modifier unit", + "enum": [ + "end_of", + "start_of" + ], + "type": "string" + }, + "modifierUnit": { + "description": "Time unit to modify the offset by", + "enum": [ + "day", + "month", + "quarter", + "week", + "year" + ], + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "offset", + "unit" + ] + }, + "end": { + "type": "object", + "properties": { + "unit": { + "description": "Time unit to offset by", + "enum": [ + "day", + "month", + "quarter", + "week", + "year" + ], + "type": "string" + }, + "offset": { + "description": "Offset distance (can be negative to offset to an earlier date)", + "type": "number" + }, + "modifier": { + "description": "Used to modify the offset by either moving the date to the start/end of the modifier unit", + "enum": [ + "end_of", + "start_of" + ], + "type": "string" + }, + "modifierUnit": { + "description": "Time unit to modify the offset by", + "enum": [ + "day", + "month", + "quarter", + "week", + "year" + ], + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "offset", + "unit" + ] + } + }, + "additionalProperties": false + }, + "exportConfig": { + "description": "Extra config options for exporting", + "type": "object", + "properties": { + "dataElementHeader": { + "description": "Sets the header for the data element in xls exports", + "type": "string" + } + }, + "additionalProperties": false + }, + "noDataMessage": { + "description": "Message which shows if no data is found", + "type": "string" + }, + "noDataFetch": { + "description": "If true, Tupaia will not fetch any data for this viz. Usually used with custom vizes of type: component, e.g. ProjectDescription.", + "default": false, + "type": "boolean" + }, + "drillDown": { + "description": "Configure drill down functionality in this viz to allow clicking through to another visual", + "type": "object", + "properties": { + "itemCode": { + "description": "The code of the dashboard item that drilling down through this viz should take you to", + "type": "string" + }, + "keyLink": { + "type": "string" + }, + "parameterLink": { + "description": "Parameter that the value which is drilled through should link to when fetching data for the drill down dashboard item", + "type": "string" + }, + "itemCodeByEntry": { + "description": "A map of series codes to dashboard item codes that drilling down each series should take you to", + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "additionalProperties": false + }, + "entityHeader": { + "description": "", + "type": "string" + }, + "reference": { + "description": "If provided shows an (i) icon next to the viz title, which allows linking to the source data", + "type": "object", + "properties": { + "link": { + "description": "url", + "type": "string" + }, + "name": { + "description": "label", + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "link", + "name" + ] + }, + "source": { + "description": "If specified allows the frontend to know where the data is coming from, so if there is no data it can show a custom no-data message e.g. \"Requires mSupply\".", + "type": "string" + }, + "weekDisplayFormat": { + "description": "Allows customising how weeks are displayed, e.g. 'W/C 6 Jan 2020' or 'ISO Week 2 2020'", + "default": "'WEEK_COMMENCING_ABBR'", + "enum": [ + "ISO_WEEK_NUMBER", + "WEEK_COMMENCING", + "WEEK_COMMENCING_ABBR", + "WEEK_ENDING", + "WEEK_ENDING_ABBR" + ], + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "matrix" + ] + }, + "dataElementColumnTitle": { + "description": "Matrix viz type can specify a column as the data element column.", + "type": "string" + }, + "hideColumnTitles": { + "description": "Like it sounds", + "type": "boolean" + }, + "presentationOptions": { + "description": "Allows for conditional styling", + "anyOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "condition" + ] + }, + "conditions": { + "type": "array", + "items": { + "additionalProperties": false, + "type": "object", + "properties": { + "color": { + "description": "Specify the color of the display item", + "type": "string" + }, + "description": { + "description": "Specify the text for the legend item. Also used in the enlarged cell view", + "type": "string" + }, + "label": { + "description": "Specify if you want a label to appear above the enlarged", + "type": "string" + }, + "key": { + "type": "string" + }, + "condition": { + "description": "the value to match against exactly, or an object with match criteria e.g. { '>=': 5.5 }", + "anyOf": [ + { + "type": "object", + "properties": { + "=": { + "type": [ + "string", + "number" + ] + }, + ">": { + "type": [ + "string", + "number" + ] + }, + "<": { + "type": [ + "string", + "number" + ] + }, + ">=": { + "type": [ + "string", + "number" + ] + }, + "<=": { + "type": [ + "string", + "number" + ] + } + }, + "additionalProperties": false + }, + { + "type": [ + "string", + "number" + ] + } + ] + }, + "legendLabel": { + "type": "string" + } + }, + "required": [ + "condition", + "key" + ] + } + }, + "showRawValue": { + "default": false, + "type": "boolean" + }, + "showNestedRows": { + "default": false, + "type": "boolean" + }, + "applyLocation": { + "description": "Specify if you want to limit where to apply the conditional presentation", + "type": "object", + "properties": { + "columnIndexes": { + "type": "array", + "items": { + "type": "number" + } + } + }, + "additionalProperties": false, + "required": [ + "columnIndexes" + ] + } + }, + "additionalProperties": false + }, + { + "additionalProperties": false, + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "range" + ] + }, + "showRawValue": { + "type": "boolean" + } + }, + "required": [ + "type" + ] + } + ] + }, + "categoryPresentationOptions": { + "description": "Category header rows can have values just like real rows, this is how you style them", + "anyOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "condition" + ] + }, + "conditions": { + "type": "array", + "items": { + "additionalProperties": false, + "type": "object", + "properties": { + "color": { + "description": "Specify the color of the display item", + "type": "string" + }, + "description": { + "description": "Specify the text for the legend item. Also used in the enlarged cell view", + "type": "string" + }, + "label": { + "description": "Specify if you want a label to appear above the enlarged", + "type": "string" + }, + "key": { + "type": "string" + }, + "condition": { + "description": "the value to match against exactly, or an object with match criteria e.g. { '>=': 5.5 }", + "anyOf": [ + { + "type": "object", + "properties": { + "=": { + "type": [ + "string", + "number" + ] + }, + ">": { + "type": [ + "string", + "number" + ] + }, + "<": { + "type": [ + "string", + "number" + ] + }, + ">=": { + "type": [ + "string", + "number" + ] + }, + "<=": { + "type": [ + "string", + "number" + ] + } + }, + "additionalProperties": false }, { "type": [ @@ -1436,6 +2103,45 @@ export const MatrixConfigSchema = { "placeholder": { "description": "A url to an image to be used when a matrix is collapsed.", "type": "string" + }, + "output": { + "description": "Configuration for rows, columns, and categories of the matrix", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "matrix" + ] + }, + "rowField": { + "description": "The column of the data-table that should be used for the row values in the matrix", + "type": "string" + }, + "categoryField": { + "description": "The column of the data-table that should be used to group the rows into categories", + "type": "string" + }, + "columns": { + "description": "The columns of the data-table that should be included as columns in the matrix.\nCan be either a list of column names, or '*' to indicate all columns", + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false, + "required": [ + "rowField", + "type" + ] } }, "required": [ @@ -1530,14 +2236,7 @@ export const ConditionalPresentationOptionsSchema = { ] } }, - "additionalProperties": false, - "required": [ - "<", - "<=", - "=", - ">", - ">=" - ] + "additionalProperties": false }, { "type": [ @@ -1660,14 +2359,7 @@ export const PresentationOptionConditionSchema = { ] } }, - "additionalProperties": false, - "required": [ - "<", - "<=", - "=", - ">", - ">=" - ] + "additionalProperties": false }, { "type": [ @@ -1712,6 +2404,43 @@ export const PresentationOptionRangeSchema = { } } +export const ConditionsObjectSchema = { + "type": "object", + "properties": { + "=": { + "type": [ + "string", + "number" + ] + }, + ">": { + "type": [ + "string", + "number" + ] + }, + "<": { + "type": [ + "string", + "number" + ] + }, + ">=": { + "type": [ + "string", + "number" + ] + }, + "<=": { + "type": [ + "string", + "number" + ] + } + }, + "additionalProperties": false +} + export const ConditionValueSchema = { "type": [ "string", @@ -1799,14 +2528,7 @@ export const MatrixPresentationOptionsSchema = { ] } }, - "additionalProperties": false, - "required": [ - "<", - "<=", - "=", - ">", - ">=" - ] + "additionalProperties": false }, { "type": [ @@ -20301,14 +21023,7 @@ export const DashboardItemConfigSchema = { ] } }, - "additionalProperties": false, - "required": [ - "<", - "<=", - "=", - ">", - ">=" - ] + "additionalProperties": false }, { "type": [ @@ -20445,14 +21160,7 @@ export const DashboardItemConfigSchema = { ] } }, - "additionalProperties": false, - "required": [ - "<", - "<=", - "=", - ">", - ">=" - ] + "additionalProperties": false }, { "type": [ @@ -28321,14 +29029,7 @@ export const PresentationOptionsSchema = { ] } }, - "additionalProperties": false, - "required": [ - "<", - "<=", - "=", - ">", - ">=" - ] + "additionalProperties": false }, { "type": [ @@ -40498,14 +41199,7 @@ export const DashboardItemSchema = { ] } }, - "additionalProperties": false, - "required": [ - "<", - "<=", - "=", - ">", - ">=" - ] + "additionalProperties": false }, { "type": [ @@ -40642,14 +41336,7 @@ export const DashboardItemSchema = { ] } }, - "additionalProperties": false, - "required": [ - "<", - "<=", - "=", - ">", - ">=" - ] + "additionalProperties": false }, { "type": [ @@ -48915,14 +49602,7 @@ export const DashboardItemCreateSchema = { ] } }, - "additionalProperties": false, - "required": [ - "<", - "<=", - "=", - ">", - ">=" - ] + "additionalProperties": false }, { "type": [ @@ -49059,14 +49739,7 @@ export const DashboardItemCreateSchema = { ] } }, - "additionalProperties": false, - "required": [ - "<", - "<=", - "=", - ">", - ">=" - ] + "additionalProperties": false }, { "type": [ @@ -57326,14 +57999,7 @@ export const DashboardItemUpdateSchema = { ] } }, - "additionalProperties": false, - "required": [ - "<", - "<=", - "=", - ">", - ">=" - ] + "additionalProperties": false }, { "type": [ @@ -57470,14 +58136,7 @@ export const DashboardItemUpdateSchema = { ] } }, - "additionalProperties": false, - "required": [ - "<", - "<=", - "=", - ">", - ">=" - ] + "additionalProperties": false }, { "type": [ @@ -83764,14 +84423,7 @@ export const DashboardWithMetadataSchema = { ] } }, - "additionalProperties": false, - "required": [ - "<", - "<=", - "=", - ">", - ">=" - ] + "additionalProperties": false }, { "type": [ @@ -83908,14 +84560,7 @@ export const DashboardWithMetadataSchema = { ] } }, - "additionalProperties": false, - "required": [ - "<", - "<=", - "=", - ">", - ">=" - ] + "additionalProperties": false }, { "type": [ diff --git a/packages/types/src/types/index.ts b/packages/types/src/types/index.ts index b826fe2f71..64fb6bd5b2 100644 --- a/packages/types/src/types/index.ts +++ b/packages/types/src/types/index.ts @@ -25,6 +25,7 @@ export { MatrixConfig, PresentationOptionCondition, MatrixPresentationOptions, + ConditionsObject, ConditionValue, ConditionType, RangePresentationOptions, diff --git a/packages/types/src/types/models-extra/dashboard-item/index.ts b/packages/types/src/types/models-extra/dashboard-item/index.ts index 21d31c978e..e6e3440156 100644 --- a/packages/types/src/types/models-extra/dashboard-item/index.ts +++ b/packages/types/src/types/models-extra/dashboard-item/index.ts @@ -7,6 +7,7 @@ import type { MatrixConfig, PresentationOptionCondition, MatrixPresentationOptions, + ConditionsObject, ConditionValue, ConditionType, RangePresentationOptions, @@ -63,6 +64,7 @@ export type { MatrixConfig, PresentationOptionCondition, MatrixPresentationOptions, + ConditionsObject, ConditionValue, ConditionType, RangePresentationOptions, diff --git a/packages/types/src/types/models-extra/dashboard-item/matricies.ts b/packages/types/src/types/models-extra/dashboard-item/matricies.ts index 6da5e309ab..3ce4d272f0 100644 --- a/packages/types/src/types/models-extra/dashboard-item/matricies.ts +++ b/packages/types/src/types/models-extra/dashboard-item/matricies.ts @@ -40,6 +40,32 @@ export type MatrixConfig = BaseConfig & { placeholder?: string; }; +export type MatrixVizBuilderConfig = MatrixConfig & { + /** + * @description Configuration for rows, columns, and categories of the matrix + */ + output?: { + type: 'matrix'; + + /** + * @description The column of the data-table that should be used for the row values in the matrix + */ + rowField: string; + + /** + * @description The column of the data-table that should be used to group the rows into categories + */ + categoryField?: string; + + /** + * @description + * The columns of the data-table that should be included as columns in the matrix. + * Can be either a list of column names, or '*' to indicate all columns + */ + columns?: string | string[]; + }; +}; + type BasePresentationOption = { /** * @description Specify the color of the display item @@ -84,7 +110,7 @@ export type PresentationOptionCondition = BasePresentationOption & { /** * @description the value to match against exactly, or an object with match criteria e.g. { '>=': 5.5 } */ - condition: ConditionValue | Record; + condition: ConditionValue | ConditionsObject; legendLabel?: string; }; @@ -93,6 +119,8 @@ export type PresentationOptionRange = BasePresentationOption & { max?: number; }; +export type ConditionsObject = { [key in ConditionType]?: ConditionValue }; + export type ConditionValue = string | number; export enum ConditionType { diff --git a/packages/types/src/types/models-extra/index.ts b/packages/types/src/types/models-extra/index.ts index 70c9331ba2..91a60a85d6 100644 --- a/packages/types/src/types/models-extra/index.ts +++ b/packages/types/src/types/models-extra/index.ts @@ -42,6 +42,7 @@ export { MatrixConfig, PresentationOptionCondition, MatrixPresentationOptions, + ConditionsObject, ConditionValue, ConditionType, RangePresentationOptions, diff --git a/packages/ui-components/src/components/Matrix/MatrixLegend.tsx b/packages/ui-components/src/components/Matrix/MatrixLegend.tsx index 4b58e46467..30d4ed5266 100644 --- a/packages/ui-components/src/components/Matrix/MatrixLegend.tsx +++ b/packages/ui-components/src/components/Matrix/MatrixLegend.tsx @@ -5,8 +5,7 @@ import React, { useContext } from 'react'; import { - ConditionType, - ConditionValue, + ConditionsObject, ConditionalPresentationOptions, PresentationOptionCondition, } from '@tupaia/types'; @@ -29,7 +28,7 @@ const Wrapper = styled.div` } `; -const convertNumberRangeToText = (condition: Record) => { +const convertNumberRangeToText = (condition: ConditionsObject) => { // sort the values so that if we have a range, it's displayed in the correct order const sortedValues = Object.values(condition).sort(); From 344c36a0a4575da42d22603315864eb78cfbd40a Mon Sep 17 00:00:00 2001 From: Rohan Port <59544282+rohan-bes@users.noreply.github.com> Date: Tue, 2 Apr 2024 10:28:40 +1100 Subject: [PATCH 03/25] MDEV-229: Add 'maintenance' entity type (#5546) --- ...ddMaintenanceEntityType-modifies-schema.js | 27 ++++ packages/types/src/schemas/schemas.ts | 117 ++++++++++++++++++ packages/types/src/types/models.ts | 1 + 3 files changed, 145 insertions(+) create mode 100644 packages/database/src/migrations/20240401222409-AddMaintenanceEntityType-modifies-schema.js diff --git a/packages/database/src/migrations/20240401222409-AddMaintenanceEntityType-modifies-schema.js b/packages/database/src/migrations/20240401222409-AddMaintenanceEntityType-modifies-schema.js new file mode 100644 index 0000000000..a677988885 --- /dev/null +++ b/packages/database/src/migrations/20240401222409-AddMaintenanceEntityType-modifies-schema.js @@ -0,0 +1,27 @@ +'use strict'; + +var dbm; +var type; +var seed; + +/** + * We receive the dbmigrate dependency from dbmigrate initially. + * This enables us to not have to rely on NODE_PATH. + */ +exports.setup = function (options, seedLink) { + dbm = options.dbmigrate; + type = dbm.dataType; + seed = seedLink; +}; + +exports.up = async function (db) { + return db.runSql(`ALTER TYPE public.entity_type ADD VALUE IF NOT EXISTS 'maintenance';`); +}; + +exports.down = function (db) { + return null; +}; + +exports._meta = { + version: 1, +}; diff --git a/packages/types/src/schemas/schemas.ts b/packages/types/src/schemas/schemas.ts index 62c7ab582c..fb2bbd8078 100644 --- a/packages/types/src/schemas/schemas.ts +++ b/packages/types/src/schemas/schemas.ts @@ -30056,6 +30056,7 @@ export const MeasureConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -30243,6 +30244,7 @@ export const EntityLevelSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -30581,6 +30583,7 @@ export const BaseMapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -30720,6 +30723,7 @@ export const BaseMapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -30849,6 +30853,7 @@ export const BaseMapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -31269,6 +31274,7 @@ export const SpectrumMapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -31408,6 +31414,7 @@ export const SpectrumMapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -31537,6 +31544,7 @@ export const SpectrumMapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -32065,6 +32073,7 @@ export const IconMapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -32204,6 +32213,7 @@ export const IconMapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -32333,6 +32343,7 @@ export const IconMapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -32783,6 +32794,7 @@ export const RadiusMapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -32922,6 +32934,7 @@ export const RadiusMapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -33051,6 +33064,7 @@ export const RadiusMapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -33475,6 +33489,7 @@ export const ColorMapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -33614,6 +33629,7 @@ export const ColorMapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -33743,6 +33759,7 @@ export const ColorMapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -34182,6 +34199,7 @@ export const ShadingMapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -34321,6 +34339,7 @@ export const ShadingMapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -34450,6 +34469,7 @@ export const ShadingMapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -34880,6 +34900,7 @@ export const MapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -35019,6 +35040,7 @@ export const MapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -35148,6 +35170,7 @@ export const MapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -35675,6 +35698,7 @@ export const MapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -35814,6 +35838,7 @@ export const MapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -35943,6 +35968,7 @@ export const MapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -36392,6 +36418,7 @@ export const MapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -36531,6 +36558,7 @@ export const MapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -36660,6 +36688,7 @@ export const MapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -37083,6 +37112,7 @@ export const MapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -37222,6 +37252,7 @@ export const MapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -37351,6 +37382,7 @@ export const MapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -37789,6 +37821,7 @@ export const MapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -37928,6 +37961,7 @@ export const MapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -38057,6 +38091,7 @@ export const MapOverlayConfigSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -38687,6 +38722,7 @@ export const EntityQuestionConfigSchema = { "institute", "larval_habitat", "local_government", + "maintenance", "medical_area", "msupply_store", "nursing_zone", @@ -38732,6 +38768,7 @@ export const EntityQuestionConfigSchema = { "institute", "larval_habitat", "local_government", + "maintenance", "medical_area", "msupply_store", "nursing_zone", @@ -39199,6 +39236,7 @@ export const SurveyScreenComponentConfigSchema = { "institute", "larval_habitat", "local_government", + "maintenance", "medical_area", "msupply_store", "nursing_zone", @@ -39244,6 +39282,7 @@ export const SurveyScreenComponentConfigSchema = { "institute", "larval_habitat", "local_government", + "maintenance", "medical_area", "msupply_store", "nursing_zone", @@ -39718,6 +39757,12 @@ export const RecentEntitiesForCountrySchema = { "items": { "type": "string" } + }, + "maintenance": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": false @@ -66137,6 +66182,7 @@ export const DashboardRelationSchema = { "institute", "larval_habitat", "local_government", + "maintenance", "medical_area", "msupply_store", "nursing_zone", @@ -66228,6 +66274,7 @@ export const DashboardRelationCreateSchema = { "institute", "larval_habitat", "local_government", + "maintenance", "medical_area", "msupply_store", "nursing_zone", @@ -66314,6 +66361,7 @@ export const DashboardRelationUpdateSchema = { "institute", "larval_habitat", "local_government", + "maintenance", "medical_area", "msupply_store", "nursing_zone", @@ -67509,6 +67557,7 @@ export const EntitySchema = { "institute", "larval_habitat", "local_government", + "maintenance", "medical_area", "msupply_store", "nursing_zone", @@ -67605,6 +67654,7 @@ export const EntityCreateSchema = { "institute", "larval_habitat", "local_government", + "maintenance", "medical_area", "msupply_store", "nursing_zone", @@ -67701,6 +67751,7 @@ export const EntityUpdateSchema = { "institute", "larval_habitat", "local_government", + "maintenance", "medical_area", "msupply_store", "nursing_zone", @@ -68931,6 +68982,7 @@ export const MapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -69070,6 +69122,7 @@ export const MapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -69199,6 +69252,7 @@ export const MapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -69726,6 +69780,7 @@ export const MapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -69865,6 +69920,7 @@ export const MapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -69994,6 +70050,7 @@ export const MapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -70443,6 +70500,7 @@ export const MapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -70582,6 +70640,7 @@ export const MapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -70711,6 +70770,7 @@ export const MapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -71134,6 +71194,7 @@ export const MapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -71273,6 +71334,7 @@ export const MapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -71402,6 +71464,7 @@ export const MapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -71840,6 +71903,7 @@ export const MapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -71979,6 +72043,7 @@ export const MapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -72108,6 +72173,7 @@ export const MapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -72599,6 +72665,7 @@ export const MapOverlayCreateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -72738,6 +72805,7 @@ export const MapOverlayCreateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -72867,6 +72935,7 @@ export const MapOverlayCreateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -73394,6 +73463,7 @@ export const MapOverlayCreateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -73533,6 +73603,7 @@ export const MapOverlayCreateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -73662,6 +73733,7 @@ export const MapOverlayCreateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -74111,6 +74183,7 @@ export const MapOverlayCreateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -74250,6 +74323,7 @@ export const MapOverlayCreateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -74379,6 +74453,7 @@ export const MapOverlayCreateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -74802,6 +74877,7 @@ export const MapOverlayCreateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -74941,6 +75017,7 @@ export const MapOverlayCreateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -75070,6 +75147,7 @@ export const MapOverlayCreateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -75508,6 +75586,7 @@ export const MapOverlayCreateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -75647,6 +75726,7 @@ export const MapOverlayCreateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -75776,6 +75856,7 @@ export const MapOverlayCreateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -76260,6 +76341,7 @@ export const MapOverlayUpdateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -76399,6 +76481,7 @@ export const MapOverlayUpdateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -76528,6 +76611,7 @@ export const MapOverlayUpdateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -77055,6 +77139,7 @@ export const MapOverlayUpdateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -77194,6 +77279,7 @@ export const MapOverlayUpdateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -77323,6 +77409,7 @@ export const MapOverlayUpdateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -77772,6 +77859,7 @@ export const MapOverlayUpdateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -77911,6 +77999,7 @@ export const MapOverlayUpdateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -78040,6 +78129,7 @@ export const MapOverlayUpdateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -78463,6 +78553,7 @@ export const MapOverlayUpdateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -78602,6 +78693,7 @@ export const MapOverlayUpdateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -78731,6 +78823,7 @@ export const MapOverlayUpdateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -79169,6 +79262,7 @@ export const MapOverlayUpdateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -79308,6 +79402,7 @@ export const MapOverlayUpdateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -79437,6 +79532,7 @@ export const MapOverlayUpdateSchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -80408,6 +80504,7 @@ export const PermissionsBasedMeditrakSyncQueueSchema = { "institute", "larval_habitat", "local_government", + "maintenance", "medical_area", "msupply_store", "nursing_zone", @@ -80486,6 +80583,7 @@ export const PermissionsBasedMeditrakSyncQueueCreateSchema = { "institute", "larval_habitat", "local_government", + "maintenance", "medical_area", "msupply_store", "nursing_zone", @@ -80561,6 +80659,7 @@ export const PermissionsBasedMeditrakSyncQueueUpdateSchema = { "institute", "larval_habitat", "local_government", + "maintenance", "medical_area", "msupply_store", "nursing_zone", @@ -82779,6 +82878,7 @@ export const EntityTypeSchema = { "institute", "larval_habitat", "local_government", + "maintenance", "medical_area", "msupply_store", "nursing_zone", @@ -83010,6 +83110,7 @@ export const MeditrakSurveyResponseRequestSchema = { "institute", "larval_habitat", "local_government", + "maintenance", "medical_area", "msupply_store", "nursing_zone", @@ -83905,6 +84006,7 @@ export const EntityResponseSchema = { "institute", "larval_habitat", "local_government", + "maintenance", "medical_area", "msupply_store", "nursing_zone", @@ -92727,6 +92829,7 @@ export const TranslatedMapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -92866,6 +92969,7 @@ export const TranslatedMapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -92995,6 +93099,7 @@ export const TranslatedMapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -93545,6 +93650,7 @@ export const TranslatedMapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -93684,6 +93790,7 @@ export const TranslatedMapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -93813,6 +93920,7 @@ export const TranslatedMapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -94285,6 +94393,7 @@ export const TranslatedMapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -94424,6 +94533,7 @@ export const TranslatedMapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -94553,6 +94663,7 @@ export const TranslatedMapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -94999,6 +95110,7 @@ export const TranslatedMapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -95138,6 +95250,7 @@ export const TranslatedMapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -95267,6 +95380,7 @@ export const TranslatedMapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -95728,6 +95842,7 @@ export const TranslatedMapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -95867,6 +95982,7 @@ export const TranslatedMapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", @@ -95996,6 +96112,7 @@ export const TranslatedMapOverlaySchema = { "Institute", "LarvalHabitat", "LocalGovernment", + "Maintenance", "MedicalArea", "MsupplyStore", "NursingZone", diff --git a/packages/types/src/types/models.ts b/packages/types/src/types/models.ts index b4db325d02..d01755d679 100644 --- a/packages/types/src/types/models.ts +++ b/packages/types/src/types/models.ts @@ -1751,6 +1751,7 @@ export enum EntityType { 'business' = 'business', 'health_clinic_boundary' = 'health_clinic_boundary', 'enumeration_area' = 'enumeration_area', + 'maintenance' = 'maintenance', } export enum DataTableType { 'analytics' = 'analytics', From 86650900bda7394b9eca9acf780c8c0ec329134b Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Tue, 2 Apr 2024 16:00:11 +1300 Subject: [PATCH 04/25] build(deps): bump express from 4.17.1 to 4.19.2 (#5545) Bumps [express](https://github.com/expressjs/express) from 4.17.1 to 4.19.2. - [Release notes](https://github.com/expressjs/express/releases) - [Changelog](https://github.com/expressjs/express/blob/master/History.md) - [Commits](https://github.com/expressjs/express/compare/4.17.1...4.19.2) --- updated-dependencies: - dependency-name: express dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- packages/admin-panel-server/package.json | 2 +- packages/central-server/package.json | 2 +- packages/data-table-server/package.json | 2 +- packages/datatrak-web-server/package.json | 2 +- packages/entity-server/package.json | 2 +- packages/lesmis-server/package.json | 2 +- packages/meditrak-app-server/package.json | 2 +- packages/psss-server/package.json | 2 +- packages/report-server/package.json | 2 +- packages/server-boilerplate/package.json | 2 +- packages/tupaia-web-server/package.json | 2 +- packages/web-config-server/package.json | 2 +- yarn.lock | 111 +++++++++++++++++++--- 13 files changed, 110 insertions(+), 25 deletions(-) mode change 100755 => 100644 packages/web-config-server/package.json diff --git a/packages/admin-panel-server/package.json b/packages/admin-panel-server/package.json index 4aed92a17d..7798587e88 100644 --- a/packages/admin-panel-server/package.json +++ b/packages/admin-panel-server/package.json @@ -39,7 +39,7 @@ "client-sessions": "^0.8.0", "cors": "^2.8.5", "dotenv": "^8.2.0", - "express": "^4.16.2", + "express": "^4.19.2", "lodash": "^4.17.4", "multer": "^1.4.3", "winston": "^3.3.3" diff --git a/packages/central-server/package.json b/packages/central-server/package.json index 3af627759b..53ce4815c6 100644 --- a/packages/central-server/package.json +++ b/packages/central-server/package.json @@ -49,7 +49,7 @@ "countrynames": "^0.1.1", "del": "^2.2.2", "dotenv": "^8.2.0", - "express": "^4.16.4", + "express": "^4.19.2", "form-data": "^2.3.3", "format-link-header": "^2.1.0", "http": "^0.0.0", diff --git a/packages/data-table-server/package.json b/packages/data-table-server/package.json index fe0e342b04..0ad2ebdfda 100644 --- a/packages/data-table-server/package.json +++ b/packages/data-table-server/package.json @@ -35,7 +35,7 @@ "@tupaia/types": "workspace:*", "@tupaia/utils": "workspace:*", "dotenv": "^8.2.0", - "express": "^4.16.2", + "express": "^4.19.2", "winston": "^3.3.3" }, "devDependencies": { diff --git a/packages/datatrak-web-server/package.json b/packages/datatrak-web-server/package.json index 6f5d916d91..000765eca6 100644 --- a/packages/datatrak-web-server/package.json +++ b/packages/datatrak-web-server/package.json @@ -33,7 +33,7 @@ "@tupaia/utils": "workspace:*", "camelcase-keys": "^6.2.2", "dotenv": "^8.2.0", - "express": "^4.16.2", + "express": "^4.19.2", "lodash.groupby": "^4.6.0", "lodash.keyby": "^4.6.0", "lodash.sortby": "^4.6.0", diff --git a/packages/entity-server/package.json b/packages/entity-server/package.json index eaad6f845f..29cc2b1f4b 100644 --- a/packages/entity-server/package.json +++ b/packages/entity-server/package.json @@ -36,7 +36,7 @@ "@tupaia/types": "workspace:*", "@tupaia/utils": "workspace:*", "dotenv": "^8.2.0", - "express": "^4.16.2", + "express": "^4.19.2", "lodash.keyby": "^4.6.0", "winston": "^3.3.3" }, diff --git a/packages/lesmis-server/package.json b/packages/lesmis-server/package.json index e46bcaaa74..af51434d34 100644 --- a/packages/lesmis-server/package.json +++ b/packages/lesmis-server/package.json @@ -35,7 +35,7 @@ "@tupaia/utils": "workspace:*", "camelcase-keys": "^6.2.2", "dotenv": "^8.2.0", - "express": "^4.16.2", + "express": "^4.19.2", "lodash.uniqby": "^4.7.0", "winston": "^3.3.3" } diff --git a/packages/meditrak-app-server/package.json b/packages/meditrak-app-server/package.json index e09201e5cf..fdb168add8 100644 --- a/packages/meditrak-app-server/package.json +++ b/packages/meditrak-app-server/package.json @@ -37,7 +37,7 @@ "@tupaia/types": "workspace:*", "@tupaia/utils": "workspace:*", "dotenv": "^8.2.0", - "express": "^4.16.2", + "express": "^4.19.2", "lodash.clonedeep": "^4.5.0", "lodash.groupby": "^4.6.0", "lodash.keyby": "^4.6.0", diff --git a/packages/psss-server/package.json b/packages/psss-server/package.json index ec85d92990..3af41917c8 100644 --- a/packages/psss-server/package.json +++ b/packages/psss-server/package.json @@ -37,7 +37,7 @@ "client-sessions": "^0.8.0", "cors": "^2.8.5", "dotenv": "^8.2.0", - "express": "^4.16.2", + "express": "^4.19.2", "lodash.groupby": "^4.6.0", "winston": "^3.3.3" } diff --git a/packages/report-server/package.json b/packages/report-server/package.json index 4d8a58e279..24452b4664 100644 --- a/packages/report-server/package.json +++ b/packages/report-server/package.json @@ -45,7 +45,7 @@ "cors": "^2.8.5", "date-fns": "^2.29.3", "dotenv": "^8.2.0", - "express": "^4.16.2", + "express": "^4.19.2", "lodash.groupby": "^4.6.0", "lodash.isplainobject": "^4.0.6", "lodash.keyby": "^4.6.0", diff --git a/packages/server-boilerplate/package.json b/packages/server-boilerplate/package.json index a15ccf6cca..f3187e8734 100644 --- a/packages/server-boilerplate/package.json +++ b/packages/server-boilerplate/package.json @@ -36,7 +36,7 @@ "client-sessions": "^0.8.0", "cors": "^2.8.5", "dotenv": "^8.2.0", - "express": "^4.16.2", + "express": "^4.19.2", "http-proxy-middleware": "^2.0.1", "i18n": "^0.13.3", "morgan": "^1.9.0", diff --git a/packages/tupaia-web-server/package.json b/packages/tupaia-web-server/package.json index 9d05411a8d..2b575bb406 100644 --- a/packages/tupaia-web-server/package.json +++ b/packages/tupaia-web-server/package.json @@ -36,7 +36,7 @@ "@tupaia/types": "workspace:*", "@tupaia/utils": "workspace:*", "dotenv": "^8.2.0", - "express": "^4.16.2", + "express": "^4.19.2", "lodash.groupby": "^4.6.0", "lodash.keyby": "^4.6.0", "winston": "^3.3.3" diff --git a/packages/web-config-server/package.json b/packages/web-config-server/package.json old mode 100755 new mode 100644 index f6357487d6..eeebae797d --- a/packages/web-config-server/package.json +++ b/packages/web-config-server/package.json @@ -40,7 +40,7 @@ "cookie-parser": "^1.4.3", "cors": "^2.8.4", "dotenv": "^8.2.0", - "express": "^4.16.2", + "express": "^4.19.2", "lodash.difference": "^4.5.0", "lodash.flatten": "^4.4.0", "lodash.flattendeep": "^4.4.0", diff --git a/yarn.lock b/yarn.lock index 09a242c004..1427396449 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11697,7 +11697,7 @@ __metadata: client-sessions: ^0.8.0 cors: ^2.8.5 dotenv: ^8.2.0 - express: ^4.16.2 + express: ^4.19.2 lodash: ^4.17.4 multer: ^1.4.3 winston: ^3.3.3 @@ -11837,7 +11837,7 @@ __metadata: deep-equal-in-any-order: ^1.0.21 del: ^2.2.2 dotenv: ^8.2.0 - express: ^4.16.4 + express: ^4.19.2 form-data: ^2.3.3 format-link-header: ^2.1.0 http: ^0.0.0 @@ -11953,7 +11953,7 @@ __metadata: "@tupaia/types": "workspace:*" "@tupaia/utils": "workspace:*" dotenv: ^8.2.0 - express: ^4.16.2 + express: ^4.19.2 mockdate: ^3.0.5 winston: ^3.3.3 languageName: unknown @@ -12003,7 +12003,7 @@ __metadata: "@types/lodash.sortby": ^4.6.0 camelcase-keys: ^6.2.2 dotenv: ^8.2.0 - express: ^4.16.2 + express: ^4.19.2 lodash.groupby: ^4.6.0 lodash.keyby: ^4.6.0 lodash.sortby: ^4.6.0 @@ -12117,7 +12117,7 @@ __metadata: "@tupaia/utils": "workspace:*" "@types/lodash.keyby": ^4.6.6 dotenv: ^8.2.0 - express: ^4.16.2 + express: ^4.19.2 lodash.keyby: ^4.6.0 winston: ^3.3.3 languageName: unknown @@ -12170,7 +12170,7 @@ __metadata: "@tupaia/utils": "workspace:*" camelcase-keys: ^6.2.2 dotenv: ^8.2.0 - express: ^4.16.2 + express: ^4.19.2 lodash.uniqby: ^4.7.0 winston: ^3.3.3 languageName: unknown @@ -12230,7 +12230,7 @@ __metadata: "@types/lodash.clonedeep": ^4.5.0 "@types/semver-compare": ^1.0.1 dotenv: ^8.2.0 - express: ^4.16.2 + express: ^4.19.2 lodash.clonedeep: ^4.5.0 lodash.groupby: ^4.6.0 lodash.keyby: ^4.6.0 @@ -12326,7 +12326,7 @@ __metadata: client-sessions: ^0.8.0 cors: ^2.8.5 dotenv: ^8.2.0 - express: ^4.16.2 + express: ^4.19.2 lodash.groupby: ^4.6.0 winston: ^3.3.3 languageName: unknown @@ -12399,7 +12399,7 @@ __metadata: cors: ^2.8.5 date-fns: ^2.29.3 dotenv: ^8.2.0 - express: ^4.16.2 + express: ^4.19.2 lodash.groupby: ^4.6.0 lodash.isplainobject: ^4.0.6 lodash.keyby: ^4.6.0 @@ -12432,7 +12432,7 @@ __metadata: client-sessions: ^0.8.0 cors: ^2.8.5 dotenv: ^8.2.0 - express: ^4.16.2 + express: ^4.19.2 http-proxy-middleware: ^2.0.1 i18n: ^0.13.3 morgan: ^1.9.0 @@ -12503,7 +12503,7 @@ __metadata: "@types/lodash.groupby": ^4.6.0 "@types/lodash.keyby": ^4.6.0 dotenv: ^8.2.0 - express: ^4.16.2 + express: ^4.19.2 lodash.groupby: ^4.6.0 lodash.keyby: ^4.6.0 winston: ^3.3.3 @@ -12735,7 +12735,7 @@ __metadata: cors: ^2.8.4 csvtojson: ^1.1.5 dotenv: ^8.2.0 - express: ^4.16.2 + express: ^4.19.2 lodash.difference: ^4.5.0 lodash.flatten: ^4.4.0 lodash.flattendeep: ^4.4.0 @@ -17120,6 +17120,26 @@ __metadata: languageName: node linkType: hard +"body-parser@npm:1.20.2": + version: 1.20.2 + resolution: "body-parser@npm:1.20.2" + dependencies: + bytes: 3.1.2 + content-type: ~1.0.5 + debug: 2.6.9 + depd: 2.0.0 + destroy: 1.2.0 + http-errors: 2.0.0 + iconv-lite: 0.4.24 + on-finished: 2.4.1 + qs: 6.11.0 + raw-body: 2.5.2 + type-is: ~1.6.18 + unpipe: 1.0.0 + checksum: 14d37ec638ab5c93f6099ecaed7f28f890d222c650c69306872e00b9efa081ff6c596cd9afb9930656aae4d6c4e1c17537bea12bb73c87a217cb3cfea8896737 + languageName: node + linkType: hard + "boolbase@npm:^1.0.0, boolbase@npm:~1.0.0": version: 1.0.0 resolution: "boolbase@npm:1.0.0" @@ -19174,6 +19194,13 @@ __metadata: languageName: node linkType: hard +"content-type@npm:~1.0.5": + version: 1.0.5 + resolution: "content-type@npm:1.0.5" + checksum: 566271e0a251642254cde0f845f9dd4f9856e52d988f4eb0d0dcffbb7a1f8ec98de7a5215fc628f3bce30fe2fb6fd2bc064b562d721658c59b544e2d34ea2766 + languageName: node + linkType: hard + "continuation-local-storage@npm:^3.2.1": version: 3.2.1 resolution: "continuation-local-storage@npm:3.2.1" @@ -19259,6 +19286,13 @@ __metadata: languageName: node linkType: hard +"cookie@npm:0.6.0": + version: 0.6.0 + resolution: "cookie@npm:0.6.0" + checksum: f56a7d32a07db5458e79c726b77e3c2eff655c36792f2b6c58d351fb5f61531e5b1ab7f46987150136e366c65213cbe31729e02a3eaed630c3bf7334635fb410 + languageName: node + linkType: hard + "cookie@npm:^0.4.1": version: 0.4.1 resolution: "cookie@npm:0.4.1" @@ -23197,7 +23231,7 @@ __metadata: languageName: node linkType: hard -"express@npm:^4.16.2, express@npm:^4.16.4, express@npm:^4.17.1": +"express@npm:^4.17.1": version: 4.17.1 resolution: "express@npm:4.17.1" dependencies: @@ -23274,6 +23308,45 @@ __metadata: languageName: node linkType: hard +"express@npm:^4.19.2": + version: 4.19.2 + resolution: "express@npm:4.19.2" + dependencies: + accepts: ~1.3.8 + array-flatten: 1.1.1 + body-parser: 1.20.2 + content-disposition: 0.5.4 + content-type: ~1.0.4 + cookie: 0.6.0 + cookie-signature: 1.0.6 + debug: 2.6.9 + depd: 2.0.0 + encodeurl: ~1.0.2 + escape-html: ~1.0.3 + etag: ~1.8.1 + finalhandler: 1.2.0 + fresh: 0.5.2 + http-errors: 2.0.0 + merge-descriptors: 1.0.1 + methods: ~1.1.2 + on-finished: 2.4.1 + parseurl: ~1.3.3 + path-to-regexp: 0.1.7 + proxy-addr: ~2.0.7 + qs: 6.11.0 + range-parser: ~1.2.1 + safe-buffer: 5.2.1 + send: 0.18.0 + serve-static: 1.15.0 + setprototypeof: 1.2.0 + statuses: 2.0.1 + type-is: ~1.6.18 + utils-merge: 1.0.1 + vary: ~1.1.2 + checksum: 212dbd6c2c222a96a61bc927639c95970a53b06257080bb9e2838adb3bffdb966856551fdad1ab5dd654a217c35db94f987d0aa88d48fb04d306340f5f34dca5 + languageName: node + linkType: hard + "extend-shallow@npm:^2.0.1": version: 2.0.1 resolution: "extend-shallow@npm:2.0.1" @@ -35822,6 +35895,18 @@ __metadata: languageName: node linkType: hard +"raw-body@npm:2.5.2": + version: 2.5.2 + resolution: "raw-body@npm:2.5.2" + dependencies: + bytes: 3.1.2 + http-errors: 2.0.0 + iconv-lite: 0.4.24 + unpipe: 1.0.0 + checksum: ba1583c8d8a48e8fbb7a873fdbb2df66ea4ff83775421bfe21ee120140949ab048200668c47d9ae3880012f6e217052690628cf679ddfbd82c9fc9358d574676 + languageName: node + linkType: hard + "raw-loader@npm:^4.0.2": version: 4.0.2 resolution: "raw-loader@npm:4.0.2" From 97c8c062a88b5ffd31a2371994f978f37c3a3e9b Mon Sep 17 00:00:00 2001 From: Rohan Port <59544282+rohan-bes@users.noreply.github.com> Date: Tue, 2 Apr 2024 15:18:09 +1100 Subject: [PATCH 05/25] ci: Add PR title check (#5548) --- .github/CONTRIBUTING.md | 43 ++++++++++++++++++++++ .github/workflows/check-pr-title.yaml | 51 +++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 .github/workflows/check-pr-title.yaml diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 1311b6e1d6..3226003be0 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -101,6 +101,49 @@ your changes and provide feedback. If your pull request has been approved by a r will then go through a round of internal testing. Once your changes have passed testing, they can be merged into main and be included in an upcoming Tupaia release. +#### Pull request title rules + +We use a squash-and-merge strategy when merging PRs into dev. This means the commit messages in dev will match the PR titles. We like to keep standardised commit messages as an easy way to track release changes and for record keeping purposes. + +The title of pull requests must be in [Conventional Commit](https://beyond-essential.slab.com/posts/pr-titles-conventional-commits-avgsj3xb) format. This is used to generate +changelogs, and to provide a consistent format for commits landing in the main branch, as pull +requests are merged in “squash” mode. + +```plain +type: +type(scope): +``` + +When a Linear card is applicable, the Linear card number should be included: + +```plain +type: TEAM-123: +type(scope): TEAM-123: +``` + +The following types are conventional: + +- `ci` for changes to the CI/CD workflows +- `db` for changes to the database schema, migrations, etc +- `deps` for changes to dependencies or dependency upgrades +- `doc` for documentation changes +- `env` for changes to the environment variables +- `feat` for new features +- `fix` for bug fixes +- `fmt` for automatic formatting changes +- `merge` for merging between branches (generally between `master` and `dev`) +- `refactor` for code refactoring +- `repo` for changes to the repository structure (e.g. `.gitignore`, `.editorconfig`, etc) +- `revert` for reverting a previous commit +- `style` for stylistic changes that do not affect the meaning of the code +- `test` for adding missing tests or correcting existing tests +- `tweak` for minor changes that do not fit into any other category + +When merging, additional change lines may be added to the squashed commit message to provide further +context to be pulled into changelogs. + +Using Conventional Commit format for actual commit messages within pull requests is not required. + ### Note on Forking While Tupaia is open-source, there is always the possibility for forking the repository, which diff --git a/.github/workflows/check-pr-title.yaml b/.github/workflows/check-pr-title.yaml new file mode 100644 index 0000000000..e462c12809 --- /dev/null +++ b/.github/workflows/check-pr-title.yaml @@ -0,0 +1,51 @@ +name: Check PR title + +on: + pull_request: + types: + - opened + - edited + - synchronize + +jobs: + check: + runs-on: ubuntu-latest + env: + PR_TITLE: ${{ github.event.pull_request.title }} + steps: + - id: release + name: Check for release PR format + if: startsWith(github.event.pull_request.title, 'Release') + shell: bash + run: | + if ! grep -qP '^Release [0-9]{4}-[0-9]{2}$' <<< "$PR_TITLE"; then + echo "::warning::Release PR title should be: 'Release -'" + exit 1 + fi + + - name: Check for Conventional Commit format + shell: bash + if: steps.release.conclusion == 'skipped' + run: | + if ! grep -qP '^\w+(\(\w+\))?:\s' <<< "$PR_TITLE"; then + echo "::warning::PR title should be in Conventional Commit style, e.g. 'feat: ...'" + exit 1 + fi + + - name: Check for conventional type allow-list + if: steps.release.conclusion == 'skipped' + shell: bash + run: | + if ! grep -qP '^(ci|db|deps|doc|env|feat|fix|fmt|merge|refactor|repo|revert|style|test|tweak)(\(\w+\))?:\s' <<< "$PR_TITLE"; then + echo "::warning::PR title Conventional Type is not on the list; refer to CONTRIBUTING.md" + exit 1 + fi + + - name: Check for Linear card number for feature branches + if: steps.release.conclusion == 'skipped' && startsWith(github.event.pull_request.title, 'feat') + shell: bash + run: | + if ! grep -qP '^\w+(\(\w+\))?:\s[A-Z]+-[0-9]+(:\s+\w+)?' <<< "$PR_TITLE"; then + echo "::warning::PR title should start with ticket number, e.g. 'feat(scope): ABC-123: ...'" + exit 1 + fi From 3a59b3a3ac7ac655b7692a97d6bf448fdacbdb0e Mon Sep 17 00:00:00 2001 From: alexd-bes <129009580+alexd-bes@users.noreply.github.com> Date: Wed, 3 Apr 2024 11:41:55 +1300 Subject: [PATCH 06/25] fix(security): RN-1084: Remove refresh token from access token (#5547) * Update userAuth.js * Update tests --- packages/auth/src/__tests__/userAuth.test.js | 30 ++++--------------- packages/auth/src/userAuth.js | 8 +---- .../ChangePasswordRoute.test.ts | 3 +- .../__integration__/UserRewardsRoute.test.ts | 3 +- .../socialFeed/SocialFeedRoute.test.ts | 2 -- .../sync/PushChangesRoute.test.ts | 2 -- .../pullChanges/ChangesMetadataRoute.test.ts | 2 -- .../pullChanges/CountChangesRoute.test.ts | 2 -- .../sync/pullChanges/PullChangesRoute.test.ts | 2 -- 9 files changed, 9 insertions(+), 45 deletions(-) diff --git a/packages/auth/src/__tests__/userAuth.test.js b/packages/auth/src/__tests__/userAuth.test.js index 01b16c7555..cc313fb896 100644 --- a/packages/auth/src/__tests__/userAuth.test.js +++ b/packages/auth/src/__tests__/userAuth.test.js @@ -14,57 +14,39 @@ describe('userAuth', () => { describe('accessToken', () => { it('can construct access token', async () => { const userId = 'user1'; - const refreshToken = 'refresh'; - return expect(constructAccessToken({ userId, refreshToken })).toBeString(); + return expect(constructAccessToken({ userId })).toBeString(); }); it('can construct access token with apiClientId', async () => { const userId = 'user1'; - const refreshToken = 'refresh'; const apiClientUserId = 'apiClient1'; - return expect(constructAccessToken({ userId, refreshToken, apiClientUserId })).toBeString(); + return expect(constructAccessToken({ userId, apiClientUserId })).toBeString(); }); it('throws error when constructing access token without userId', async () => { - const refreshToken = 'refresh'; const apiClientUserId = 'apiClient1'; - return expect(() => constructAccessToken({ refreshToken, apiClientUserId })).toThrow( + return expect(() => constructAccessToken({ apiClientUserId })).toThrow( 'Cannot construct accessToken: missing userId', ); }); - it('throws error when constructing access token without refreshToken', async () => { - const userId = 'user1'; - const apiClientUserId = 'apiClient1'; - - return expect(() => constructAccessToken({ userId, apiClientUserId })).toThrow( - 'Cannot construct accessToken: missing refreshToken', - ); - }); - it('can decrypt access token claims', async () => { const userId = 'user1'; - const refreshToken = 'refresh'; const apiClientUserId = 'apiClient1'; - const accessToken = constructAccessToken({ userId, refreshToken, apiClientUserId }); + const accessToken = constructAccessToken({ userId, apiClientUserId }); const authHeader = createBearerHeader(accessToken); - const { - userId: decryptedUserId, - refreshToken: decryptedRefreshToken, - apiClientUserId: decryptedApiClientUserId, - } = getTokenClaimsFromBearerAuth(authHeader); + const { userId: decryptedUserId, apiClientUserId: decryptedApiClientUserId } = + getTokenClaimsFromBearerAuth(authHeader); return expect({ userId: decryptedUserId, - refreshToken: decryptedRefreshToken, apiClientUserId: decryptedApiClientUserId, }).toEqual({ userId, - refreshToken, apiClientUserId, }); }); diff --git a/packages/auth/src/userAuth.js b/packages/auth/src/userAuth.js index b64b93b2ba..1142b10484 100644 --- a/packages/auth/src/userAuth.js +++ b/packages/auth/src/userAuth.js @@ -9,18 +9,13 @@ import { getJwtToken } from './security'; const ACCESS_TOKEN_EXPIRY_SECONDS = 15 * 60; // User's access expires every 15 mins -export const constructAccessToken = ({ userId, apiClientUserId, refreshToken }) => { +export const constructAccessToken = ({ userId, apiClientUserId }) => { if (!userId) { throw new Error('Cannot construct accessToken: missing userId'); } - if (!refreshToken) { - throw new Error('Cannot construct accessToken: missing refreshToken'); - } - const jwtPayload = { userId, - refreshToken, }; if (apiClientUserId) { @@ -96,7 +91,6 @@ export const getAuthorizationObject = async ({ }) => { const tokenClaims = { userId: user.id, - refreshToken, }; if (apiClientUser) { diff --git a/packages/meditrak-app-server/src/__tests__/__integration__/ChangePasswordRoute.test.ts b/packages/meditrak-app-server/src/__tests__/__integration__/ChangePasswordRoute.test.ts index 7de8049188..dd934d01d4 100644 --- a/packages/meditrak-app-server/src/__tests__/__integration__/ChangePasswordRoute.test.ts +++ b/packages/meditrak-app-server/src/__tests__/__integration__/ChangePasswordRoute.test.ts @@ -8,7 +8,7 @@ import { clearTestData, getTestDatabase } from '@tupaia/database'; import { TestableServer } from '@tupaia/server-boilerplate'; import { createBearerHeader } from '@tupaia/utils'; import { grantUserAccess, revokeAccess, setupTestApp, setupTestUser } from '../utilities'; -import { CAT_USER, CAT_USER_SESSION } from './fixtures'; +import { CAT_USER } from './fixtures'; const mockResponseMsg = 'Successfully changed password'; @@ -37,7 +37,6 @@ describe('me/changePassword', () => { authHeader = createBearerHeader( constructAccessToken({ userId: user.id, - refreshToken: CAT_USER_SESSION.refresh_token, apiClientUserId: undefined, }), ); diff --git a/packages/meditrak-app-server/src/__tests__/__integration__/UserRewardsRoute.test.ts b/packages/meditrak-app-server/src/__tests__/__integration__/UserRewardsRoute.test.ts index f18be92034..9d4538848a 100644 --- a/packages/meditrak-app-server/src/__tests__/__integration__/UserRewardsRoute.test.ts +++ b/packages/meditrak-app-server/src/__tests__/__integration__/UserRewardsRoute.test.ts @@ -15,7 +15,7 @@ import { TestableServer } from '@tupaia/server-boilerplate'; import { createBearerHeader } from '@tupaia/utils'; import { TestModelRegistry } from '../types'; import { grantUserAccess, revokeAccess, setupTestApp, setupTestUser } from '../utilities'; -import { CAT_QUESTION, CAT_SURVEY, CAT_USER_SESSION } from './fixtures'; +import { CAT_QUESTION, CAT_SURVEY } from './fixtures'; describe('me/rewards', () => { const numberOfSurveyResponses = 100; @@ -30,7 +30,6 @@ describe('me/rewards', () => { authHeader = createBearerHeader( constructAccessToken({ userId: user.id, - refreshToken: CAT_USER_SESSION.refresh_token, apiClientUserId: undefined, }), ); diff --git a/packages/meditrak-app-server/src/__tests__/__integration__/socialFeed/SocialFeedRoute.test.ts b/packages/meditrak-app-server/src/__tests__/__integration__/socialFeed/SocialFeedRoute.test.ts index 4a2c76e05f..fd9b019bc3 100644 --- a/packages/meditrak-app-server/src/__tests__/__integration__/socialFeed/SocialFeedRoute.test.ts +++ b/packages/meditrak-app-server/src/__tests__/__integration__/socialFeed/SocialFeedRoute.test.ts @@ -23,7 +23,6 @@ import { replaceItemsCountryWithCountryId, } from './helper'; import { COUNTRIES, FEED_ITEMS, GONDOR } from './SocialFeedRoute.fixtures'; -import { CAT_USER_SESSION } from '../fixtures'; describe('socialFeed', () => { const CURRENT_DATE_STUB = '2020-12-15T00:00:00.000Z'; @@ -81,7 +80,6 @@ describe('socialFeed', () => { authHeader = createBearerHeader( constructAccessToken({ userId: user.id, - refreshToken: CAT_USER_SESSION.refresh_token, apiClientUserId: undefined, }), ); diff --git a/packages/meditrak-app-server/src/__tests__/__integration__/sync/PushChangesRoute.test.ts b/packages/meditrak-app-server/src/__tests__/__integration__/sync/PushChangesRoute.test.ts index b36f1006b1..2ed11e451b 100644 --- a/packages/meditrak-app-server/src/__tests__/__integration__/sync/PushChangesRoute.test.ts +++ b/packages/meditrak-app-server/src/__tests__/__integration__/sync/PushChangesRoute.test.ts @@ -25,7 +25,6 @@ import { grantUserAccess, revokeAccess, } from '../../utilities'; -import { CAT_USER_SESSION } from '../fixtures'; import { TEST_IMAGE_DATA } from './testImageData'; import { upsertQuestion, @@ -127,7 +126,6 @@ describe('changes (POST)', () => { authHeader = createBearerHeader( constructAccessToken({ userId, - refreshToken: CAT_USER_SESSION.refresh_token, apiClientUserId: undefined, }), ); diff --git a/packages/meditrak-app-server/src/__tests__/__integration__/sync/pullChanges/ChangesMetadataRoute.test.ts b/packages/meditrak-app-server/src/__tests__/__integration__/sync/pullChanges/ChangesMetadataRoute.test.ts index cdd910dea6..d10e72714a 100644 --- a/packages/meditrak-app-server/src/__tests__/__integration__/sync/pullChanges/ChangesMetadataRoute.test.ts +++ b/packages/meditrak-app-server/src/__tests__/__integration__/sync/pullChanges/ChangesMetadataRoute.test.ts @@ -11,7 +11,6 @@ import { SyncableChangeEnqueuer, createPermissionsBasedMeditrakSyncQueue } from import { MeditrakAppServerModelRegistry } from '../../../../types'; import { TestModelRegistry } from '../../../types'; import { grantUserAccess, revokeAccess, setupTestApp, setupTestUser } from '../../../utilities'; -import { CAT_USER_SESSION } from '../../fixtures'; import { PERMISSIONS_BASED_SYNC_MIN_APP_VERSION } from '../../../../routes/sync/pullChanges/supportsPermissionsBasedSync'; import { BASIC_ACCESS } from '../../../utilities/grantUserAccess'; @@ -55,7 +54,6 @@ describe('changes/metadata', () => { authHeader = createBearerHeader( constructAccessToken({ userId: user.id, - refreshToken: CAT_USER_SESSION.refresh_token, apiClientUserId: undefined, }), ); diff --git a/packages/meditrak-app-server/src/__tests__/__integration__/sync/pullChanges/CountChangesRoute.test.ts b/packages/meditrak-app-server/src/__tests__/__integration__/sync/pullChanges/CountChangesRoute.test.ts index 21d36b3db6..fc1aa20022 100644 --- a/packages/meditrak-app-server/src/__tests__/__integration__/sync/pullChanges/CountChangesRoute.test.ts +++ b/packages/meditrak-app-server/src/__tests__/__integration__/sync/pullChanges/CountChangesRoute.test.ts @@ -11,7 +11,6 @@ import { SyncableChangeEnqueuer, createPermissionsBasedMeditrakSyncQueue } from import { MeditrakAppServerModelRegistry } from '../../../../types'; import { TestModelRegistry } from '../../../types'; import { grantUserAccess, revokeAccess, setupTestApp, setupTestUser } from '../../../utilities'; -import { CAT_USER_SESSION } from '../../fixtures'; import { upsertDummyQuestion } from '../upsertDummyQuestion'; describe('changes/count', () => { @@ -32,7 +31,6 @@ describe('changes/count', () => { authHeader = createBearerHeader( constructAccessToken({ userId: user.id, - refreshToken: CAT_USER_SESSION.refresh_token, apiClientUserId: undefined, }), ); diff --git a/packages/meditrak-app-server/src/__tests__/__integration__/sync/pullChanges/PullChangesRoute.test.ts b/packages/meditrak-app-server/src/__tests__/__integration__/sync/pullChanges/PullChangesRoute.test.ts index 51f82be521..2c6f5e1495 100644 --- a/packages/meditrak-app-server/src/__tests__/__integration__/sync/pullChanges/PullChangesRoute.test.ts +++ b/packages/meditrak-app-server/src/__tests__/__integration__/sync/pullChanges/PullChangesRoute.test.ts @@ -24,7 +24,6 @@ import { import { MeditrakAppServerModelRegistry } from '../../../../types'; import { TestModelRegistry } from '../../../types'; import { grantUserAccess, revokeAccess, setupTestApp, setupTestUser } from '../../../utilities'; -import { CAT_USER_SESSION } from '../../fixtures'; import { upsertDummyQuestion } from '../upsertDummyQuestion'; import { findRecordsWithPermissions, @@ -125,7 +124,6 @@ describe('changes (GET)', () => { authHeader = createBearerHeader( constructAccessToken({ userId, - refreshToken: CAT_USER_SESSION.refresh_token, apiClientUserId: undefined, }), ); From 7fc05ab89f0e30c4c1f70d7a1413a4b95d0b9ebd Mon Sep 17 00:00:00 2001 From: alexd-bes <129009580+alexd-bes@users.noreply.github.com> Date: Thu, 4 Apr 2024 09:01:20 +1300 Subject: [PATCH 07/25] feat(adminPanel): RN-1242: Export entity hierarchies (#5530) * Export hierarchies * Run yarn * Export with entity hierarchy name instead of code --------- Co-authored-by: Andrew --- packages/admin-panel-server/package.json | 3 +- .../admin-panel-server/src/app/createApp.ts | 6 ++ .../routes/ExportEntityHierarchiesRoute.ts | 56 +++++++++++++++++++ .../admin-panel-server/src/routes/index.ts | 1 + .../EntityHierarchyExportModal.jsx | 32 +++++++++++ .../admin-panel/src/importExport/index.js | 1 + .../pages/resources/EntityHierarchiesPage.jsx | 2 + .../src/pages/resources/TreeResourcePage.jsx | 18 +++++- yarn.lock | 1 + 9 files changed, 117 insertions(+), 3 deletions(-) create mode 100644 packages/admin-panel-server/src/routes/ExportEntityHierarchiesRoute.ts create mode 100644 packages/admin-panel/src/importExport/EntityHierarchyExportModal.jsx diff --git a/packages/admin-panel-server/package.json b/packages/admin-panel-server/package.json index 7798587e88..24d434d58f 100644 --- a/packages/admin-panel-server/package.json +++ b/packages/admin-panel-server/package.json @@ -42,7 +42,8 @@ "express": "^4.19.2", "lodash": "^4.17.4", "multer": "^1.4.3", - "winston": "^3.3.3" + "winston": "^3.3.3", + "xlsx": "^0.10.9" }, "devDependencies": { "@types/multer": "^1.4.7" diff --git a/packages/admin-panel-server/src/app/createApp.ts b/packages/admin-panel-server/src/app/createApp.ts index 15c93dad9f..a9916ebf67 100644 --- a/packages/admin-panel-server/src/app/createApp.ts +++ b/packages/admin-panel-server/src/app/createApp.ts @@ -42,6 +42,8 @@ import { FetchTransformSchemasRoute, FetchDataTableBuiltInParamsRequest, FetchDataTableBuiltInParamsRoute, + ExportEntityHierarchiesRequest, + ExportEntityHierarchiesRoute, } from '../routes'; import { authHandlerProvider } from '../auth'; @@ -105,6 +107,10 @@ export async function createApp() { 'export/mapOverlayVisualisation/:mapOverlayVisualisationId', handleWith(ExportMapOverlayVisualisationRoute), ) + .get( + 'export/hierarchies', + handleWith(ExportEntityHierarchiesRoute), + ) .post( 'export/dashboardVisualisation', handleWith(ExportDashboardVisualisationRoute), diff --git a/packages/admin-panel-server/src/routes/ExportEntityHierarchiesRoute.ts b/packages/admin-panel-server/src/routes/ExportEntityHierarchiesRoute.ts new file mode 100644 index 0000000000..b7fa815e99 --- /dev/null +++ b/packages/admin-panel-server/src/routes/ExportEntityHierarchiesRoute.ts @@ -0,0 +1,56 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2023 Beyond Essential Systems Pty Ltd + */ + +import { Request } from 'express'; +import xlsx from 'xlsx'; +import { Route } from '@tupaia/server-boilerplate'; + +export type ExportEntityHierarchiesRequest = Request< + { hierarchies: string }, + { contents: Buffer; filePath: string; type: string }, + Record, + Record +>; + +export class ExportEntityHierarchiesRoute extends Route { + protected readonly type = 'download'; + + public async buildResponse() { + const { hierarchies } = this.req.query; + const { entity: entityApi } = this.req.ctx.services; + + const hierarchiesArray = Array.isArray(hierarchies) ? hierarchies : [hierarchies]; + + const workbook = xlsx.utils.book_new(); + + for (const hierarchy of hierarchiesArray) { + const descendants = await entityApi.getDescendantsOfEntity( + hierarchy, + hierarchy, + { + fields: ['name', 'code', 'parent_code'], + }, + false, + false, + ); + + const projectEntity = await entityApi.getEntity(hierarchy, hierarchy, { + fields: ['name'], + }); + + const sheetName = projectEntity?.name || hierarchy; + const sheet = xlsx.utils.json_to_sheet(descendants); + xlsx.utils.book_append_sheet(workbook, sheet, sheetName); + } + + const buffer = xlsx.write(workbook, { type: 'buffer', bookType: 'xlsx' }); + + return { + contents: buffer, + filePath: `entity_hierarchies_export_${Date.now()}.xlsx`, + type: '.xlsx', + }; + } +} diff --git a/packages/admin-panel-server/src/routes/index.ts b/packages/admin-panel-server/src/routes/index.ts index b033d89f67..31d9f4f378 100644 --- a/packages/admin-panel-server/src/routes/index.ts +++ b/packages/admin-panel-server/src/routes/index.ts @@ -12,3 +12,4 @@ export * from './FetchReportPreviewDataRoute'; export * from './FetchReportSchemasRoute'; export * from './UploadTestDataRoute'; export * from './UserRoute'; +export * from './ExportEntityHierarchiesRoute'; diff --git a/packages/admin-panel/src/importExport/EntityHierarchyExportModal.jsx b/packages/admin-panel/src/importExport/EntityHierarchyExportModal.jsx new file mode 100644 index 0000000000..81075997d7 --- /dev/null +++ b/packages/admin-panel/src/importExport/EntityHierarchyExportModal.jsx @@ -0,0 +1,32 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2020 Beyond Essential Systems Pty Ltd + */ + +import React, { useState } from 'react'; + +import { ExportModal } from './ExportModal'; +import { ReduxAutocomplete } from '../autocomplete'; + +export const EntityHierarchyExportModal = () => { + const [hierarchies, setHierarchies] = useState(null); + + return ( + + + + ); +}; diff --git a/packages/admin-panel/src/importExport/index.js b/packages/admin-panel/src/importExport/index.js index 57dab7495c..b162b77b1d 100644 --- a/packages/admin-panel/src/importExport/index.js +++ b/packages/admin-panel/src/importExport/index.js @@ -7,3 +7,4 @@ export { ExportButton } from './ExportButton'; export { ImportModal } from './ImportModal'; export { ExportModal } from './ExportModal'; export { SurveyResponsesExportModal } from './SurveyResponsesExportModal'; +export { EntityHierarchyExportModal } from './EntityHierarchyExportModal'; diff --git a/packages/admin-panel/src/pages/resources/EntityHierarchiesPage.jsx b/packages/admin-panel/src/pages/resources/EntityHierarchiesPage.jsx index 208c4524e2..bd435a48ab 100644 --- a/packages/admin-panel/src/pages/resources/EntityHierarchiesPage.jsx +++ b/packages/admin-panel/src/pages/resources/EntityHierarchiesPage.jsx @@ -10,6 +10,7 @@ import PropTypes from 'prop-types'; import { getSortByKey } from '@tupaia/utils'; import { get } from '../../VizBuilderApp/api'; import { TreeResourcePage } from './TreeResourcePage'; +import { EntityHierarchyExportModal } from '../../importExport'; const fetchRoot = async () => get('hierarchies'); @@ -44,6 +45,7 @@ export const EntityHierarchiesPage = ({ getHeaderEl }) => ( fetchRoot={fetchRoot} fetchBranch={fetchBranch} getHeaderEl={getHeaderEl} + ExportModalComponent={EntityHierarchyExportModal} /> ); diff --git a/packages/admin-panel/src/pages/resources/TreeResourcePage.jsx b/packages/admin-panel/src/pages/resources/TreeResourcePage.jsx index ea0cbe2a01..fef1c7e223 100644 --- a/packages/admin-panel/src/pages/resources/TreeResourcePage.jsx +++ b/packages/admin-panel/src/pages/resources/TreeResourcePage.jsx @@ -28,8 +28,17 @@ const StyledHorizontalTree = styled(HorizontalTree)` } `; -export const TreeResourcePage = ({ title, getHeaderEl, fetchRoot, fetchBranch }) => { - const HeaderPortal = usePortalWithCallback(
, getHeaderEl); +export const TreeResourcePage = ({ + title, + getHeaderEl, + fetchRoot, + fetchBranch, + ExportModalComponent, +}) => { + const HeaderPortal = usePortalWithCallback( +
, + getHeaderEl, + ); return ( <> @@ -42,9 +51,14 @@ export const TreeResourcePage = ({ title, getHeaderEl, fetchRoot, fetchBranch }) ); }; +TreeResourcePage.defaultProps = { + ExportModalComponent: null, +}; + TreeResourcePage.propTypes = { getHeaderEl: PropTypes.func.isRequired, title: PropTypes.string.isRequired, fetchRoot: PropTypes.func.isRequired, fetchBranch: PropTypes.func.isRequired, + ExportModalComponent: PropTypes.elementType, }; diff --git a/yarn.lock b/yarn.lock index 1427396449..73704e6f42 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11701,6 +11701,7 @@ __metadata: lodash: ^4.17.4 multer: ^1.4.3 winston: ^3.3.3 + xlsx: ^0.10.9 languageName: unknown linkType: soft From f64998d372a972b3fc96b18358bdb1f87b485b38 Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Thu, 4 Apr 2024 13:53:07 +1300 Subject: [PATCH 08/25] fix(vizBuilder): RN-1054: picks wrong date (#5447) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * adjust `PreviewOptions`’ date picking logic * make start/end dates deselectable again * query API with only date, no time * circumvent unpredictable Yup date coercion * move `getIsoDate` to tsutils * reusable `ISO_DATE_PATTERN` constant * refactor datepicker change handlers * `getIsoDate` → `getIsoDateString` * finish renaming `getIsoDate` → `getIsoDateString` * add missing import (and flip a ternary) * assume valid input avoids typing issues where return type could be `null` * prettier * fix string slicing bug * fix tests * remove `tsutils` dep from `admin-panel` Importing from `tsutils` to front-end packages is problematic * remove erroneous `export` * update lockfile * more robust `getIsoDateString` * refactor change handlers * organise imports * more robust `convertDateToIsoString ` * fix date picker picking previous day from selected * comment date correcting helper function * clearer comment * use datetime helper * tidy up import * fix header comment reference * simplify explainer comment --------- Co-authored-by: Andrew --- .../PreviewOptions/DateRangeField.jsx | 25 ++++++++++- .../AnalyticsDataTableService.test.ts | 8 ++-- .../services/EventsDataTableService.test.ts | 8 ++-- .../services/AnalyticsDataTableService.ts | 42 +++++++++++-------- .../services/EventsDataTableService.ts | 32 +++++++++----- .../services/utils/getDefaultDates.ts | 8 +++- packages/tsutils/src/datetime.ts | 10 ++++- 7 files changed, 93 insertions(+), 40 deletions(-) diff --git a/packages/admin-panel/src/VizBuilderApp/components/PreviewOptions/DateRangeField.jsx b/packages/admin-panel/src/VizBuilderApp/components/PreviewOptions/DateRangeField.jsx index 8bfc949baa..37c4b8026a 100644 --- a/packages/admin-panel/src/VizBuilderApp/components/PreviewOptions/DateRangeField.jsx +++ b/packages/admin-panel/src/VizBuilderApp/components/PreviewOptions/DateRangeField.jsx @@ -5,6 +5,7 @@ import React, { useEffect } from 'react'; import styled from 'styled-components'; import { DatePicker as DatePickerComponent } from '@tupaia/ui-components'; +import { getIsoDateString } from '@tupaia/tsutils'; import { useVizConfigContext } from '../../context'; const DatePicker = styled(DatePickerComponent)` @@ -40,7 +41,29 @@ export const DateRangeField = () => { } }, [defaultStartDate, startDate, defaultEndDate, endDate]); - const convertDateToIsoString = date => (date ? date.toISOString() : null); + /** + * When you select a date in the date picker (either with the interactive calendar or by inserting + * a valid date string), the picker actually selects a specific moment in time on that date with + * the normal, millisecond granularity of a Date object. The time of day it selects is the current + * local time on the system where VizBuilder is running. + * + * When inputting the date, the date picker interprets it in the user’s time zone. Under the hood, + * this is converted to UTC, which can cause the picked date to be different from what the user + * input (±1 day). + * + * This helper function accounts for that discrepancy. + * + * @param date A valid date object + * @returns {Date} + */ + const shiftEpoch = date => + new Date(date.setMinutes(date.getMinutes() - date.getTimezoneOffset())); + + const convertDateToIsoString = date => { + if (!date || isNaN(new Date(date).getTime())) return null; + const correctedDate = shiftEpoch(date); + return getIsoDateString(correctedDate); + }; const handleChangeStartDate = date => { const newDate = convertDateToIsoString(date); diff --git a/packages/data-table-server/src/__tests__/dataTableService/services/AnalyticsDataTableService.test.ts b/packages/data-table-server/src/__tests__/dataTableService/services/AnalyticsDataTableService.test.ts index 6208064bf5..a5612aef15 100644 --- a/packages/data-table-server/src/__tests__/dataTableService/services/AnalyticsDataTableService.test.ts +++ b/packages/data-table-server/src/__tests__/dataTableService/services/AnalyticsDataTableService.test.ts @@ -93,7 +93,7 @@ describe('AnalyticsDataTableService', () => { dataElementCodes: ['PSSS_AFR_Cases'], startDate: 'cat', }, - 'startDate must be a `date` type', + 'startDate must be a valid ISO 8601 date: YYYY-MM-DD', ], [ 'endDate wrong format', @@ -103,7 +103,7 @@ describe('AnalyticsDataTableService', () => { dataElementCodes: ['PSSS_AFR_Cases'], endDate: 'dog', }, - 'endDate must be a `date` type', + 'endDate must be a valid ISO 8601 date: YYYY-MM-DD', ], [ 'aggregations wrong format', @@ -142,8 +142,8 @@ describe('AnalyticsDataTableService', () => { }, name: 'dataElementCodes', }, - { config: { defaultValue: new Date('2018-12-01'), type: 'date' }, name: 'startDate' }, - { config: { defaultValue: new Date('2023-12-31'), type: 'date' }, name: 'endDate' }, + { config: { defaultValue: '2018-12-01', type: 'string' }, name: 'startDate' }, + { config: { defaultValue: '2023-12-31', type: 'string' }, name: 'endDate' }, ]); }); diff --git a/packages/data-table-server/src/__tests__/dataTableService/services/EventsDataTableService.test.ts b/packages/data-table-server/src/__tests__/dataTableService/services/EventsDataTableService.test.ts index 1971bcd151..3ac8f39d45 100644 --- a/packages/data-table-server/src/__tests__/dataTableService/services/EventsDataTableService.test.ts +++ b/packages/data-table-server/src/__tests__/dataTableService/services/EventsDataTableService.test.ts @@ -159,7 +159,7 @@ describe('EventsDataTableService', () => { dataGroupCode: 'PSSS_WNR', startDate: 'cat', }, - 'startDate must be a `date` type', + 'startDate must be a valid ISO 8601 date: YYYY-MM-DD', ], [ 'endDate wrong format', @@ -169,7 +169,7 @@ describe('EventsDataTableService', () => { dataGroupCode: 'PSSS_WNR', endDate: 'dog', }, - 'endDate must be a `date` type', + 'endDate must be a valid ISO 8601 date: YYYY-MM-DD', ], [ 'aggregations wrong format', @@ -208,8 +208,8 @@ describe('EventsDataTableService', () => { config: { innerType: { required: true, type: 'string' }, type: 'dataElementCodes' }, name: 'dataElementCodes', }, - { config: { defaultValue: new Date('2018-12-01'), type: 'date' }, name: 'startDate' }, - { config: { defaultValue: new Date('2023-12-31'), type: 'date' }, name: 'endDate' }, + { config: { defaultValue: '2018-12-01', type: 'string' }, name: 'startDate' }, + { config: { defaultValue: '2023-12-31', type: 'string' }, name: 'endDate' }, ]); }); diff --git a/packages/data-table-server/src/dataTableService/services/AnalyticsDataTableService.ts b/packages/data-table-server/src/dataTableService/services/AnalyticsDataTableService.ts index 81493346da..44aff26c7d 100644 --- a/packages/data-table-server/src/dataTableService/services/AnalyticsDataTableService.ts +++ b/packages/data-table-server/src/dataTableService/services/AnalyticsDataTableService.ts @@ -8,16 +8,31 @@ import { TupaiaApiClient } from '@tupaia/api-client'; import { Aggregator } from '@tupaia/aggregator'; import { DataBroker } from '@tupaia/data-broker'; import { yup } from '@tupaia/utils'; +import { ISO_DATE_PATTERN } from '@tupaia/tsutils'; import { DataTableService } from '../DataTableService'; import { orderParametersByName } from '../utils'; -import { getDefaultEndDate, getDefaultStartDate, mapProjectEntitiesToCountries } from './utils'; +import { + getDefaultEndDateString, + getDefaultStartDateString, + mapProjectEntitiesToCountries, +} from './utils'; const requiredParamsSchema = yup.object().shape({ hierarchy: yup.string().default('explore'), dataElementCodes: yup.array().of(yup.string().required()).required(), organisationUnitCodes: yup.array().of(yup.string().required()).required(), - startDate: yup.date().default(getDefaultStartDate), - endDate: yup.date().default(getDefaultEndDate), + startDate: yup + .string() + .matches(ISO_DATE_PATTERN, { + message: 'startDate must be a valid ISO 8601 date: YYYY-MM-DD', + }) + .default(getDefaultStartDateString), + endDate: yup + .string() + .matches(ISO_DATE_PATTERN, { + message: 'endDate must be a valid ISO 8601 date: YYYY-MM-DD', + }) + .default(getDefaultEndDateString), aggregations: yup.array().of( yup.object().shape({ type: yup.string().required(), @@ -50,21 +65,12 @@ export class AnalyticsDataTableService extends DataTableService< hierarchy: string; dataElementCodes: string[]; organisationUnitCodes: string[]; - startDate: Date; - endDate: Date; + startDate: string; + endDate: string; aggregations?: { type: string; config?: Record }[]; }) { - const { - hierarchy, - dataElementCodes, - organisationUnitCodes, - startDate, - endDate, - aggregations, - } = params; - - const startDateString = startDate.toISOString(); - const endDateString = endDate.toISOString(); + const { hierarchy, dataElementCodes, organisationUnitCodes, startDate, endDate, aggregations } = + params; // Ensure that if fetching for project, we map it to the underlying countries const entityCodesForFetch = await mapProjectEntitiesToCountries( @@ -85,8 +91,8 @@ export class AnalyticsDataTableService extends DataTableService< { organisationUnitCodes: entityCodesForFetch, hierarchy, - startDate: startDateString, - endDate: endDateString, + startDate, + endDate, detectDataServices: true, }, { aggregations }, diff --git a/packages/data-table-server/src/dataTableService/services/EventsDataTableService.ts b/packages/data-table-server/src/dataTableService/services/EventsDataTableService.ts index f2bd4579b5..19763b0094 100644 --- a/packages/data-table-server/src/dataTableService/services/EventsDataTableService.ts +++ b/packages/data-table-server/src/dataTableService/services/EventsDataTableService.ts @@ -8,17 +8,32 @@ import { Aggregator } from '@tupaia/aggregator'; import { TupaiaApiClient } from '@tupaia/api-client'; import { DataBroker } from '@tupaia/data-broker'; import { yup } from '@tupaia/utils'; +import { ISO_DATE_PATTERN } from '@tupaia/tsutils'; import { DataTableService } from '../DataTableService'; import { orderParametersByName } from '../utils'; -import { getDefaultEndDate, getDefaultStartDate, mapProjectEntitiesToCountries } from './utils'; +import { + getDefaultEndDateString, + getDefaultStartDateString, + mapProjectEntitiesToCountries, +} from './utils'; const requiredParamsSchema = yup.object().shape({ hierarchy: yup.string().default('explore'), dataGroupCode: yup.string().required(), dataElementCodes: yup.array().of(yup.string().required()).min(1), organisationUnitCodes: yup.array().of(yup.string().required()).strict().required(), - startDate: yup.date().default(getDefaultStartDate), - endDate: yup.date().default(getDefaultEndDate), + startDate: yup + .string() + .matches(ISO_DATE_PATTERN, { + message: 'startDate must be a valid ISO 8601 date: YYYY-MM-DD', + }) + .default(getDefaultStartDateString), + endDate: yup + .string() + .matches(ISO_DATE_PATTERN, { + message: 'endDate must be a valid ISO 8601 date: YYYY-MM-DD', + }) + .default(getDefaultEndDateString), aggregations: yup.array().of( yup.object().shape({ type: yup.string().required(), @@ -73,8 +88,8 @@ export class EventsDataTableService extends DataTableService< dataGroupCode: string; dataElementCodes?: string[]; organisationUnitCodes: string[]; - startDate?: Date; - endDate?: Date; + startDate?: string; + endDate?: string; aggregations?: { type: string; config?: Record }[]; }) { const { @@ -94,9 +109,6 @@ export class EventsDataTableService extends DataTableService< }), ); - const startDateString = startDate ? startDate.toISOString() : undefined; - const endDateString = endDate ? endDate.toISOString() : undefined; - // Ensure that if fetching for project, we map it to the underlying countries const entityCodesForFetch = await mapProjectEntitiesToCountries( this.ctx.apiClient, @@ -112,8 +124,8 @@ export class EventsDataTableService extends DataTableService< { hierarchy, organisationUnitCodes: entityCodesForFetch, - startDate: startDateString, - endDate: endDateString, + startDate, + endDate, dataElementCodes: dataElementCodesForFetch, }, { aggregations }, diff --git a/packages/data-table-server/src/dataTableService/services/utils/getDefaultDates.ts b/packages/data-table-server/src/dataTableService/services/utils/getDefaultDates.ts index 0b1fe5c8f6..a32ca02456 100644 --- a/packages/data-table-server/src/dataTableService/services/utils/getDefaultDates.ts +++ b/packages/data-table-server/src/dataTableService/services/utils/getDefaultDates.ts @@ -1,11 +1,15 @@ -/** +/* * Tupaia - * Copyright (c) 2017 - 2023 Beyond Essential Systems Pty Ltd + * Copyright (c) 2017 - 2024 Beyond Essential Systems Pty Ltd */ import { convertPeriodStringToDateRange, getDefaultPeriod } from '@tupaia/utils'; +import { getIsoDateString } from '@tupaia/tsutils'; export const getDefaultStartDate = () => new Date(convertPeriodStringToDateRange(getDefaultPeriod())[0]); export const getDefaultEndDate = () => new Date(convertPeriodStringToDateRange(getDefaultPeriod())[1]); + +export const getDefaultStartDateString = () => getIsoDateString(getDefaultStartDate()); +export const getDefaultEndDateString = () => getIsoDateString(getDefaultEndDate()); diff --git a/packages/tsutils/src/datetime.ts b/packages/tsutils/src/datetime.ts index de1652edb2..5863b864d3 100644 --- a/packages/tsutils/src/datetime.ts +++ b/packages/tsutils/src/datetime.ts @@ -6,6 +6,14 @@ import moment from 'moment'; import momentTimezone from 'moment-timezone'; +export const ISO_DATE_PATTERN = /^\d{4}-\d{2}-\d{2}$/; + +/** + * @returns ISO date string in the format "yyyy-mm-dd", discarding the timestamp. + * @remarks Assumes the input date object is valid. + */ +export const getIsoDateString = (date: Date) => date.toISOString().slice(0, 10); + /** * @returns utcOffset in format: "+05:00" */ @@ -19,4 +27,4 @@ export const getTimezoneNameFromTimestamp = (timestamp: string) => .names() .find(name => getUtcOffsetFromTimestamp(timestamp) === momentTimezone.tz(name).format('Z')); -export const utcMoment = (...args: Parameters) => moment.utc(...args); +export const utcMoment = (...args: Parameters<(typeof moment)['utc']>) => moment.utc(...args); From b2c0d4be13cbbc55aaf5b1adf77ef128c8d1ba15 Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Thu, 4 Apr 2024 14:23:16 +1300 Subject: [PATCH 09/25] tweak(centralServer): remove legacy `requestCountryAccess` endpoint (#5509) * remove legacy `requestCountryAccess` endpoint * remove handling of legacy param * attempt fixing test * remove dangling options object property * use correct endpoint --- packages/central-server/src/apiV2/index.js | 8 -------- packages/central-server/src/apiV2/requestCountryAccess.js | 3 +-- .../central-server/src/tests/apiV2/accessRequests.test.js | 2 +- 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/packages/central-server/src/apiV2/index.js b/packages/central-server/src/apiV2/index.js index 94d05f6c49..d2d1a488b9 100644 --- a/packages/central-server/src/apiV2/index.js +++ b/packages/central-server/src/apiV2/index.js @@ -154,14 +154,6 @@ apiV2.use(logApiRequest); // log every request to the api_request_log table apiV2.use(ensurePermissionCheck); // ensure permissions checking is handled by each endpoint -/** - * Legacy routes to be eventually removed - */ -apiV2.post( - '/user/:userId/requestCountryAccess', // TODO not used from app version 1.7.93. Once usage stops, remove - allowAnyone(requestCountryAccess), -); - /** * /export and /import routes */ diff --git a/packages/central-server/src/apiV2/requestCountryAccess.js b/packages/central-server/src/apiV2/requestCountryAccess.js index 068b4cced7..540bbfcaf3 100644 --- a/packages/central-server/src/apiV2/requestCountryAccess.js +++ b/packages/central-server/src/apiV2/requestCountryAccess.js @@ -66,14 +66,13 @@ const fetchEntities = async (models, entityIds, countryIds) => { }; export const requestCountryAccess = async (req, res) => { - const { body: requestBody = {}, userId: requestUserId, params, models } = req; + const { body: requestBody = {}, userId, models } = req; const { countryIds, entityIds, message = '', projectCode } = requestBody; if ((!countryIds || countryIds.length === 0) && (!entityIds || entityIds.length === 0)) { throw new ValidationError('Please select at least one country'); } const entities = await fetchEntities(models, entityIds, countryIds); - const userId = requestUserId || params.userId; try { checkUserPermission(req, userId); diff --git a/packages/central-server/src/tests/apiV2/accessRequests.test.js b/packages/central-server/src/tests/apiV2/accessRequests.test.js index fa40febc8c..2398bd4ce9 100644 --- a/packages/central-server/src/tests/apiV2/accessRequests.test.js +++ b/packages/central-server/src/tests/apiV2/accessRequests.test.js @@ -27,7 +27,7 @@ describe('Access Requests', () => { }; const requestCountryAccess = async (userId, entityId, projectCode) => { - return app.post(`user/${userId}/requestCountryAccess`, { + return app.post('me/requestCountryAccess', { body: { entityIds: [entityId], message: "E rab'a te kaitiboo! / Pleased to meet you", From 7616b19bf7be477a6189820794912101117ae75a Mon Sep 17 00:00:00 2001 From: alexd-bes <129009580+alexd-bes@users.noreply.github.com> Date: Thu, 4 Apr 2024 14:55:00 +1300 Subject: [PATCH 10/25] refactor: RN-1225: Update how env vars are handled (#5533) * Set default env vars * WIP * WIP * Update configureEnv.js * Update mergeEnvForDB.sh * Update mergeEnvForDB.sh * Rename env vars and fix scripts * Make sure tests work * Add configure env logic to web-config-server * Put server-utils before data-api in getInternalDependencies * Remove install of dotenv from packages that don't need it * Get script paths * Test * Correct paths * Update scripts/bash/mergeEnvForDB.sh Co-authored-by: Jasper Lai <33956381+jaskfla@users.noreply.github.com> * Remove auth env example * Remove auth and db from having env files * Add quotation marks * remove comma * Update packages/data-table-server/jest.setup.ts Co-authored-by: Jasper Lai <33956381+jaskfla@users.noreply.github.com> * Update packages/data-table-server/.env.example Co-authored-by: Jasper Lai <33956381+jaskfla@users.noreply.github.com> * Merge * Update env handling * Don't download types package --------- Co-authored-by: Jasper Lai <33956381+jaskfla@users.noreply.github.com> --- .gitignore | 1 + env/aggregation.env.example | 1 + env/api-client.env.example | 7 +++ env/aws.env.example | 3 + env/data-lake.env.example | 4 ++ .../auth/.env.example => env/db.env.example | 1 - env/dhis.env.example | 8 +++ env/external-db-connections.env.example | 15 +++++ env/mail.env.example | 5 ++ env/pg.env.example | 4 ++ env/servers.env.example | 5 ++ env/superset.env.example | 3 + env/weatherbit.env.example | 1 + jest.setup.js | 2 +- package.json | 2 +- packages/admin-panel-server/.env.example | 14 +---- packages/admin-panel-server/package.json | 2 +- packages/admin-panel-server/src/index.ts | 11 +++- .../admin-panel/src/VizBuilderApp/api/api.js | 2 +- packages/api-client/src/constants.ts | 15 ++++- packages/auth/jest.setup.js | 9 ++- packages/auth/package.json | 1 + packages/central-server/.env.example | 57 ------------------- packages/central-server/package.json | 1 - packages/central-server/src/configureEnv.js | 24 ++++++++ .../src/createMeditrakSyncView.js | 6 +- packages/central-server/src/index.js | 6 +- .../src/tests/testUtilities/TestableApp.js | 5 +- .../database/addBaselineTestData.js | 4 +- packages/data-api/.env.example | 7 --- packages/data-api/jest.setup.ts | 3 + packages/data-api/package.json | 2 +- .../scripts/buildAnalyticsMaterializedView.sh | 2 +- .../scripts/dropAnalyticsMaterializedView.sh | 2 +- .../scripts/fastRefreshAnalyticsTable.sh | 2 +- .../scripts/fullRefreshAnalyticsTable.sh | 2 +- .../scripts/installMvRefreshModule.sh | 2 +- packages/data-api/scripts/patchMvRefresh.ts | 4 +- .../data-api/scripts/patchMvRefreshModule.sh | 8 +-- .../scripts/uninstallMvRefreshModule.sh | 2 +- packages/data-api/src/configureEnv.ts | 15 +++++ packages/data-api/tsconfig.json | 2 +- packages/data-lake-api/.env.example | 2 +- packages/data-lake-api/package.json | 1 - .../scripts/checkTestDataLakeExists.sh | 4 +- .../scripts/setupTestDataLake.sh | 2 +- packages/data-table-server/.env.example | 45 --------------- packages/data-table-server/jest.config.ts | 2 +- packages/data-table-server/jest.setup.ts | 8 +++ packages/data-table-server/package.json | 2 +- .../data-table-server/src/configureEnv.ts | 22 +++++++ packages/data-table-server/src/index.ts | 5 +- packages/database/.env.example | 7 --- packages/database/jest.setup.js | 3 + packages/database/package.json | 2 +- .../scripts/checkTestDatabaseExists.sh | 2 +- packages/database/scripts/dumpDatabase.sh | 5 +- packages/database/scripts/refreshDatabase.js | 8 ++- .../database/scripts/setupTestDatabase.sh | 2 +- packages/database/src/configureEnv.js | 17 ++++++ packages/database/src/migrate.js | 5 +- .../src/testUtilities/getTestDatabase.js | 4 +- packages/database/tsconfig.json | 9 ++- packages/datatrak-web-server/.env.example | 16 +----- packages/datatrak-web-server/package.json | 2 +- packages/datatrak-web-server/src/index.ts | 12 +++- .../ci/validateTypesAndDbSchemaInSync.sh | 10 +++- .../deployment/buildDeployablePackages.sh | 2 +- packages/entity-server/.env.example | 8 --- packages/entity-server/jest.setup.ts | 3 + packages/entity-server/package.json | 2 +- packages/entity-server/src/configureEnv.ts | 14 +++++ packages/entity-server/src/index.ts | 5 +- packages/lesmis-server/.env.example | 18 +----- packages/lesmis-server/package.json | 1 - .../SessionHandlingApiConnection.ts | 7 +-- packages/lesmis-server/src/index.ts | 10 +++- packages/meditrak-app-server/.env.example | 18 +----- packages/meditrak-app-server/jest.config.ts | 6 +- packages/meditrak-app-server/jest.setup.ts | 3 + packages/meditrak-app-server/package.json | 1 - .../src/__tests__/__integration__/fixtures.ts | 3 - .../src/__tests__/setupTestEnvVars.ts | 9 --- .../meditrak-app-server/src/configureEnv.ts | 17 ++++++ packages/meditrak-app-server/src/index.ts | 4 +- .../src/sync/createMeditrakSyncView.ts | 5 +- packages/meditrak-app/app/Meditrak.jsx | 33 +++++++---- packages/meditrak-app/app/version/isBeta.jsx | 3 +- packages/psss-server/.env.example | 13 +---- packages/psss-server/package.json | 2 +- packages/psss-server/src/index.ts | 11 +++- packages/psss/src/api/api.js | 2 +- packages/report-server/.env.example | 21 +------ packages/report-server/package.json | 2 +- packages/report-server/src/index.ts | 14 ++++- packages/server-boilerplate/.env.example | 3 - packages/server-boilerplate/package.json | 2 +- packages/server-utils/package.json | 1 + packages/server-utils/src/configureDotEnv.ts | 10 ++++ packages/server-utils/src/index.ts | 1 + packages/tupaia-web-server/.env.example | 15 +---- packages/tupaia-web-server/examples.http | 4 +- packages/tupaia-web-server/package.json | 1 - packages/tupaia-web-server/src/index.ts | 12 +++- packages/types/generate-models.ts | 5 +- packages/types/package.json | 2 +- packages/web-config-server/.env.example | 29 +--------- packages/web-config-server/jest.setup.js | 3 + packages/web-config-server/package.json | 1 - packages/web-config-server/src/app.js | 4 +- .../requestHelpers/fetchFromCentralServer.js | 6 +- .../src/authSession/authSession.js | 5 +- .../web-config-server/src/configureEnv.js | 21 +++++++ .../export/requestFromTupaiaConfigServer.js | 4 +- packages/web-config-server/src/index.js | 5 +- scripts/bash/downloadEnvironmentVariables.sh | 54 ++++++++++++++---- scripts/bash/getInternalDependencies.sh | 2 +- scripts/bash/getPackagesWithEnvFiles.sh | 2 +- scripts/bash/mergeCurrentEnvWithEnvFile.sh | 5 -- scripts/bash/mergeEnvForDB.sh | 51 +++++++++++++++++ yarn.lock | 46 +++++++-------- 121 files changed, 567 insertions(+), 433 deletions(-) create mode 100644 env/aggregation.env.example create mode 100644 env/api-client.env.example create mode 100644 env/aws.env.example create mode 100644 env/data-lake.env.example rename packages/auth/.env.example => env/db.env.example (70%) create mode 100644 env/dhis.env.example create mode 100644 env/external-db-connections.env.example create mode 100644 env/mail.env.example create mode 100644 env/pg.env.example create mode 100644 env/servers.env.example create mode 100644 env/superset.env.example create mode 100644 env/weatherbit.env.example create mode 100644 packages/central-server/src/configureEnv.js create mode 100644 packages/data-api/src/configureEnv.ts create mode 100644 packages/data-table-server/jest.setup.ts create mode 100644 packages/data-table-server/src/configureEnv.ts delete mode 100644 packages/database/.env.example create mode 100644 packages/database/src/configureEnv.js delete mode 100644 packages/entity-server/.env.example create mode 100644 packages/entity-server/src/configureEnv.ts delete mode 100644 packages/meditrak-app-server/src/__tests__/setupTestEnvVars.ts create mode 100644 packages/meditrak-app-server/src/configureEnv.ts delete mode 100644 packages/server-boilerplate/.env.example create mode 100644 packages/server-utils/src/configureDotEnv.ts create mode 100644 packages/web-config-server/src/configureEnv.js delete mode 100755 scripts/bash/mergeCurrentEnvWithEnvFile.sh create mode 100755 scripts/bash/mergeEnvForDB.sh diff --git a/.gitignore b/.gitignore index e4ec26f7d2..7b8a839263 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.env* !*.env.example* +!*.env.*.example* !*.env.encrypted* .DS_Store **/node_modules diff --git a/env/aggregation.env.example b/env/aggregation.env.example new file mode 100644 index 0000000000..eec3f69c7a --- /dev/null +++ b/env/aggregation.env.example @@ -0,0 +1 @@ +AGGREGATION_URL_PREFIX= diff --git a/env/api-client.env.example b/env/api-client.env.example new file mode 100644 index 0000000000..e9893627bb --- /dev/null +++ b/env/api-client.env.example @@ -0,0 +1,7 @@ +AUTH_API_URL= +DATA_TABLE_API_URL= +CENTRAL_API_URL= +ENTITY_API_URL= +REPORT_API_URL= +WEB_CONFIG_API_URL= +API_CLIENT_SALT= diff --git a/env/aws.env.example b/env/aws.env.example new file mode 100644 index 0000000000..6b0f4de4fe --- /dev/null +++ b/env/aws.env.example @@ -0,0 +1,3 @@ +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +AWS_REGION= diff --git a/env/data-lake.env.example b/env/data-lake.env.example new file mode 100644 index 0000000000..704cf59d1c --- /dev/null +++ b/env/data-lake.env.example @@ -0,0 +1,4 @@ +DATA_LAKE_DB_NAME= +DATA_LAKE_DB_PASSWORD= +DATA_LAKE_DB_URL= +DATA_LAKE_DB_USER= diff --git a/packages/auth/.env.example b/env/db.env.example similarity index 70% rename from packages/auth/.env.example rename to env/db.env.example index 1bd703139a..3572aac4c4 100644 --- a/packages/auth/.env.example +++ b/env/db.env.example @@ -1,4 +1,3 @@ -DB_DISABLE_SSL= DB_NAME= DB_PASSWORD= DB_URL= diff --git a/env/dhis.env.example b/env/dhis.env.example new file mode 100644 index 0000000000..ab3193b43f --- /dev/null +++ b/env/dhis.env.example @@ -0,0 +1,8 @@ +PALAU_DHIS_CLIENT_SECRET= +PALAU_DHIS_PASSWORD= +PALAU_DHIS_API_URL= +DHIS_CLIENT_ID= +DHIS_CLIENT_SECRET= +DHIS_PASSWORD= +DHIS_USERNAME= +IS_PRODUCTION_ENVIRONMENT= diff --git a/env/external-db-connections.env.example b/env/external-db-connections.env.example new file mode 100644 index 0000000000..dd8e9fe540 --- /dev/null +++ b/env/external-db-connections.env.example @@ -0,0 +1,15 @@ +EXT_DB_niwa_PORT= +EXT_DB_niwa_DATABASE= +EXT_DB_niwa_USER= +EXT_DB_niwa_PASSWORD= +EXT_DB_niwa_HOST= +EXT_DB_data_lake_HOST= +EXT_DB_data_lake_PORT= +EXT_DB_data_lake_DATABASE= +EXT_DB_data_lake_USER= +EXT_DB_data_lake_PASSWORD= +EXT_DB_msupply_scheduled_HOST= +EXT_DB_msupply_scheduled_PORT= +EXT_DB_msupply_scheduled_DATABASE= +EXT_DB_msupply_scheduled_USER= +EXT_DB_msupply_scheduled_PASSWORD= diff --git a/env/mail.env.example b/env/mail.env.example new file mode 100644 index 0000000000..508bb48c61 --- /dev/null +++ b/env/mail.env.example @@ -0,0 +1,5 @@ +SITE_EMAIL_ADDRESS= +SMTP_HOST= +SMTP_PASSWORD= +SMTP_USER= +DEV_EMAIL_ADDRESS= diff --git a/env/pg.env.example b/env/pg.env.example new file mode 100644 index 0000000000..624889d64a --- /dev/null +++ b/env/pg.env.example @@ -0,0 +1,4 @@ +DB_PG_USER= +DB_PG_PASSWORD= +# used in CI/CD, and for running analytics refresh via CLI +DB_ENABLE_SSL= diff --git a/env/servers.env.example b/env/servers.env.example new file mode 100644 index 0000000000..d878afa1e4 --- /dev/null +++ b/env/servers.env.example @@ -0,0 +1,5 @@ + +JWT_SECRET= +SESSION_COOKIE_SECRET= + + diff --git a/env/superset.env.example b/env/superset.env.example new file mode 100644 index 0000000000..2163502a0f --- /dev/null +++ b/env/superset.env.example @@ -0,0 +1,3 @@ +SUPERSET_API_PROXY_URL= +SUPERSET_API_USERNAME= +SUPERSET_API_PASSWORD= diff --git a/env/weatherbit.env.example b/env/weatherbit.env.example new file mode 100644 index 0000000000..76432ce1ac --- /dev/null +++ b/env/weatherbit.env.example @@ -0,0 +1 @@ +WEATHERBIT_API_KEY= diff --git a/jest.setup.js b/jest.setup.js index 8da307bb4d..8a03c8e6bf 100644 --- a/jest.setup.js +++ b/jest.setup.js @@ -1,6 +1,6 @@ /** * Tupaia - * Copyright (c) 2017 - 2020 Beyond Essential Systems Pty Ltd + * Copyright (c) 2017 - 2024 Beyond Essential Systems Pty Ltd */ /* eslint-env jest */ diff --git a/package.json b/package.json index 53299770bb..3cc5dfb880 100644 --- a/package.json +++ b/package.json @@ -72,7 +72,7 @@ "babel-preset-react-app": "^10.0.0", "concurrently": "^5.2.0", "cypress-dotenv": "^1.2.2", - "dotenv": "^16.0.3", + "dotenv": "16.4.5", "eslint": "^7.9.0", "eslint-import-resolver-babel-module": "^5.3.1", "eslint-plugin-cypress": "^2.11.1", diff --git a/packages/admin-panel-server/.env.example b/packages/admin-panel-server/.env.example index d7e43b99a6..df525d89bd 100644 --- a/packages/admin-panel-server/.env.example +++ b/packages/admin-panel-server/.env.example @@ -1,16 +1,4 @@ PORT= API_CLIENT_NAME= -API_CLIENT_PASSWORD= -API_CLIENT_SALT= - -DB_URL= -DB_NAME= -DB_PASSWORD= -DB_USER= - -SESSION_COOKIE_SECRET= - -CENTRAL_API_URL= -ENTITY_API_URL= -REPORT_API_URL= +API_CLIENT_PASSWORD= diff --git a/packages/admin-panel-server/package.json b/packages/admin-panel-server/package.json index 24d434d58f..df2797970c 100644 --- a/packages/admin-panel-server/package.json +++ b/packages/admin-panel-server/package.json @@ -30,6 +30,7 @@ "@tupaia/api-client": "workspace:*", "@tupaia/database": "workspace:*", "@tupaia/server-boilerplate": "workspace:*", + "@tupaia/server-utils": "workspace:*", "@tupaia/types": "workspace:*", "@tupaia/utils": "workspace:*", "api-error-handler": "^1.0.0", @@ -38,7 +39,6 @@ "case": "^1.6.3", "client-sessions": "^0.8.0", "cors": "^2.8.5", - "dotenv": "^8.2.0", "express": "^4.19.2", "lodash": "^4.17.4", "multer": "^1.4.3", diff --git a/packages/admin-panel-server/src/index.ts b/packages/admin-panel-server/src/index.ts index 632cf44019..70b230004f 100644 --- a/packages/admin-panel-server/src/index.ts +++ b/packages/admin-panel-server/src/index.ts @@ -4,14 +4,21 @@ */ import http from 'http'; -import * as dotenv from 'dotenv'; +import path from 'path'; import winston from 'winston'; import { configureWinston } from '@tupaia/server-boilerplate'; +import { configureDotEnv } from '@tupaia/server-utils'; import { createApp } from './app'; configureWinston(); -dotenv.config(); // Load the environment variables into process.env + +configureDotEnv([ + path.resolve(__dirname, '../../../env/servers.env'), + path.resolve(__dirname, '../../../env/db.env'), + path.resolve(__dirname, '../../../env/api-client.env'), + path.resolve(__dirname, '../.env'), +]); (async () => { /** diff --git a/packages/admin-panel/src/VizBuilderApp/api/api.js b/packages/admin-panel/src/VizBuilderApp/api/api.js index ebba5d5338..92f7cb9fc9 100644 --- a/packages/admin-panel/src/VizBuilderApp/api/api.js +++ b/packages/admin-panel/src/VizBuilderApp/api/api.js @@ -7,7 +7,7 @@ import axios from 'axios'; import { saveAs } from 'file-saver'; import FetchError from './fetchError'; -const baseUrl = import.meta.env.REACT_APP_VIZ_BUILDER_API_URL; +const baseUrl = import.meta.env.REACT_APP_VIZ_BUILDER_API_URL || 'http://localhost:8070/v1'; const timeout = 45 * 1000; // 45 seconds // withCredentials needs to be set for cookies to save @see https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/withCredentials diff --git a/packages/api-client/src/constants.ts b/packages/api-client/src/constants.ts index 38b1b818b7..c87a28b7c3 100644 --- a/packages/api-client/src/constants.ts +++ b/packages/api-client/src/constants.ts @@ -31,36 +31,47 @@ const SERVICES = { subdomain: 'api', version: 'v2', localPort: '8090', + prefix: null, }, entity: { subdomain: 'entity-api', version: 'v1', localPort: '8050', + prefix: null, }, central: { subdomain: 'api', version: 'v2', localPort: '8090', + prefix: null, }, report: { subdomain: 'report-api', version: 'v1', localPort: '8030', + prefix: null, }, dataTable: { subdomain: 'data-table-api', version: 'v1', localPort: '8010', + prefix: null, }, webConfig: { subdomain: 'config', + prefix: 'api', version: 'v1', localPort: '8000', }, }; -const getLocalUrl = (service: ServiceName): string => - `http://localhost:${SERVICES[service].localPort}/${SERVICES[service].version}`; +const getLocalUrl = (service: ServiceName): string => { + const { prefix, localPort, version } = SERVICES[service]; + const base = `http://localhost:${localPort}`; + const prefixPath = prefix ? `/${prefix}` : ''; + return `${base}${prefixPath}/${version}`; +}; + export const LOCALHOST_BASE_URLS: ServiceBaseUrlSet = { auth: getLocalUrl('auth'), entity: getLocalUrl('entity'), diff --git a/packages/auth/jest.setup.js b/packages/auth/jest.setup.js index e816993d02..6e07d01190 100644 --- a/packages/auth/jest.setup.js +++ b/packages/auth/jest.setup.js @@ -1,9 +1,16 @@ /** * Tupaia - * Copyright (c) 2017 - 2020 Beyond Essential Systems Pty Ltd + * Copyright (c) 2017 - 2024 Beyond Essential Systems Pty Ltd */ +import path from 'path'; import { getTestDatabase, clearTestData } from '@tupaia/database'; +import { configureDotEnv } from '@tupaia/server-utils'; + +configureDotEnv([ + path.resolve(__dirname, '../../env/db.env'), + path.resolve(__dirname, '../../env/servers.env'), +]); afterAll(async () => { const database = getTestDatabase(); diff --git a/packages/auth/package.json b/packages/auth/package.json index 8c216a938c..3d9a3b2ac0 100644 --- a/packages/auth/package.json +++ b/packages/auth/package.json @@ -23,6 +23,7 @@ "test:coverage": "yarn test --coverage" }, "dependencies": { + "@tupaia/server-utils": "workspace:*", "@tupaia/utils": "workspace:*", "jsonwebtoken": "^7.4.3", "rand-token": "^1.0.1", diff --git a/packages/central-server/.env.example b/packages/central-server/.env.example index 1d9f7c43d4..1c494cac30 100644 --- a/packages/central-server/.env.example +++ b/packages/central-server/.env.example @@ -1,71 +1,14 @@ -PORT= - -AGGREGATION_URL_PREFIX= -API_CLIENT_SALT= - AUTOMATED_TEST_USER= AUTOMATED_TEST_PASSWORD= -AWS_ACCESS_KEY_ID= -AWS_REGION= -AWS_SECRET_ACCESS_KEY= - CLIENT_SECRET= CLIENT_USERNAME= -DB_URL= -DB_NAME= -DB_USER= -DB_PASSWORD= - -DEV_EMAIL_ADDRESS= - -DHIS_CLIENT_ID= -DHIS_CLIENT_SECRET= -DHIS_PASSWORD= -DHIS_USERNAME= - -EXT_DB_data_lake_HOST= -EXT_DB_data_lake_PORT= -EXT_DB_data_lake_DATABASE= -EXT_DB_data_lake_USER= -EXT_DB_data_lake_PASSWORD= - -EXT_DB_msupply_scheduled_HOST= -EXT_DB_msupply_scheduled_PORT= -EXT_DB_msupply_scheduled_DATABASE= -EXT_DB_msupply_scheduled_USER= -EXT_DB_msupply_scheduled_PASSWORD= - -EXT_DB_niwa_HOST= -EXT_DB_niwa_PORT= -EXT_DB_niwa_DATABASE= -EXT_DB_niwa_USER= -EXT_DB_niwa_PASSWORD= - -IS_PRODUCTION_ENVIRONMENT= - -JWT_SECRET= MS1_PASSWORD= MS1_URL= MS1_USERNAME= -PALAU_DHIS_API_URL= -PALAU_DHIS_CLIENT_SECRET= -PALAU_DHIS_PASSWORD= - -SITE_EMAIL_ADDRESS= - -SMTP_HOST= -SMTP_PASSWORD= -SMTP_USER= - -TUPAIA_ADMIN_EMAIL_ADDRESS= - -VERIFY_EMAIL_URL= -PASSWORD_RESET_URL= - DHIS_SYNC_DISABLE= KOBO_SYNC_DISABLE= FEED_SCRAPER_DISABLE= diff --git a/packages/central-server/package.json b/packages/central-server/package.json index 53ce4815c6..6fe6c9fd86 100644 --- a/packages/central-server/package.json +++ b/packages/central-server/package.json @@ -48,7 +48,6 @@ "cors": "^2.8.5", "countrynames": "^0.1.1", "del": "^2.2.2", - "dotenv": "^8.2.0", "express": "^4.19.2", "form-data": "^2.3.3", "format-link-header": "^2.1.0", diff --git a/packages/central-server/src/configureEnv.js b/packages/central-server/src/configureEnv.js new file mode 100644 index 0000000000..f64abfbb5c --- /dev/null +++ b/packages/central-server/src/configureEnv.js @@ -0,0 +1,24 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2024 Beyond Essential Systems Pty Ltd + */ + +import path from 'path'; +import { configureDotEnv } from '@tupaia/server-utils'; + +const envFilePaths = [ + path.resolve(__dirname, '../../../env/db.env'), + path.resolve(__dirname, '../../../env/servers.env'), + path.resolve(__dirname, '../../../env/pg.env'), + path.resolve(__dirname, '../../../env/dhis.env'), + path.resolve(__dirname, '../../../env/external-db-connections.env'), + path.resolve(__dirname, '../../../env/mail.env'), + path.resolve(__dirname, '../../../env/aws.env'), + path.resolve(__dirname, '../../../env/aggregation.env'), + path.resolve(__dirname, '../../../env/api-client.env'), + path.resolve(__dirname, '../.env'), +]; + +export const configureEnv = () => { + configureDotEnv(envFilePaths); +}; diff --git a/packages/central-server/src/createMeditrakSyncView.js b/packages/central-server/src/createMeditrakSyncView.js index 2f270bd5b9..95a89abfb4 100644 --- a/packages/central-server/src/createMeditrakSyncView.js +++ b/packages/central-server/src/createMeditrakSyncView.js @@ -8,18 +8,20 @@ import '@babel/polyfill'; -import {} from 'dotenv/config'; // Load the environment variables into process.env - import { TupaiaDatabase } from '@tupaia/database'; import { isFeatureEnabled } from '@tupaia/utils'; import winston from './log'; import { createPermissionsBasedMeditrakSyncQueue } from './database'; +import { configureEnv } from './configureEnv'; + +configureEnv(); (async () => { /** * Set up database */ + const database = new TupaiaDatabase(); if (!isFeatureEnabled('MEDITRAK_SYNC_QUEUE')) { diff --git a/packages/central-server/src/index.js b/packages/central-server/src/index.js index 68432e82cc..40296a5943 100644 --- a/packages/central-server/src/index.js +++ b/packages/central-server/src/index.js @@ -4,9 +4,6 @@ */ import '@babel/polyfill'; - -import {} from 'dotenv/config'; // Load the environment variables into process.env - import http from 'http'; import { AnalyticsRefresher, @@ -27,6 +24,9 @@ import { startFeedScraper } from './social'; import { createApp } from './createApp'; import winston from './log'; +import { configureEnv } from './configureEnv'; + +configureEnv(); (async () => { /** diff --git a/packages/central-server/src/tests/testUtilities/TestableApp.js b/packages/central-server/src/tests/testUtilities/TestableApp.js index c64186eb51..71ae8cb192 100644 --- a/packages/central-server/src/tests/testUtilities/TestableApp.js +++ b/packages/central-server/src/tests/testUtilities/TestableApp.js @@ -2,7 +2,7 @@ * Tupaia MediTrak * Copyright (c) 2017 Beyond Essential Systems Pty Ltd */ -import {} from 'dotenv/config'; // Load the environment variables into process.env + import supertest from 'supertest'; import autobind from 'react-autobind'; import sinon from 'sinon'; @@ -15,6 +15,9 @@ import { BES_ADMIN_PERMISSION_GROUP } from '../../permissions'; import { createApp } from '../../createApp'; import { getModels } from './database'; import { TEST_USER_EMAIL } from './constants'; +import { configureEnv } from '../../configureEnv'; + +configureEnv(); const DEFAULT_API_VERSION = 2; const getVersionedEndpoint = (endpoint, apiVersion = DEFAULT_API_VERSION) => diff --git a/packages/central-server/src/tests/testUtilities/database/addBaselineTestData.js b/packages/central-server/src/tests/testUtilities/database/addBaselineTestData.js index bdbe4bcbc2..b63bce5a63 100644 --- a/packages/central-server/src/tests/testUtilities/database/addBaselineTestData.js +++ b/packages/central-server/src/tests/testUtilities/database/addBaselineTestData.js @@ -3,15 +3,17 @@ * Copyright (c) 2019 Beyond Essential Systems Pty Ltd */ -import {} from 'dotenv/config'; // Load the environment variables into process.env import { encryptPassword } from '@tupaia/auth'; import { generateId } from '@tupaia/database'; import { createUser as createUserAccessor } from '../../../dataAccessors'; +import { configureEnv } from '../../../configureEnv'; import { getModels } from './getModels'; import { TEST_USER_EMAIL } from '../constants'; const models = getModels(); +configureEnv(); + export async function addBaselineTestData() { // if there's a pre-existing Demo Land in the DB, use that, otherwise create // one with a test ID so it'll get cleaned up later diff --git a/packages/data-api/.env.example b/packages/data-api/.env.example index 1a2344c2b3..f0333b699f 100644 --- a/packages/data-api/.env.example +++ b/packages/data-api/.env.example @@ -1,9 +1,2 @@ -DB_ENABLE_SSL= -DB_NAME= -DB_PASSWORD= -DB_URL= -DB_USER= -DB_PG_USER= -DB_PG_PASSWORD= DB_MV_USER= DB_MV_PASSWORD= diff --git a/packages/data-api/jest.setup.ts b/packages/data-api/jest.setup.ts index fa88883bfd..236d57b22f 100644 --- a/packages/data-api/jest.setup.ts +++ b/packages/data-api/jest.setup.ts @@ -5,6 +5,9 @@ import { getTestModels, setupTest, clearTestData } from '@tupaia/database'; import { SETUP } from './src/__tests__/TupaiaDataApi.fixtures'; +import { configureEnv } from './src/configureEnv'; + +configureEnv(); const models = getTestModels(); diff --git a/packages/data-api/package.json b/packages/data-api/package.json index a2ffbd4758..ebfbceaceb 100644 --- a/packages/data-api/package.json +++ b/packages/data-api/package.json @@ -28,13 +28,13 @@ }, "dependencies": { "@tupaia/database": "workspace:*", + "@tupaia/server-utils": "workspace:*", "@tupaia/tsutils": "workspace:*", "@tupaia/utils": "workspace:*", "@types/lodash.groupby": "^4.6.0", "db-migrate": "^0.11.5", "db-migrate-pg": "^1.2.2", "deep-equal-in-any-order": "^1.0.27", - "dotenv": "^8.2.0", "lodash.groupby": "^4.6.0", "moment": "^2.24.0" }, diff --git a/packages/data-api/scripts/buildAnalyticsMaterializedView.sh b/packages/data-api/scripts/buildAnalyticsMaterializedView.sh index 34e110e229..05a4638d62 100755 --- a/packages/data-api/scripts/buildAnalyticsMaterializedView.sh +++ b/packages/data-api/scripts/buildAnalyticsMaterializedView.sh @@ -1,7 +1,7 @@ #!/bin/bash -e DIR=$(pwd "$0") -source $DIR/../../scripts/bash/mergeCurrentEnvWithEnvFile.sh +source "$DIR/../../scripts/bash/mergeEnvForDB.sh" # Set default port in case it wasn't in .env : "${DB_PORT:=5432}" diff --git a/packages/data-api/scripts/dropAnalyticsMaterializedView.sh b/packages/data-api/scripts/dropAnalyticsMaterializedView.sh index ef74ea958f..c961a67eaa 100755 --- a/packages/data-api/scripts/dropAnalyticsMaterializedView.sh +++ b/packages/data-api/scripts/dropAnalyticsMaterializedView.sh @@ -1,7 +1,7 @@ #!/bin/bash -e DIR=$(pwd "$0") -source $DIR/../../scripts/bash/mergeCurrentEnvWithEnvFile.sh +source "$DIR/../../scripts/bash/mergeEnvForDB.sh" # Set default port in case it wasn't in .env : "${DB_PORT:=5432}" diff --git a/packages/data-api/scripts/fastRefreshAnalyticsTable.sh b/packages/data-api/scripts/fastRefreshAnalyticsTable.sh index 2996f4e341..5df9fd9863 100755 --- a/packages/data-api/scripts/fastRefreshAnalyticsTable.sh +++ b/packages/data-api/scripts/fastRefreshAnalyticsTable.sh @@ -2,7 +2,7 @@ echo "Fast refreshing analytics table" DIR=$(pwd "$0") -source $DIR/../../scripts/bash/mergeCurrentEnvWithEnvFile.sh +source "$DIR/../../scripts/bash/mergeEnvForDB.sh" # Set default port in case it wasn't in .env : "${DB_PORT:=5432}" diff --git a/packages/data-api/scripts/fullRefreshAnalyticsTable.sh b/packages/data-api/scripts/fullRefreshAnalyticsTable.sh index 139459aa6d..65da86c341 100755 --- a/packages/data-api/scripts/fullRefreshAnalyticsTable.sh +++ b/packages/data-api/scripts/fullRefreshAnalyticsTable.sh @@ -2,7 +2,7 @@ echo "Fully refreshing analytics table" DIR=$(pwd "$0") -source $DIR/../../scripts/bash/mergeCurrentEnvWithEnvFile.sh +source "$DIR/../../scripts/bash/mergeEnvForDB.sh" # Set default port in case it wasn't in .env : "${DB_PORT:=5432}" diff --git a/packages/data-api/scripts/installMvRefreshModule.sh b/packages/data-api/scripts/installMvRefreshModule.sh index d799686eca..f6800f1ece 100755 --- a/packages/data-api/scripts/installMvRefreshModule.sh +++ b/packages/data-api/scripts/installMvRefreshModule.sh @@ -3,7 +3,7 @@ DIR=$(dirname "$0") DIR=$(pwd "$0") -source $DIR/../../scripts/bash/mergeCurrentEnvWithEnvFile.sh +source "$DIR/../../scripts/bash/mergeEnvForDB.sh" # Set default port in case it wasn't in .env : "${DB_PORT:=5432}" diff --git a/packages/data-api/scripts/patchMvRefresh.ts b/packages/data-api/scripts/patchMvRefresh.ts index bfa6f7f8e2..05f1ab8030 100644 --- a/packages/data-api/scripts/patchMvRefresh.ts +++ b/packages/data-api/scripts/patchMvRefresh.ts @@ -5,11 +5,11 @@ // @ts-expect-error db-migrate has no types unfortunately import DBMigrate from 'db-migrate'; -import * as dotenv from 'dotenv'; import { requireEnv } from '@tupaia/utils'; import { getConnectionConfig } from '@tupaia/database'; +import { configureEnv } from '../src/configureEnv'; -dotenv.config(); // Load the environment variables into process.env +configureEnv(); // Load the environment variables into process.env const exitWithError = (error: Error) => { console.error(error.message); diff --git a/packages/data-api/scripts/patchMvRefreshModule.sh b/packages/data-api/scripts/patchMvRefreshModule.sh index 2008491e29..d38e9d73c9 100755 --- a/packages/data-api/scripts/patchMvRefreshModule.sh +++ b/packages/data-api/scripts/patchMvRefreshModule.sh @@ -1,5 +1,8 @@ #!/bin/bash +DIR=$(pwd "$0") +source "$DIR/../../scripts/bash/mergeEnvForDB.sh" + COMMAND=$1 if [[ "$COMMAND" == "" ]]; then @@ -16,11 +19,6 @@ VERSION=$2 if [[ "$VERSION" == "" ]]; then echo "Version unspecified, defaulting to database mvrefresh version" - - # Use whatever existing .env vars have been specified - curenv=$(declare -p -x) - test -f .env && source .env - eval "$curenv" # Set default port in case it wasn't in .env : "${DB_PORT:=5432}" diff --git a/packages/data-api/scripts/uninstallMvRefreshModule.sh b/packages/data-api/scripts/uninstallMvRefreshModule.sh index 58c9e260f0..9880e0ab60 100755 --- a/packages/data-api/scripts/uninstallMvRefreshModule.sh +++ b/packages/data-api/scripts/uninstallMvRefreshModule.sh @@ -1,7 +1,7 @@ #!/bin/bash DIR=$(pwd "$0") -source $DIR/../../scripts/bash/mergeCurrentEnvWithEnvFile.sh +source "$DIR/../../scripts/bash/mergeEnvForDB.sh" # Set default port in case it wasn't in .env : "${DB_PORT:=5432}" diff --git a/packages/data-api/src/configureEnv.ts b/packages/data-api/src/configureEnv.ts new file mode 100644 index 0000000000..91d5887544 --- /dev/null +++ b/packages/data-api/src/configureEnv.ts @@ -0,0 +1,15 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2024 Beyond Essential Systems Pty Ltd + */ + +import path from 'path'; +import { configureDotEnv } from '@tupaia/server-utils'; + +export const configureEnv = () => + configureDotEnv([ + path.resolve(__dirname, '../../../env/servers.env'), + path.resolve(__dirname, '../../../env/db.env'), + path.resolve(__dirname, '../../../env/pg.env'), + path.resolve(__dirname, '../.env'), + ]); // Load the environment variables into process.env diff --git a/packages/data-api/tsconfig.json b/packages/data-api/tsconfig.json index 02b9f73dcb..8916e052f4 100644 --- a/packages/data-api/tsconfig.json +++ b/packages/data-api/tsconfig.json @@ -8,6 +8,6 @@ "jest.config.ts", "jest.setup.ts", "global.d.ts", - "scripts/patchMvRefresh.ts" + "scripts/patchMvRefresh.ts", ] } diff --git a/packages/data-lake-api/.env.example b/packages/data-lake-api/.env.example index c2615d32f0..788c6cdbde 100644 --- a/packages/data-lake-api/.env.example +++ b/packages/data-lake-api/.env.example @@ -6,4 +6,4 @@ DATA_LAKE_DB_URL= DATA_LAKE_DB_USER= DB_PG_USER= -DB_PG_PASSWORD= \ No newline at end of file +DB_PG_PASSWORD= diff --git a/packages/data-lake-api/package.json b/packages/data-lake-api/package.json index 907faf88c7..4feecb3646 100644 --- a/packages/data-lake-api/package.json +++ b/packages/data-lake-api/package.json @@ -29,7 +29,6 @@ "db-migrate": "^0.11.5", "db-migrate-pg": "^1.2.2", "deep-equal-in-any-order": "^1.0.27", - "dotenv": "^8.2.0", "knex": "0.14.6", "lodash.groupby": "^4.6.0", "moment": "^2.24.0", diff --git a/packages/data-lake-api/scripts/checkTestDataLakeExists.sh b/packages/data-lake-api/scripts/checkTestDataLakeExists.sh index 1ffbaa78b1..927ee9aef7 100755 --- a/packages/data-lake-api/scripts/checkTestDataLakeExists.sh +++ b/packages/data-lake-api/scripts/checkTestDataLakeExists.sh @@ -1,7 +1,7 @@ #!/bin/bash -e DIR=$(pwd "$0") -source $DIR/../../scripts/bash/mergeCurrentEnvWithEnvFile.sh +source "$DIR/../../scripts/bash/mergeEnvForDB.sh" # Set default port in case it wasn't in .env : "${DATA_LAKE_DB_PORT:=5432}" @@ -12,4 +12,4 @@ then fi echo -e "Error: $DATA_LAKE_DB_NAME database does not exist!\n\nTo create it, please get the .env file from lastpass then run:\nyarn workspace @tupaia/data-lake-api setup-test-data-lake\n" -exit 1 \ No newline at end of file +exit 1 diff --git a/packages/data-lake-api/scripts/setupTestDataLake.sh b/packages/data-lake-api/scripts/setupTestDataLake.sh index 3ad0590389..f9563a4065 100755 --- a/packages/data-lake-api/scripts/setupTestDataLake.sh +++ b/packages/data-lake-api/scripts/setupTestDataLake.sh @@ -1,7 +1,7 @@ #!/bin/bash -e DIR=$(pwd "$0") -source $DIR/../../scripts/bash/mergeCurrentEnvWithEnvFile.sh +source "$DIR/../../scripts/bash/mergeEnvForDB.sh" # Set default port in case it wasn't in .env : "${DATA_LAKE_DB_PORT:=5432}" diff --git a/packages/data-table-server/.env.example b/packages/data-table-server/.env.example index 5f7f04ca36..3da70491fa 100644 --- a/packages/data-table-server/.env.example +++ b/packages/data-table-server/.env.example @@ -1,46 +1 @@ PORT= - -AGGREGATION_URL_PREFIX= -API_CLIENT_SALT= - -DATA_LAKE_DB_URL= -DATA_LAKE_DB_NAME= -DATA_LAKE_DB_USER= -DATA_LAKE_DB_PASSWORD= - -DB_URL= -DB_NAME= -DB_USER= -DB_PASSWORD= - -DHIS_CLIENT_ID= -DHIS_CLIENT_SECRET= -DHIS_PASSWORD= -DHIS_USERNAME= - -EXT_DB_data_lake_HOST= -EXT_DB_data_lake_PORT= -EXT_DB_data_lake_DATABASE= -EXT_DB_data_lake_USER= -EXT_DB_data_lake_PASSWORD= - -EXT_DB_msupply_scheduled_HOST= -EXT_DB_msupply_scheduled_PORT= -EXT_DB_msupply_scheduled_DATABASE= -EXT_DB_msupply_scheduled_USER= -EXT_DB_msupply_scheduled_PASSWORD= - -EXT_DB_niwa_HOST= -EXT_DB_niwa_PORT= -EXT_DB_niwa_DATABASE= -EXT_DB_niwa_USER= -EXT_DB_niwa_PASSWORD= - -JWT_SECRET= - -PALAU_DHIS_CLIENT_SECRET= -PALAU_DHIS_PASSWORD= - -SUPERSET_API_PASSWORD= -SUPERSET_API_PROXY_URL= -SUPERSET_API_USERNAME= diff --git a/packages/data-table-server/jest.config.ts b/packages/data-table-server/jest.config.ts index bb9a27fcda..7b1e7002dc 100644 --- a/packages/data-table-server/jest.config.ts +++ b/packages/data-table-server/jest.config.ts @@ -8,5 +8,5 @@ import baseConfig from '../../jest.config-ts.json'; module.exports = async () => ({ ...baseConfig, rootDir: '.', - setupFilesAfterEnv: ['../../jest.setup.js'], + setupFilesAfterEnv: ['../../jest.setup.js', './jest.setup.ts'], }); diff --git a/packages/data-table-server/jest.setup.ts b/packages/data-table-server/jest.setup.ts new file mode 100644 index 0000000000..71bb096faa --- /dev/null +++ b/packages/data-table-server/jest.setup.ts @@ -0,0 +1,8 @@ +/* + * Tupaia + * Copyright (c) 2017 - 2024 Beyond Essential Systems Pty Ltd + */ + +import { configureEnv } from './src/configureEnv'; + +configureEnv(); diff --git a/packages/data-table-server/package.json b/packages/data-table-server/package.json index 0ad2ebdfda..0c82df806d 100644 --- a/packages/data-table-server/package.json +++ b/packages/data-table-server/package.json @@ -31,10 +31,10 @@ "@tupaia/data-broker": "workspace:*", "@tupaia/database": "workspace:*", "@tupaia/server-boilerplate": "workspace:*", + "@tupaia/server-utils": "workspace:*", "@tupaia/tsutils": "workspace:*", "@tupaia/types": "workspace:*", "@tupaia/utils": "workspace:*", - "dotenv": "^8.2.0", "express": "^4.19.2", "winston": "^3.3.3" }, diff --git a/packages/data-table-server/src/configureEnv.ts b/packages/data-table-server/src/configureEnv.ts new file mode 100644 index 0000000000..3527e3c2e8 --- /dev/null +++ b/packages/data-table-server/src/configureEnv.ts @@ -0,0 +1,22 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2020 Beyond Essential Systems Pty Ltd + */ + +import path from 'path'; +import { configureDotEnv } from '@tupaia/server-utils'; + +export const configureEnv = () => { + configureDotEnv([ + path.resolve(__dirname, '../../../env/aggregation.env'), + path.resolve(__dirname, '../../../env/api-client.env'), + path.resolve(__dirname, '../../../env/servers.env'), + path.resolve(__dirname, '../../../env/db.env'), + path.resolve(__dirname, '../../../env/dhis.env'), + path.resolve(__dirname, '../../../env/data-lake.env'), + path.resolve(__dirname, '../../../env/external-db-connections.env'), + path.resolve(__dirname, '../../../env/superset.env'), + path.resolve(__dirname, '../../../env/weatherbit.env'), + '../.env', + ]); // Load the environment variables into process.env +}; diff --git a/packages/data-table-server/src/index.ts b/packages/data-table-server/src/index.ts index aa904d11ff..6fdbda18ef 100644 --- a/packages/data-table-server/src/index.ts +++ b/packages/data-table-server/src/index.ts @@ -4,15 +4,14 @@ */ import http from 'http'; -import * as dotenv from 'dotenv'; - import winston from 'winston'; import { TupaiaDatabase } from '@tupaia/database'; import { configureWinston } from '@tupaia/server-boilerplate'; import { createApp } from './app'; +import { configureEnv } from './configureEnv'; configureWinston(); -dotenv.config(); // Load the environment variables into process.env +configureEnv(); (async () => { const database = new TupaiaDatabase(); diff --git a/packages/database/.env.example b/packages/database/.env.example deleted file mode 100644 index 4cc37e4c19..0000000000 --- a/packages/database/.env.example +++ /dev/null @@ -1,7 +0,0 @@ -DB_NAME= -DB_PASSWORD= -DB_URL= -DB_USER= -DB_DISABLE_SSL= -DB_PG_USER= -DB_PG_PASSWORD= diff --git a/packages/database/jest.setup.js b/packages/database/jest.setup.js index 26f006a067..f6289a7032 100644 --- a/packages/database/jest.setup.js +++ b/packages/database/jest.setup.js @@ -3,8 +3,11 @@ * Copyright (c) 2017 - 2020 Beyond Essential Systems Pty Ltd */ +import { configureEnv } from './src/configureEnv'; import { clearTestData, getTestDatabase } from './src/testUtilities'; +configureEnv(); + afterAll(async () => { const database = getTestDatabase(); await clearTestData(database); diff --git a/packages/database/package.json b/packages/database/package.json index 7c3dc44e9e..02df7a6df8 100644 --- a/packages/database/package.json +++ b/packages/database/package.json @@ -37,7 +37,7 @@ "@tupaia/utils": "workspace:*", "db-migrate": "^0.11.5", "db-migrate-pg": "^1.2.2", - "dotenv": "^8.2.0", + "dotenv": "^16.4.5", "knex": "0.14.6", "lodash.clonedeep": "^4.5.0", "lodash.groupby": "^4.6.0", diff --git a/packages/database/scripts/checkTestDatabaseExists.sh b/packages/database/scripts/checkTestDatabaseExists.sh index 8c92ff1e8e..0d40d2da14 100755 --- a/packages/database/scripts/checkTestDatabaseExists.sh +++ b/packages/database/scripts/checkTestDatabaseExists.sh @@ -1,7 +1,7 @@ #!/bin/bash -e DIR=$(pwd "$0") -source $DIR/../../scripts/bash/mergeCurrentEnvWithEnvFile.sh +source "$DIR/../../scripts/bash/mergeEnvForDB.sh" # Set default port in case it wasn't in .env : "${DB_PORT:=5432}" diff --git a/packages/database/scripts/dumpDatabase.sh b/packages/database/scripts/dumpDatabase.sh index cb767adb0e..7b028cb779 100755 --- a/packages/database/scripts/dumpDatabase.sh +++ b/packages/database/scripts/dumpDatabase.sh @@ -29,7 +29,8 @@ function show_loading_spinner() { echo "" # reset prompt } -source ".env" +DIR=$(pwd "$0") +source "$DIR/../../scripts/bash/mergeEnvForDB.sh" DUMP_FILE_NAME="dump.sql" @@ -87,4 +88,4 @@ show_loading_spinner "Unzipping $target_zip_path" "gunzip -f $target_zip_path" echo "Dump file available at $target_path" -echo "Done!" \ No newline at end of file +echo "Done!" diff --git a/packages/database/scripts/refreshDatabase.js b/packages/database/scripts/refreshDatabase.js index 2614f8649a..32beb3ec52 100644 --- a/packages/database/scripts/refreshDatabase.js +++ b/packages/database/scripts/refreshDatabase.js @@ -5,7 +5,13 @@ * Copyright (c) 2017 - 2020 Beyond Essential Systems Pty Ltd */ -require('dotenv/config'); +require('dotenv').config({ + path: [ + require('path').resolve(__dirname, '../../../env/db.env'), + require('path').resolve(__dirname, '../../../env/pg.env'), + require('path').resolve(__dirname, '../.env'), + ], +}); const fs = require('fs'); const path = require('path'); diff --git a/packages/database/scripts/setupTestDatabase.sh b/packages/database/scripts/setupTestDatabase.sh index 296e695666..b0cb07538c 100755 --- a/packages/database/scripts/setupTestDatabase.sh +++ b/packages/database/scripts/setupTestDatabase.sh @@ -1,7 +1,7 @@ #!/bin/bash -e DIR=$(pwd "$0") -source $DIR/../../scripts/bash/mergeCurrentEnvWithEnvFile.sh +source "$DIR/../../scripts/bash/mergeEnvForDB.sh" # Set default port in case it wasn't in .env : "${DB_PORT:=5432}" diff --git a/packages/database/src/configureEnv.js b/packages/database/src/configureEnv.js new file mode 100644 index 0000000000..d9d22fcd40 --- /dev/null +++ b/packages/database/src/configureEnv.js @@ -0,0 +1,17 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2024 Beyond Essential Systems Pty Ltd + */ + +const path = require('path'); +const dotenv = require('dotenv'); + +export const configureEnv = () => { + dotenv.config({ + path: [ + path.resolve(__dirname, '../../../env/db.env'), + path.resolve(__dirname, '../../../env/pg.env'), + path.resolve(__dirname, '../.env'), + ], + }); +}; diff --git a/packages/database/src/migrate.js b/packages/database/src/migrate.js index c57c22427d..5da41c6595 100644 --- a/packages/database/src/migrate.js +++ b/packages/database/src/migrate.js @@ -2,8 +2,11 @@ * Tupaia * Copyright (c) 2017 - 2020 Beyond Essential Systems Pty Ltd */ -import {} from 'dotenv/config'; // Load the environment variables into process.env + import { getDbMigrator } from './getDbMigrator'; +import { configureEnv } from './configureEnv'; + +configureEnv(); const migrator = getDbMigrator(true); migrator.run(); diff --git a/packages/database/src/testUtilities/getTestDatabase.js b/packages/database/src/testUtilities/getTestDatabase.js index 3d472e9249..867363bb5e 100644 --- a/packages/database/src/testUtilities/getTestDatabase.js +++ b/packages/database/src/testUtilities/getTestDatabase.js @@ -2,12 +2,14 @@ * Tupaia * Copyright (c) 2017 - 2020 Beyond Essential Systems Pty Ltd */ -import {} from 'dotenv/config'; import { ModelRegistry } from '../ModelRegistry'; import { TupaiaDatabase } from '../TupaiaDatabase'; +import { configureEnv } from '../configureEnv'; let database = null; +configureEnv(); + export function getTestDatabase() { if (!database) { database = new TupaiaDatabase(); diff --git a/packages/database/tsconfig.json b/packages/database/tsconfig.json index ada1054457..3715a46e63 100644 --- a/packages/database/tsconfig.json +++ b/packages/database/tsconfig.json @@ -3,6 +3,11 @@ "compilerOptions": { "outDir": "dist" }, - "include": ["src/**/*.js"], - "exclude": ["src/__tests__", "src/migrations"] + "include": [ + "src/**/*.js" + ], + "exclude": [ + "src/__tests__", + "src/migrations" + ] } diff --git a/packages/datatrak-web-server/.env.example b/packages/datatrak-web-server/.env.example index 10946cf3fc..df525d89bd 100644 --- a/packages/datatrak-web-server/.env.example +++ b/packages/datatrak-web-server/.env.example @@ -1,18 +1,4 @@ PORT= API_CLIENT_NAME= -API_CLIENT_PASSWORD= -API_CLIENT_SALT= - -DB_NAME= -DB_PASSWORD= -DB_URL= -DB_USER= - -SESSION_COOKIE_SECRET= - -CENTRAL_API_URL= -ENTITY_API_URL= -REPORT_API_URL= -WEB_CONFIG_API_URL= - +API_CLIENT_PASSWORD= diff --git a/packages/datatrak-web-server/package.json b/packages/datatrak-web-server/package.json index 000765eca6..e1a21a1d44 100644 --- a/packages/datatrak-web-server/package.json +++ b/packages/datatrak-web-server/package.json @@ -29,10 +29,10 @@ "@tupaia/api-client": "workspace:*", "@tupaia/database": "workspace:*", "@tupaia/server-boilerplate": "workspace:*", + "@tupaia/server-utils": "workspace:*", "@tupaia/types": "workspace:*", "@tupaia/utils": "workspace:*", "camelcase-keys": "^6.2.2", - "dotenv": "^8.2.0", "express": "^4.19.2", "lodash.groupby": "^4.6.0", "lodash.keyby": "^4.6.0", diff --git a/packages/datatrak-web-server/src/index.ts b/packages/datatrak-web-server/src/index.ts index 3c3ff27671..1e7466b297 100644 --- a/packages/datatrak-web-server/src/index.ts +++ b/packages/datatrak-web-server/src/index.ts @@ -3,16 +3,22 @@ * Copyright (c) 2017 - 2023 Beyond Essential Systems Pty Ltd */ -import * as dotenv from 'dotenv'; - +import path from 'path'; import http from 'http'; import winston from 'winston'; +import { configureDotEnv } from '@tupaia/server-utils'; import { configureWinston } from '@tupaia/server-boilerplate'; import { createApp } from './app'; configureWinston(); -dotenv.config(); // Load the environment variables into process.env + +configureDotEnv([ + path.resolve(__dirname, '../../../env/servers.env'), + path.resolve(__dirname, '../../../env/db.env'), + path.resolve(__dirname, '../../../env/api-client.env'), + path.resolve(__dirname, '../.env'), +]); // Load the environment variables into process.env from the common .env file and this server's .env file (async () => { /** diff --git a/packages/devops/scripts/ci/validateTypesAndDbSchemaInSync.sh b/packages/devops/scripts/ci/validateTypesAndDbSchemaInSync.sh index b364f6933f..baf9446d62 100755 --- a/packages/devops/scripts/ci/validateTypesAndDbSchemaInSync.sh +++ b/packages/devops/scripts/ci/validateTypesAndDbSchemaInSync.sh @@ -1,11 +1,17 @@ #!/bin/bash -ex + + +SCRIPT_DIR=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P ) + +# Load environment variables from .env files +source $SCRIPT_DIR/../../../../scripts/bash/mergeEnvForDB.sh -SCRIPT_DIR=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P ) cd "$SCRIPT_DIR" + cd ../../../.. echo "Connected to postgres server: $DB_URL, starting to setup database" yarn workspace @tupaia/database setup-test-database # Run check -yarn workspace @tupaia/types assert-no-changes \ No newline at end of file +yarn workspace @tupaia/types assert-no-changes diff --git a/packages/devops/scripts/deployment/buildDeployablePackages.sh b/packages/devops/scripts/deployment/buildDeployablePackages.sh index c33dbda963..1500eceb2b 100755 --- a/packages/devops/scripts/deployment/buildDeployablePackages.sh +++ b/packages/devops/scripts/deployment/buildDeployablePackages.sh @@ -19,7 +19,7 @@ yarn install --immutable # packages' dists are not rebuilt. This will be fixed by changing to a single yarn:build command in a future PR. yarn build:internal-dependencies -# Inject environment variables from LastPass +# Inject environment variables from Bitwarden BITWARDEN_EMAIL=$($DIR/fetchParameterStoreValue.sh BITWARDEN_EMAIL) BITWARDEN_PASSWORD=$($DIR/fetchParameterStoreValue.sh BITWARDEN_PASSWORD) BITWARDEN_EMAIL=$BITWARDEN_EMAIL BITWARDEN_PASSWORD=$BITWARDEN_PASSWORD yarn download-env-vars $DEPLOYMENT_NAME diff --git a/packages/entity-server/.env.example b/packages/entity-server/.env.example deleted file mode 100644 index 8759fa661e..0000000000 --- a/packages/entity-server/.env.example +++ /dev/null @@ -1,8 +0,0 @@ -PORT= - -API_CLIENT_SALT= -DB_NAME= -DB_PASSWORD= -DB_URL= -DB_USER= -JWT_SECRET= diff --git a/packages/entity-server/jest.setup.ts b/packages/entity-server/jest.setup.ts index 0207580d7e..70c54af5fc 100644 --- a/packages/entity-server/jest.setup.ts +++ b/packages/entity-server/jest.setup.ts @@ -5,6 +5,9 @@ import { getTestDatabase, clearTestData } from '@tupaia/database'; import { setupTestData } from './src/__tests__/testUtilities'; +import { configureEnv } from './src/configureEnv'; + +configureEnv(); beforeAll(async () => { await setupTestData(); diff --git a/packages/entity-server/package.json b/packages/entity-server/package.json index 29cc2b1f4b..e75a0e12e7 100644 --- a/packages/entity-server/package.json +++ b/packages/entity-server/package.json @@ -32,10 +32,10 @@ "@tupaia/auth": "workspace:*", "@tupaia/database": "workspace:*", "@tupaia/server-boilerplate": "workspace:*", + "@tupaia/server-utils": "workspace:*", "@tupaia/tsutils": "workspace:*", "@tupaia/types": "workspace:*", "@tupaia/utils": "workspace:*", - "dotenv": "^8.2.0", "express": "^4.19.2", "lodash.keyby": "^4.6.0", "winston": "^3.3.3" diff --git a/packages/entity-server/src/configureEnv.ts b/packages/entity-server/src/configureEnv.ts new file mode 100644 index 0000000000..f475f40daa --- /dev/null +++ b/packages/entity-server/src/configureEnv.ts @@ -0,0 +1,14 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2024 Beyond Essential Systems Pty Ltd + */ + +import path from 'path'; +import { configureDotEnv } from '@tupaia/server-utils'; + +export const configureEnv = () => + configureDotEnv([ + path.resolve(__dirname, '../../../env/servers.env'), + path.resolve(__dirname, '../../../env/db.env'), + path.resolve(__dirname, '../.env'), + ]); // Load the environment variables into process.env diff --git a/packages/entity-server/src/index.ts b/packages/entity-server/src/index.ts index 19fdf57ca9..d66ce59a63 100644 --- a/packages/entity-server/src/index.ts +++ b/packages/entity-server/src/index.ts @@ -4,14 +4,13 @@ */ import http from 'http'; -import * as dotenv from 'dotenv'; - import winston from 'winston'; import { configureWinston } from '@tupaia/server-boilerplate'; import { createApp } from './app'; +import { configureEnv } from './configureEnv'; configureWinston(); -dotenv.config(); // Load the environment variables into process.env +configureEnv(); (async () => { /** diff --git a/packages/lesmis-server/.env.example b/packages/lesmis-server/.env.example index dcb052c536..1fc1bad6d9 100644 --- a/packages/lesmis-server/.env.example +++ b/packages/lesmis-server/.env.example @@ -1,21 +1,5 @@ PORT= API_CLIENT_NAME= -API_CLIENT_PASSWORD= -API_CLIENT_SALT= - -DB_NAME= -DB_PASSWORD= -DB_URL= -DB_USER= - -MICROSERVICE_CLIENT_SECRET= -MICROSERVICE_CLIENT_USERNAME= - -SESSION_COOKIE_SECRET= - -CENTRAL_API_URL= -ENTITY_API_URL= -REPORT_API_URL= -WEB_CONFIG_API_URL= +API_CLIENT_PASSWORD= diff --git a/packages/lesmis-server/package.json b/packages/lesmis-server/package.json index af51434d34..7cd377168d 100644 --- a/packages/lesmis-server/package.json +++ b/packages/lesmis-server/package.json @@ -34,7 +34,6 @@ "@tupaia/tsutils": "workspace:*", "@tupaia/utils": "workspace:*", "camelcase-keys": "^6.2.2", - "dotenv": "^8.2.0", "express": "^4.19.2", "lodash.uniqby": "^4.7.0", "winston": "^3.3.3" diff --git a/packages/lesmis-server/src/connections/SessionHandlingApiConnection.ts b/packages/lesmis-server/src/connections/SessionHandlingApiConnection.ts index c6803706f9..6eddd66c96 100644 --- a/packages/lesmis-server/src/connections/SessionHandlingApiConnection.ts +++ b/packages/lesmis-server/src/connections/SessionHandlingApiConnection.ts @@ -6,11 +6,8 @@ import { ApiConnection, AuthHandler, SessionRecord } from '@tupaia/server-boilerplate'; import { createBasicHeader } from '@tupaia/utils'; -const { MICROSERVICE_CLIENT_USERNAME, MICROSERVICE_CLIENT_SECRET } = process.env; -const DEFAULT_AUTH_HEADER = createBasicHeader( - MICROSERVICE_CLIENT_USERNAME, - MICROSERVICE_CLIENT_SECRET, -); +const { API_CLIENT_NAME, API_CLIENT_SECRET } = process.env; +const DEFAULT_AUTH_HEADER = createBasicHeader(API_CLIENT_NAME, API_CLIENT_SECRET); class SessionSwitchingAuthHandler implements AuthHandler { session?: SessionRecord; diff --git a/packages/lesmis-server/src/index.ts b/packages/lesmis-server/src/index.ts index 3dae31598d..5e1c708aa4 100644 --- a/packages/lesmis-server/src/index.ts +++ b/packages/lesmis-server/src/index.ts @@ -4,14 +4,20 @@ */ import http from 'http'; -import * as dotenv from 'dotenv'; +import path from 'path'; import winston from 'winston'; +import { configureDotEnv } from '@tupaia/server-utils'; import { configureWinston } from '@tupaia/server-boilerplate'; import { createApp } from './app'; configureWinston(); -dotenv.config(); // Load the environment variables into process.env +configureDotEnv([ + path.resolve(__dirname, '../../../env/servers.env'), + path.resolve(__dirname, '../../../env/db.env'), + path.resolve(__dirname, '../../../env/api-client.env'), + path.resolve(__dirname, '../.env'), +]); // Load the environment variables into process.env (async () => { /** diff --git a/packages/meditrak-app-server/.env.example b/packages/meditrak-app-server/.env.example index 83812d3e0f..46c182d047 100644 --- a/packages/meditrak-app-server/.env.example +++ b/packages/meditrak-app-server/.env.example @@ -1,18 +1,4 @@ PORT= - + API_CLIENT_NAME= -API_CLIENT_PASSWORD= - -AWS_ACCESS_KEY_ID= -AWS_REGION= -AWS_SECRET_ACCESS_KEY= - -DB_NAME= -DB_PASSWORD= -DB_URL= -DB_USER= - -JWT_SECRET= - -AUTH_API_URL= -CENTRAL_API_URL= +API_CLIENT_PASSWORD= diff --git a/packages/meditrak-app-server/jest.config.ts b/packages/meditrak-app-server/jest.config.ts index 3de5f5998e..7b1e7002dc 100644 --- a/packages/meditrak-app-server/jest.config.ts +++ b/packages/meditrak-app-server/jest.config.ts @@ -8,9 +8,5 @@ import baseConfig from '../../jest.config-ts.json'; module.exports = async () => ({ ...baseConfig, rootDir: '.', - setupFilesAfterEnv: [ - '../../jest.setup.js', - './jest.setup.ts', - './src/__tests__/setupTestEnvVars.ts', - ], + setupFilesAfterEnv: ['../../jest.setup.js', './jest.setup.ts'], }); diff --git a/packages/meditrak-app-server/jest.setup.ts b/packages/meditrak-app-server/jest.setup.ts index c77eb90ac0..fa7a1fbdf8 100644 --- a/packages/meditrak-app-server/jest.setup.ts +++ b/packages/meditrak-app-server/jest.setup.ts @@ -4,6 +4,9 @@ */ import { getTestDatabase, clearTestData } from '@tupaia/database'; +import { configureEnv } from './src/configureEnv'; + +configureEnv(); beforeAll(async () => { const database = getTestDatabase(); diff --git a/packages/meditrak-app-server/package.json b/packages/meditrak-app-server/package.json index fdb168add8..487353813d 100644 --- a/packages/meditrak-app-server/package.json +++ b/packages/meditrak-app-server/package.json @@ -36,7 +36,6 @@ "@tupaia/tsutils": "workspace:*", "@tupaia/types": "workspace:*", "@tupaia/utils": "workspace:*", - "dotenv": "^8.2.0", "express": "^4.19.2", "lodash.clonedeep": "^4.5.0", "lodash.groupby": "^4.6.0", diff --git a/packages/meditrak-app-server/src/__tests__/__integration__/fixtures.ts b/packages/meditrak-app-server/src/__tests__/__integration__/fixtures.ts index 1bd920ec5c..f0eb56d28e 100644 --- a/packages/meditrak-app-server/src/__tests__/__integration__/fixtures.ts +++ b/packages/meditrak-app-server/src/__tests__/__integration__/fixtures.ts @@ -3,9 +3,6 @@ * Copyright (c) 2017 - 2022 Beyond Essential Systems Pty Ltd */ -export const API_CLIENT_NAME = 'meditrak-app-server@tupaia.org'; -export const API_CLIENT_PASSWORD = 'test_api_client_password'; - export const CAT_USER = { firstName: 'Cat', lastName: 'Meow', diff --git a/packages/meditrak-app-server/src/__tests__/setupTestEnvVars.ts b/packages/meditrak-app-server/src/__tests__/setupTestEnvVars.ts deleted file mode 100644 index 528827d783..0000000000 --- a/packages/meditrak-app-server/src/__tests__/setupTestEnvVars.ts +++ /dev/null @@ -1,9 +0,0 @@ -/** - * Tupaia - * Copyright (c) 2017 - 2022 Beyond Essential Systems Pty Ltd - */ - -import { API_CLIENT_NAME, API_CLIENT_PASSWORD } from './__integration__/fixtures'; - -process.env.API_CLIENT_NAME = API_CLIENT_NAME; -process.env.API_CLIENT_PASSWORD = API_CLIENT_PASSWORD; diff --git a/packages/meditrak-app-server/src/configureEnv.ts b/packages/meditrak-app-server/src/configureEnv.ts new file mode 100644 index 0000000000..fcc4e7109c --- /dev/null +++ b/packages/meditrak-app-server/src/configureEnv.ts @@ -0,0 +1,17 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2024 Beyond Essential Systems Pty Ltd + */ + +import path from 'path'; +import { configureDotEnv } from '@tupaia/server-utils'; + +export const configureEnv = () => { + configureDotEnv([ + path.resolve(__dirname, '../../../env/servers.env'), + path.resolve(__dirname, '../../../env/db.env'), + path.resolve(__dirname, '../../../env/aws.env'), + path.resolve(__dirname, '../../../env/api-client.env'), + path.resolve(__dirname, '../.env'), + ]); +}; diff --git a/packages/meditrak-app-server/src/index.ts b/packages/meditrak-app-server/src/index.ts index 1268b7e8e4..6535159d36 100644 --- a/packages/meditrak-app-server/src/index.ts +++ b/packages/meditrak-app-server/src/index.ts @@ -4,7 +4,6 @@ */ import http from 'http'; -import * as dotenv from 'dotenv'; import winston from 'winston'; import { ModelRegistry, TupaiaDatabase } from '@tupaia/database'; @@ -13,9 +12,10 @@ import { isFeatureEnabled } from '@tupaia/utils'; import { createApp } from './app'; import { MeditrakAppServerModelRegistry } from './types'; import { SyncableChangeEnqueuer } from './sync'; +import { configureEnv } from './configureEnv'; configureWinston(); -dotenv.config(); // Load the environment variables into process.env +configureEnv(); (async () => { const database = new TupaiaDatabase(); diff --git a/packages/meditrak-app-server/src/sync/createMeditrakSyncView.ts b/packages/meditrak-app-server/src/sync/createMeditrakSyncView.ts index 4b9bb69a54..a7048bb958 100644 --- a/packages/meditrak-app-server/src/sync/createMeditrakSyncView.ts +++ b/packages/meditrak-app-server/src/sync/createMeditrakSyncView.ts @@ -6,16 +6,15 @@ // Exact copy of: @tupaia/central-server/src/createMeditrakSyncView.js // TODO: Tidy this up as part of RN-502 -import * as dotenv from 'dotenv'; - import winston from 'winston'; import { TupaiaDatabase } from '@tupaia/database'; import { configureWinston } from '@tupaia/server-boilerplate'; import { isFeatureEnabled } from '@tupaia/utils'; +import { configureEnv } from '../configureEnv'; import { createPermissionsBasedMeditrakSyncQueue } from './createPermissionsBasedMeditrakSyncQueue'; configureWinston(); -dotenv.config(); // Load the environment variables into process.env +configureEnv(); (async () => { if (!isFeatureEnabled('SERVER_CHANGE_ENQUEUER')) { diff --git a/packages/meditrak-app/app/Meditrak.jsx b/packages/meditrak-app/app/Meditrak.jsx index 88f2f31a37..f09722f591 100644 --- a/packages/meditrak-app/app/Meditrak.jsx +++ b/packages/meditrak-app/app/Meditrak.jsx @@ -4,17 +4,21 @@ */ import React from 'react'; -import { StyleSheet, Text, View, BackHandler } from 'react-native'; +import {StyleSheet, Text, View, BackHandler} from 'react-native'; import PropTypes from 'prop-types'; -import { connect } from 'react-redux'; +import {connect} from 'react-redux'; -import { database } from './database'; -import { MessageOverlay } from './messages/MessageOverlay'; -import { isBeta, betaBranch, centralApiUrl } from './version'; -import { NavigationMenuContainer, goBack } from './navigation'; -import { DEFAULT_PADDING, THEME_COLOR_THREE, THEME_COLOR_ONE } from './globalStyles'; +import {database} from './database'; +import {MessageOverlay} from './messages/MessageOverlay'; +import {isBeta, betaBranch, centralApiUrl} from './version'; +import {NavigationMenuContainer, goBack} from './navigation'; +import { + DEFAULT_PADDING, + THEME_COLOR_THREE, + THEME_COLOR_ONE, +} from './globalStyles'; -import { requestLocationPermission } from './utilities/userLocation/permission'; +import {requestLocationPermission} from './utilities/userLocation/permission'; class MeditrakContainer extends React.Component { componentDidMount() { @@ -29,14 +33,14 @@ class MeditrakContainer extends React.Component { } getCanNavigateBack = () => { - const { nav } = this.props; + const {nav} = this.props; return nav.index !== 0; }; onBackPress = () => { if (!this.getCanNavigateBack()) return false; - const { onGoBack } = this.props; + const {onGoBack} = this.props; onGoBack(); return true; }; @@ -44,7 +48,9 @@ class MeditrakContainer extends React.Component { renderBetaBanner() { return ( - {betaBranch.toUpperCase() || centralApiUrl} + + {betaBranch?.toUpperCase() || centralApiUrl} + ); } @@ -94,4 +100,7 @@ const mapDispatchToProps = dispatch => ({ onGoBack: () => dispatch(goBack()), }); -export const Meditrak = connect(mapStateToProps, mapDispatchToProps)(MeditrakContainer); +export const Meditrak = connect( + mapStateToProps, + mapDispatchToProps, +)(MeditrakContainer); diff --git a/packages/meditrak-app/app/version/isBeta.jsx b/packages/meditrak-app/app/version/isBeta.jsx index eae2e1af1f..b400ef0626 100644 --- a/packages/meditrak-app/app/version/isBeta.jsx +++ b/packages/meditrak-app/app/version/isBeta.jsx @@ -7,4 +7,5 @@ import Config from 'react-native-config'; export const isBeta = !!Config.BETA_BRANCH; export const betaBranch = Config.BETA_BRANCH; -export const centralApiUrl = Config.CENTRAL_API_URL; +export const centralApiUrl = + Config.CENTRAL_API_URL || 'http://10.0.2.2:8090/v2'; diff --git a/packages/psss-server/.env.example b/packages/psss-server/.env.example index ccc907caa2..1fc1bad6d9 100644 --- a/packages/psss-server/.env.example +++ b/packages/psss-server/.env.example @@ -1,16 +1,5 @@ PORT= API_CLIENT_NAME= -API_CLIENT_PASSWORD= -API_CLIENT_SALT= +API_CLIENT_PASSWORD= -DB_NAME= -DB_PASSWORD= -DB_URL= -DB_USER= - -SESSION_COOKIE_SECRET= - -CENTRAL_API_URL= -ENTITY_API_URL= -REPORT_API_URL= diff --git a/packages/psss-server/package.json b/packages/psss-server/package.json index 3af41917c8..07c8492069 100644 --- a/packages/psss-server/package.json +++ b/packages/psss-server/package.json @@ -30,13 +30,13 @@ "@tupaia/access-policy": "workspace:*", "@tupaia/database": "workspace:*", "@tupaia/server-boilerplate": "workspace:*", + "@tupaia/server-utils": "workspace:*", "@tupaia/types": "workspace:*", "@tupaia/utils": "workspace:*", "api-error-handler": "^1.0.0", "body-parser": "^1.18.3", "client-sessions": "^0.8.0", "cors": "^2.8.5", - "dotenv": "^8.2.0", "express": "^4.19.2", "lodash.groupby": "^4.6.0", "winston": "^3.3.3" diff --git a/packages/psss-server/src/index.ts b/packages/psss-server/src/index.ts index 61df4d2603..6843fc3bd2 100644 --- a/packages/psss-server/src/index.ts +++ b/packages/psss-server/src/index.ts @@ -4,14 +4,19 @@ */ import http from 'http'; -import * as dotenv from 'dotenv'; - +import path from 'path'; import winston from 'winston'; import { configureWinston } from '@tupaia/server-boilerplate'; +import { configureDotEnv } from '@tupaia/server-utils'; import { createApp } from './app'; configureWinston(); -dotenv.config(); // Load the environment variables into process.env +configureDotEnv([ + path.resolve(__dirname, '../../../env/servers.env'), + path.resolve(__dirname, '../../../env/db.env'), + path.resolve(__dirname, '../../../env/api-client.env'), + path.resolve(__dirname, '../.env'), +]); // Load the environment variables into process.env (async () => { /** diff --git a/packages/psss/src/api/api.js b/packages/psss/src/api/api.js index b8080614d9..793fb4a181 100644 --- a/packages/psss/src/api/api.js +++ b/packages/psss/src/api/api.js @@ -4,7 +4,7 @@ */ import axios from 'axios'; -const PSSS_API_URL = process.env.REACT_APP_PSSS_API_URL; +const PSSS_API_URL = process.env.REACT_APP_PSSS_API_URL || 'http://localhost:8040/v1'; const timeout = 45 * 1000; // withCredentials needs to be set for cookies to save @see https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/withCredentials diff --git a/packages/report-server/.env.example b/packages/report-server/.env.example index 15ba5c6a63..a6d40ecfd6 100644 --- a/packages/report-server/.env.example +++ b/packages/report-server/.env.example @@ -1,20 +1,3 @@ PORT= - -AGGREGATION_URL_PREFIX= -API_CLIENT_SALT= -DATA_LAKE_DB_NAME= -DATA_LAKE_DB_PASSWORD= -DATA_LAKE_DB_URL= -DATA_LAKE_DB_USER= -DB_NAME= -DB_PASSWORD= -DB_URL= -DB_USER= -DHIS_CLIENT_ID= -DHIS_CLIENT_SECRET= -DHIS_PASSWORD= -DHIS_USERNAME= -JWT_SECRET= -SUPERSET_API_PASSWORD= -SUPERSET_API_PROXY_URL= -SUPERSET_API_USERNAME= + + \ No newline at end of file diff --git a/packages/report-server/package.json b/packages/report-server/package.json index 24452b4664..b2b7511102 100644 --- a/packages/report-server/package.json +++ b/packages/report-server/package.json @@ -36,6 +36,7 @@ "@tupaia/database": "workspace:*", "@tupaia/expression-parser": "workspace:*", "@tupaia/server-boilerplate": "workspace:*", + "@tupaia/server-utils": "workspace:*", "@tupaia/tsutils": "workspace:*", "@tupaia/types": "workspace:*", "@tupaia/utils": "workspace:*", @@ -44,7 +45,6 @@ "body-parser": "^1.18.3", "cors": "^2.8.5", "date-fns": "^2.29.3", - "dotenv": "^8.2.0", "express": "^4.19.2", "lodash.groupby": "^4.6.0", "lodash.isplainobject": "^4.0.6", diff --git a/packages/report-server/src/index.ts b/packages/report-server/src/index.ts index d920f10b1f..6a96cef3ec 100644 --- a/packages/report-server/src/index.ts +++ b/packages/report-server/src/index.ts @@ -4,15 +4,25 @@ */ import http from 'http'; -import * as dotenv from 'dotenv'; +import path from 'path'; import winston from 'winston'; import { configureWinston } from '@tupaia/server-boilerplate'; +import { configureDotEnv } from '@tupaia/server-utils'; import { createApp } from './app'; configureWinston(); -dotenv.config(); // Load the environment variables into process.env +configureDotEnv([ + path.resolve(__dirname, '../../../env/aggregation.env'), + path.resolve(__dirname, '../../../env/dhis.env'), + path.resolve(__dirname, '../../../env/data-lake.env'), + path.resolve(__dirname, '../../../env/superset.env'), + path.resolve(__dirname, '../../../env/servers.env'), + path.resolve(__dirname, '../../../env/db.env'), + path.resolve(__dirname, '../../../env/api-client.env'), + path.resolve(__dirname, '../.env'), +]); // Load the environment variables into process.env (async () => { /** diff --git a/packages/server-boilerplate/.env.example b/packages/server-boilerplate/.env.example deleted file mode 100644 index e7f142e498..0000000000 --- a/packages/server-boilerplate/.env.example +++ /dev/null @@ -1,3 +0,0 @@ -REPORT_API_URL=http://dev-report-api.tupaia.org/v1 -ENTITY_API_URL=https://dev-entity-api.tupaia.org/v1 -CENTRAL_API_URL=https://dev-api.tupaia.org/v2 \ No newline at end of file diff --git a/packages/server-boilerplate/package.json b/packages/server-boilerplate/package.json index f3187e8734..4b38a2d183 100644 --- a/packages/server-boilerplate/package.json +++ b/packages/server-boilerplate/package.json @@ -35,7 +35,7 @@ "body-parser": "^1.18.3", "client-sessions": "^0.8.0", "cors": "^2.8.5", - "dotenv": "^8.2.0", + "dotenv": "^16.4.5", "express": "^4.19.2", "http-proxy-middleware": "^2.0.1", "i18n": "^0.13.3", diff --git a/packages/server-utils/package.json b/packages/server-utils/package.json index 9c8bd0e302..547359c2cd 100644 --- a/packages/server-utils/package.json +++ b/packages/server-utils/package.json @@ -25,6 +25,7 @@ "@aws-sdk/lib-storage": "^3.348.0", "@tupaia/utils": "workspace:*", "cookie": "^0.5.0", + "dotenv": "^16.4.5", "nodemailer": "^6.9.12", "puppeteer": "^15.4.0", "sha256": "^0.2.0" diff --git a/packages/server-utils/src/configureDotEnv.ts b/packages/server-utils/src/configureDotEnv.ts new file mode 100644 index 0000000000..eb634e5443 --- /dev/null +++ b/packages/server-utils/src/configureDotEnv.ts @@ -0,0 +1,10 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2024 Beyond Essential Systems Pty Ltd + */ + +import dotenv from 'dotenv'; + +export const configureDotEnv = (envFiles: string[]) => { + dotenv.config({ path: envFiles }); +}; diff --git a/packages/server-utils/src/index.ts b/packages/server-utils/src/index.ts index cee17c115e..bc76e358e4 100644 --- a/packages/server-utils/src/index.ts +++ b/packages/server-utils/src/index.ts @@ -2,3 +2,4 @@ export { downloadPageAsPDF } from './downloadPageAsPDF'; export * from './s3'; export { sendEmail } from './sendEmail'; export { generateUnsubscribeToken, verifyUnsubscribeToken } from './unsubscribeToken'; +export { configureDotEnv } from './configureDotEnv'; diff --git a/packages/tupaia-web-server/.env.example b/packages/tupaia-web-server/.env.example index 8dc7efba45..c4c4f23d07 100644 --- a/packages/tupaia-web-server/.env.example +++ b/packages/tupaia-web-server/.env.example @@ -1,18 +1,7 @@ PORT= API_CLIENT_NAME= -API_CLIENT_PASSWORD= -API_CLIENT_SALT= +API_CLIENT_PASSWORD= -DB_NAME= -DB_PASSWORD= -DB_URL= -DB_USER= - -SESSION_COOKIE_SECRET= -WEB_CONFIG_API_URL= - -CENTRAL_API_URL= -REPORT_API_URL= -ENTITY_API_URL= +UNSUBSCRIBE_TOKEN_SECRET= diff --git a/packages/tupaia-web-server/examples.http b/packages/tupaia-web-server/examples.http index f4597b5378..3987715cc6 100644 --- a/packages/tupaia-web-server/examples.http +++ b/packages/tupaia-web-server/examples.http @@ -44,7 +44,7 @@ GET {{host}}/countryAccessList/pacmossi content-type: {{contentType}} ### Fetch legacy report -GET {{host}}/report/28?dashboardCode=explore_General&itemCode=28&legacy=true&organisationUnitCode=explore&projectCode=explore&timeZone=Pacific%2FAuckland +GET {{host}}/legacyDashboardReport/8?dashboardCode=explore_General&itemCode=8&projectCode=explore&organisationUnitCode=explore&timeZone=Pacific%2FAuckland content-type: {{contentType}} ### Fetch report @@ -53,4 +53,4 @@ content-type: {{contentType}} ### Fetch entities GET {{host}}/entities/strive/PG_Lae_Abb?includeRootEntity=true&fields[]=parent_code&fields[]=code&fields[]=name&fields[]=type&fields[]=point&fields[]=image_url&fields[]=attributes&fields[]=child_codes -content-type: {{contentType}} \ No newline at end of file +content-type: {{contentType}} diff --git a/packages/tupaia-web-server/package.json b/packages/tupaia-web-server/package.json index 2b575bb406..c92fbe499e 100644 --- a/packages/tupaia-web-server/package.json +++ b/packages/tupaia-web-server/package.json @@ -35,7 +35,6 @@ "@tupaia/tsutils": "workspace:*", "@tupaia/types": "workspace:*", "@tupaia/utils": "workspace:*", - "dotenv": "^8.2.0", "express": "^4.19.2", "lodash.groupby": "^4.6.0", "lodash.keyby": "^4.6.0", diff --git a/packages/tupaia-web-server/src/index.ts b/packages/tupaia-web-server/src/index.ts index 45c9545c64..7278fae7c7 100644 --- a/packages/tupaia-web-server/src/index.ts +++ b/packages/tupaia-web-server/src/index.ts @@ -4,14 +4,20 @@ */ import http from 'http'; -import * as dotenv from 'dotenv'; - +import path from 'path'; import winston from 'winston'; +import { configureDotEnv } from '@tupaia/server-utils'; import { configureWinston } from '@tupaia/server-boilerplate'; import { createApp } from './app'; configureWinston(); -dotenv.config(); // Load the environment variables into process.env +configureDotEnv([ + path.resolve(__dirname, '../../../env/servers.env'), + path.resolve(__dirname, '../../../env/db.env'), + path.resolve(__dirname, '../../../env/mail.env'), + path.resolve(__dirname, '../../../env/api-client.env'), + path.resolve(__dirname, '../.env'), +]); // Load the environment variables into process.env (async () => { /** diff --git a/packages/types/generate-models.ts b/packages/types/generate-models.ts index afe7d2ca83..be339f8cfc 100644 --- a/packages/types/generate-models.ts +++ b/packages/types/generate-models.ts @@ -6,6 +6,7 @@ // See https://rmp135.github.io/sql-ts/#/?id=totypescript import sqlts, { Table } from '@rmp135/sql-ts'; +import path from 'path'; import Knex from 'knex'; // @ts-ignore @@ -13,7 +14,9 @@ import config from './config/models/config.json'; import * as dotenv from 'dotenv'; import * as fs from 'fs'; -dotenv.config(); +dotenv.config({ + path: [path.resolve(__dirname, '../../../env/db.env'), path.resolve(__dirname, '.env')], +}); const db = Knex({ client: 'postgresql', diff --git a/packages/types/package.json b/packages/types/package.json index 834ea5e8b4..52686a1954 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -27,7 +27,7 @@ }, "devDependencies": { "@rmp135/sql-ts": "^1.15.1", - "dotenv": "^16.0.3", + "dotenv": "^16.4.5", "knex": "^2.3.0", "ts-node": "^10.9.1", "typescript-json-schema": "^0.55.0" diff --git a/packages/web-config-server/.env.example b/packages/web-config-server/.env.example index 29116a765c..7ab01aaa31 100644 --- a/packages/web-config-server/.env.example +++ b/packages/web-config-server/.env.example @@ -1,20 +1,7 @@ PORT= - -AGGREGATION_URL_PREFIX= -API_CLIENT_SALT= -CONFIG_SERVER_BASE_URL= -COOKIE_SECRET= -DB_NAME= -DB_PASSWORD= -DB_URL= -DB_USER= -DHIS_CLIENT_ID= -DHIS_CLIENT_SECRET= -DHIS_PASSWORD= -DHIS_USERNAME= + EXPORT_COOKIE_URL= -EXPORT_URL= -JWT_SECRET= +EXPORT_URL= LAO-PEOPLES-DEMOCRATIC-REPUBLIC_DHIS_AUTH_TOKEN_HOST= LAO-PEOPLES-DEMOCRATIC-REPUBLIC_DHIS_AUTH_TOKEN_PATH= LAO-PEOPLES-DEMOCRATIC-REPUBLIC_DHIS_CLIENT_ID= @@ -22,19 +9,9 @@ LAO-PEOPLES-DEMOCRATIC-REPUBLIC_DHIS_CLIENT_SECRET= LAO-PEOPLES-DEMOCRATIC-REPUBLIC_DHIS_PASSWORD= LAO-PEOPLES-DEMOCRATIC-REPUBLIC_DHIS_USERNAME= LAST_USER_SESSION_COOKIE_SECRET= -LOG_FILE_NAME= -MICROSERVICE_CLIENT_SECRET= +LOG_FILE_NAME= MICROSERVICE_CLIENT_USERNAME= -PALAU_DHIS_API_URL= -PALAU_DHIS_CLIENT_SECRET= -PALAU_DHIS_PASSWORD= -REACT_APP_CONFIG_SERVER_BASE_URL= -SITE_EMAIL_ADDRESS= -SMTP_HOST= -SMTP_PASSWORD= -SMTP_USER= SUPERSET_API_PROXY_URL= TUPAIA_APP_SERVER_AUTH= TUPAIA_APP_SERVER_URL= USER_SESSION_COOKIE_SECRET= -WEATHERBIT_API_KEY= diff --git a/packages/web-config-server/jest.setup.js b/packages/web-config-server/jest.setup.js index e80049e612..06aa4e8eb3 100644 --- a/packages/web-config-server/jest.setup.js +++ b/packages/web-config-server/jest.setup.js @@ -4,6 +4,9 @@ */ import { clearTestData, getTestDatabase } from '@tupaia/database'; +import { configureEnv } from './src/configureEnv'; + +configureEnv(); afterAll(async () => { const database = getTestDatabase(); diff --git a/packages/web-config-server/package.json b/packages/web-config-server/package.json index eeebae797d..1b0997c704 100644 --- a/packages/web-config-server/package.json +++ b/packages/web-config-server/package.json @@ -39,7 +39,6 @@ "compression": "^1.7.2", "cookie-parser": "^1.4.3", "cors": "^2.8.4", - "dotenv": "^8.2.0", "express": "^4.19.2", "lodash.difference": "^4.5.0", "lodash.flatten": "^4.4.0", diff --git a/packages/web-config-server/src/app.js b/packages/web-config-server/src/app.js index 841abd9bdc..5f240f9bdf 100755 --- a/packages/web-config-server/src/app.js +++ b/packages/web-config-server/src/app.js @@ -1,4 +1,3 @@ -import {} from 'dotenv/config'; // Load the environment variables into process.env import '@babel/polyfill'; import http from 'http'; import express from 'express'; @@ -14,6 +13,9 @@ import { bindUserSessions } from './authSession'; import { modelClasses } from './models'; import { handleError, logApiRequest } from './utils'; import './log'; +import { configureEnv } from './configureEnv'; + +configureEnv(); export async function createApp() { const app = express(); diff --git a/packages/web-config-server/src/appServer/requestHelpers/fetchFromCentralServer.js b/packages/web-config-server/src/appServer/requestHelpers/fetchFromCentralServer.js index a08f6b97ec..2c9440d652 100644 --- a/packages/web-config-server/src/appServer/requestHelpers/fetchFromCentralServer.js +++ b/packages/web-config-server/src/appServer/requestHelpers/fetchFromCentralServer.js @@ -28,7 +28,11 @@ export const fetchFromCentralServer = async ( queryParameters, authHeader = DEFAULT_AUTH_HEADER, ) => { - const url = stringifyQuery(process.env.TUPAIA_APP_SERVER_URL, endpoint, queryParameters); + const url = stringifyQuery( + process.env.TUPAIA_APP_SERVER_URL || 'http://localhost:8090/v2', + endpoint, + queryParameters, + ); const config = { method: payload ? 'POST' : 'GET', headers: { diff --git a/packages/web-config-server/src/authSession/authSession.js b/packages/web-config-server/src/authSession/authSession.js index ed92478802..0946422d67 100644 --- a/packages/web-config-server/src/authSession/authSession.js +++ b/packages/web-config-server/src/authSession/authSession.js @@ -1,12 +1,13 @@ -import {} from 'dotenv/config'; // Load the environment variables into process.env import session from 'client-sessions'; import { UnauthenticatedError } from '@tupaia/utils'; - +import { configureEnv } from '../configureEnv'; import { getUserFromAuthHeader } from './getUserFromAuthHeader'; import { getAccessPolicyForUser } from './getAccessPolicyForUser'; import { PUBLIC_USER_NAME } from './publicAccess'; +configureEnv(); + // auth is a middleware that runs on every request const auth = () => async (req, res, next) => { const { authenticator } = req; diff --git a/packages/web-config-server/src/configureEnv.js b/packages/web-config-server/src/configureEnv.js new file mode 100644 index 0000000000..b03dc8e1b2 --- /dev/null +++ b/packages/web-config-server/src/configureEnv.js @@ -0,0 +1,21 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2024 Beyond Essential Systems Pty Ltd + */ + +import path from 'path'; +import { configureDotEnv } from '@tupaia/server-utils'; + +export const configureEnv = () => { + configureDotEnv([ + path.resolve(__dirname, '../../../env/aggregation.env'), + path.resolve(__dirname, '../../../env/db.env'), + path.resolve(__dirname, '../../../env/dhis.env'), + path.resolve(__dirname, '../../../env/mail.env'), + path.resolve(__dirname, '../../../env/pg.env'), + path.resolve(__dirname, '../../../env/servers.env'), + path.resolve(__dirname, '../../../env/superset.env'), + path.resolve(__dirname, '../../../env/api-client.env'), + path.resolve(__dirname, '../.env'), + ]); +}; diff --git a/packages/web-config-server/src/export/requestFromTupaiaConfigServer.js b/packages/web-config-server/src/export/requestFromTupaiaConfigServer.js index 227353aebb..592d17fef7 100644 --- a/packages/web-config-server/src/export/requestFromTupaiaConfigServer.js +++ b/packages/web-config-server/src/export/requestFromTupaiaConfigServer.js @@ -1,6 +1,6 @@ import { fetchWithTimeout } from '@tupaia/utils'; -const BASE_URL = process.env.CONFIG_SERVER_BASE_URL || 'http://localhost:8080/api/v1'; +const BASE_URL = process.env.WEB_CONFIG_API_URL || 'http://localhost:8000/api/v1'; /** * Requests a URL, returning a promise @@ -23,7 +23,7 @@ export const requestFromTupaiaConfigServer = async ( try { const headers = { cookie: `${sessionCookieName}=${sessionCookie}`, - ...additionalHeaders + ...additionalHeaders, }; const response = await fetchWithTimeout( diff --git a/packages/web-config-server/src/index.js b/packages/web-config-server/src/index.js index 689c71682f..dcf896e451 100644 --- a/packages/web-config-server/src/index.js +++ b/packages/web-config-server/src/index.js @@ -1,6 +1,9 @@ import winston from 'winston'; import { createApp } from './app'; import { runPreaggregation } from './preaggregation/runPreaggregation'; +import { configureEnv } from './configureEnv'; + +configureEnv(); async function start() { if (process.env.RUN_PREAGGREGATION) { @@ -9,7 +12,7 @@ async function start() { const app = await createApp(); // process.env.PORT as per run command PORT=XXXX npm run dev - const port = process.env.PORT || 8080; + const port = process.env.PORT || 8000; app.server.listen(port); winston.debug('Logging at debug level'); winston.info(`Running on port ${port}`); diff --git a/scripts/bash/downloadEnvironmentVariables.sh b/scripts/bash/downloadEnvironmentVariables.sh index a67e2938ae..68a2d94cb0 100755 --- a/scripts/bash/downloadEnvironmentVariables.sh +++ b/scripts/bash/downloadEnvironmentVariables.sh @@ -5,6 +5,7 @@ DEPLOYMENT_NAME=$1 DIR=$(dirname "$0") COLLECTION_PATH="Engineering/Tupaia General/Environment Variables" # Collection in BitWarden where .env vars are kept + # can provide one or more packages as command line arguments, or will default to all if [ -z $2 ]; then echo "Fetching all .env files" @@ -20,26 +21,36 @@ eval "$(bw unlock $BITWARDEN_PASSWORD | grep -o -m 1 'export BW_SESSION=.*$')" COLLECTION_ID=$(bw get collection "$COLLECTION_PATH" | jq .id) -for PACKAGE in $PACKAGES; do - ENV_FILE_PATH=${DIR}/../../packages/${PACKAGE}/.env +load_env_file_from_bw () { + FILE_NAME=$1 + BASE_FILE_PATH=$2 + NEW_FILE_NAME=$3 + ENV_FILE_PATH=${BASE_FILE_PATH}/${NEW_FILE_NAME}.env + + echo "Fetching environment variables for $FILE_NAME: $ENV_FILE_PATH" + + echo "Fetching environment variables for $FILE_NAME" # checkout deployment specific env vars, or dev as fallback - DEPLOYMENT_ENV_VARS=$(bw list items --search ${PACKAGE}.${DEPLOYMENT_NAME}.env | jq --raw-output "map(select(.collectionIds[] | contains ($COLLECTION_ID))) | .[] .notes") + DEPLOYMENT_ENV_VARS=$(bw list items --search ${FILE_NAME}.${DEPLOYMENT_NAME}.env | jq --raw-output "map(select(.collectionIds[] | contains ($COLLECTION_ID))) | .[] .notes") + + if [ -n "$DEPLOYMENT_ENV_VARS" ]; then echo "$DEPLOYMENT_ENV_VARS" > ${ENV_FILE_PATH} else - DEV_ENV_VARS=$(bw list items --search ${PACKAGE}.dev.env | jq --raw-output "map(select(.collectionIds[] | contains ($COLLECTION_ID))) | .[] .notes") + DEV_ENV_VARS=$(bw list items --search ${FILE_NAME}.dev.env | jq --raw-output "map(select(.collectionIds[] | contains ($COLLECTION_ID))) | .[] .notes") echo "$DEV_ENV_VARS" > ${ENV_FILE_PATH} fi # Replace any instances of the placeholder [deployment-name] in the .env file with the actual deployment # name (e.g. [deployment-name]-api.tupaia.org -> specific-deployment-api.tupaia.org) - sed -i -e "s/\[deployment-name\]/${DEPLOYMENT_NAME}/g" ${ENV_FILE_PATH} + sed -i "" "s/\[deployment-name\]/${DEPLOYMENT_NAME}/g" "${ENV_FILE_PATH}" + if [[ "${DEPLOYMENT_NAME}" == *-e2e || "${DEPLOYMENT_NAME}" == e2e ]]; then # Update e2e environment variables - if [[ ${PACKAGE} == "central-server" || ${PACKAGE} == "web-config-server" ]]; then - sed -i -E 's/^AGGREGATION_URL_PREFIX="?dev-"?$/AGGREGATION_URL_PREFIX=e2e-/g' ${ENV_FILE_PATH} + if [[ ${FILE_NAME} == "aggregation" ]]; then + sed -i "" 's/^AGGREGATION_URL_PREFIX="?dev-"?$/AGGREGATION_URL_PREFIX=e2e-/g' ${ENV_FILE_PATH} fi fi @@ -47,10 +58,33 @@ for PACKAGE in $PACKAGES; do # Update dev specific environment variables # (removes ###DEV_ONLY### prefixes, leaving the key=value pair uncommented) # (after removing prefix, if there are duplicate keys, dotenv uses the last one in the file) - sed -i -E 's/^###DEV_ONLY###//g' ${ENV_FILE_PATH} + sed -i "" 's/^###DEV_ONLY###//g' ${ENV_FILE_PATH} fi + - echo "downloaded .env vars for $PACKAGE" + echo "downloaded .env vars for $FILE_NAME" +} + +for PACKAGE in $PACKAGES; do + # only download the env file if there is an example file in the package. If there isn't, this means it is a package that doesn't need env vars + has_example_env_in_package=$(find $DIR/../../packages/$PACKAGE -type f -name '*.env.example' | wc -l) + if [ $has_example_env_in_package -eq 1 ]; then + load_env_file_from_bw $PACKAGE $DIR/../../packages/$PACKAGE "" + fi +done + + +# get all .env.*.example files in the env directory +file_names=$(find $DIR/../../env -type f -name '*.env.example' -exec basename {} \;) + + +# for each file, get the extract the filename without the .example extension +for file_name in $file_names; do + env_name=$(echo $file_name | sed 's/\.env.example//') + load_env_file_from_bw $env_name $DIR/../../env $env_name done -bw logout \ No newline at end of file + + + +bw logout diff --git a/scripts/bash/getInternalDependencies.sh b/scripts/bash/getInternalDependencies.sh index 82b73a594c..c38095854e 100755 --- a/scripts/bash/getInternalDependencies.sh +++ b/scripts/bash/getInternalDependencies.sh @@ -12,7 +12,7 @@ dependencies_already_visited=($@) # if no package.json entrypoint is specified, just return all internal dependencies if [ -z ${package_path} ]; then - echo "types" "utils" "tsutils" "ui-components" "ui-chart-components" "ui-map-components" "access-policy" "admin-panel" "aggregator" "api-client" "auth" "database" "data-api" "dhis-api" "data-lake-api" "expression-parser" "indicators" "weather-api" "kobo-api" "superset-api" "data-broker" "server-boilerplate" "server-utils" + echo "types" "utils" "tsutils" "ui-components" "ui-chart-components" "ui-map-components" "server-utils" "access-policy" "admin-panel" "aggregator" "api-client" "auth" "database" "data-api" "dhis-api" "data-lake-api" "expression-parser" "indicators" "weather-api" "kobo-api" "superset-api" "data-broker" "server-boilerplate" exit 0 fi diff --git a/scripts/bash/getPackagesWithEnvFiles.sh b/scripts/bash/getPackagesWithEnvFiles.sh index 4a283e9b3c..27c7cebe38 100755 --- a/scripts/bash/getPackagesWithEnvFiles.sh +++ b/scripts/bash/getPackagesWithEnvFiles.sh @@ -4,6 +4,6 @@ DIR=$(dirname "$0") # packages with .env files are (currently) all deployable, plus auth, data-api, and database PACKAGES=$(${DIR}/getDeployablePackages.sh) -PACKAGES+=" auth data-api database" +PACKAGES+=" data-api" echo $PACKAGES exit 0 diff --git a/scripts/bash/mergeCurrentEnvWithEnvFile.sh b/scripts/bash/mergeCurrentEnvWithEnvFile.sh deleted file mode 100755 index fa297c92c7..0000000000 --- a/scripts/bash/mergeCurrentEnvWithEnvFile.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -e - -curenv=$(declare -p -x) -test -f .env && source .env -eval "$curenv" \ No newline at end of file diff --git a/scripts/bash/mergeEnvForDB.sh b/scripts/bash/mergeEnvForDB.sh new file mode 100755 index 0000000000..b8db6343c2 --- /dev/null +++ b/scripts/bash/mergeEnvForDB.sh @@ -0,0 +1,51 @@ +#!/bin/bash -e + +# Function to get the directory of the package that's calling this script +get_caller_package_directory() { + local dir + dir=$(dirname "$(readlink -f "$0")") + while [[ "$dir" != "/" ]]; do + if [[ -f "$dir/package.json" ]]; then + echo "$dir" + return + fi + dir=$(dirname "$dir") + done +} + +# Get the directory of the package that's calling this script +CALLING_SCRIPT_DIR=$(get_caller_package_directory) + +# Get the directory of this script +CURRENT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Fixed paths to the .env files for the test db +file1="$CURRENT_DIR/../../env/db.env" +file2="$CURRENT_DIR/../../env/pg.env" +file3="$CURRENT_DIR/../../env/data-lake.env" +file4="$CALLING_SCRIPT_DIR/.env" + +common_files="$file1 $file2 $file3 $file4" + + # Remove files that don't exist +for file in $common_files; do + if [ ! -f "$file" ]; then + common_files=$(echo "$common_files" | sed "s|$file||g") + fi +done + +# Load environment variables from .env files +merged_content="$(cat $common_files)" + +# Process command line arguments, overwriting values if present +for var in $(env); do + if [[ "$var" == *=* ]]; then + key="${var%%=*}" + value="${var#*=}" + # Override values from command line + merged_content+=" $key=\"$value\"" + fi +done + +# Evaluate merged content to set variables +eval "$merged_content" diff --git a/yarn.lock b/yarn.lock index 73704e6f42..0901f5522d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11687,6 +11687,7 @@ __metadata: "@tupaia/api-client": "workspace:*" "@tupaia/database": "workspace:*" "@tupaia/server-boilerplate": "workspace:*" + "@tupaia/server-utils": "workspace:*" "@tupaia/types": "workspace:*" "@tupaia/utils": "workspace:*" "@types/multer": ^1.4.7 @@ -11696,7 +11697,6 @@ __metadata: case: ^1.6.3 client-sessions: ^0.8.0 cors: ^2.8.5 - dotenv: ^8.2.0 express: ^4.19.2 lodash: ^4.17.4 multer: ^1.4.3 @@ -11797,6 +11797,7 @@ __metadata: "@beyondessential/tupaia-access-policy": ^2.5.1 "@tupaia/access-policy": "workspace:*" "@tupaia/database": "workspace:*" + "@tupaia/server-utils": "workspace:*" "@tupaia/utils": "workspace:*" jsonwebtoken: ^7.4.3 npm-run-all: ^4.1.5 @@ -11837,7 +11838,6 @@ __metadata: cross-env: ^7.0.2 deep-equal-in-any-order: ^1.0.21 del: ^2.2.2 - dotenv: ^8.2.0 express: ^4.19.2 form-data: ^2.3.3 format-link-header: ^2.1.0 @@ -11875,13 +11875,13 @@ __metadata: resolution: "@tupaia/data-api@workspace:packages/data-api" dependencies: "@tupaia/database": "workspace:*" + "@tupaia/server-utils": "workspace:*" "@tupaia/tsutils": "workspace:*" "@tupaia/utils": "workspace:*" "@types/lodash.groupby": ^4.6.0 db-migrate: ^0.11.5 db-migrate-pg: ^1.2.2 deep-equal-in-any-order: ^1.0.27 - dotenv: ^8.2.0 lodash.groupby: ^4.6.0 moment: ^2.24.0 ts-node: ^10.7.0 @@ -11931,7 +11931,6 @@ __metadata: db-migrate: ^0.11.5 db-migrate-pg: ^1.2.2 deep-equal-in-any-order: ^1.0.27 - dotenv: ^8.2.0 jest: ^29.7.0 knex: 0.14.6 lodash.groupby: ^4.6.0 @@ -11950,10 +11949,10 @@ __metadata: "@tupaia/data-broker": "workspace:*" "@tupaia/database": "workspace:*" "@tupaia/server-boilerplate": "workspace:*" + "@tupaia/server-utils": "workspace:*" "@tupaia/tsutils": "workspace:*" "@tupaia/types": "workspace:*" "@tupaia/utils": "workspace:*" - dotenv: ^8.2.0 express: ^4.19.2 mockdate: ^3.0.5 winston: ^3.3.3 @@ -11971,7 +11970,7 @@ __metadata: "@tupaia/utils": "workspace:*" db-migrate: ^0.11.5 db-migrate-pg: ^1.2.2 - dotenv: ^8.2.0 + dotenv: ^16.4.5 knex: 0.14.6 lodash.clonedeep: ^4.5.0 lodash.groupby: ^4.6.0 @@ -11997,13 +11996,13 @@ __metadata: "@tupaia/api-client": "workspace:*" "@tupaia/database": "workspace:*" "@tupaia/server-boilerplate": "workspace:*" + "@tupaia/server-utils": "workspace:*" "@tupaia/types": "workspace:*" "@tupaia/utils": "workspace:*" "@types/lodash.groupby": ^4.6.0 "@types/lodash.keyby": ^4.6.0 "@types/lodash.sortby": ^4.6.0 camelcase-keys: ^6.2.2 - dotenv: ^8.2.0 express: ^4.19.2 lodash.groupby: ^4.6.0 lodash.keyby: ^4.6.0 @@ -12113,11 +12112,11 @@ __metadata: "@tupaia/auth": "workspace:*" "@tupaia/database": "workspace:*" "@tupaia/server-boilerplate": "workspace:*" + "@tupaia/server-utils": "workspace:*" "@tupaia/tsutils": "workspace:*" "@tupaia/types": "workspace:*" "@tupaia/utils": "workspace:*" "@types/lodash.keyby": ^4.6.6 - dotenv: ^8.2.0 express: ^4.19.2 lodash.keyby: ^4.6.0 winston: ^3.3.3 @@ -12170,7 +12169,6 @@ __metadata: "@tupaia/tsutils": "workspace:*" "@tupaia/utils": "workspace:*" camelcase-keys: ^6.2.2 - dotenv: ^8.2.0 express: ^4.19.2 lodash.uniqby: ^4.7.0 winston: ^3.3.3 @@ -12230,7 +12228,6 @@ __metadata: "@tupaia/utils": "workspace:*" "@types/lodash.clonedeep": ^4.5.0 "@types/semver-compare": ^1.0.1 - dotenv: ^8.2.0 express: ^4.19.2 lodash.clonedeep: ^4.5.0 lodash.groupby: ^4.6.0 @@ -12320,13 +12317,13 @@ __metadata: "@tupaia/access-policy": "workspace:*" "@tupaia/database": "workspace:*" "@tupaia/server-boilerplate": "workspace:*" + "@tupaia/server-utils": "workspace:*" "@tupaia/types": "workspace:*" "@tupaia/utils": "workspace:*" api-error-handler: ^1.0.0 body-parser: ^1.18.3 client-sessions: ^0.8.0 cors: ^2.8.5 - dotenv: ^8.2.0 express: ^4.19.2 lodash.groupby: ^4.6.0 winston: ^3.3.3 @@ -12388,6 +12385,7 @@ __metadata: "@tupaia/database": "workspace:*" "@tupaia/expression-parser": "workspace:*" "@tupaia/server-boilerplate": "workspace:*" + "@tupaia/server-utils": "workspace:*" "@tupaia/tsutils": "workspace:*" "@tupaia/types": "workspace:*" "@tupaia/utils": "workspace:*" @@ -12399,7 +12397,6 @@ __metadata: body-parser: ^1.18.3 cors: ^2.8.5 date-fns: ^2.29.3 - dotenv: ^8.2.0 express: ^4.19.2 lodash.groupby: ^4.6.0 lodash.isplainobject: ^4.0.6 @@ -12432,7 +12429,7 @@ __metadata: body-parser: ^1.18.3 client-sessions: ^0.8.0 cors: ^2.8.5 - dotenv: ^8.2.0 + dotenv: ^16.4.5 express: ^4.19.2 http-proxy-middleware: ^2.0.1 i18n: ^0.13.3 @@ -12452,6 +12449,7 @@ __metadata: "@types/nodemailer": ^6.4.13 "@types/sha256": ^0.2.2 cookie: ^0.5.0 + dotenv: ^16.4.5 nodemailer: ^6.9.12 puppeteer: ^15.4.0 sha256: ^0.2.0 @@ -12503,7 +12501,6 @@ __metadata: "@tupaia/utils": "workspace:*" "@types/lodash.groupby": ^4.6.0 "@types/lodash.keyby": ^4.6.0 - dotenv: ^8.2.0 express: ^4.19.2 lodash.groupby: ^4.6.0 lodash.keyby: ^4.6.0 @@ -12566,7 +12563,7 @@ __metadata: resolution: "@tupaia/types@workspace:packages/types" dependencies: "@rmp135/sql-ts": ^1.15.1 - dotenv: ^16.0.3 + dotenv: ^16.4.5 knex: ^2.3.0 ts-node: ^10.9.1 typescript-json-schema: ^0.55.0 @@ -12735,7 +12732,6 @@ __metadata: cookie-parser: ^1.4.3 cors: ^2.8.4 csvtojson: ^1.1.5 - dotenv: ^8.2.0 express: ^4.19.2 lodash.difference: ^4.5.0 lodash.flatten: ^4.4.0 @@ -21334,6 +21330,13 @@ __metadata: languageName: node linkType: hard +"dotenv@npm:16.4.5, dotenv@npm:^16.4.5": + version: 16.4.5 + resolution: "dotenv@npm:16.4.5" + checksum: 301a12c3d44fd49888b74eb9ccf9f07a1f5df43f489e7fcb89647a2edcd84c42d6bc349dc8df099cd18f07c35c7b04685c1a4f3e6a6a9e6b30f8d48c15b7f49c + languageName: node + linkType: hard + "dotenv@npm:^16.0.0": version: 16.3.1 resolution: "dotenv@npm:16.3.1" @@ -21341,13 +21344,6 @@ __metadata: languageName: node linkType: hard -"dotenv@npm:^16.0.3": - version: 16.0.3 - resolution: "dotenv@npm:16.0.3" - checksum: afcf03f373d7a6d62c7e9afea6328e62851d627a4e73f2e12d0a8deae1cd375892004f3021883f8aec85932cd2834b091f568ced92b4774625b321db83b827f8 - languageName: node - linkType: hard - "dotenv@npm:^5.0.1": version: 5.0.1 resolution: "dotenv@npm:5.0.1" @@ -21362,7 +21358,7 @@ __metadata: languageName: node linkType: hard -"dotenv@npm:^8.0.0, dotenv@npm:^8.2.0": +"dotenv@npm:^8.0.0": version: 8.2.0 resolution: "dotenv@npm:8.2.0" checksum: ad4c8e0df3e24b4811c8e93377d048a10a9b213dcd9f062483b4a2d3168f08f10ec9c618c23f5639060d230ccdb174c08761479e9baa29610aa978e1ee66df76 @@ -41662,7 +41658,7 @@ __metadata: babel-preset-react-app: ^10.0.0 concurrently: ^5.2.0 cypress-dotenv: ^1.2.2 - dotenv: ^16.0.3 + dotenv: 16.4.5 eslint: ^7.9.0 eslint-import-resolver-babel-module: ^5.3.1 eslint-plugin-cypress: ^2.11.1 From 9cb809e823001a03439ac9a36bd82ea59a05cfce Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Thu, 4 Apr 2024 16:05:24 +1300 Subject: [PATCH 11/25] deps: bump `vite` to 4.5.3 Bumps [vite](https://github.com/vitejs/vite/tree/HEAD/packages/vite) from 4.4.5 to 4.5.3. - [Release notes](https://github.com/vitejs/vite/releases) - [Changelog](https://github.com/vitejs/vite/blob/v4.5.3/packages/vite/CHANGELOG.md) - [Commits](https://github.com/vitejs/vite/commits/v4.5.3/packages/vite) --- updated-dependencies: - dependency-name: vite dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package.json | 2 +- packages/admin-panel/package.json | 2 +- packages/datatrak-web/package.json | 2 +- packages/lesmis/package.json | 2 +- packages/psss/package.json | 2 +- packages/tupaia-web/package.json | 2 +- yarn.lock | 108 +++++++++++------------------ 7 files changed, 47 insertions(+), 73 deletions(-) mode change 100755 => 100644 packages/lesmis/package.json diff --git a/package.json b/package.json index 3cc5dfb880..bb6790791b 100644 --- a/package.json +++ b/package.json @@ -87,7 +87,7 @@ "ts-jest": "^29.1.2", "ts-node": "^10.7.0", "typescript": "^5.2.2", - "vite": "^4.4.8", + "vite": "^4.5.3", "vite-plugin-compression": "^0.5.1", "vite-plugin-ejs": "^1.6.4", "yargs": "15.4.1" diff --git a/packages/admin-panel/package.json b/packages/admin-panel/package.json index 9619e9c5e0..c70595fc1c 100644 --- a/packages/admin-panel/package.json +++ b/packages/admin-panel/package.json @@ -77,7 +77,7 @@ "redux-thunk": "^2.2.0", "styled-components": "^5.1.0", "uuid": "^3.2.1", - "vite": "^4.4.8" + "vite": "^4.5.3" }, "devDependencies": { "npm-run-all": "^4.1.5" diff --git a/packages/datatrak-web/package.json b/packages/datatrak-web/package.json index fccbde075c..7e8bcd5ae7 100644 --- a/packages/datatrak-web/package.json +++ b/packages/datatrak-web/package.json @@ -48,7 +48,7 @@ "@vitejs/plugin-react": "^4.0.0", "msw": "^1.3.1", "npm-run-all": "^4.1.5", - "vite": "^4.3.2" + "vite": "^4.5.3" }, "scripts": { "build": "yarn package:build:types && yarn package:build:vite", diff --git a/packages/lesmis/package.json b/packages/lesmis/package.json old mode 100755 new mode 100644 index c6db73fd66..09f09457a2 --- a/packages/lesmis/package.json +++ b/packages/lesmis/package.json @@ -56,6 +56,6 @@ }, "devDependencies": { "npm-run-all": "^4.1.5", - "vite": "^4.3.2" + "vite": "^4.5.3" } } diff --git a/packages/psss/package.json b/packages/psss/package.json index 1dc2afe866..11f78305f5 100644 --- a/packages/psss/package.json +++ b/packages/psss/package.json @@ -60,6 +60,6 @@ "cross-env": "^7.0.2", "mockdate": "^3.0.5", "npm-run-all": "^4.1.5", - "vite": "^4.3.2" + "vite": "^4.5.3" } } diff --git a/packages/tupaia-web/package.json b/packages/tupaia-web/package.json index a0a7a6b5cf..3af37f3225 100644 --- a/packages/tupaia-web/package.json +++ b/packages/tupaia-web/package.json @@ -64,7 +64,7 @@ "msw": "^2.0.9", "npm-run-all": "^4.1.5", "storybook": "^7.0.18", - "vite": "^4.3.2", + "vite": "^4.5.3", "vite-plugin-html": "^3.2.0" }, "msw": { diff --git a/yarn.lock b/yarn.lock index 0901f5522d..6eb4393473 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11760,7 +11760,7 @@ __metadata: redux-thunk: ^2.2.0 styled-components: ^5.1.0 uuid: ^3.2.1 - vite: ^4.4.8 + vite: ^4.5.3 languageName: unknown linkType: soft @@ -12050,7 +12050,7 @@ __metadata: react-router: 6.3.0 react-router-dom: 6.3.0 styled-components: ^5.1.0 - vite: ^4.3.2 + vite: ^4.5.3 yup: ^1.3.2 languageName: unknown linkType: soft @@ -12209,7 +12209,7 @@ __metadata: react-redux: ^5.0.6 react-router-dom: ^5.2.0 styled-components: ^5.1.0 - vite: ^4.3.2 + vite: ^4.5.3 languageName: unknown linkType: soft @@ -12368,7 +12368,7 @@ __metadata: redux-thunk: ^2.3.0 reselect: ^4.0.0 styled-components: ^5.1.0 - vite: ^4.3.2 + vite: ^4.5.3 whatwg-fetch: ^3.0.0 languageName: unknown linkType: soft @@ -12551,7 +12551,7 @@ __metadata: react-router-dom: 6.3.0 storybook: ^7.0.18 styled-components: ^5.1.0 - vite: ^4.3.2 + vite: ^4.5.3 vite-plugin-ejs: ^1.6.4 vite-plugin-env-compatible: ^1.1.1 vite-plugin-html: ^3.2.0 @@ -32245,12 +32245,12 @@ __metadata: languageName: node linkType: hard -"nanoid@npm:^3.3.6": - version: 3.3.6 - resolution: "nanoid@npm:3.3.6" +"nanoid@npm:^3.3.7": + version: 3.3.7 + resolution: "nanoid@npm:3.3.7" bin: nanoid: bin/nanoid.cjs - checksum: 7d0eda657002738aa5206107bd0580aead6c95c460ef1bdd0b1a87a9c7ae6277ac2e9b945306aaa5b32c6dcb7feaf462d0f552e7f8b5718abfc6ead5c94a71b3 + checksum: d36c427e530713e4ac6567d488b489a36582ef89da1d6d4e3b87eded11eb10d7042a877958c6f104929809b2ab0bafa17652b076cdf84324aa75b30b722204f2 languageName: node linkType: hard @@ -34985,14 +34985,14 @@ __metadata: languageName: node linkType: hard -"postcss@npm:^8.4.26": - version: 8.4.26 - resolution: "postcss@npm:8.4.26" +"postcss@npm:^8.4.27": + version: 8.4.38 + resolution: "postcss@npm:8.4.38" dependencies: - nanoid: ^3.3.6 + nanoid: ^3.3.7 picocolors: ^1.0.0 - source-map-js: ^1.0.2 - checksum: 1cf08ee10d58cbe98f94bf12ac49a5e5ed1588507d333d2642aacc24369ca987274e1f60ff4cbf0081f70d2ab18a5cd3a4a273f188d835b8e7f3ba381b184e57 + source-map-js: ^1.2.0 + checksum: 649f9e60a763ca4b5a7bbec446a069edf07f057f6d780a5a0070576b841538d1ecf7dd888f2fbfd1f76200e26c969e405aeeae66332e6927dbdc8bdcb90b9451 languageName: node linkType: hard @@ -38463,7 +38463,7 @@ __metadata: languageName: node linkType: hard -"rollup@npm:^2.25.0 || ^3.3.0, rollup@npm:^3.25.2": +"rollup@npm:^2.25.0 || ^3.3.0": version: 3.26.3 resolution: "rollup@npm:3.26.3" dependencies: @@ -38477,6 +38477,20 @@ __metadata: languageName: node linkType: hard +"rollup@npm:^3.27.1": + version: 3.29.4 + resolution: "rollup@npm:3.29.4" + dependencies: + fsevents: ~2.3.2 + dependenciesMeta: + fsevents: + optional: true + bin: + rollup: dist/bin/rollup + checksum: 8bb20a39c8d91130825159c3823eccf4dc2295c9a0a5c4ed851a5bf2167dbf24d9a29f23461a54c955e5506395e6cc188eafc8ab0e20399d7489fb33793b184e + languageName: node + linkType: hard + "rsvp@npm:^4.8.4": version: 4.8.5 resolution: "rsvp@npm:4.8.5" @@ -39552,10 +39566,10 @@ __metadata: languageName: node linkType: hard -"source-map-js@npm:^1.0.2": - version: 1.0.2 - resolution: "source-map-js@npm:1.0.2" - checksum: c049a7fc4deb9a7e9b481ae3d424cc793cb4845daa690bc5a05d428bf41bf231ced49b4cf0c9e77f9d42fdb3d20d6187619fc586605f5eabe995a316da8d377c +"source-map-js@npm:^1.2.0": + version: 1.2.0 + resolution: "source-map-js@npm:1.2.0" + checksum: 791a43306d9223792e84293b00458bf102a8946e7188f3db0e4e22d8d530b5f80a4ce468eb5ec0bf585443ad55ebbd630bf379c98db0b1f317fd902500217f97 languageName: node linkType: hard @@ -41673,7 +41687,7 @@ __metadata: ts-jest: ^29.1.2 ts-node: ^10.7.0 typescript: ^5.2.2 - vite: ^4.4.8 + vite: ^4.5.3 vite-plugin-compression: ^0.5.1 vite-plugin-ejs: ^1.6.4 yargs: 15.4.1 @@ -42908,54 +42922,14 @@ __metadata: languageName: node linkType: hard -"vite@npm:^4.3.2": - version: 4.4.5 - resolution: "vite@npm:4.4.5" - dependencies: - esbuild: ^0.18.10 - fsevents: ~2.3.2 - postcss: ^8.4.26 - rollup: ^3.25.2 - peerDependencies: - "@types/node": ">= 14" - less: "*" - lightningcss: ^1.21.0 - sass: "*" - stylus: "*" - sugarss: "*" - terser: ^5.4.0 - dependenciesMeta: - fsevents: - optional: true - peerDependenciesMeta: - "@types/node": - optional: true - less: - optional: true - lightningcss: - optional: true - sass: - optional: true - stylus: - optional: true - sugarss: - optional: true - terser: - optional: true - bin: - vite: bin/vite.js - checksum: 825d8c13511b764710ca503e603746e20a7cb963194033dc606c46b1650a0d8986b577ba92507b54143cbf467032e9868ea04913a86a8fcb88b69c5b8551938b - languageName: node - linkType: hard - -"vite@npm:^4.4.8": - version: 4.4.8 - resolution: "vite@npm:4.4.8" +"vite@npm:^4.5.3": + version: 4.5.3 + resolution: "vite@npm:4.5.3" dependencies: esbuild: ^0.18.10 fsevents: ~2.3.2 - postcss: ^8.4.26 - rollup: ^3.25.2 + postcss: ^8.4.27 + rollup: ^3.27.1 peerDependencies: "@types/node": ">= 14" less: "*" @@ -42984,7 +42958,7 @@ __metadata: optional: true bin: vite: bin/vite.js - checksum: e8ffe688f8a7396b1357778f00cb06d1f3dadad200823c47a1955cf52774a0cbff5ac4d6a8f8d09e26c1d4e588e5815956f9eba02ae301e77a36c3d181a1bc86 + checksum: fd3f512ce48ca2a1fe60ad0376283b832de9272725fdbc65064ae9248f792de87b0f27a89573115e23e26784800daca329f8a9234d298ba6f60e808a9c63883c languageName: node linkType: hard From cb316edb4f04637074781c2c185070d17fabbd55 Mon Sep 17 00:00:00 2001 From: alexd-bes <129009580+alexd-bes@users.noreply.github.com> Date: Thu, 4 Apr 2024 16:11:59 +1300 Subject: [PATCH 12/25] Fix issue with mac/linux (#5562) --- scripts/bash/downloadEnvironmentVariables.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/bash/downloadEnvironmentVariables.sh b/scripts/bash/downloadEnvironmentVariables.sh index 68a2d94cb0..7ddbd61b0f 100755 --- a/scripts/bash/downloadEnvironmentVariables.sh +++ b/scripts/bash/downloadEnvironmentVariables.sh @@ -44,13 +44,13 @@ load_env_file_from_bw () { # Replace any instances of the placeholder [deployment-name] in the .env file with the actual deployment # name (e.g. [deployment-name]-api.tupaia.org -> specific-deployment-api.tupaia.org) - sed -i "" "s/\[deployment-name\]/${DEPLOYMENT_NAME}/g" "${ENV_FILE_PATH}" + sed -i -e "s/\[deployment-name\]/${DEPLOYMENT_NAME}/g" "${ENV_FILE_PATH}" if [[ "${DEPLOYMENT_NAME}" == *-e2e || "${DEPLOYMENT_NAME}" == e2e ]]; then # Update e2e environment variables if [[ ${FILE_NAME} == "aggregation" ]]; then - sed -i "" 's/^AGGREGATION_URL_PREFIX="?dev-"?$/AGGREGATION_URL_PREFIX=e2e-/g' ${ENV_FILE_PATH} + sed -i -e 's/^AGGREGATION_URL_PREFIX="?dev-"?$/AGGREGATION_URL_PREFIX=e2e-/g' ${ENV_FILE_PATH} fi fi @@ -58,7 +58,7 @@ load_env_file_from_bw () { # Update dev specific environment variables # (removes ###DEV_ONLY### prefixes, leaving the key=value pair uncommented) # (after removing prefix, if there are duplicate keys, dotenv uses the last one in the file) - sed -i "" 's/^###DEV_ONLY###//g' ${ENV_FILE_PATH} + sed -i -e 's/^###DEV_ONLY###//g' ${ENV_FILE_PATH} fi From edf5ad1a0d4f1bd297331816601990b04aba313d Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Thu, 4 Apr 2024 16:23:32 +1300 Subject: [PATCH 13/25] repo: mark lockfile as generated (#5555) mark lockfile as generated --- .gitattributes | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitattributes b/.gitattributes index 506bce338b..1bd553dba5 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,3 +2,4 @@ *.pbxproj -text packages/types/src/schemas/schemas.ts linguist-generated=true packages/types/src/types/models.ts linguist-generated=true +yarn.lock linguist-generated=true From 1287db291c33d52df584f646aa0c2da8fb1213f3 Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Fri, 5 Apr 2024 09:34:23 +1300 Subject: [PATCH 14/25] deps: RN-1112: bump `knex` to 3.1.0 (#5483) * bump `knex` to 3.1.0 * remove unused `@ts-expect-error` * selectively upgrade `knex` version * update copyright message * organise imports * organise imports * experiment: bump `knex` for `database` * comment rationale for `#>>` * parameterise query helper Co-authored-by: Rohan Port <59544282+rohan-bes@users.noreply.github.com> * use shallow copy * fox knex binding --------- Co-authored-by: Rohan Port <59544282+rohan-bes@users.noreply.github.com> --- packages/data-lake-api/package.json | 2 +- .../data-lake-api/src/DataLakeDatabase.ts | 1 - packages/database/package.json | 2 +- packages/database/src/TupaiaDatabase.js | 38 +- .../modelClasses/DashboardRelation.test.js | 4 +- packages/types/package.json | 2 +- yarn.lock | 388 ++---------------- 7 files changed, 51 insertions(+), 386 deletions(-) diff --git a/packages/data-lake-api/package.json b/packages/data-lake-api/package.json index 4feecb3646..cd38b9e406 100644 --- a/packages/data-lake-api/package.json +++ b/packages/data-lake-api/package.json @@ -29,7 +29,7 @@ "db-migrate": "^0.11.5", "db-migrate-pg": "^1.2.2", "deep-equal-in-any-order": "^1.0.27", - "knex": "0.14.6", + "knex": "^3.1.0", "lodash.groupby": "^4.6.0", "moment": "^2.24.0", "pg": "8.5.1" diff --git a/packages/data-lake-api/src/DataLakeDatabase.ts b/packages/data-lake-api/src/DataLakeDatabase.ts index ff7c64703c..e7f26ac1d3 100644 --- a/packages/data-lake-api/src/DataLakeDatabase.ts +++ b/packages/data-lake-api/src/DataLakeDatabase.ts @@ -5,7 +5,6 @@ // @ts-expect-error pg has no types import { types as pgTypes } from 'pg'; -// @ts-expect-error must upgrade knex to get types. Note: last upgrade attempt broke json field querying import knex from 'knex'; import { getConnectionConfig } from './getConnectionConfig'; diff --git a/packages/database/package.json b/packages/database/package.json index 02df7a6df8..dc83fe448e 100644 --- a/packages/database/package.json +++ b/packages/database/package.json @@ -38,7 +38,7 @@ "db-migrate": "^0.11.5", "db-migrate-pg": "^1.2.2", "dotenv": "^16.4.5", - "knex": "0.14.6", + "knex": "^3.1.0", "lodash.clonedeep": "^4.5.0", "lodash.groupby": "^4.6.0", "lodash.keyby": "^4.6.0", diff --git a/packages/database/src/TupaiaDatabase.js b/packages/database/src/TupaiaDatabase.js index 9d6400f2b1..4f4767eb01 100644 --- a/packages/database/src/TupaiaDatabase.js +++ b/packages/database/src/TupaiaDatabase.js @@ -1,6 +1,6 @@ -/** +/* * Tupaia - * Copyright (c) 2017-2020 Beyond Essential Systems Pty Ltd + * Copyright (c) 2017-2024 Beyond Essential Systems Pty Ltd */ import autobind from 'react-autobind'; @@ -9,13 +9,12 @@ import knex from 'knex'; import winston from 'winston'; import { Multilock } from '@tupaia/utils'; import { hashStringToInt } from '@tupaia/tsutils'; - import { getConnectionConfig } from './getConnectionConfig'; import { DatabaseChangeChannel } from './DatabaseChangeChannel'; -import { generateId } from './utilities/generateId'; +import { generateId } from './utilities'; import { - runDatabaseFunctionInBatches, MAX_BINDINGS_PER_QUERY, + runDatabaseFunctionInBatches, } from './utilities/runDatabaseFunctionInBatches'; const QUERY_METHODS = { @@ -683,26 +682,13 @@ function addJoin(baseQuery, recordType, joinOptions) { } function getColSelector(connection, inputColStr) { - const regexp = new RegExp(/->>?/, 'g'); - if (regexp.test(inputColStr)) { - // Shorthand way of querying json property - // TODO: Replace with knex json where functions, eg. whereJsonPath - const [first, ...rest] = inputColStr.split(regexp); - if (rest.length === 1) { - // get the first separator (there will only be 1 in this case) - const separator = inputColStr.match(regexp)[0]; - // e.g. 'config->>colour' is converted to config->>'colour' and 'config->colour' is converted to config->'colour' - return connection.raw(`??${separator}?`, [first, ...rest]); - } - // e.g. 'config->item->>colour' is converted to config->'item'->>'colour' - const last = rest.slice(-1); - const middle = rest.slice(0, rest.length - 1); - return connection.raw(`??->${middle.map(i => '?').join('->')}->>?`, [ - first, - ...middle, - ...last, - ]); - } + const jsonOperatorPattern = /->>?/g; + if (!jsonOperatorPattern.test(inputColStr)) return inputColStr; + + const params = inputColStr.split(jsonOperatorPattern); + const allButFirst = params.slice(1); - return inputColStr; + // Turn `config->item->>colour` into `config #>> '{item,colour}'` + // For some reason, Knex fails when we try to convert it to `config->'item'->>'colour'` + return connection.raw(`?? #>> '{${allButFirst.map(() => '??').join(',')}}'`, params); } diff --git a/packages/database/src/__tests__/modelClasses/DashboardRelation.test.js b/packages/database/src/__tests__/modelClasses/DashboardRelation.test.js index 13800f80f0..03c411c7a2 100644 --- a/packages/database/src/__tests__/modelClasses/DashboardRelation.test.js +++ b/packages/database/src/__tests__/modelClasses/DashboardRelation.test.js @@ -4,9 +4,9 @@ */ import { - getTestModels, - findOrCreateDummyRecord, findOrCreateDummyCountryEntity, + findOrCreateDummyRecord, + getTestModels, } from '../../testUtilities'; const DASHBOARDS = [ diff --git a/packages/types/package.json b/packages/types/package.json index 52686a1954..83f4934d57 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -28,7 +28,7 @@ "devDependencies": { "@rmp135/sql-ts": "^1.15.1", "dotenv": "^16.4.5", - "knex": "^2.3.0", + "knex": "^3.1.0", "ts-node": "^10.9.1", "typescript-json-schema": "^0.55.0" } diff --git a/yarn.lock b/yarn.lock index 6eb4393473..58dab47362 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11932,7 +11932,7 @@ __metadata: db-migrate-pg: ^1.2.2 deep-equal-in-any-order: ^1.0.27 jest: ^29.7.0 - knex: 0.14.6 + knex: ^3.1.0 lodash.groupby: ^4.6.0 moment: ^2.24.0 pg: 8.5.1 @@ -11971,7 +11971,7 @@ __metadata: db-migrate: ^0.11.5 db-migrate-pg: ^1.2.2 dotenv: ^16.4.5 - knex: 0.14.6 + knex: ^3.1.0 lodash.clonedeep: ^4.5.0 lodash.groupby: ^4.6.0 lodash.keyby: ^4.6.0 @@ -12564,7 +12564,7 @@ __metadata: dependencies: "@rmp135/sql-ts": ^1.15.1 dotenv: ^16.4.5 - knex: ^2.3.0 + knex: ^3.1.0 ts-node: ^10.9.1 typescript-json-schema: ^0.55.0 languageName: unknown @@ -15638,13 +15638,6 @@ __metadata: languageName: node linkType: hard -"array-each@npm:^1.0.1": - version: 1.0.1 - resolution: "array-each@npm:1.0.1" - checksum: eb2393c1200003993d97dab2b280aa01e6ca339b383198e5d250cc8cd31f8012a0c22b66f275401a80e89e21bfab420e0f4c77c295637dea525fe0e152ba2300 - languageName: node - linkType: hard - "array-flatten@npm:1.1.1": version: 1.1.1 resolution: "array-flatten@npm:1.1.1" @@ -15676,13 +15669,6 @@ __metadata: languageName: node linkType: hard -"array-slice@npm:^1.0.0": - version: 1.1.0 - resolution: "array-slice@npm:1.1.0" - checksum: 3c8ecc7eefe104c97e2207e1d5644be160924c89e08b1807f3cad77f4a8fb10150fc275ebfab90dc02064d178b010cad31b69c9386769d172da270be5e233c51 - languageName: node - linkType: hard - "array-union@npm:^1.0.1, array-union@npm:^1.0.2": version: 1.0.2 resolution: "array-union@npm:1.0.2" @@ -17058,7 +17044,7 @@ __metadata: languageName: node linkType: hard -"bluebird@npm:^3.1.1, bluebird@npm:^3.3.5, bluebird@npm:^3.5.0, bluebird@npm:^3.5.1, bluebird@npm:^3.5.5, bluebird@npm:^3.7.2": +"bluebird@npm:^3.1.1, bluebird@npm:^3.3.5, bluebird@npm:^3.5.0, bluebird@npm:^3.5.5, bluebird@npm:^3.7.2": version: 3.7.2 resolution: "bluebird@npm:3.7.2" checksum: 869417503c722e7dc54ca46715f70e15f4d9c602a423a02c825570862d12935be59ed9c7ba34a9b31f186c017c23cac6b54e35446f8353059c101da73eac22ef @@ -18081,17 +18067,6 @@ __metadata: languageName: node linkType: hard -"chalk@npm:2.3.2": - version: 2.3.2 - resolution: "chalk@npm:2.3.2" - dependencies: - ansi-styles: ^3.2.1 - escape-string-regexp: ^1.0.5 - supports-color: ^5.3.0 - checksum: de2d4db6904539999b1bce771477fed3ff449022be089bb6d8cc9198e33396f7640b07e165b714718fd62ef0d99e4208939b90c91cc120e37f470e559fbc7ccf - languageName: node - linkType: hard - "chalk@npm:2.4.2, chalk@npm:^2.0.0, chalk@npm:^2.0.1, chalk@npm:^2.4.1, chalk@npm:^2.4.2": version: 2.4.2 resolution: "chalk@npm:2.4.2" @@ -18868,7 +18843,14 @@ __metadata: languageName: node linkType: hard -"commander@npm:^2.15.1, commander@npm:^2.19.0, commander@npm:^2.20.0": +"commander@npm:^10.0.0": + version: 10.0.1 + resolution: "commander@npm:10.0.1" + checksum: 436901d64a818295803c1996cd856621a74f30b9f9e28a588e726b2b1670665bccd7c1a77007ebf328729f0139838a88a19265858a0fa7a8728c4656796db948 + languageName: node + linkType: hard + +"commander@npm:^2.19.0, commander@npm:^2.20.0": version: 2.20.3 resolution: "commander@npm:2.20.3" checksum: ab8c07884e42c3a8dbc5dd9592c606176c7eb5c1ca5ff274bcf907039b2c41de3626f684ea75ccf4d361ba004bbaff1f577d5384c155f3871e456bdf27becf9e @@ -20784,13 +20766,6 @@ __metadata: languageName: node linkType: hard -"detect-file@npm:^1.0.0": - version: 1.0.0 - resolution: "detect-file@npm:1.0.0" - checksum: 1861e4146128622e847abe0e1ed80fef01e78532665858a792267adf89032b7a9c698436137707fcc6f02956c2a6a0052d6a0cef5be3d4b76b1ff0da88e2158a - languageName: node - linkType: hard - "detect-indent@npm:^4.0.0": version: 4.0.0 resolution: "detect-indent@npm:4.0.0" @@ -23168,15 +23143,6 @@ __metadata: languageName: node linkType: hard -"expand-tilde@npm:^2.0.0, expand-tilde@npm:^2.0.2": - version: 2.0.2 - resolution: "expand-tilde@npm:2.0.2" - dependencies: - homedir-polyfill: ^1.0.1 - checksum: 2efe6ed407d229981b1b6ceb552438fbc9e5c7d6a6751ad6ced3e0aa5cf12f0b299da695e90d6c2ac79191b5c53c613e508f7149e4573abfbb540698ddb7301a - languageName: node - linkType: hard - "expect@npm:^24.1.0": version: 24.9.0 resolution: "expect@npm:24.9.0" @@ -23942,38 +23908,6 @@ __metadata: languageName: node linkType: hard -"findup-sync@npm:^2.0.0": - version: 2.0.0 - resolution: "findup-sync@npm:2.0.0" - dependencies: - detect-file: ^1.0.0 - is-glob: ^3.1.0 - micromatch: ^3.0.4 - resolve-dir: ^1.0.1 - checksum: af2849f4006208c7c0940ab87a5f816187becf30c430a735377f6163cff8e95f405db504f5435728663099878f2e8002da1bf1976132458c23f5d73f540b1fcc - languageName: node - linkType: hard - -"fined@npm:^1.0.1": - version: 1.2.0 - resolution: "fined@npm:1.2.0" - dependencies: - expand-tilde: ^2.0.2 - is-plain-object: ^2.0.3 - object.defaults: ^1.1.0 - object.pick: ^1.2.0 - parse-filepath: ^1.0.1 - checksum: 9c76fb17e9f7e3f21e65b563cf49aed944c6b257a46b04306cef8883d60e295e904f57514443e60c64874914d13557b2f464071181d8d80a37cd9d8565075b7f - languageName: node - linkType: hard - -"flagged-respawn@npm:^1.0.0": - version: 1.0.1 - resolution: "flagged-respawn@npm:1.0.1" - checksum: 73596ca037dba21455937a27e7efe6aa12074ff653a930abec238db80d65b7129aaae58cc686e1ac5ede718c18c14207ee0f265c542425afc396f2b8ca675f78 - languageName: node - linkType: hard - "flat-cache@npm:^3.0.4": version: 3.0.4 resolution: "flat-cache@npm:3.0.4" @@ -24087,22 +24021,13 @@ __metadata: languageName: node linkType: hard -"for-in@npm:^1.0.1, for-in@npm:^1.0.2": +"for-in@npm:^1.0.2": version: 1.0.2 resolution: "for-in@npm:1.0.2" checksum: 09f4ae93ce785d253ac963d94c7f3432d89398bf25ac7a24ed034ca393bf74380bdeccc40e0f2d721a895e54211b07c8fad7132e8157827f6f7f059b70b4043d languageName: node linkType: hard -"for-own@npm:^1.0.0": - version: 1.0.0 - resolution: "for-own@npm:1.0.0" - dependencies: - for-in: ^1.0.1 - checksum: 233238f6e9060f61295a7f7c7e3e9de11aaef57e82a108e7f350dc92ae84fe2189848077ac4b8db47fd8edd45337ed8d9f66bd0b1efa4a6a1b3f38aa21b7ab2e - languageName: node - linkType: hard - "foreground-child@npm:^2.0.0": version: 2.0.0 resolution: "foreground-child@npm:2.0.0" @@ -25038,30 +24963,6 @@ __metadata: languageName: node linkType: hard -"global-modules@npm:^1.0.0": - version: 1.0.0 - resolution: "global-modules@npm:1.0.0" - dependencies: - global-prefix: ^1.0.1 - is-windows: ^1.0.1 - resolve-dir: ^1.0.0 - checksum: 10be68796c1e1abc1e2ba87ec4ea507f5629873b119ab0cd29c07284ef2b930f1402d10df01beccb7391dedd9cd479611dd6a24311c71be58937beaf18edf85e - languageName: node - linkType: hard - -"global-prefix@npm:^1.0.1": - version: 1.0.2 - resolution: "global-prefix@npm:1.0.2" - dependencies: - expand-tilde: ^2.0.2 - homedir-polyfill: ^1.0.1 - ini: ^1.3.4 - is-windows: ^1.0.1 - which: ^1.2.14 - checksum: 061b43470fe498271bcd514e7746e8a8535032b17ab9570517014ae27d700ff0dca749f76bbde13ba384d185be4310d8ba5712cb0e74f7d54d59390db63dd9a0 - languageName: node - linkType: hard - "global-prefix@npm:^3.0.0": version: 3.0.0 resolution: "global-prefix@npm:3.0.0" @@ -26524,13 +26425,6 @@ __metadata: languageName: node linkType: hard -"interpret@npm:^1.1.0": - version: 1.4.0 - resolution: "interpret@npm:1.4.0" - checksum: 2e5f51268b5941e4a17e4ef0575bc91ed0ab5f8515e3cf77486f7c14d13f3010df9c0959f37063dcc96e78d12dc6b0bb1b9e111cdfe69771f4656d2993d36155 - languageName: node - linkType: hard - "interpret@npm:^2.2.0": version: 2.2.0 resolution: "interpret@npm:2.2.0" @@ -26613,16 +26507,6 @@ __metadata: languageName: node linkType: hard -"is-absolute@npm:^1.0.0": - version: 1.0.0 - resolution: "is-absolute@npm:1.0.0" - dependencies: - is-relative: ^1.0.0 - is-windows: ^1.0.1 - checksum: 9d16b2605eda3f3ce755410f1d423e327ad3a898bcb86c9354cf63970ed3f91ba85e9828aa56f5d6a952b9fae43d0477770f78d37409ae8ecc31e59ebc279b27 - languageName: node - linkType: hard - "is-accessor-descriptor@npm:^0.1.6": version: 0.1.6 resolution: "is-accessor-descriptor@npm:0.1.6" @@ -26826,15 +26710,6 @@ __metadata: languageName: node linkType: hard -"is-core-module@npm:^2.9.0": - version: 2.11.0 - resolution: "is-core-module@npm:2.11.0" - dependencies: - has: ^1.0.3 - checksum: f96fd490c6b48eb4f6d10ba815c6ef13f410b0ba6f7eb8577af51697de523e5f2cd9de1c441b51d27251bf0e4aebc936545e33a5d26d5d51f28d25698d4a8bab - languageName: node - linkType: hard - "is-data-descriptor@npm:^0.1.4": version: 0.1.4 resolution: "is-data-descriptor@npm:0.1.4" @@ -27361,15 +27236,6 @@ __metadata: languageName: node linkType: hard -"is-relative@npm:^1.0.0": - version: 1.0.0 - resolution: "is-relative@npm:1.0.0" - dependencies: - is-unc-path: ^1.0.0 - checksum: 3271a0df109302ef5e14a29dcd5d23d9788e15ade91a40b942b035827ffbb59f7ce9ff82d036ea798541a52913cbf9d2d0b66456340887b51f3542d57b5a4c05 - languageName: node - linkType: hard - "is-retry-allowed@npm:^1.1.0": version: 1.2.0 resolution: "is-retry-allowed@npm:1.2.0" @@ -27462,15 +27328,6 @@ __metadata: languageName: node linkType: hard -"is-unc-path@npm:^1.0.0": - version: 1.0.0 - resolution: "is-unc-path@npm:1.0.0" - dependencies: - unc-path-regex: ^0.1.2 - checksum: e8abfde203f7409f5b03a5f1f8636e3a41e78b983702ef49d9343eb608cdfe691429398e8815157519b987b739bcfbc73ae7cf4c8582b0ab66add5171088eab6 - languageName: node - linkType: hard - "is-unicode-supported@npm:^0.1.0": version: 0.1.0 resolution: "is-unicode-supported@npm:0.1.0" @@ -27525,7 +27382,7 @@ __metadata: languageName: node linkType: hard -"is-windows@npm:^1.0.1, is-windows@npm:^1.0.2": +"is-windows@npm:^1.0.2": version: 1.0.2 resolution: "is-windows@npm:1.0.2" checksum: 438b7e52656fe3b9b293b180defb4e448088e7023a523ec21a91a80b9ff8cdb3377ddb5b6e60f7c7de4fa8b63ab56e121b6705fe081b3cf1b828b0a380009ad7 @@ -29381,34 +29238,6 @@ __metadata: languageName: node linkType: hard -"knex@npm:0.14.6": - version: 0.14.6 - resolution: "knex@npm:0.14.6" - dependencies: - babel-runtime: ^6.26.0 - bluebird: ^3.5.1 - chalk: 2.3.2 - commander: ^2.15.1 - debug: 3.1.0 - inherits: ~2.0.3 - interpret: ^1.1.0 - liftoff: 2.5.0 - lodash: ^4.17.5 - minimist: 1.2.0 - mkdirp: ^0.5.1 - pg-connection-string: 2.0.0 - readable-stream: 2.3.6 - safe-buffer: ^5.1.1 - tarn: ^1.1.4 - tildify: 1.2.0 - uuid: ^3.2.1 - v8flags: ^3.0.2 - bin: - knex: ./bin/cli.js - checksum: b9ad07288c2ec372c40fb497f471e172705b22eacb29e83b1f367c7b38b8a9d945f24a5802f5163f716562ca76c407ecc2d77e90126778364e83e969b1ab4b30 - languageName: node - linkType: hard - "knex@npm:^1.0.3": version: 1.0.7 resolution: "knex@npm:1.0.7" @@ -29448,12 +29277,12 @@ __metadata: languageName: node linkType: hard -"knex@npm:^2.3.0": - version: 2.3.0 - resolution: "knex@npm:2.3.0" +"knex@npm:^3.1.0": + version: 3.1.0 + resolution: "knex@npm:3.1.0" dependencies: colorette: 2.0.19 - commander: ^9.1.0 + commander: ^10.0.0 debug: 4.3.4 escalade: ^3.1.1 esm: ^3.2.25 @@ -29461,7 +29290,7 @@ __metadata: getopts: 2.3.0 interpret: ^2.2.0 lodash: ^4.17.21 - pg-connection-string: 2.5.0 + pg-connection-string: 2.6.2 rechoir: ^0.8.0 resolve-from: ^5.0.0 tarn: ^3.0.2 @@ -29483,7 +29312,7 @@ __metadata: optional: true bin: knex: bin/cli.js - checksum: ec00da6dd622a386507881e9d8ad44b9c5a4d9f272889b181cafb856fd26d7fdb19ef009cecdbbdedaf7c76f985a8fd14348ae47a0bb77b3dbec15f2c03c0dfe + checksum: 3905f8d27960975f7f57f3f488d1ef3ccf47784acc8eb627e8a28cbbe1f296c6879c8ef0cbd9e17e867be80117d305cd948545f3fbd4c74b24c90d2413bbc021 languageName: node linkType: hard @@ -29609,22 +29438,6 @@ __metadata: languageName: node linkType: hard -"liftoff@npm:2.5.0": - version: 2.5.0 - resolution: "liftoff@npm:2.5.0" - dependencies: - extend: ^3.0.0 - findup-sync: ^2.0.0 - fined: ^1.0.1 - flagged-respawn: ^1.0.0 - is-plain-object: ^2.0.4 - object.map: ^1.0.0 - rechoir: ^0.6.2 - resolve: ^1.1.7 - checksum: d9c1bde7dff136541bfef97e4c2b7e326092f9d18800dbcde432522628f0972c7142d1863d5e0cda517d4d2ca45c2289bb771ebcdb45efb9de042ddb3a8898c2 - languageName: node - linkType: hard - "lines-and-columns@npm:^1.1.6": version: 1.1.6 resolution: "lines-and-columns@npm:1.1.6" @@ -30676,15 +30489,6 @@ __metadata: languageName: node linkType: hard -"make-iterator@npm:^1.0.0": - version: 1.0.1 - resolution: "make-iterator@npm:1.0.1" - dependencies: - kind-of: ^6.0.2 - checksum: d38afc388f4374b15c0622d4fa4d3e8c3154e3a6ba35b01e9a5179c127d7dd09a91fa571056aa9e041981b39f80bdbab035c05475e56ef675a18bdf550f0cb6a - languageName: node - linkType: hard - "make-plural@npm:^4.3.0": version: 4.3.0 resolution: "make-plural@npm:4.3.0" @@ -30724,7 +30528,7 @@ __metadata: languageName: node linkType: hard -"map-cache@npm:^0.2.0, map-cache@npm:^0.2.2": +"map-cache@npm:^0.2.2": version: 0.2.2 resolution: "map-cache@npm:0.2.2" checksum: 3067cea54285c43848bb4539f978a15dedc63c03022abeec6ef05c8cb6829f920f13b94bcaf04142fc6a088318e564c4785704072910d120d55dbc2e0c421969 @@ -31416,7 +31220,7 @@ __metadata: languageName: node linkType: hard -"micromatch@npm:^3.0.4, micromatch@npm:^3.1.10, micromatch@npm:^3.1.4": +"micromatch@npm:^3.1.10, micromatch@npm:^3.1.4": version: 3.1.10 resolution: "micromatch@npm:3.1.10" dependencies: @@ -31694,7 +31498,7 @@ __metadata: languageName: node linkType: hard -"minimist@npm:1.2.0, minimist@npm:^1.1.0, minimist@npm:^1.1.1, minimist@npm:^1.2.0": +"minimist@npm:^1.1.0, minimist@npm:^1.1.1, minimist@npm:^1.2.0": version: 1.2.0 resolution: "minimist@npm:1.2.0" checksum: 72473f0fce6692cf1e134dfdccfcfddd64d354d465dac3e43053e0c6d398eb9684c9d964f666e3c1be93829de47cb1ddf3cd26d4071322ed25fbaa625441dd85 @@ -33198,18 +33002,6 @@ __metadata: languageName: node linkType: hard -"object.defaults@npm:^1.1.0": - version: 1.1.0 - resolution: "object.defaults@npm:1.1.0" - dependencies: - array-each: ^1.0.1 - array-slice: ^1.0.0 - for-own: ^1.0.0 - isobject: ^3.0.0 - checksum: 25468e06132af866bffedf9889b8180a31b9915776dbb660106866c5dd70cd0c0ad54f17e34de8ab99e6f548d579678de2e558390f56bd4ee61899fa6057f946 - languageName: node - linkType: hard - "object.entries@npm:^1.1.0": version: 1.1.1 resolution: "object.entries@npm:1.1.1" @@ -33287,17 +33079,7 @@ __metadata: languageName: node linkType: hard -"object.map@npm:^1.0.0": - version: 1.0.1 - resolution: "object.map@npm:1.0.1" - dependencies: - for-own: ^1.0.0 - make-iterator: ^1.0.0 - checksum: 3c9cf1a417f78915c7cf34054188193d4506b3d28f60ffd57aaf035fb34f19688fdf91a1af0ff9b81092270de7d3538ebe6783ae742663ea28a2b19d5eb6c6d9 - languageName: node - linkType: hard - -"object.pick@npm:^1.2.0, object.pick@npm:^1.3.0": +"object.pick@npm:^1.3.0": version: 1.3.0 resolution: "object.pick@npm:1.3.0" dependencies: @@ -33937,17 +33719,6 @@ __metadata: languageName: node linkType: hard -"parse-filepath@npm:^1.0.1": - version: 1.0.2 - resolution: "parse-filepath@npm:1.0.2" - dependencies: - is-absolute: ^1.0.0 - map-cache: ^0.2.0 - path-root: ^0.1.1 - checksum: 6794c3f38d3921f0f7cc63fb1fb0c4d04cd463356ad389c8ce6726d3c50793b9005971f4138975a6d7025526058d5e65e9bfe634d0765e84c4e2571152665a69 - languageName: node - linkType: hard - "parse-json@npm:^2.2.0": version: 2.2.0 resolution: "parse-json@npm:2.2.0" @@ -34153,22 +33924,6 @@ __metadata: languageName: node linkType: hard -"path-root-regex@npm:^0.1.0": - version: 0.1.2 - resolution: "path-root-regex@npm:0.1.2" - checksum: dcd75d1f8e93faabe35a58e875b0f636839b3658ff2ad8c289463c40bc1a844debe0dab73c3398ef9dc8f6ec6c319720aff390cf4633763ddcf3cf4b1bbf7e8b - languageName: node - linkType: hard - -"path-root@npm:^0.1.1": - version: 0.1.1 - resolution: "path-root@npm:0.1.1" - dependencies: - path-root-regex: ^0.1.0 - checksum: ff88aebfc1c59ace510cc06703d67692a11530989920427625e52b66a303ca9b3d4059b0b7d0b2a73248d1ad29bcb342b8b786ec00592f3101d38a45fd3b2e08 - languageName: node - linkType: hard - "path-scurry@npm:^1.10.1": version: 1.10.1 resolution: "path-scurry@npm:1.10.1" @@ -34293,13 +34048,6 @@ __metadata: languageName: node linkType: hard -"pg-connection-string@npm:2.0.0": - version: 2.0.0 - resolution: "pg-connection-string@npm:2.0.0" - checksum: 442086b413e0e8304fbb7374bc0a50afbe161ec52431552fe80161ba354a8f81bfcf9e4a3b4b6acfaae4ec1cc059be5abacd388528650160d51811308b356eca - languageName: node - linkType: hard - "pg-connection-string@npm:2.5.0": version: 2.5.0 resolution: "pg-connection-string@npm:2.5.0" @@ -34307,6 +34055,13 @@ __metadata: languageName: node linkType: hard +"pg-connection-string@npm:2.6.2, pg-connection-string@npm:^2.6.2": + version: 2.6.2 + resolution: "pg-connection-string@npm:2.6.2" + checksum: 22265882c3b6f2320785378d0760b051294a684989163d5a1cde4009e64e84448d7bf67d9a7b9e7f69440c3ee9e2212f9aa10dd17ad6773f6143c6020cebbcb5 + languageName: node + linkType: hard + "pg-connection-string@npm:^2.3.0": version: 2.3.0 resolution: "pg-connection-string@npm:2.3.0" @@ -34321,13 +34076,6 @@ __metadata: languageName: node linkType: hard -"pg-connection-string@npm:^2.6.2": - version: 2.6.2 - resolution: "pg-connection-string@npm:2.6.2" - checksum: 22265882c3b6f2320785378d0760b051294a684989163d5a1cde4009e64e84448d7bf67d9a7b9e7f69440c3ee9e2212f9aa10dd17ad6773f6143c6020cebbcb5 - languageName: node - linkType: hard - "pg-format@npm:^1.0.2": version: 1.0.4 resolution: "pg-format@npm:1.0.4" @@ -37309,7 +37057,7 @@ __metadata: languageName: node linkType: hard -"readable-stream@npm:1 || 2, readable-stream@npm:2.3.6, readable-stream@npm:^2.0.0, readable-stream@npm:^2.0.1, readable-stream@npm:^2.0.2, readable-stream@npm:^2.0.6, readable-stream@npm:^2.1.5, readable-stream@npm:^2.2.2, readable-stream@npm:^2.3.3, readable-stream@npm:^2.3.5, readable-stream@npm:^2.3.6, readable-stream@npm:~2.3.6": +"readable-stream@npm:1 || 2, readable-stream@npm:^2.0.0, readable-stream@npm:^2.0.1, readable-stream@npm:^2.0.2, readable-stream@npm:^2.0.6, readable-stream@npm:^2.1.5, readable-stream@npm:^2.2.2, readable-stream@npm:^2.3.3, readable-stream@npm:^2.3.5, readable-stream@npm:^2.3.6, readable-stream@npm:~2.3.6": version: 2.3.6 resolution: "readable-stream@npm:2.3.6" dependencies: @@ -37489,15 +37237,6 @@ __metadata: languageName: node linkType: hard -"rechoir@npm:^0.6.2": - version: 0.6.2 - resolution: "rechoir@npm:0.6.2" - dependencies: - resolve: ^1.1.6 - checksum: fe76bf9c21875ac16e235defedd7cbd34f333c02a92546142b7911a0f7c7059d2e16f441fe6fb9ae203f459c05a31b2bcf26202896d89e390eda7514d5d2702b - languageName: node - linkType: hard - "rechoir@npm:^0.8.0": version: 0.8.0 resolution: "rechoir@npm:0.8.0" @@ -38129,16 +37868,6 @@ __metadata: languageName: node linkType: hard -"resolve-dir@npm:^1.0.0, resolve-dir@npm:^1.0.1": - version: 1.0.1 - resolution: "resolve-dir@npm:1.0.1" - dependencies: - expand-tilde: ^2.0.0 - global-modules: ^1.0.0 - checksum: ef736b8ed60d6645c3b573da17d329bfb50ec4e1d6c5ffd6df49e3497acef9226f9810ea6823b8ece1560e01dcb13f77a9f6180d4f242d00cc9a8f4de909c65c - languageName: node - linkType: hard - "resolve-from@npm:^3.0.0": version: 3.0.0 resolution: "resolve-from@npm:3.0.0" @@ -38197,19 +37926,6 @@ __metadata: languageName: node linkType: hard -"resolve@npm:^1.1.7": - version: 1.22.1 - resolution: "resolve@npm:1.22.1" - dependencies: - is-core-module: ^2.9.0 - path-parse: ^1.0.7 - supports-preserve-symlinks-flag: ^1.0.0 - bin: - resolve: bin/resolve - checksum: 07af5fc1e81aa1d866cbc9e9460fbb67318a10fa3c4deadc35c3ad8a898ee9a71a86a65e4755ac3195e0ea0cfbe201eb323ebe655ce90526fd61917313a34e4e - languageName: node - linkType: hard - "resolve@npm:^1.14.2, resolve@npm:^1.19.0, resolve@npm:^1.20.0": version: 1.20.0 resolution: "resolve@npm:1.20.0" @@ -38274,19 +37990,6 @@ __metadata: languageName: node linkType: hard -"resolve@patch:resolve@^1.1.7#~builtin": - version: 1.22.1 - resolution: "resolve@patch:resolve@npm%3A1.22.1#~builtin::version=1.22.1&hash=07638b" - dependencies: - is-core-module: ^2.9.0 - path-parse: ^1.0.7 - supports-preserve-symlinks-flag: ^1.0.0 - bin: - resolve: bin/resolve - checksum: 5656f4d0bedcf8eb52685c1abdf8fbe73a1603bb1160a24d716e27a57f6cecbe2432ff9c89c2bd57542c3a7b9d14b1882b73bfe2e9d7849c9a4c0b8b39f02b8b - languageName: node - linkType: hard - "resolve@patch:resolve@^1.14.2#~builtin, resolve@patch:resolve@^1.19.0#~builtin, resolve@patch:resolve@^1.20.0#~builtin": version: 1.20.0 resolution: "resolve@patch:resolve@npm%3A1.20.0#~builtin::version=1.20.0&hash=07638b" @@ -40787,13 +40490,6 @@ __metadata: languageName: node linkType: hard -"tarn@npm:^1.1.4": - version: 1.1.5 - resolution: "tarn@npm:1.1.5" - checksum: 1a0734b5def0459ab86c162fbb137efa1ca916aa06d37e8b6b8bee9cc82c84aa956686677065478cf250bad9309b528ac24810e781416230e704bbdefae445ec - languageName: node - linkType: hard - "tarn@npm:^3.0.2": version: 3.0.2 resolution: "tarn@npm:3.0.2" @@ -41030,15 +40726,6 @@ __metadata: languageName: node linkType: hard -"tildify@npm:1.2.0": - version: 1.2.0 - resolution: "tildify@npm:1.2.0" - dependencies: - os-homedir: ^1.0.0 - checksum: 20eb23ae40e0cfbe599e74257383d35f2b2e131edb74857acb7b4ed5aed0b6622b2cc94dddea87ac08c8067255558e4c7aaf5a5b0f3bee783bf2d33fb9912fbc - languageName: node - linkType: hard - "tildify@npm:2.0.0": version: 2.0.0 resolution: "tildify@npm:2.0.0" @@ -42007,13 +41694,6 @@ __metadata: languageName: node linkType: hard -"unc-path-regex@npm:^0.1.2": - version: 0.1.2 - resolution: "unc-path-regex@npm:0.1.2" - checksum: a05fa2006bf4606051c10fc7968f08ce7b28fa646befafa282813aeb1ac1a56f65cb1b577ca7851af2726198d59475bb49b11776036257b843eaacee2860a4ec - languageName: node - linkType: hard - "undefsafe@npm:^2.0.2": version: 2.0.2 resolution: "undefsafe@npm:2.0.2" @@ -42764,7 +42444,7 @@ __metadata: languageName: node linkType: hard -"v8flags@npm:^3.0.2, v8flags@npm:^3.1.1": +"v8flags@npm:^3.1.1": version: 3.2.0 resolution: "v8flags@npm:3.2.0" dependencies: @@ -43368,7 +43048,7 @@ __metadata: languageName: node linkType: hard -"which@npm:^1.2.14, which@npm:^1.2.9, which@npm:^1.3.1": +"which@npm:^1.2.9, which@npm:^1.3.1": version: 1.3.1 resolution: "which@npm:1.3.1" dependencies: From 56032fec80347f7f63d6b4bcd1344814af9c5a4a Mon Sep 17 00:00:00 2001 From: alexd-bes <129009580+alexd-bes@users.noreply.github.com> Date: Fri, 5 Apr 2024 09:40:19 +1300 Subject: [PATCH 15/25] tweak(environment): RN-1225: Override with local env if exists (#5564) --- packages/database/src/configureEnv.js | 1 - packages/server-utils/src/configureDotEnv.ts | 10 +++++++++- packages/types/generate-models.ts | 3 ++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/database/src/configureEnv.js b/packages/database/src/configureEnv.js index d9d22fcd40..1dc2d2efae 100644 --- a/packages/database/src/configureEnv.js +++ b/packages/database/src/configureEnv.js @@ -11,7 +11,6 @@ export const configureEnv = () => { path: [ path.resolve(__dirname, '../../../env/db.env'), path.resolve(__dirname, '../../../env/pg.env'), - path.resolve(__dirname, '../.env'), ], }); }; diff --git a/packages/server-utils/src/configureDotEnv.ts b/packages/server-utils/src/configureDotEnv.ts index eb634e5443..d7692d333f 100644 --- a/packages/server-utils/src/configureDotEnv.ts +++ b/packages/server-utils/src/configureDotEnv.ts @@ -6,5 +6,13 @@ import dotenv from 'dotenv'; export const configureDotEnv = (envFiles: string[]) => { - dotenv.config({ path: envFiles }); + const filesThatExistInSystem = envFiles.filter(file => { + try { + require.resolve(file); + return true; + } catch (error) { + return false; + } + }); + dotenv.config({ path: filesThatExistInSystem, override: true }); }; diff --git a/packages/types/generate-models.ts b/packages/types/generate-models.ts index be339f8cfc..419811614d 100644 --- a/packages/types/generate-models.ts +++ b/packages/types/generate-models.ts @@ -15,7 +15,8 @@ import config from './config/models/config.json'; import * as dotenv from 'dotenv'; import * as fs from 'fs'; dotenv.config({ - path: [path.resolve(__dirname, '../../../env/db.env'), path.resolve(__dirname, '.env')], + path: [path.resolve(__dirname, '../../env/db.env'), path.resolve(__dirname, '.env')], + override: true, }); const db = Knex({ From 998c8207776a05d306658be6bcb16b695bf29bbc Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Fri, 5 Apr 2024 10:51:51 +1300 Subject: [PATCH 16/25] fix(vizBuilder): vizBuilder crashes Admin Panel (#5565) remove admin-panel dependency on tsutils --- packages/admin-panel/package.json | 1 - .../components/PreviewOptions/DateRangeField.jsx | 5 +++-- yarn.lock | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/admin-panel/package.json b/packages/admin-panel/package.json index c70595fc1c..74fb739795 100644 --- a/packages/admin-panel/package.json +++ b/packages/admin-panel/package.json @@ -32,7 +32,6 @@ "@material-ui/lab": "^4.0.0-alpha.47", "@material-ui/styles": "^4.9.10", "@tupaia/access-policy": "workspace:*", - "@tupaia/tsutils": "workspace:*", "@tupaia/types": "workspace:*", "@tupaia/ui-chart-components": "workspace:*", "@tupaia/ui-components": "workspace:*", diff --git a/packages/admin-panel/src/VizBuilderApp/components/PreviewOptions/DateRangeField.jsx b/packages/admin-panel/src/VizBuilderApp/components/PreviewOptions/DateRangeField.jsx index 37c4b8026a..517a7e32ed 100644 --- a/packages/admin-panel/src/VizBuilderApp/components/PreviewOptions/DateRangeField.jsx +++ b/packages/admin-panel/src/VizBuilderApp/components/PreviewOptions/DateRangeField.jsx @@ -5,7 +5,6 @@ import React, { useEffect } from 'react'; import styled from 'styled-components'; import { DatePicker as DatePickerComponent } from '@tupaia/ui-components'; -import { getIsoDateString } from '@tupaia/tsutils'; import { useVizConfigContext } from '../../context'; const DatePicker = styled(DatePickerComponent)` @@ -62,7 +61,9 @@ export const DateRangeField = () => { const convertDateToIsoString = date => { if (!date || isNaN(new Date(date).getTime())) return null; const correctedDate = shiftEpoch(date); - return getIsoDateString(correctedDate); + + // Slice to discard timestamp, keeping only "yyyy-mm-dd" + return correctedDate.toISOString().slice(0, 10); }; const handleChangeStartDate = date => { diff --git a/yarn.lock b/yarn.lock index 58dab47362..0c6e39c121 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11714,7 +11714,6 @@ __metadata: "@material-ui/lab": ^4.0.0-alpha.47 "@material-ui/styles": ^4.9.10 "@tupaia/access-policy": "workspace:*" - "@tupaia/tsutils": "workspace:*" "@tupaia/types": "workspace:*" "@tupaia/ui-chart-components": "workspace:*" "@tupaia/ui-components": "workspace:*" From 0c15d3d9cdb066a4df4266f32f96b61e5e140f08 Mon Sep 17 00:00:00 2001 From: Rohan Port <59544282+rohan-bes@users.noreply.github.com> Date: Fri, 5 Apr 2024 10:21:45 +1100 Subject: [PATCH 17/25] db(entityTypes): MDEV-230: Add larval_sample entity type (#5559) Co-authored-by: Rohan Port --- ...dLarvalSampleEntityType-modifies-schema.js | 27 ++++ packages/types/src/schemas/schemas.ts | 117 ++++++++++++++++++ packages/types/src/types/models.ts | 1 + 3 files changed, 145 insertions(+) create mode 100644 packages/database/src/migrations/20240404003956-AddLarvalSampleEntityType-modifies-schema.js diff --git a/packages/database/src/migrations/20240404003956-AddLarvalSampleEntityType-modifies-schema.js b/packages/database/src/migrations/20240404003956-AddLarvalSampleEntityType-modifies-schema.js new file mode 100644 index 0000000000..fe890c696f --- /dev/null +++ b/packages/database/src/migrations/20240404003956-AddLarvalSampleEntityType-modifies-schema.js @@ -0,0 +1,27 @@ +'use strict'; + +var dbm; +var type; +var seed; + +/** + * We receive the dbmigrate dependency from dbmigrate initially. + * This enables us to not have to rely on NODE_PATH. + */ +exports.setup = function (options, seedLink) { + dbm = options.dbmigrate; + type = dbm.dataType; + seed = seedLink; +}; + +exports.up = async function (db) { + return db.runSql(`ALTER TYPE public.entity_type ADD VALUE IF NOT EXISTS 'larval_sample';`); +}; + +exports.down = function (db) { + return null; +}; + +exports._meta = { + version: 1, +}; diff --git a/packages/types/src/schemas/schemas.ts b/packages/types/src/schemas/schemas.ts index fb2bbd8078..5515de1149 100644 --- a/packages/types/src/schemas/schemas.ts +++ b/packages/types/src/schemas/schemas.ts @@ -30055,6 +30055,7 @@ export const MeasureConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -30243,6 +30244,7 @@ export const EntityLevelSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -30582,6 +30584,7 @@ export const BaseMapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -30722,6 +30725,7 @@ export const BaseMapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -30852,6 +30856,7 @@ export const BaseMapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -31273,6 +31278,7 @@ export const SpectrumMapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -31413,6 +31419,7 @@ export const SpectrumMapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -31543,6 +31550,7 @@ export const SpectrumMapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -32072,6 +32080,7 @@ export const IconMapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -32212,6 +32221,7 @@ export const IconMapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -32342,6 +32352,7 @@ export const IconMapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -32793,6 +32804,7 @@ export const RadiusMapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -32933,6 +32945,7 @@ export const RadiusMapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -33063,6 +33076,7 @@ export const RadiusMapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -33488,6 +33502,7 @@ export const ColorMapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -33628,6 +33643,7 @@ export const ColorMapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -33758,6 +33774,7 @@ export const ColorMapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -34198,6 +34215,7 @@ export const ShadingMapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -34338,6 +34356,7 @@ export const ShadingMapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -34468,6 +34487,7 @@ export const ShadingMapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -34899,6 +34919,7 @@ export const MapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -35039,6 +35060,7 @@ export const MapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -35169,6 +35191,7 @@ export const MapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -35697,6 +35720,7 @@ export const MapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -35837,6 +35861,7 @@ export const MapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -35967,6 +35992,7 @@ export const MapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -36417,6 +36443,7 @@ export const MapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -36557,6 +36584,7 @@ export const MapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -36687,6 +36715,7 @@ export const MapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -37111,6 +37140,7 @@ export const MapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -37251,6 +37281,7 @@ export const MapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -37381,6 +37412,7 @@ export const MapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -37820,6 +37852,7 @@ export const MapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -37960,6 +37993,7 @@ export const MapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -38090,6 +38124,7 @@ export const MapOverlayConfigSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -38721,6 +38756,7 @@ export const EntityQuestionConfigSchema = { "individual", "institute", "larval_habitat", + "larval_sample", "local_government", "maintenance", "medical_area", @@ -38767,6 +38803,7 @@ export const EntityQuestionConfigSchema = { "individual", "institute", "larval_habitat", + "larval_sample", "local_government", "maintenance", "medical_area", @@ -39235,6 +39272,7 @@ export const SurveyScreenComponentConfigSchema = { "individual", "institute", "larval_habitat", + "larval_sample", "local_government", "maintenance", "medical_area", @@ -39281,6 +39319,7 @@ export const SurveyScreenComponentConfigSchema = { "individual", "institute", "larval_habitat", + "larval_sample", "local_government", "maintenance", "medical_area", @@ -39763,6 +39802,12 @@ export const RecentEntitiesForCountrySchema = { "items": { "type": "string" } + }, + "larval_sample": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": false @@ -66181,6 +66226,7 @@ export const DashboardRelationSchema = { "individual", "institute", "larval_habitat", + "larval_sample", "local_government", "maintenance", "medical_area", @@ -66273,6 +66319,7 @@ export const DashboardRelationCreateSchema = { "individual", "institute", "larval_habitat", + "larval_sample", "local_government", "maintenance", "medical_area", @@ -66360,6 +66407,7 @@ export const DashboardRelationUpdateSchema = { "individual", "institute", "larval_habitat", + "larval_sample", "local_government", "maintenance", "medical_area", @@ -67556,6 +67604,7 @@ export const EntitySchema = { "individual", "institute", "larval_habitat", + "larval_sample", "local_government", "maintenance", "medical_area", @@ -67653,6 +67702,7 @@ export const EntityCreateSchema = { "individual", "institute", "larval_habitat", + "larval_sample", "local_government", "maintenance", "medical_area", @@ -67750,6 +67800,7 @@ export const EntityUpdateSchema = { "individual", "institute", "larval_habitat", + "larval_sample", "local_government", "maintenance", "medical_area", @@ -68981,6 +69032,7 @@ export const MapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -69121,6 +69173,7 @@ export const MapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -69251,6 +69304,7 @@ export const MapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -69779,6 +69833,7 @@ export const MapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -69919,6 +69974,7 @@ export const MapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -70049,6 +70105,7 @@ export const MapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -70499,6 +70556,7 @@ export const MapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -70639,6 +70697,7 @@ export const MapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -70769,6 +70828,7 @@ export const MapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -71193,6 +71253,7 @@ export const MapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -71333,6 +71394,7 @@ export const MapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -71463,6 +71525,7 @@ export const MapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -71902,6 +71965,7 @@ export const MapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -72042,6 +72106,7 @@ export const MapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -72172,6 +72237,7 @@ export const MapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -72664,6 +72730,7 @@ export const MapOverlayCreateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -72804,6 +72871,7 @@ export const MapOverlayCreateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -72934,6 +73002,7 @@ export const MapOverlayCreateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -73462,6 +73531,7 @@ export const MapOverlayCreateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -73602,6 +73672,7 @@ export const MapOverlayCreateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -73732,6 +73803,7 @@ export const MapOverlayCreateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -74182,6 +74254,7 @@ export const MapOverlayCreateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -74322,6 +74395,7 @@ export const MapOverlayCreateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -74452,6 +74526,7 @@ export const MapOverlayCreateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -74876,6 +74951,7 @@ export const MapOverlayCreateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -75016,6 +75092,7 @@ export const MapOverlayCreateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -75146,6 +75223,7 @@ export const MapOverlayCreateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -75585,6 +75663,7 @@ export const MapOverlayCreateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -75725,6 +75804,7 @@ export const MapOverlayCreateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -75855,6 +75935,7 @@ export const MapOverlayCreateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -76340,6 +76421,7 @@ export const MapOverlayUpdateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -76480,6 +76562,7 @@ export const MapOverlayUpdateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -76610,6 +76693,7 @@ export const MapOverlayUpdateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -77138,6 +77222,7 @@ export const MapOverlayUpdateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -77278,6 +77363,7 @@ export const MapOverlayUpdateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -77408,6 +77494,7 @@ export const MapOverlayUpdateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -77858,6 +77945,7 @@ export const MapOverlayUpdateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -77998,6 +78086,7 @@ export const MapOverlayUpdateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -78128,6 +78217,7 @@ export const MapOverlayUpdateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -78552,6 +78642,7 @@ export const MapOverlayUpdateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -78692,6 +78783,7 @@ export const MapOverlayUpdateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -78822,6 +78914,7 @@ export const MapOverlayUpdateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -79261,6 +79354,7 @@ export const MapOverlayUpdateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -79401,6 +79495,7 @@ export const MapOverlayUpdateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -79531,6 +79626,7 @@ export const MapOverlayUpdateSchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -80503,6 +80599,7 @@ export const PermissionsBasedMeditrakSyncQueueSchema = { "individual", "institute", "larval_habitat", + "larval_sample", "local_government", "maintenance", "medical_area", @@ -80582,6 +80679,7 @@ export const PermissionsBasedMeditrakSyncQueueCreateSchema = { "individual", "institute", "larval_habitat", + "larval_sample", "local_government", "maintenance", "medical_area", @@ -80658,6 +80756,7 @@ export const PermissionsBasedMeditrakSyncQueueUpdateSchema = { "individual", "institute", "larval_habitat", + "larval_sample", "local_government", "maintenance", "medical_area", @@ -82877,6 +82976,7 @@ export const EntityTypeSchema = { "individual", "institute", "larval_habitat", + "larval_sample", "local_government", "maintenance", "medical_area", @@ -83109,6 +83209,7 @@ export const MeditrakSurveyResponseRequestSchema = { "individual", "institute", "larval_habitat", + "larval_sample", "local_government", "maintenance", "medical_area", @@ -84005,6 +84106,7 @@ export const EntityResponseSchema = { "individual", "institute", "larval_habitat", + "larval_sample", "local_government", "maintenance", "medical_area", @@ -92828,6 +92930,7 @@ export const TranslatedMapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -92968,6 +93071,7 @@ export const TranslatedMapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -93098,6 +93202,7 @@ export const TranslatedMapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -93649,6 +93754,7 @@ export const TranslatedMapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -93789,6 +93895,7 @@ export const TranslatedMapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -93919,6 +94026,7 @@ export const TranslatedMapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -94392,6 +94500,7 @@ export const TranslatedMapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -94532,6 +94641,7 @@ export const TranslatedMapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -94662,6 +94772,7 @@ export const TranslatedMapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -95109,6 +95220,7 @@ export const TranslatedMapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -95249,6 +95361,7 @@ export const TranslatedMapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -95379,6 +95492,7 @@ export const TranslatedMapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -95841,6 +95955,7 @@ export const TranslatedMapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -95981,6 +96096,7 @@ export const TranslatedMapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", @@ -96111,6 +96227,7 @@ export const TranslatedMapOverlaySchema = { "Individual", "Institute", "LarvalHabitat", + "LarvalSample", "LocalGovernment", "Maintenance", "MedicalArea", diff --git a/packages/types/src/types/models.ts b/packages/types/src/types/models.ts index d01755d679..0d91bb067e 100644 --- a/packages/types/src/types/models.ts +++ b/packages/types/src/types/models.ts @@ -1752,6 +1752,7 @@ export enum EntityType { 'health_clinic_boundary' = 'health_clinic_boundary', 'enumeration_area' = 'enumeration_area', 'maintenance' = 'maintenance', + 'larval_sample' = 'larval_sample', } export enum DataTableType { 'analytics' = 'analytics', From 63c6b5961f73f25396eafc175fe6cf5e6953d84c Mon Sep 17 00:00:00 2001 From: Rohan Port <59544282+rohan-bes@users.noreply.github.com> Date: Fri, 5 Apr 2024 10:27:12 +1100 Subject: [PATCH 18/25] refactor(permissions): Removed un-used 'Viz Builder User' permissions checking code (#5558) refactor: Removed un-used 'Viz Builder User' permissions checking code Co-authored-by: Rohan Port --- packages/admin-panel-server/src/constants.ts | 1 - .../src/utils/hasVizBuilderUserAccess.ts | 19 ------------ .../admin-panel-server/src/utils/index.ts | 1 - .../src/permissions/assertions.js | 17 +--------- .../src/permissions/constants.js | 1 - .../dashboardItems/EditDashboardItems.test.js | 31 ------------------- 6 files changed, 1 insertion(+), 69 deletions(-) delete mode 100644 packages/admin-panel-server/src/utils/hasVizBuilderUserAccess.ts diff --git a/packages/admin-panel-server/src/constants.ts b/packages/admin-panel-server/src/constants.ts index 7d75c2b462..5544cc7415 100644 --- a/packages/admin-panel-server/src/constants.ts +++ b/packages/admin-panel-server/src/constants.ts @@ -6,4 +6,3 @@ export const BES_ADMIN_PERMISSION_GROUP = 'BES Admin'; export const TUPAIA_ADMIN_PANEL_PERMISSION_GROUP = 'Tupaia Admin Panel'; -export const VIZ_BUILDER_USER_PERMISSION_GROUP = 'Viz Builder User'; diff --git a/packages/admin-panel-server/src/utils/hasVizBuilderUserAccess.ts b/packages/admin-panel-server/src/utils/hasVizBuilderUserAccess.ts deleted file mode 100644 index f76162e896..0000000000 --- a/packages/admin-panel-server/src/utils/hasVizBuilderUserAccess.ts +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Tupaia - * Copyright (c) 2017 - 2021 Beyond Essential Systems Pty Ltd - * - */ - -import { AccessPolicy } from '@tupaia/access-policy'; -import { PermissionsError } from '@tupaia/utils'; - -import { VIZ_BUILDER_USER_PERMISSION_GROUP } from '../constants'; - -export const hasVizBuilderUserAccess = (accessPolicy: AccessPolicy) => { - const hasAccess = accessPolicy.allowsSome(undefined, VIZ_BUILDER_USER_PERMISSION_GROUP); - if (!hasAccess) { - throw new PermissionsError(`Need ${VIZ_BUILDER_USER_PERMISSION_GROUP} access`); - } - - return hasAccess; -}; diff --git a/packages/admin-panel-server/src/utils/index.ts b/packages/admin-panel-server/src/utils/index.ts index ae823007d2..b7617a5468 100644 --- a/packages/admin-panel-server/src/utils/index.ts +++ b/packages/admin-panel-server/src/utils/index.ts @@ -5,5 +5,4 @@ */ export { hasTupaiaAdminPanelAccess } from './hasTupaiaAdminPanelAccess'; -export { hasVizBuilderUserAccess } from './hasVizBuilderUserAccess'; export { readFileContent } from './readFileContent'; diff --git a/packages/central-server/src/permissions/assertions.js b/packages/central-server/src/permissions/assertions.js index 9e4e8f5989..9c40c265cd 100644 --- a/packages/central-server/src/permissions/assertions.js +++ b/packages/central-server/src/permissions/assertions.js @@ -3,11 +3,7 @@ * Copyright (c) 2017 - 2020 Beyond Essential Systems Pty Ltd */ -import { - BES_ADMIN_PERMISSION_GROUP, - TUPAIA_ADMIN_PANEL_PERMISSION_GROUP, - VIZ_BUILDER_USER_PERMISSION_GROUP, -} from './constants'; +import { BES_ADMIN_PERMISSION_GROUP, TUPAIA_ADMIN_PANEL_PERMISSION_GROUP } from './constants'; /** * Returns true all the time. This is for any route handlers that do not need permissions. @@ -58,9 +54,6 @@ export const assertAnyPermissions = (assertions, errorMessage) => async accessPo export const hasBESAdminAccess = accessPolicy => accessPolicy.allowsSome(undefined, BES_ADMIN_PERMISSION_GROUP); -export const hasVizBuilderAccess = accessPolicy => - accessPolicy.allowsSome(undefined, VIZ_BUILDER_USER_PERMISSION_GROUP); - export const hasPermissionGroupAccess = (accessPolicy, permissionGroup) => accessPolicy.allowsSome(undefined, permissionGroup); @@ -106,14 +99,6 @@ export const assertAdminPanelAccess = accessPolicy => { throw new Error(`Need ${TUPAIA_ADMIN_PANEL_PERMISSION_GROUP} access`); }; -export const assertVizBuilderAccess = accessPolicy => { - if (hasVizBuilderAccess(accessPolicy)) { - return true; - } - - throw new Error(`Need ${VIZ_BUILDER_USER_PERMISSION_GROUP} access`); -}; - export const assertPermissionGroupAccess = (accessPolicy, permissionGroupName) => { if (hasPermissionGroupAccess(accessPolicy, permissionGroupName)) { return true; diff --git a/packages/central-server/src/permissions/constants.js b/packages/central-server/src/permissions/constants.js index 0d68dccf2e..b5675017c3 100644 --- a/packages/central-server/src/permissions/constants.js +++ b/packages/central-server/src/permissions/constants.js @@ -5,4 +5,3 @@ export const BES_ADMIN_PERMISSION_GROUP = 'BES Admin'; export const TUPAIA_ADMIN_PANEL_PERMISSION_GROUP = 'Tupaia Admin Panel'; -export const VIZ_BUILDER_USER_PERMISSION_GROUP = 'Viz Builder User'; diff --git a/packages/central-server/src/tests/apiV2/dashboardItems/EditDashboardItems.test.js b/packages/central-server/src/tests/apiV2/dashboardItems/EditDashboardItems.test.js index 5603105200..35db5413e7 100644 --- a/packages/central-server/src/tests/apiV2/dashboardItems/EditDashboardItems.test.js +++ b/packages/central-server/src/tests/apiV2/dashboardItems/EditDashboardItems.test.js @@ -8,7 +8,6 @@ import { findOrCreateDummyRecord, findOrCreateDummyCountryEntity } from '@tupaia import { TUPAIA_ADMIN_PANEL_PERMISSION_GROUP, BES_ADMIN_PERMISSION_GROUP, - VIZ_BUILDER_USER_PERMISSION_GROUP, } from '../../../permissions'; import { TestableApp } from '../../testUtilities'; @@ -26,10 +25,6 @@ describe('EditDashboardItems', async () => { SB: [BES_ADMIN_PERMISSION_GROUP], }; - const VIZ_BUILDER_USER_ADMIN_POLICY = { - SB: [VIZ_BUILDER_USER_PERMISSION_GROUP], - }; - const app = new TestableApp(); const { models } = app; let dashboardDL; @@ -237,32 +232,6 @@ describe('EditDashboardItems', async () => { expect(result.config.name).to.equal(newName); }); - - it('Does not allow editing of a dashboard item if we have Viz Builder User access in any country, and where the user we are editing does not have access to that country', async () => { - const newName = 'Where to find cool pokemon'; - await app.grantAccess(VIZ_BUILDER_USER_ADMIN_POLICY); - const { body: result } = await app.put( - `dashboardItems/${dashboardItemDLPublicLAAdmin.id}`, - { - body: { config: { name: newName } }, - }, - ); - - expect(result).to.have.keys('error'); - }); - - it('Does not allow editing of a dashboard item if user has Viz Builder User access in the requested country but not BES Admin or the dashboard specific permission', async () => { - const newName = 'Where to find cool pokemon'; - await app.grantAccess(VIZ_BUILDER_USER_ADMIN_POLICY); - const { body: result } = await app.put( - `dashboardItems/${dashboardItemDLPublicSBAdmin.id}`, - { - body: { config: { name: newName } }, - }, - ); - - expect(result).to.have.keys('error'); - }); }); describe('Validation', async () => { From cac1e43158e1fc8ccae41f2355dcec3db8b1587c Mon Sep 17 00:00:00 2001 From: alexd-bes <129009580+alexd-bes@users.noreply.github.com> Date: Mon, 8 Apr 2024 18:56:09 +1200 Subject: [PATCH 19/25] fix(environment): use requireEnv and catch errors when missing an env var (#5573) * fix(environment): use requireEnv and catch errors when missing an env var * Updated .env vars needed for tests * PR Fixups * Fixed build issues --------- Co-authored-by: Rohan Port --- .github/workflows/ci.yaml | 1 + .../admin-panel-server/src/app/createApp.ts | 9 +++------ .../central-server/src/apiV2/deleteAccount.js | 4 ++-- .../src/apiV2/middleware/auth/clientAuth.js | 4 ++-- .../src/apiV2/requestCountryAccess.js | 4 ++-- packages/central-server/src/ms1/api/Ms1Api.js | 5 +++-- .../datatrak-web-server/src/app/createApp.ts | 11 ++++++----- .../src/app/middleware/attachAccessPolicy.ts | 4 ++-- packages/lesmis-server/src/app/createApp.ts | 4 ++-- .../src/connections/CentralConnection.ts | 5 ++--- .../src/connections/EntityConnection.ts | 5 ++--- .../src/connections/ReportConnection.ts | 5 ++--- .../SessionHandlingApiConnection.ts | 9 ++++----- .../src/connections/WebConfigConnection.ts | 5 ++--- .../src/connections/CentralConnection.ts | 10 ++++++---- .../src/connections/EntityConnection.ts | 6 ++---- .../src/connections/ReportConnection.ts | 6 ++---- .../src/orchestrator/auth/AuthConnection.ts | 17 +++++++++-------- packages/server-utils/src/sendEmail.ts | 7 +++++-- .../tupaia-web-server/src/app/createApp.ts | 11 ++++++----- .../requestHelpers/fetchFromCentralServer.js | 16 ++++++---------- .../src/connections/ReportConnection.js | 18 +++++++++++------- 22 files changed, 82 insertions(+), 84 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f5b7b2b5de..d16c5dd6fb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -95,6 +95,7 @@ jobs: API_CLIENT_PASSWORD: api-client API_CLIENT_SALT: abc132 JWT_SECRET: abc132 + TUPAIA_ADMIN_EMAIL_ADDRESS: test@tupaia.org # test database DB_PG_USER: postgres diff --git a/packages/admin-panel-server/src/app/createApp.ts b/packages/admin-panel-server/src/app/createApp.ts index a9916ebf67..3c6fc8bf5c 100644 --- a/packages/admin-panel-server/src/app/createApp.ts +++ b/packages/admin-panel-server/src/app/createApp.ts @@ -4,7 +4,7 @@ */ import { TupaiaDatabase } from '@tupaia/database'; import { OrchestratorApiBuilder, forwardRequest, handleWith } from '@tupaia/server-boilerplate'; - +import { getEnvVarOrDefault } from '@tupaia/utils'; import { AdminPanelSessionModel } from '../models'; import { hasTupaiaAdminPanelAccess } from '../utils'; import { upload } from '../middleware'; @@ -47,15 +47,12 @@ import { } from '../routes'; import { authHandlerProvider } from '../auth'; -const { - CENTRAL_API_URL = 'http://localhost:8090/v2', - ENTITY_API_URL = 'http://localhost:8050/v1', -} = process.env; - /** * Set up express server with middleware, */ export async function createApp() { + const CENTRAL_API_URL = getEnvVarOrDefault('CENTRAL_API_URL', 'http://localhost:8090/v2'); + const ENTITY_API_URL = getEnvVarOrDefault('ENTITY_API_URL', 'http://localhost:8050/v1'); const forwardToEntityApi = forwardRequest(ENTITY_API_URL); const builder = new OrchestratorApiBuilder(new TupaiaDatabase(), 'admin-panel') .attachApiClientToContext(authHandlerProvider) diff --git a/packages/central-server/src/apiV2/deleteAccount.js b/packages/central-server/src/apiV2/deleteAccount.js index 54ee6963d2..6b6b14cb1f 100644 --- a/packages/central-server/src/apiV2/deleteAccount.js +++ b/packages/central-server/src/apiV2/deleteAccount.js @@ -2,12 +2,12 @@ * Tupaia MediTrak * Copyright (c) 2017 - 2022 Beyond Essential Systems Pty Ltd */ -import { respond } from '@tupaia/utils'; +import { requireEnv, respond } from '@tupaia/utils'; import { sendEmail } from '@tupaia/server-utils'; import { getUserInfoInString } from './utilities'; const sendRequest = userInfo => { - const { TUPAIA_ADMIN_EMAIL_ADDRESS } = process.env; + const TUPAIA_ADMIN_EMAIL_ADDRESS = requireEnv('TUPAIA_ADMIN_EMAIL_ADDRESS'); const emailText = `${userInfo} has requested to delete their account`; return sendEmail(TUPAIA_ADMIN_EMAIL_ADDRESS, { diff --git a/packages/central-server/src/apiV2/middleware/auth/clientAuth.js b/packages/central-server/src/apiV2/middleware/auth/clientAuth.js index c7fe433468..0af89b46d1 100644 --- a/packages/central-server/src/apiV2/middleware/auth/clientAuth.js +++ b/packages/central-server/src/apiV2/middleware/auth/clientAuth.js @@ -3,7 +3,7 @@ * Copyright (c) 2017 Beyond Essential Systems Pty Ltd */ -import { UnauthenticatedError } from '@tupaia/utils'; +import { UnauthenticatedError, requireEnv } from '@tupaia/utils'; import { encryptPassword, getUserAndPassFromBasicAuth } from '@tupaia/auth'; export async function getAPIClientUser(authHeader, models) { @@ -12,7 +12,7 @@ export async function getAPIClientUser(authHeader, models) { throw new UnauthenticatedError('The provided basic authorization header is invalid'); } - const { API_CLIENT_SALT } = process.env; + const API_CLIENT_SALT = requireEnv('API_CLIENT_SALT'); // We always need a valid client; throw if none is found const secretKeyHash = encryptPassword(secretKey, API_CLIENT_SALT); diff --git a/packages/central-server/src/apiV2/requestCountryAccess.js b/packages/central-server/src/apiV2/requestCountryAccess.js index 540bbfcaf3..d3bf7c158e 100644 --- a/packages/central-server/src/apiV2/requestCountryAccess.js +++ b/packages/central-server/src/apiV2/requestCountryAccess.js @@ -3,7 +3,7 @@ * Copyright (c) 2017 - 2024 Beyond Essential Systems Pty Ltd */ -import { respond, UnauthenticatedError, ValidationError } from '@tupaia/utils'; +import { requireEnv, respond, UnauthenticatedError, ValidationError } from '@tupaia/utils'; import { sendEmail } from '@tupaia/server-utils'; import { getTokenClaimsFromBearerAuth } from '@tupaia/auth'; import { getUserInfoInString } from './utilities'; @@ -18,7 +18,7 @@ const checkUserPermission = (req, userId) => { }; const sendRequest = (userInfo, countryNames, message, project) => { - const { TUPAIA_ADMIN_EMAIL_ADDRESS } = process.env; + const TUPAIA_ADMIN_EMAIL_ADDRESS = requireEnv('TUPAIA_ADMIN_EMAIL_ADDRESS'); const emailText = ` ${userInfo} has requested access to countries: diff --git a/packages/central-server/src/ms1/api/Ms1Api.js b/packages/central-server/src/ms1/api/Ms1Api.js index af9a6ce6fa..535124f6f1 100644 --- a/packages/central-server/src/ms1/api/Ms1Api.js +++ b/packages/central-server/src/ms1/api/Ms1Api.js @@ -2,7 +2,7 @@ * Tupaia MediTrak * Copyright (c) 2019 Beyond Essential Systems Pty Ltd */ -import { fetchWithTimeout, HttpError, createBearerHeader } from '@tupaia/utils'; +import { fetchWithTimeout, HttpError, createBearerHeader, requireEnv } from '@tupaia/utils'; export class Ms1Api { constructor() { @@ -12,7 +12,8 @@ export class Ms1Api { async getAccessToken() { const currentServerTime = Date.now(); if (!this.accessToken || currentServerTime > this.accessTokenExpiry) { - const { MS1_USERNAME, MS1_PASSWORD } = process.env; + const MS1_USERNAME = requireEnv('MS1_USERNAME'); + const MS1_PASSWORD = requireEnv('MS1_PASSWORD'); const body = JSON.stringify({ username: MS1_USERNAME, password: MS1_PASSWORD, diff --git a/packages/datatrak-web-server/src/app/createApp.ts b/packages/datatrak-web-server/src/app/createApp.ts index 82d0cb02fd..c88026b745 100644 --- a/packages/datatrak-web-server/src/app/createApp.ts +++ b/packages/datatrak-web-server/src/app/createApp.ts @@ -12,6 +12,7 @@ import { SessionSwitchingAuthHandler, forwardRequest, } from '@tupaia/server-boilerplate'; +import { getEnvVarOrDefault } from '@tupaia/utils'; import { DataTrakSessionModel } from '../models'; import { UserRoute, @@ -47,14 +48,14 @@ import { } from '../routes'; import { attachAccessPolicy } from './middleware'; -const { - WEB_CONFIG_API_URL = 'http://localhost:8000/api/v1', - CENTRAL_API_URL = 'http://localhost:8090/v2', -} = process.env; - const authHandlerProvider = (req: Request) => new SessionSwitchingAuthHandler(req); export async function createApp() { + const WEB_CONFIG_API_URL = getEnvVarOrDefault( + 'WEB_CONFIG_API_URL', + 'http://localhost:8000/api/v1', + ); + const CENTRAL_API_URL = getEnvVarOrDefault('CENTRAL_API_URL', 'http://localhost:8090/v2'); const builder = new OrchestratorApiBuilder(new TupaiaDatabase(), 'datatrak-web-server') .useSessionModel(DataTrakSessionModel) .useAttachSession(attachSessionIfAvailable) diff --git a/packages/datatrak-web-server/src/app/middleware/attachAccessPolicy.ts b/packages/datatrak-web-server/src/app/middleware/attachAccessPolicy.ts index e3e8acfad3..6a8e93c0aa 100644 --- a/packages/datatrak-web-server/src/app/middleware/attachAccessPolicy.ts +++ b/packages/datatrak-web-server/src/app/middleware/attachAccessPolicy.ts @@ -6,10 +6,10 @@ import { RequestHandler } from 'express'; import { AccessPolicyBuilder, mergeAccessPolicies } from '@tupaia/auth'; import { AccessPolicy } from '@tupaia/access-policy'; - -const { API_CLIENT_NAME } = process.env; +import { requireEnv } from '@tupaia/utils'; export const attachAccessPolicy: RequestHandler = async (req, res, next) => { + const API_CLIENT_NAME = requireEnv('API_CLIENT_NAME'); const accessPolicyBuilder = new AccessPolicyBuilder(req.models); const apiUser = await req.models.user.findOne({ email: API_CLIENT_NAME }); if (!apiUser) { diff --git a/packages/lesmis-server/src/app/createApp.ts b/packages/lesmis-server/src/app/createApp.ts index 875624eb63..48b6e04eb6 100644 --- a/packages/lesmis-server/src/app/createApp.ts +++ b/packages/lesmis-server/src/app/createApp.ts @@ -4,6 +4,7 @@ */ import { TupaiaDatabase } from '@tupaia/database'; import { OrchestratorApiBuilder, forwardRequest, handleWith } from '@tupaia/server-boilerplate'; +import { getEnvVarOrDefault } from '@tupaia/utils'; import { LesmisSessionModel } from '../models'; import { DashboardRoute, @@ -28,8 +29,6 @@ import { VerifyEmailRequest } from '../routes/VerifyEmailRoute'; import { RegisterRequest } from '../routes/RegisterRoute'; import { PDFExportRequest } from '../routes/PDFExportRoute'; -const { CENTRAL_API_URL = 'http://localhost:8090/v2' } = process.env; - // eslint-disable-next-line @typescript-eslint/no-var-requires const path = require('path'); @@ -37,6 +36,7 @@ const path = require('path'); * Set up express server with middleware, */ export async function createApp() { + const CENTRAL_API_URL = getEnvVarOrDefault('CENTRAL_API_URL', 'http://localhost:8090/v2'); const builder = new OrchestratorApiBuilder(new TupaiaDatabase(), 'lesmis') .useSessionModel(LesmisSessionModel) .useAttachSession(attachSession) diff --git a/packages/lesmis-server/src/connections/CentralConnection.ts b/packages/lesmis-server/src/connections/CentralConnection.ts index 4b87c3f0e4..db814e99d3 100644 --- a/packages/lesmis-server/src/connections/CentralConnection.ts +++ b/packages/lesmis-server/src/connections/CentralConnection.ts @@ -3,18 +3,17 @@ * Copyright (c) 2017 - 2020 Beyond Essential Systems Pty Ltd */ import camelcaseKeys from 'camelcase-keys'; +import { getEnvVarOrDefault } from '@tupaia/utils'; import { hasAdminPanelAccess } from '../utils'; import { SessionHandlingApiConnection } from './SessionHandlingApiConnection'; -const { CENTRAL_API_URL = 'http://localhost:8090/v2' } = process.env; - type RequestBody = Record | Record[]; /** * @deprecated use @tupaia/api-client */ export class CentralConnection extends SessionHandlingApiConnection { - public baseUrl = CENTRAL_API_URL; + public baseUrl = getEnvVarOrDefault('CENTRAL_API_URL', 'http://localhost:8090/v2'); public async getUser() { // if user is not logged in, return null rather than fetching the api client user details diff --git a/packages/lesmis-server/src/connections/EntityConnection.ts b/packages/lesmis-server/src/connections/EntityConnection.ts index 25caedd971..4166ab2c77 100644 --- a/packages/lesmis-server/src/connections/EntityConnection.ts +++ b/packages/lesmis-server/src/connections/EntityConnection.ts @@ -5,16 +5,15 @@ */ import camelcaseKeys from 'camelcase-keys'; import { QueryParameters } from '@tupaia/server-boilerplate'; +import { getEnvVarOrDefault } from '@tupaia/utils'; import { SessionHandlingApiConnection } from './SessionHandlingApiConnection'; import { LESMIS_PROJECT_NAME } from '../constants'; -const { ENTITY_API_URL = 'http://localhost:8050/v1' } = process.env; - /** * @deprecated use @tupaia/api-client */ export class EntityConnection extends SessionHandlingApiConnection { - public baseUrl = ENTITY_API_URL; + public baseUrl = getEnvVarOrDefault('ENTITY_API_URL', 'http://localhost:8050/v1'); public async getEntities(entityCode: string, queryParameters: QueryParameters = {}) { const response = await this.get( diff --git a/packages/lesmis-server/src/connections/ReportConnection.ts b/packages/lesmis-server/src/connections/ReportConnection.ts index 34ae5c7a5f..5d94569493 100644 --- a/packages/lesmis-server/src/connections/ReportConnection.ts +++ b/packages/lesmis-server/src/connections/ReportConnection.ts @@ -5,8 +5,7 @@ import { QueryParameters } from '@tupaia/server-boilerplate'; import { SessionHandlingApiConnection } from './SessionHandlingApiConnection'; - -const { REPORT_API_URL = 'http://localhost:8030/v1' } = process.env; +import { getEnvVarOrDefault } from '@tupaia/utils'; type ReportObject = { results: Record[]; @@ -16,7 +15,7 @@ type ReportObject = { * @deprecated use @tupaia/api-client */ export class ReportConnection extends SessionHandlingApiConnection { - public baseUrl = REPORT_API_URL; + public baseUrl = getEnvVarOrDefault('REPORT_API_URL', 'http://localhost:8030/v1'); public async fetchReport(reportCode: string, query: QueryParameters): Promise { return this.get(`fetchReport/${reportCode}`, query); diff --git a/packages/lesmis-server/src/connections/SessionHandlingApiConnection.ts b/packages/lesmis-server/src/connections/SessionHandlingApiConnection.ts index 6eddd66c96..0500c60cbd 100644 --- a/packages/lesmis-server/src/connections/SessionHandlingApiConnection.ts +++ b/packages/lesmis-server/src/connections/SessionHandlingApiConnection.ts @@ -4,10 +4,7 @@ */ import { ApiConnection, AuthHandler, SessionRecord } from '@tupaia/server-boilerplate'; -import { createBasicHeader } from '@tupaia/utils'; - -const { API_CLIENT_NAME, API_CLIENT_SECRET } = process.env; -const DEFAULT_AUTH_HEADER = createBasicHeader(API_CLIENT_NAME, API_CLIENT_SECRET); +import { createBasicHeader, requireEnv } from '@tupaia/utils'; class SessionSwitchingAuthHandler implements AuthHandler { session?: SessionRecord; @@ -24,7 +21,9 @@ class SessionSwitchingAuthHandler implements AuthHandler { if (this.session) { return this.session.getAuthHeader(); } - + const API_CLIENT_NAME = requireEnv('API_CLIENT_NAME'); + const API_CLIENT_SECRET = requireEnv('API_CLIENT_SECRET'); + const DEFAULT_AUTH_HEADER = createBasicHeader(API_CLIENT_NAME, API_CLIENT_SECRET); return DEFAULT_AUTH_HEADER; } } diff --git a/packages/lesmis-server/src/connections/WebConfigConnection.ts b/packages/lesmis-server/src/connections/WebConfigConnection.ts index 30947e9b45..2af6cc245d 100644 --- a/packages/lesmis-server/src/connections/WebConfigConnection.ts +++ b/packages/lesmis-server/src/connections/WebConfigConnection.ts @@ -4,15 +4,14 @@ */ import { QueryParameters } from '@tupaia/server-boilerplate'; +import { getEnvVarOrDefault } from '@tupaia/utils'; import { SessionHandlingApiConnection } from './SessionHandlingApiConnection'; -const { WEB_CONFIG_API_URL = 'http://localhost:8000/api/v1' } = process.env; - /** * @deprecated use @tupaia/api-client */ export class WebConfigConnection extends SessionHandlingApiConnection { - public baseUrl = WEB_CONFIG_API_URL; + public baseUrl = getEnvVarOrDefault('WEB_CONFIG_API_URL', 'http://localhost:8000/api/v1'); public async fetchDashboard(query: QueryParameters) { return this.get('dashboards', query); diff --git a/packages/psss-server/src/connections/CentralConnection.ts b/packages/psss-server/src/connections/CentralConnection.ts index a832a200e5..b97a8bd943 100644 --- a/packages/psss-server/src/connections/CentralConnection.ts +++ b/packages/psss-server/src/connections/CentralConnection.ts @@ -4,12 +4,14 @@ */ import { generateId } from '@tupaia/database'; -import { convertPeriodStringToDateRange, stripTimezoneFromDate } from '@tupaia/utils'; +import { + convertPeriodStringToDateRange, + getEnvVarOrDefault, + stripTimezoneFromDate, +} from '@tupaia/utils'; import { Entity, Survey } from '@tupaia/types'; import { ApiConnection } from './ApiConnection'; -const { CENTRAL_API_URL = 'http://localhost:8090/v2' } = process.env; - type SurveyResponseObject = { 'entity.code': string; 'survey.code': string; @@ -26,7 +28,7 @@ type Answer = { * @deprecated use @tupaia/api-client */ export class CentralConnection extends ApiConnection { - public baseUrl = CENTRAL_API_URL; + public baseUrl = getEnvVarOrDefault('CENTRAL_API_URL', 'http://localhost:8090/v2'); public async updateOrCreateSurveyResponse( surveyCode: string, diff --git a/packages/psss-server/src/connections/EntityConnection.ts b/packages/psss-server/src/connections/EntityConnection.ts index 22d76e84c6..6a030af4fa 100644 --- a/packages/psss-server/src/connections/EntityConnection.ts +++ b/packages/psss-server/src/connections/EntityConnection.ts @@ -3,12 +3,10 @@ * Copyright (c) 2017 - 2021 Beyond Essential Systems Pty Ltd */ -import { removeAt } from '@tupaia/utils'; +import { getEnvVarOrDefault, removeAt } from '@tupaia/utils'; import { PSSS_ENTITY, PSSS_HIERARCHY } from '../constants'; import { ApiConnection } from './ApiConnection'; -const { ENTITY_API_URL = 'http://localhost:8050/v1' } = process.env; - interface Entity { id: string; code: string; @@ -76,7 +74,7 @@ const getRelationParams = (options: RelationOptions) => { * @deprecated use @tupaia/api-client */ export class EntityConnection extends ApiConnection { - baseUrl = ENTITY_API_URL; + baseUrl = getEnvVarOrDefault('ENTITY_API_URL', 'http://localhost:8050/v1'); public fetchCountries = async () => this.fetchDescendants(PSSS_ENTITY, { diff --git a/packages/psss-server/src/connections/ReportConnection.ts b/packages/psss-server/src/connections/ReportConnection.ts index 6de88c2c88..db57a56691 100644 --- a/packages/psss-server/src/connections/ReportConnection.ts +++ b/packages/psss-server/src/connections/ReportConnection.ts @@ -3,12 +3,10 @@ * Copyright (c) 2017 - 2020 Beyond Essential Systems Pty Ltd */ -import { convertPeriodStringToDateRange } from '@tupaia/utils'; +import { convertPeriodStringToDateRange, getEnvVarOrDefault } from '@tupaia/utils'; import { ApiConnection } from './ApiConnection'; import { PSSS_HIERARCHY } from '../constants'; -const { REPORT_API_URL = 'http://localhost:8030/v1' } = process.env; - const buildEmptyReport = (periods: string[]) => ({ results: [], metadata: { dataElementCodeToName: {} }, @@ -27,7 +25,7 @@ type ReportObject = { * @deprecated use @tupaia/api-client */ export class ReportConnection extends ApiConnection { - public baseUrl = REPORT_API_URL; + public baseUrl = getEnvVarOrDefault('REPORT_API_URL', 'http://localhost:8030/v1'); public async fetchReport( reportCode: string, diff --git a/packages/server-boilerplate/src/orchestrator/auth/AuthConnection.ts b/packages/server-boilerplate/src/orchestrator/auth/AuthConnection.ts index ad9eb74a51..51742c2a63 100644 --- a/packages/server-boilerplate/src/orchestrator/auth/AuthConnection.ts +++ b/packages/server-boilerplate/src/orchestrator/auth/AuthConnection.ts @@ -4,20 +4,13 @@ * */ -import { createBasicHeader } from '@tupaia/utils'; +import { createBasicHeader, requireEnv } from '@tupaia/utils'; import { AccessPolicyObject } from '../../types'; import { Credentials, OneTimeCredentials } from '../types'; import { ApiConnection } from '../../connections'; const DEFAULT_NAME = 'TUPAIA-SERVER'; -const basicAuthHandler = { - getAuthHeader: async () => { - const { API_CLIENT_NAME, API_CLIENT_PASSWORD } = process.env; - return createBasicHeader(API_CLIENT_NAME, API_CLIENT_PASSWORD); - }, -}; - export interface AuthResponse { accessToken?: string; refreshToken?: string; @@ -27,6 +20,14 @@ export interface AuthResponse { }; } +const basicAuthHandler = { + getAuthHeader: async () => { + const API_CLIENT_NAME = requireEnv('API_CLIENT_NAME'); + const API_CLIENT_PASSWORD = requireEnv('API_CLIENT_PASSWORD'); + return createBasicHeader(API_CLIENT_NAME, API_CLIENT_PASSWORD); + }, +}; + export class AuthConnection extends ApiConnection { public baseUrl = process.env.CENTRAL_API_URL || 'http://localhost:8090/v2'; // auth server is actually just central server diff --git a/packages/server-utils/src/sendEmail.ts b/packages/server-utils/src/sendEmail.ts index d422776a75..79e0b775cb 100644 --- a/packages/server-utils/src/sendEmail.ts +++ b/packages/server-utils/src/sendEmail.ts @@ -4,7 +4,7 @@ */ import nodemailer from 'nodemailer'; -import { getIsProductionEnvironment, requireEnv } from '@tupaia/utils'; +import { getEnvVarOrDefault, getIsProductionEnvironment, requireEnv } from '@tupaia/utils'; import Mail from 'nodemailer/lib/mailer'; const TEXT_SIGN_OFF = 'Cheers,\n\nThe Tupaia Team'; @@ -26,7 +26,10 @@ export const sendEmail = async (to: string | string[], mailOptions: MailOptions attachments, signOff = html ? HTML_SIGN_OFF : TEXT_SIGN_OFF, } = mailOptions; - const { SMTP_HOST, SMTP_USER, SMTP_PASSWORD, SITE_EMAIL_ADDRESS } = process.env; + const SMTP_HOST = getEnvVarOrDefault('SMTP_HOST', undefined); + const SMTP_USER = getEnvVarOrDefault('SMTP_USER', undefined); + const SMTP_PASSWORD = getEnvVarOrDefault('SMTP_PASSWORD', undefined); + const SITE_EMAIL_ADDRESS = getEnvVarOrDefault('SITE_EMAIL_ADDRESS', undefined); if (text && html) { throw new Error('Only text or HTML can be sent in an email, not both'); diff --git a/packages/tupaia-web-server/src/app/createApp.ts b/packages/tupaia-web-server/src/app/createApp.ts index f8ba2343d7..0c39aa4727 100644 --- a/packages/tupaia-web-server/src/app/createApp.ts +++ b/packages/tupaia-web-server/src/app/createApp.ts @@ -12,17 +12,18 @@ import { SessionSwitchingAuthHandler, forwardRequest, } from '@tupaia/server-boilerplate'; +import { getEnvVarOrDefault } from '@tupaia/utils'; import { TupaiaWebSessionModel } from '../models'; import * as routes from '../routes'; -const { - WEB_CONFIG_API_URL = 'http://localhost:8000/api/v1', - CENTRAL_API_URL = 'http://localhost:8090/v2', -} = process.env; - const authHandlerProvider = (req: Request) => new SessionSwitchingAuthHandler(req); export async function createApp(db: TupaiaDatabase = new TupaiaDatabase()) { + const WEB_CONFIG_API_URL = getEnvVarOrDefault( + 'WEB_CONFIG_API_URL', + 'http://localhost:8000/api/v1', + ); + const CENTRAL_API_URL = getEnvVarOrDefault('CENTRAL_API_URL', 'http://localhost:8090/v2'); const builder = new OrchestratorApiBuilder(db, 'tupaia-web') .useSessionModel(TupaiaWebSessionModel) .useAttachSession(attachSessionIfAvailable) diff --git a/packages/web-config-server/src/appServer/requestHelpers/fetchFromCentralServer.js b/packages/web-config-server/src/appServer/requestHelpers/fetchFromCentralServer.js index 2c9440d652..65c668bee2 100644 --- a/packages/web-config-server/src/appServer/requestHelpers/fetchFromCentralServer.js +++ b/packages/web-config-server/src/appServer/requestHelpers/fetchFromCentralServer.js @@ -4,12 +4,10 @@ import { stringifyQuery, createBasicHeader, createBearerHeader, + requireEnv, } from '@tupaia/utils'; import { refreshAndSaveAccessToken } from './refreshAndSaveAccessToken'; -const { API_CLIENT_NAME, API_CLIENT_PASSWORD } = process.env; -const DEFAULT_AUTH_HEADER = createBasicHeader(API_CLIENT_NAME, API_CLIENT_PASSWORD); - /** * Send request to Central server and handle responses. * @@ -22,12 +20,10 @@ const DEFAULT_AUTH_HEADER = createBasicHeader(API_CLIENT_NAME, API_CLIENT_PASSWO * @param {object} authHeader * To overwrite the default Authorization header, e.g. if using an access token */ -export const fetchFromCentralServer = async ( - endpoint, - payload, - queryParameters, - authHeader = DEFAULT_AUTH_HEADER, -) => { +export const fetchFromCentralServer = async (endpoint, payload, queryParameters, authHeader) => { + const API_CLIENT_NAME = requireEnv('API_CLIENT_NAME'); + const API_CLIENT_PASSWORD = requireEnv('API_CLIENT_PASSWORD'); + const DEFAULT_AUTH_HEADER = createBasicHeader(API_CLIENT_NAME, API_CLIENT_PASSWORD); const url = stringifyQuery( process.env.TUPAIA_APP_SERVER_URL || 'http://localhost:8090/v2', endpoint, @@ -36,7 +32,7 @@ export const fetchFromCentralServer = async ( const config = { method: payload ? 'POST' : 'GET', headers: { - Authorization: authHeader, + Authorization: authHeader || DEFAULT_AUTH_HEADER, 'Content-Type': 'application/json', }, }; diff --git a/packages/web-config-server/src/connections/ReportConnection.js b/packages/web-config-server/src/connections/ReportConnection.js index 5f38590a08..2fbb8bf8de 100644 --- a/packages/web-config-server/src/connections/ReportConnection.js +++ b/packages/web-config-server/src/connections/ReportConnection.js @@ -4,21 +4,21 @@ */ import { ApiConnection } from '@tupaia/server-boilerplate'; -import { createBasicHeader, createBearerHeader } from '@tupaia/utils'; +import { + createBasicHeader, + createBearerHeader, + getEnvVarOrDefault, + requireEnv, +} from '@tupaia/utils'; import { refreshAndSaveAccessToken } from '/appServer/requestHelpers/refreshAndSaveAccessToken'; -const { API_CLIENT_NAME, API_CLIENT_PASSWORD } = process.env; - const PUBLIC_USER_NAME = 'public'; -const PUBLIC_USER_AUTH_HEADER = createBasicHeader(API_CLIENT_NAME, API_CLIENT_PASSWORD); - -const { REPORT_API_URL = 'http://localhost:8030/v1' } = process.env; /** * @deprecated use @tupaia/api-client */ export class ReportConnection extends ApiConnection { - baseUrl = REPORT_API_URL; + baseUrl = getEnvVarOrDefault('REPORT_API_URL', 'http://localhost:8030/v1'); constructor(req) { const userName = req?.userJson?.userName; @@ -31,6 +31,10 @@ export class ReportConnection extends ApiConnection { } if (userName === PUBLIC_USER_NAME) { + const API_CLIENT_NAME = requireEnv('API_CLIENT_NAME'); + const API_CLIENT_PASSWORD = requireEnv('API_CLIENT_PASSWORD'); + + const PUBLIC_USER_AUTH_HEADER = createBasicHeader(API_CLIENT_NAME, API_CLIENT_PASSWORD); return PUBLIC_USER_AUTH_HEADER; } From 080b317a737072aade41c9a1d7b6d7fe33b2fc54 Mon Sep 17 00:00:00 2001 From: alexd-bes <129009580+alexd-bes@users.noreply.github.com> Date: Tue, 9 Apr 2024 10:18:39 +1200 Subject: [PATCH 20/25] fix(centralServer): Update to check for FE env vars --- .../src/apiV2/utilities/emailVerification.js | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/packages/central-server/src/apiV2/utilities/emailVerification.js b/packages/central-server/src/apiV2/utilities/emailVerification.js index 6686eeedfb..96080eba74 100644 --- a/packages/central-server/src/apiV2/utilities/emailVerification.js +++ b/packages/central-server/src/apiV2/utilities/emailVerification.js @@ -5,32 +5,31 @@ import { encryptPassword } from '@tupaia/auth'; import { sendEmail } from '@tupaia/server-utils'; - -const { TUPAIA_FRONT_END_URL, LESMIS_FRONT_END_URL, DATATRAK_FRONT_END_URL } = process.env; +import { requireEnv } from '@tupaia/utils'; const EMAILS = { tupaia: { subject: 'Tupaia email verification', - body: token => + body: (token, url) => 'Thank you for registering with tupaia.org.\n' + 'Please click on the following link to register your email address.\n\n' + - `${TUPAIA_FRONT_END_URL}/verify-email?verifyEmailToken=${token}\n\n` + + `${url}/verify-email?verifyEmailToken=${token}\n\n` + 'If you believe this email was sent to you in error, please contact us immediately at admin@tupaia.org.\n', }, datatrak: { subject: 'Tupaia Datatrak email verification', - body: token => + body: (token, url) => 'Thank you for registering with datatrak.tupaia.org.\n' + 'Please click on the following link to register your email address.\n\n' + - `${DATATRAK_FRONT_END_URL}/verify-email?verifyEmailToken=${token}\n\n` + + `${url}/verify-email?verifyEmailToken=${token}\n\n` + 'If you believe this email was sent to you in error, please contact us immediately at admin@tupaia.org.\n', }, lesmis: { subject: 'LESMIS email verification', - body: token => + body: (token, url) => 'Thank you for registering with lesmis.la.\n' + 'Please click on the following link to register your email address.\n\n' + - `${LESMIS_FRONT_END_URL}/en/verify-email?verifyEmailToken=${token}\n\n` + + `${url}/en/verify-email?verifyEmailToken=${token}\n\n` + 'If you believe this email was sent to you in error, please contact us immediately at admin@tupaia.org.\n', signOff: 'Best regards,\nThe LESMIS Team', }, @@ -40,8 +39,17 @@ export const sendEmailVerification = async user => { const token = encryptPassword(user.email + user.password_hash, user.password_salt); const platform = user.primary_platform ? user.primary_platform : 'tupaia'; const { subject, body, signOff } = EMAILS[platform]; + const TUPAIA_FRONT_END_URL = requireEnv('TUPAIA_FRONT_END_URL'); + const LESMIS_FRONT_END_URL = requireEnv('LESMIS_FRONT_END_URL'); + const DATATRAK_FRONT_END_URL = requireEnv('DATATRAK_FRONT_END_URL'); + + const url = { + tupaia: TUPAIA_FRONT_END_URL, + datatrak: DATATRAK_FRONT_END_URL, + lesmis: LESMIS_FRONT_END_URL, + }[platform]; - return sendEmail(user.email, { subject, text: body(token), signOff }); + return sendEmail(user.email, { subject, text: body(token, url), signOff }); }; export const verifyEmailHelper = async (models, searchCondition, token) => { From e7cbff60f1db592dea554521b4c0adcea96c02dd Mon Sep 17 00:00:00 2001 From: alexd-bes <129009580+alexd-bes@users.noreply.github.com> Date: Tue, 9 Apr 2024 10:31:12 +1200 Subject: [PATCH 21/25] fix(tests): Fix central-server env tests --- .../src/tests/apiV2/authenticate/authenticate.test.js | 4 +++- .../src/tests/apiV2/authenticate/oneTimeLogin.test.js | 4 +++- .../src/tests/apiV2/requestPasswordReset.test.js | 4 +++- packages/central-server/src/tests/apiV2/verifyEmail.test.js | 4 ++-- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/central-server/src/tests/apiV2/authenticate/authenticate.test.js b/packages/central-server/src/tests/apiV2/authenticate/authenticate.test.js index 30a65b954c..4d0084bbef 100644 --- a/packages/central-server/src/tests/apiV2/authenticate/authenticate.test.js +++ b/packages/central-server/src/tests/apiV2/authenticate/authenticate.test.js @@ -3,7 +3,6 @@ * Copyright (c) 2017 Beyond Essential Systems Pty Ltd */ -import {} from 'dotenv/config'; // Load the environment variables into process.env import { expect } from 'chai'; import { encryptPassword, hashAndSaltPassword, getTokenClaims } from '@tupaia/auth'; @@ -11,6 +10,9 @@ import { findOrCreateDummyRecord, findOrCreateDummyCountryEntity } from '@tupaia import { createBasicHeader } from '@tupaia/utils'; import { TestableApp } from '../../testUtilities'; +import { configureEnv } from '../../../configureEnv'; + +configureEnv(); const app = new TestableApp(); const { models } = app; diff --git a/packages/central-server/src/tests/apiV2/authenticate/oneTimeLogin.test.js b/packages/central-server/src/tests/apiV2/authenticate/oneTimeLogin.test.js index fc69ebd59e..241a6da45a 100644 --- a/packages/central-server/src/tests/apiV2/authenticate/oneTimeLogin.test.js +++ b/packages/central-server/src/tests/apiV2/authenticate/oneTimeLogin.test.js @@ -3,12 +3,14 @@ * Copyright (c) 2017 Beyond Essential Systems Pty Ltd */ -import {} from 'dotenv/config'; // Load the environment variables into process.env import { expect } from 'chai'; import moment from 'moment'; import { randomEmail } from '@tupaia/utils'; import { getAuthorizationHeader, TestableApp } from '../../testUtilities'; +import { configureEnv } from '../../../configureEnv'; + +configureEnv(); describe('One Time Login', function () { const app = new TestableApp(); diff --git a/packages/central-server/src/tests/apiV2/requestPasswordReset.test.js b/packages/central-server/src/tests/apiV2/requestPasswordReset.test.js index bd8fbad421..e9ebdf2aed 100644 --- a/packages/central-server/src/tests/apiV2/requestPasswordReset.test.js +++ b/packages/central-server/src/tests/apiV2/requestPasswordReset.test.js @@ -3,12 +3,14 @@ * Copyright (c) 2017 Beyond Essential Systems Pty Ltd */ -import {} from 'dotenv/config'; // Load the environment variables into process.env import { expect } from 'chai'; import { createBearerHeader, randomEmail, randomString } from '@tupaia/utils'; import { getAuthorizationHeader, TestableApp } from '../testUtilities'; +import { configureEnv } from '../../configureEnv'; + +configureEnv(); describe('Reset Password', () => { const app = new TestableApp(); diff --git a/packages/central-server/src/tests/apiV2/verifyEmail.test.js b/packages/central-server/src/tests/apiV2/verifyEmail.test.js index d0fd15a0da..fab279bd50 100644 --- a/packages/central-server/src/tests/apiV2/verifyEmail.test.js +++ b/packages/central-server/src/tests/apiV2/verifyEmail.test.js @@ -2,14 +2,14 @@ * Tupaia MediTrak * Copyright (c) 2017 Beyond Essential Systems Pty Ltd */ - -import {} from 'dotenv/config'; // Load the environment variables into process.env import { expect } from 'chai'; import { encryptPassword } from '@tupaia/auth'; import { randomEmail } from '@tupaia/utils'; import { getAuthorizationHeader, TestableApp } from '../testUtilities'; +import { configureEnv } from '../../configureEnv'; +configureEnv(); describe('Verify Email', () => { const app = new TestableApp(); const { models } = app; From dcbabb904fee1b9122cacd8110dbb6ca5c07502f Mon Sep 17 00:00:00 2001 From: alexd-bes <129009580+alexd-bes@users.noreply.github.com> Date: Tue, 9 Apr 2024 10:42:17 +1200 Subject: [PATCH 22/25] fix(devops): Update env vars used in tests --- .github/workflows/ci.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d16c5dd6fb..a57bbfee07 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -96,6 +96,9 @@ jobs: API_CLIENT_SALT: abc132 JWT_SECRET: abc132 TUPAIA_ADMIN_EMAIL_ADDRESS: test@tupaia.org + TUPAIA_FRONTEND_URL: dev.tupaia.org + LESMIS_FRONTEND_URL: dev-lesmis.tupaia.org + DATATRAK_FRONTEND_URL: dev-datatrak.tupaia.org # test database DB_PG_USER: postgres From 28bc363f653adf15eca68de1da737a2a6dee36af Mon Sep 17 00:00:00 2001 From: alexd-bes <129009580+alexd-bes@users.noreply.github.com> Date: Tue, 9 Apr 2024 10:49:47 +1200 Subject: [PATCH 23/25] fix(devops): Update env vars used in tests --- .github/workflows/ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a57bbfee07..ca421a9994 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -96,9 +96,9 @@ jobs: API_CLIENT_SALT: abc132 JWT_SECRET: abc132 TUPAIA_ADMIN_EMAIL_ADDRESS: test@tupaia.org - TUPAIA_FRONTEND_URL: dev.tupaia.org - LESMIS_FRONTEND_URL: dev-lesmis.tupaia.org - DATATRAK_FRONTEND_URL: dev-datatrak.tupaia.org + TUPAIA_FRONT_END_URL: dev.tupaia.org + LESMIS_FRONT_END_URL: dev-lesmis.tupaia.org + DATATRAK_FRONT_END_URL: dev-datatrak.tupaia.org # test database DB_PG_USER: postgres From 54d4c16a4a27630c30a16f4ba7a2c6c1abd9af9b Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Thu, 11 Apr 2024 11:26:10 +1200 Subject: [PATCH 24/25] fix: `/report` endpoint date validation (#5575) fix `/report` endpoint date param validation --- packages/tsutils/src/datetime.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/tsutils/src/datetime.ts b/packages/tsutils/src/datetime.ts index 5863b864d3..b9f55b8b48 100644 --- a/packages/tsutils/src/datetime.ts +++ b/packages/tsutils/src/datetime.ts @@ -6,7 +6,7 @@ import moment from 'moment'; import momentTimezone from 'moment-timezone'; -export const ISO_DATE_PATTERN = /^\d{4}-\d{2}-\d{2}$/; +export const ISO_DATE_PATTERN = /\d{4}-\d{2}-\d{2}/; /** * @returns ISO date string in the format "yyyy-mm-dd", discarding the timestamp. From 3683505c16bcbed8c488076390aee65fab49997d Mon Sep 17 00:00:00 2001 From: Rohan Port <59544282+rohan-bes@users.noreply.github.com> Date: Fri, 12 Apr 2024 14:01:58 +1000 Subject: [PATCH 25/25] fix(adminPanel): Fix UsedBy queries for data elements and data-tables (#5585) Co-authored-by: Rohan Port --- packages/database/src/TupaiaDatabase.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/database/src/TupaiaDatabase.js b/packages/database/src/TupaiaDatabase.js index 4f4767eb01..f035d9945a 100644 --- a/packages/database/src/TupaiaDatabase.js +++ b/packages/database/src/TupaiaDatabase.js @@ -687,8 +687,11 @@ function getColSelector(connection, inputColStr) { const params = inputColStr.split(jsonOperatorPattern); const allButFirst = params.slice(1); + const lastIndexOfLookupAsText = inputColStr.lastIndexOf('->>'); + const lastIndexOfLookupAsJson = inputColStr.lastIndexOf('->'); + const selector = lastIndexOfLookupAsText >= lastIndexOfLookupAsJson ? '#>>' : '#>'; // Turn `config->item->>colour` into `config #>> '{item,colour}'` // For some reason, Knex fails when we try to convert it to `config->'item'->>'colour'` - return connection.raw(`?? #>> '{${allButFirst.map(() => '??').join(',')}}'`, params); + return connection.raw(`?? ${selector} '{${allButFirst.map(() => '??').join(',')}}'`, params); }