Merge branch 'dev' into odds-and-sods

packages/admin-panel-server/src/viz-builder/mapOverlayVisualisation/MapOverlayVisualisationExtractor.ts

@@ -15,7 +15,7 @@ import { getVizOutputConfig } from '../utils';
 
 export class MapOverlayVisualisationExtractor<
   MapOverlayValidator extends yup.AnyObjectSchema,
-  ReportValidator extends yup.AnyObjectSchema
+  ReportValidator extends yup.AnyObjectSchema,
 > {
   private readonly visualisation: ExpandType<yup.InferType<typeof baseVisualisationValidator>>;
   private readonly mapOverlayValidator: MapOverlayValidator;
@@ -56,6 +56,7 @@ export class MapOverlayVisualisationExtractor<
       countryCodes,
       linkedMeasures,
       mapOverlayPermissionGroup: permissionGroup,
+      entityAttributesFilter,
     } = this.visualisation;
     const { output, ...presentation } = this.visualisation.presentation;
 
@@ -71,6 +72,7 @@ export class MapOverlayVisualisationExtractor<
       countryCodes,
       linkedMeasures,
       permissionGroup,
+      entityAttributesFilter,
     };
   }
 

packages/admin-panel/src/VizBuilderApp/components/MapOverlay/MapOverlayMetadataForm.jsx

@@ -28,10 +28,8 @@ export const MapOverlayMetadataForm = ({ Header, Body, Footer, onSubmit }) => {
   } = defaults;
   const [searchInput, setSearchInput] = useState(mapOverlayPermissionGroup || '');
   const debouncedSearchInput = useDebounce(searchInput, 200);
-  const {
-    data: permissionGroups = [],
-    isLoading: isLoadingPermissionGroups,
-  } = useSearchPermissionGroups({ search: debouncedSearchInput });
+  const { data: permissionGroups = [], isLoading: isLoadingPermissionGroups } =
+    useSearchPermissionGroups({ search: debouncedSearchInput });
   const [projectCodes, setProjectCodes] = useState(inputProjectCodes);
   const [countryCodes, setCountryCodes] = useState(inputCountryCodes);
 

packages/admin-panel/src/pages/resources/EntitiesPage.jsx

@@ -10,25 +10,33 @@ import { SURVEY_RESPONSE_COLUMNS, ANSWER_COLUMNS } from './SurveyResponsesPage';
 
 const ENTITIES_ENDPOINT = 'entities';
 
-export const FIELDS = [
-  { source: 'id', show: false },
-  {
+export const FIELDS = {
+  id: { source: 'id', show: false },
+  code: {
     Header: 'Code',
     source: 'code',
   },
-  {
+  name: {
     Header: 'Name',
     source: 'name',
     type: 'tooltip',
   },
-  {
+  type: {
     Header: 'Type',
     source: 'type',
   },
-];
+  attributes: {
+    Header: 'Attributes',
+    source: 'attributes',
+    type: 'jsonTooltip',
+    editConfig: {
+      type: 'jsonEditor',
+    },
+  },
+};
 
 export const COLUMNS = [
-  ...FIELDS,
+  ...Object.values(FIELDS),
   {
     Header: 'Country',
     source: 'country_code',
@@ -40,12 +48,7 @@ export const COLUMNS = [
     actionConfig: {
       editEndpoint: ENTITIES_ENDPOINT,
       title: 'Edit Entity',
-      fields: [
-        {
-          Header: 'Name',
-          source: 'name',
-        },
-      ],
+      fields: [FIELDS.name, FIELDS.attributes],
     },
   },
   {

packages/admin-panel/src/pages/resources/MapOverlaysPage.jsx

@@ -69,6 +69,17 @@ const FIELDS = [
     width: 200,
     editConfig: { type: 'jsonEditor' },
   },
+  {
+    Header: 'Entity attributes filters',
+    source: 'entity_attributes_filter',
+    type: 'jsonTooltip',
+    width: 200,
+    editConfig: {
+      type: 'jsonEditor',
+      secondaryLabel:
+        'This field will be used to filter the entities that this map overlay will have data for. This field is case sensitive. It is an extension of `config.measureLevel`. E.g. {"facility_type": "Hospital"}',
+    },
+  },
   {
     Header: 'Country Codes',
     source: 'country_codes',

packages/admin-panel/src/widgets/InputField/JsonEditor.jsx

@@ -38,7 +38,7 @@ const HelperText = styled(Typography)`
   line-height: 1.66;
 `;
 
-export const JsonEditor = ({ inputKey, label, secondaryLabel, value, onChange, stringify }) => {
+export const JsonEditor = ({ inputKey, label, helperText, value, onChange, stringify }) => {
   if (!value) {
     return null;
   }
@@ -60,7 +60,7 @@ export const JsonEditor = ({ inputKey, label, secondaryLabel, value, onChange, s
         onChange={json => onChange(inputKey, stringify ? JSON.stringify(json) : json)}
         value={editorValue}
       />
-      {secondaryLabel && <HelperText>{secondaryLabel}</HelperText>}
+      {helperText && <HelperText>{helperText}</HelperText>}
     </Container>
   );
 };
@@ -70,12 +70,12 @@ JsonEditor.propTypes = {
   label: PropTypes.string.isRequired,
   value: PropTypes.oneOfType([PropTypes.string, PropTypes.object, PropTypes.array]),
   onChange: PropTypes.func.isRequired,
-  secondaryLabel: PropTypes.string,
+  helperText: PropTypes.string,
   stringify: PropTypes.bool,
 };
 
 JsonEditor.defaultProps = {
-  secondaryLabel: null,
+  helperText: null,
   value: null,
   stringify: true,
 };

packages/admin-panel/src/widgets/InputField/index.js

@@ -2,4 +2,6 @@ import { registerInputFields } from './registerInputFields';
 
 export { InputField } from './InputField';
 
+export { JsonEditor as JsonEditorInputField } from './JsonEditor';
+
 registerInputFields();

packages/admin-panel/src/widgets/index.js

@@ -4,7 +4,7 @@
  */
 
 export { IconButton } from './IconButton';
-export { InputField } from './InputField';
+export { InputField, JsonEditorInputField } from './InputField';
 export { Navbar } from './Navbar';
 export { Tabs } from './Tabs';
 export { Header } from './Header';

packages/api-client/package.json

@@ -25,7 +25,7 @@
     "@tupaia/utils": "workspace:*",
     "lodash.pick": "^4.4.0",
     "node-fetch": "^1.7.3",
-    "qs": "^6.10.1"
+    "qs": "^6.12.0"
   },
   "devDependencies": {
     "@tupaia/types": "workspace:*"

packages/central-server/src/tests/apiV2/mapOverlayVisualisations/GETMapOverlayVisualisations.test.js

@@ -78,6 +78,7 @@ describe('GET map overlay visualisations', () => {
             reportCode: 'Modern_Report',
             dataServices: [{ isDataRegional: true }],
             legacy: false,
+            entityAttributesFilter: {},
           },
           report: {
             code: modernReport.code,

packages/database/src/migrations/20240303205520-AddEntityAttributesFilterToMapOverlaysTable-modifies-schema.js

@@ -0,0 +1,31 @@
+'use strict';
+
+var dbm;
+var type;
+var seed;
+
+/**
+ * We receive the dbmigrate dependency from dbmigrate initially.
+ * This enables us to not have to rely on NODE_PATH.
+ */
+exports.setup = function (options, seedLink) {
+  dbm = options.dbmigrate;
+  type = dbm.dataType;
+  seed = seedLink;
+};
+
+exports.up = function (db) {
+  return db.addColumn('map_overlay', 'entity_attributes_filter', {
+    type: 'jsonb',
+    defaultValue: '{}',
+    notNull: true,
+  });
+};
+
+exports.down = function (db) {
+  return db.removeColumn('map_overlay', 'entity_attributes_filter');
+};
+
+exports._meta = {
+  version: 1,
+};

packages/database/src/migrations/20240305204428-ChangeNestedDelimiterReportConfigs-modifies-data.js

@@ -0,0 +1,100 @@
+'use strict';
+
+var dbm;
+var type;
+var seed;
+
+/**
+ * We receive the dbmigrate dependency from dbmigrate initially.
+ * This enables us to not have to rely on NODE_PATH.
+ */
+exports.setup = function (options, seedLink) {
+  dbm = options.dbmigrate;
+  type = dbm.dataType;
+  seed = seedLink;
+};
+
+const getReportsWithDataSourceEntityFilter = async db => {
+  // Get all reports with dataSourceEntityFilter set
+  const reports = await db.runSql(
+    `SELECT * from report where config->>'transform' LIKE '%dataSourceEntityFilter%';`,
+  );
+  return reports.rows;
+};
+
+const reformatReportConfig = (report, initialDelimiter, newDelimiter) => {
+  const { config } = report;
+
+  const newTransforms = config.transform.map(transform => {
+    // If the transform doesn't have aggregations, return it as is
+    if (!transform.parameters?.aggregations) return transform;
+
+    // If the transform has aggregations, update the dataSourceEntityFilter to use standard jsonb syntax
+    const updatedAggregations = transform.parameters.aggregations.map(aggregation => {
+      if (aggregation.config?.dataSourceEntityFilter) {
+        return {
+          ...aggregation,
+          config: {
+            ...aggregation.config,
+            dataSourceEntityFilter: Object.entries(
+              aggregation.config.dataSourceEntityFilter,
+            ).reduce((result, [key, value]) => {
+              const newKey = key.replace(initialDelimiter, newDelimiter);
+              return {
+                ...result,
+                [newKey]: value,
+              };
+            }, {}),
+          },
+        };
+      }
+      return aggregation;
+    });
+
+    return {
+      ...transform,
+      parameters: {
+        ...transform.parameters,
+        aggregations: updatedAggregations,
+      },
+    };
+  });
+  return {
+    ...config,
+    transform: newTransforms,
+  };
+};
+
+const updateReport = async (db, report, newConfig) => {
+  await db.runSql(
+    `UPDATE report SET config = '${JSON.stringify(newConfig).replaceAll(
+      "'",
+      "''",
+    )}'::json WHERE id = '${report.id}'`,
+  );
+};
+
+exports.up = async function (db) {
+  const reportsToChange = await getReportsWithDataSourceEntityFilter(db);
+  await Promise.all(
+    reportsToChange.map(report => {
+      const newConfig = reformatReportConfig(report, '_', '->>');
+      return updateReport(db, report, newConfig);
+    }),
+  );
+  return null;
+};
+
+exports.down = async function (db) {
+  const reportsToChange = await getReportsWithDataSourceEntityFilter(db);
+  await Promise.all(
+    reportsToChange.map(report => {
+      const newConfig = reformatReportConfig(report, '->>', '_');
+      return updateReport(db, report, newConfig);
+    }),
+  );
+};
+
+exports._meta = {
+  version: 1,
+};

packages/datatrak-web-server/examples.http

@@ -30,8 +30,10 @@ content-type: {{contentType}}
 GET {{host}}/projects HTTP/1.1
 content-type: {{contentType}}
 
+### Get user
+GET {{host}}/getUser HTTP/1.1
+content-type: {{contentType}}
 
-
 ### Get survey
 GET {{host}}/surveys/TAR HTTP/1.1
 content-type: {{contentType}}

packages/datatrak-web-server/src/app/createApp.ts

@@ -45,6 +45,7 @@ import {
   GenerateLoginTokenRoute,
   GenerateLoginTokenRequest,
 } from '../routes';
+import { attachAccessPolicy } from './middleware';
 
 const {
   WEB_CONFIG_API_URL = 'http://localhost:8000/api/v1',
@@ -60,6 +61,7 @@ export async function createApp() {
     .useSessionModel(DataTrakSessionModel)
 
     .useAttachSession(attachSessionIfAvailable)
+    .use('*', attachAccessPolicy)
     .attachApiClientToContext(authHandlerProvider)
     .post<SubmitSurveyRequest>('submitSurvey', handleWith(SubmitSurveyRoute))
     .post<GenerateLoginTokenRequest>('generateLoginToken', handleWith(GenerateLoginTokenRoute))

packages/datatrak-web-server/src/app/middleware/attachAccessPolicy.ts

@@ -0,0 +1,29 @@
+/**
+ * Tupaia
+ * Copyright (c) 2017 - 2023 Beyond Essential Systems Pty Ltd
+ */
+
+import { RequestHandler } from 'express';
+import { AccessPolicyBuilder, mergeAccessPolicies } from '@tupaia/auth';
+import { AccessPolicy } from '@tupaia/access-policy';
+
+const { API_CLIENT_NAME } = process.env;
+
+export const attachAccessPolicy: RequestHandler = async (req, res, next) => {
+  const accessPolicyBuilder = new AccessPolicyBuilder(req.models);
+  const apiUser = await req.models.user.findOne({ email: API_CLIENT_NAME });
+  if (!apiUser) {
+    throw new Error('API Client not found');
+  }
+  const apiAccessPolicy = await accessPolicyBuilder.getPolicyForUser(apiUser.id);
+
+  // If we have a session, merge it with the api user
+  // Otherwise just use the api user policy
+  req.accessPolicy = new AccessPolicy(
+    req.session
+      ? mergeAccessPolicies(req.session.accessPolicy.policy, apiAccessPolicy)
+      : apiAccessPolicy,
+  );
+
+  next();
+};

packages/datatrak-web-server/src/app/middleware/index.ts

@@ -0,0 +1,6 @@
+/**
+ * Tupaia
+ * Copyright (c) 2017 - 2023 Beyond Essential Systems Pty Ltd
+ */
+
+export { attachAccessPolicy } from './attachAccessPolicy';

packages/datatrak-web-server/src/models/User.ts

@@ -3,10 +3,7 @@
  * Copyright (c) 2017 - 2023 Beyond Essential Systems Pty Ltd
  */
 
-import {
-  UserModel as BaseUserModel,
-  UserType as BaseUserType,
-} from '@tupaia/database';
+import { UserModel as BaseUserModel, UserType as BaseUserType } from '@tupaia/database';
 import { Model } from '@tupaia/server-boilerplate';
 import { UserAccount, NullableKeysToOptional } from '@tupaia/types';