RN-857: Set missing parameter values to NULL in SQL data-tables (#4469)

beyondessential · Apr 18, 2023 · ef5011b · ef5011b
1 parent 8a890f0
commit ef5011b
Show file tree

Hide file tree

Showing 18 changed files with 91 additions and 74 deletions.
diff --git a/packages/admin-panel/src/dataTables/DataTableEditFields.js b/packages/admin-panel/src/dataTables/DataTableEditFields.js
@@ -91,8 +91,6 @@ export const DataTableEditFields = React.memo(
       error,
     } = useDataTablePreview({
       previewConfig: recordData,
-      builtInParams,
-      additionalParams,
       runtimeParams,
       onSettled: () => {
         setFetchDisabled(false);

diff --git a/packages/admin-panel/src/dataTables/components/PreviewFilters/filters/ArrayField.js b/packages/admin-panel/src/dataTables/components/PreviewFilters/filters/ArrayField.js
@@ -8,10 +8,11 @@ import PropTypes from 'prop-types';
 
 import { ParameterType } from '../../editing';
 import { Autocomplete } from '../../../../autocomplete';
+import { getArrayFieldValue } from './utils';
 
 export const ArrayField = ({ name, onChange, value, config }) => {
   const [searchTerm, setSearchTerm] = useState('');
-  const { defaultValue = [] } = config || {};
+  const { defaultValue } = config || {};
   const placeholder = value.length === 0 ? JSON.stringify(defaultValue) : 'type to add more';
 
   return (
@@ -20,9 +21,7 @@ export const ArrayField = ({ name, onChange, value, config }) => {
       value={value}
       label={name}
       options={[]}
-      onChangeSelection={(event, selectedValues) => {
-        onChange(selectedValues);
-      }}
+      onChangeSelection={(event, selectedValues) => onChange(getArrayFieldValue(selectedValues))}
       getOptionSelected={(option, selected) => {
         return option === selected;
       }}

diff --git a/packages/admin-panel/src/dataTables/components/PreviewFilters/filters/BooleanField.js b/packages/admin-panel/src/dataTables/components/PreviewFilters/filters/BooleanField.js
@@ -18,24 +18,27 @@ const getBooleanValue = value => {
     return value;
   }
 
-  return '';
+  return undefined;
 };
 
 export const BooleanField = ({ name, value, onChange, config }) => {
-  const defaultValue = getBooleanValue(config?.hasDefaultValue && config?.defaultValue);
+  const defaultValue =
+    typeof config?.defaultValue === 'boolean' ? `${config?.defaultValue}` : undefined;
+
   const booleanValue = getBooleanValue(value);
 
   return (
     <Select
       id={name}
-      placeholder={defaultValue.toString()}
+      placeholder={defaultValue}
       options={[
+        { label: 'none', value: undefined },
         { label: 'true', value: true },
         { label: 'false', value: false },
       ]}
       name={name}
       label={name}
-      value={typeof booleanValue === 'boolean' ? booleanValue : defaultValue}
+      value={booleanValue}
       onChange={event => {
         onChange(event.target.value);
       }}

diff --git a/...ges/admin-panel/src/dataTables/components/PreviewFilters/filters/DataElementCodesField.js b/...ges/admin-panel/src/dataTables/components/PreviewFilters/filters/DataElementCodesField.js
@@ -9,6 +9,7 @@ import PropTypes from 'prop-types';
 
 import { ParameterType } from '../../editing';
 import { ReduxAutocomplete } from '../../../../autocomplete';
+import { getArrayFieldValue } from './utils';
 
 export const DataElementCodesField = ({ name, onChange }) => {
   return (
@@ -17,7 +18,7 @@ export const DataElementCodesField = ({ name, onChange }) => {
       key="dataElementCodes"
       inputKey="dataElementCodesField"
       label={name}
-      onChange={selectedValues => onChange(selectedValues)}
+      onChange={selectedValues => onChange(getArrayFieldValue(selectedValues))}
       id="inputField-dataElementCodes"
       reduxId="dataTableEditFields-dataElementCodesField"
       endpoint="dataElements"

diff --git a/packages/admin-panel/src/dataTables/components/PreviewFilters/filters/DataGroupCodeField.js b/packages/admin-panel/src/dataTables/components/PreviewFilters/filters/DataGroupCodeField.js
@@ -9,14 +9,15 @@ import PropTypes from 'prop-types';
 
 import { ParameterType } from '../../editing';
 import { ReduxAutocomplete } from '../../../../autocomplete';
+import { getTextFieldValue } from './utils';
 
 export const DataGroupCodeField = ({ name, onChange }) => {
   return (
     <ReduxAutocomplete
       key="dataGroupCode"
       inputKey="dataGroupCodeField"
       label={name}
-      onChange={selectedValues => onChange(selectedValues)}
+      onChange={selectedValue => onChange(getTextFieldValue(selectedValue))}
       id="inputField-dataGroupCode"
       reduxId="dataTableEditFields-dataGroupCodeField"
       endpoint="dataGroups"

diff --git a/packages/admin-panel/src/dataTables/components/PreviewFilters/filters/DatePicker.js b/packages/admin-panel/src/dataTables/components/PreviewFilters/filters/DatePicker.js
@@ -30,7 +30,7 @@ const getDateValue = value => {
 
 export const DatePicker = ({ name, value, onChange, config }) => {
   const dateValue = getDateValue(value);
-  const defaultDateValue = getDateValue(config?.hasDefaultValue && config?.defaultValue);
+  const defaultDateValue = getDateValue(config?.defaultValue);
   // Convert date to UTC as server uses UTC timezone
   const onChangeDate = localDate => {
     if (!localDate) {

diff --git a/packages/admin-panel/src/dataTables/components/PreviewFilters/filters/HierarchyField.js b/packages/admin-panel/src/dataTables/components/PreviewFilters/filters/HierarchyField.js
@@ -10,6 +10,7 @@ import { Autocomplete } from '@tupaia/ui-components';
 
 import { ParameterType } from '../../editing';
 import { useProjects } from '../../../../VizBuilderApp/api';
+import { getTextFieldValue } from './utils';
 
 export const HierarchyField = ({ name, value, onChange, config }) => {
   const { data: hierarchies = [], isLoading } = useProjects();
@@ -23,7 +24,7 @@ export const HierarchyField = ({ name, value, onChange, config }) => {
       placeholder={config?.defaultValue}
       options={hierarchies.map(p => p['project.code'])}
       disabled={isLoading}
-      onChange={(event, selectedValue) => onChange(selectedValue)}
+      onChange={(event, selectedValue) => onChange(getTextFieldValue(selectedValue))}
     />
   );
 };

diff --git a/packages/admin-panel/src/dataTables/components/PreviewFilters/filters/NumberField.js b/packages/admin-panel/src/dataTables/components/PreviewFilters/filters/NumberField.js
@@ -17,7 +17,7 @@ const getNumberValue = value => {
 };
 
 export const NumberField = ({ id, name, value, onChange, config }) => {
-  const defaultValue = getNumberValue(config?.hasDefaultValue && config?.defaultValue);
+  const defaultValue = getNumberValue(config?.defaultValue);
   const numberValue = getNumberValue(value);
 
   return (
@@ -29,7 +29,12 @@ export const NumberField = ({ id, name, value, onChange, config }) => {
       label={name}
       value={numberValue}
       onChange={event => {
-        onChange(+event.target.value);
+        const { value: newValue } = event.target;
+        if (newValue === '') {
+          onChange(undefined);
+        } else {
+          onChange(+newValue);
+        }
       }}
     />
   );

diff --git a/...dmin-panel/src/dataTables/components/PreviewFilters/filters/OrganisationUnitCodesField.js b/...dmin-panel/src/dataTables/components/PreviewFilters/filters/OrganisationUnitCodesField.js
@@ -10,6 +10,7 @@ import PropTypes from 'prop-types';
 import { ParameterType } from '../../editing';
 import { useLocations } from '../../../../VizBuilderApp/api';
 import { Autocomplete } from '../../../../autocomplete';
+import { getArrayFieldValue } from './utils';
 
 export const OrganisationUnitCodesField = ({ name, onChange, runtimeParams }) => {
   const { hierarchy = 'explore' } = runtimeParams;
@@ -35,7 +36,7 @@ export const OrganisationUnitCodesField = ({ name, onChange, runtimeParams }) =>
       isLoading={isLoading}
       onChangeSelection={(event, selectedValues) => {
         setSelectedOptions(selectedValues);
-        onChange(selectedValues.map(v => v.code));
+        onChange(getArrayFieldValue(selectedValues.map(v => v.code)));
       }}
       onChangeSearchTerm={setSearchTerm}
       searchTerm={searchTerm}

diff --git a/packages/admin-panel/src/dataTables/components/PreviewFilters/filters/TextField.js b/packages/admin-panel/src/dataTables/components/PreviewFilters/filters/TextField.js
@@ -7,9 +7,10 @@ import PropTypes from 'prop-types';
 import React from 'react';
 import { TextField as BaseTextField } from '@tupaia/ui-components';
 import { ParameterType } from '../../editing';
+import { getTextFieldValue } from './utils';
 
 export const TextField = ({ name, value, onChange, config }) => {
-  const defaultValue = config?.hasDefaultValue ? config?.defaultValue : '';
+  const defaultValue = config?.defaultValue || '';
 
   return (
     <BaseTextField
@@ -19,7 +20,7 @@ export const TextField = ({ name, value, onChange, config }) => {
       label={name}
       value={value}
       onChange={event => {
-        onChange(event.target.value);
+        onChange(getTextFieldValue(event.target.value));
       }}
     />
   );

diff --git a/packages/admin-panel/src/dataTables/components/PreviewFilters/filters/utils.js b/packages/admin-panel/src/dataTables/components/PreviewFilters/filters/utils.js
@@ -0,0 +1,8 @@
+/**
+ * Tupaia
+ * Copyright (c) 2017 - 2023 Beyond Essential Systems Pty Ltd
+ */
+
+export const getArrayFieldValue = array => (array.length === 0 ? undefined : array);
+
+export const getTextFieldValue = text => (text === '' || text === null ? undefined : text);
diff --git a/packages/admin-panel/src/dataTables/components/editing/ParameterItem.js b/packages/admin-panel/src/dataTables/components/editing/ParameterItem.js
@@ -4,7 +4,7 @@
  */
 
 import PropTypes from 'prop-types';
-import React from 'react';
+import React, { useState } from 'react';
 import styled from 'styled-components';
 import { Divider as BaseDivider } from '@material-ui/core';
 import Grid from '@material-ui/core/Grid';
@@ -42,18 +42,9 @@ const DeleteOutlinedIcon = styled(BaseDeleteOutlinedIcon)`
 `;
 
 export const ParameterItem = props => {
-  const {
-    id,
-    name,
-    type,
-    hasDefaultValue,
-    defaultValue,
-    hasError,
-    error,
-    onDelete,
-    onChange,
-  } = props;
+  const { id, name, type, defaultValue, hasError, error, onDelete, onChange } = props;
 
+  const [hasDefaultValue, setHasDefaultValue] = useState(defaultValue !== undefined);
   const option = FilterTypeOptions.find(t => t.value === type) || {};
   const { FilterComponent } = option;
 
@@ -97,7 +88,7 @@ export const ParameterItem = props => {
             color="primary"
             checked={hasDefaultValue}
             onChange={event => {
-              onChange(id, 'hasDefaultValue', event.target.checked);
+              setHasDefaultValue(event.target.checked);
               if (!event.target.checked) {
                 onChange(id, 'defaultValue', undefined);
               }
@@ -129,7 +120,6 @@ export const ParameterItem = props => {
 ParameterItem.propTypes = {
   defaultValue: DefaultValueType,
   error: PropTypes.string,
-  hasDefaultValue: PropTypes.bool,
   hasError: PropTypes.bool,
   id: PropTypes.string.isRequired,
   name: PropTypes.string,
@@ -141,7 +131,6 @@ ParameterItem.propTypes = {
 ParameterItem.defaultProps = {
   defaultValue: undefined,
   error: '',
-  hasDefaultValue: false,
   hasError: false,
   name: '',
   type: '',

diff --git a/packages/admin-panel/src/dataTables/query/useDataTablePreview.js b/packages/admin-panel/src/dataTables/query/useDataTablePreview.js
@@ -6,50 +6,25 @@ import { useQuery } from 'react-query';
 import { post } from '../../VizBuilderApp/api/api';
 import { DEFAULT_REACT_QUERY_OPTIONS } from '../../VizBuilderApp/api/constants';
 
-const assignDefaultValueToRuntimeParam = ({ builtInParams, additionalParams, runtimeParams }) => {
-  const newRuntimeParams = { ...runtimeParams };
-
-  [...builtInParams, ...additionalParams].forEach(p => {
-    const runtimeParameterValue = runtimeParams[p.name];
-    if (
-      p?.config?.hasDefaultValue &&
-      (runtimeParameterValue === undefined ||
-        runtimeParameterValue === null ||
-        runtimeParameterValue === '' ||
-        (Array.isArray(runtimeParameterValue) && runtimeParameterValue.length === 0))
-    ) {
-      newRuntimeParams[p.name] = p?.config?.defaultValue;
-    }
-  });
-
-  const trimmedRuntimeParams = Object.fromEntries(
-    Object.entries(newRuntimeParams).map(([key, value]) => {
+const trimWhitespace = object => {
+  const trimmedObject = Object.fromEntries(
+    Object.entries(object).map(([key, value]) => {
       return [key, typeof value === 'string' ? value.trim() : value];
     }),
   );
 
-  return trimmedRuntimeParams;
+  return trimmedObject;
 };
 
-export const useDataTablePreview = ({
-  previewConfig,
-  builtInParams,
-  additionalParams,
-  runtimeParams,
-  onSettled,
-}) =>
+export const useDataTablePreview = ({ previewConfig, runtimeParams, onSettled }) =>
   useQuery(
     ['fetchDataTablePreviewData'],
     async () => {
       const response = await post('fetchDataTablePreviewData', {
         data: {
           previewConfig: {
             ...previewConfig,
-            runtimeParams: assignDefaultValueToRuntimeParam({
-              builtInParams,
-              additionalParams,
-              runtimeParams,
-            }),
+            runtimeParams: trimWhitespace(runtimeParams),
           },
         },
       });

diff --git a/packages/admin-panel/src/dataTables/query/useFetchDataTableBuiltInParams.js b/packages/admin-panel/src/dataTables/query/useFetchDataTableBuiltInParams.js
@@ -21,7 +21,6 @@ export const useFetchDataTableBuiltInParams = dataTableType =>
       return parameters.map((p, index) => ({
         ...p,
         id: `builtIn_parameter_${index}`,
-        config: { hasDefaultValue: !!p.config?.defaultValue, ...p.config },
       }));
     },
     {

diff --git a/packages/admin-panel/src/dataTables/useParams.js b/packages/admin-panel/src/dataTables/useParams.js
@@ -9,7 +9,7 @@ import generateId from 'uuid/v1';
 import { useFetchDataTableBuiltInParams } from './query/useFetchDataTableBuiltInParams';
 import { useRuntimeParams } from './useRuntimeParams';
 
-const convertRecordDataToFronendConfig = (additionalParams = []) => {
+const convertRecordDataToFrontendConfig = (additionalParams = []) => {
   return additionalParams.map(p => ({
     ...p,
     id: generateId(),
@@ -28,7 +28,7 @@ export const useParams = ({ recordData, onEditField }) => {
   });
 
   useEffect(() => {
-    const newAdditionalParams = convertRecordDataToFronendConfig(config.additionalParams);
+    const newAdditionalParams = convertRecordDataToFrontendConfig(config.additionalParams);
     setAdditionalParams(newAdditionalParams);
     const defaultRuntimeParams = Object.fromEntries(
       [...newAdditionalParams, ...builtInParams].map(p => [p.name, undefined]),

diff --git a/...ges/data-table-server/src/__tests__/dataTableService/services/SqlDataTableService.test.ts b/...ges/data-table-server/src/__tests__/dataTableService/services/SqlDataTableService.test.ts
@@ -69,12 +69,33 @@ describe('SqlDataTableService', () => {
         .setConfig({
           sql,
           externalDatabaseConnectionCode: externalDatabaseConnection.code,
-          additionalParameters: [{ name: 'entityCode', config: { type: 'string' } }],
+          additionalParams: [{ name: 'entityCode', config: { type: 'string' } }],
         })
         .build();
 
       const [result] = await service.fetchData(params);
       expect(result).toEqual({ sql, params });
     });
+
+    it('converts params with missing values to nulls', async () => {
+      const sql =
+        'SELECT * FROM analytics WHERE entity_code = :entityCode AND hierarchy = :hierarchy';
+      const params = { entityCode: 'TO' };
+      const service = new DataTableServiceBuilder()
+        .setServiceType('sql')
+        .setContext({ models: modelsStub })
+        .setConfig({
+          sql,
+          externalDatabaseConnectionCode: externalDatabaseConnection.code,
+          additionalParams: [
+            { name: 'entityCode', config: { type: 'string' } },
+            { name: 'hierarchy', config: { type: 'string' } },
+          ],
+        })
+        .build();
+
+      const [result] = await service.fetchData(params);
+      expect(result).toEqual({ sql, params: { ...params, hierarchy: null } });
+    });
   });
 });