diff --git a/superset-frontend/spec/javascripts/filters/utils_spec.ts b/superset-frontend/spec/javascripts/filters/utils_spec.ts index 13258a348cf6b..dcc02b7053185 100644 --- a/superset-frontend/spec/javascripts/filters/utils_spec.ts +++ b/superset-frontend/spec/javascripts/filters/utils_spec.ts @@ -18,9 +18,18 @@ */ import { + GenericDataType, + getNumberFormatter, + getTimeFormatter, + NumberFormats, + TimeFormats, +} from '@superset-ui/core'; +import { + getDataRecordFormatter, getRangeExtraFormData, getSelectExtraFormData, -} from '../../../src/filters/utils'; +} from 'src/filters/utils'; +import { FALSE_STRING, NULL_STRING, TRUE_STRING } from 'src/utils/common'; describe('Filter utils', () => { describe('getRangeExtraFormData', () => { @@ -157,4 +166,85 @@ describe('Filter utils', () => { }); }); }); + + describe('getDataRecordFormatter', () => { + it('default formatter returns expected values', () => { + const formatter = getDataRecordFormatter(); + expect(formatter(null, GenericDataType.STRING)).toEqual(NULL_STRING); + expect(formatter(null, GenericDataType.NUMERIC)).toEqual(NULL_STRING); + expect(formatter(null, GenericDataType.TEMPORAL)).toEqual(NULL_STRING); + expect(formatter(null, GenericDataType.BOOLEAN)).toEqual(NULL_STRING); + expect(formatter('foo', GenericDataType.STRING)).toEqual('foo'); + expect(formatter('foo', GenericDataType.NUMERIC)).toEqual('foo'); + expect(formatter('foo', GenericDataType.TEMPORAL)).toEqual('foo'); + expect(formatter('foo', GenericDataType.BOOLEAN)).toEqual(FALSE_STRING); + expect(formatter(true, GenericDataType.BOOLEAN)).toEqual(TRUE_STRING); + expect(formatter(false, GenericDataType.BOOLEAN)).toEqual(FALSE_STRING); + expect(formatter('true', GenericDataType.BOOLEAN)).toEqual(TRUE_STRING); + expect(formatter('false', GenericDataType.BOOLEAN)).toEqual(FALSE_STRING); + expect(formatter('TRUE', GenericDataType.BOOLEAN)).toEqual(TRUE_STRING); + expect(formatter('FALSE', GenericDataType.BOOLEAN)).toEqual(FALSE_STRING); + expect(formatter(0, GenericDataType.BOOLEAN)).toEqual(FALSE_STRING); + expect(formatter(1, GenericDataType.BOOLEAN)).toEqual(TRUE_STRING); + expect(formatter(2, GenericDataType.BOOLEAN)).toEqual(TRUE_STRING); + expect(formatter(0, GenericDataType.STRING)).toEqual('0'); + expect(formatter(0, GenericDataType.NUMERIC)).toEqual('0'); + expect(formatter(0, GenericDataType.TEMPORAL)).toEqual('0'); + expect(formatter(1234567.89, GenericDataType.STRING)).toEqual( + '1234567.89', + ); + expect(formatter(1234567.89, GenericDataType.NUMERIC)).toEqual( + '1234567.89', + ); + expect(formatter(1234567.89, GenericDataType.TEMPORAL)).toEqual( + '1234567.89', + ); + expect(formatter(1234567.89, GenericDataType.BOOLEAN)).toEqual( + TRUE_STRING, + ); + }); + + it('formatter with defined formatters returns expected values', () => { + const formatter = getDataRecordFormatter({ + timeFormatter: getTimeFormatter(TimeFormats.DATABASE_DATETIME), + numberFormatter: getNumberFormatter(NumberFormats.SMART_NUMBER), + }); + expect(formatter(null, GenericDataType.STRING)).toEqual(NULL_STRING); + expect(formatter(null, GenericDataType.NUMERIC)).toEqual(NULL_STRING); + expect(formatter(null, GenericDataType.TEMPORAL)).toEqual(NULL_STRING); + expect(formatter(null, GenericDataType.BOOLEAN)).toEqual(NULL_STRING); + expect(formatter('foo', GenericDataType.STRING)).toEqual('foo'); + expect(formatter('foo', GenericDataType.NUMERIC)).toEqual('foo'); + expect(formatter('foo', GenericDataType.TEMPORAL)).toEqual('foo'); + expect(formatter('foo', GenericDataType.BOOLEAN)).toEqual(FALSE_STRING); + expect(formatter(0, GenericDataType.STRING)).toEqual('0'); + expect(formatter(0, GenericDataType.NUMERIC)).toEqual('0'); + expect(formatter(0, GenericDataType.TEMPORAL)).toEqual( + '1970-01-01 00:00:00', + ); + expect(formatter(0, GenericDataType.BOOLEAN)).toEqual(FALSE_STRING); + expect(formatter(1234567.89, GenericDataType.STRING)).toEqual( + '1234567.89', + ); + expect(formatter(1234567.89, GenericDataType.NUMERIC)).toEqual('1.23M'); + expect(formatter(1234567.89, GenericDataType.TEMPORAL)).toEqual( + '1970-01-01 00:20:34', + ); + expect(formatter(1234567.89, GenericDataType.BOOLEAN)).toEqual( + TRUE_STRING, + ); + expect(formatter('1970-01-01 00:00:00', GenericDataType.STRING)).toEqual( + '1970-01-01 00:00:00', + ); + expect(formatter('1970-01-01 00:00:00', GenericDataType.NUMERIC)).toEqual( + '1970-01-01 00:00:00', + ); + expect(formatter('1970-01-01 00:00:00', GenericDataType.BOOLEAN)).toEqual( + FALSE_STRING, + ); + expect( + formatter('1970-01-01 00:00:00', GenericDataType.TEMPORAL), + ).toEqual('1970-01-01 00:00:00'); + }); + }); }); diff --git a/superset-frontend/src/filters/components/Select/SelectFilterPlugin.tsx b/superset-frontend/src/filters/components/Select/SelectFilterPlugin.tsx index 7238c2faa8125..c56d25a6a1f54 100644 --- a/superset-frontend/src/filters/components/Select/SelectFilterPlugin.tsx +++ b/superset-frontend/src/filters/components/Select/SelectFilterPlugin.tsx @@ -16,17 +16,48 @@ * specific language governing permissions and limitations * under the License. */ -import { Behavior, DataMask, t, tn, ensureIsArray } from '@superset-ui/core'; +import { + createMultiFormatter, + Behavior, + DataMask, + ensureIsArray, + GenericDataType, + t, + tn, +} from '@superset-ui/core'; import React, { useEffect, useState } from 'react'; import { Select } from 'src/common/components'; import { PluginFilterSelectProps } from './types'; -import { Styles, StyledSelect } from '../common'; -import { getSelectExtraFormData } from '../../utils'; +import { StyledSelect, Styles } from '../common'; +import { getDataRecordFormatter, getSelectExtraFormData } from '../../utils'; const { Option } = Select; +const timeFormatter = createMultiFormatter({ + id: 'smart_date_verbose', + label: 'Adaptive temporal formatter', + formats: { + millisecond: '%Y-%m-%d %H:%M:%S.%L', + second: '%Y-%m-%d %H:%M:%S', + minute: '%Y-%m-%d %H:%M', + hour: '%Y-%m-%d %H:%M:%M', + day: '%Y-%m-%d', + week: '%Y-%m-%d', + month: '%Y-%m-%d', + year: '%Y-%m-%d', + }, +}); + export default function PluginFilterSelect(props: PluginFilterSelectProps) { - const { data, formData, height, width, behaviors, setDataMask } = props; + const { + coltypeMap, + data, + formData, + height, + width, + behaviors, + setDataMask, + } = props; const { defaultValue, enableEmptyFilter, @@ -37,10 +68,16 @@ export default function PluginFilterSelect(props: PluginFilterSelectProps) { inputRef, } = formData; - const [values, setValues] = useState<(string | number)[]>(defaultValue ?? []); + const [values, setValues] = useState<(string | number | boolean)[]>( + defaultValue ?? [], + ); + const groupby = ensureIsArray(formData.groupby); - let { groupby = [] } = formData; - groupby = Array.isArray(groupby) ? groupby : [groupby]; + const [col] = groupby; + const datatype: GenericDataType = coltypeMap[col]; + const labelFormatter = getDataRecordFormatter({ + timeFormatter, + }); const handleChange = ( value?: (number | string)[] | number | string | null, @@ -50,7 +87,6 @@ export default function PluginFilterSelect(props: PluginFilterSelectProps) { ); setValues(resultValue); - const [col] = groupby; const emptyFilter = enableEmptyFilter && !inverseSelection && resultValue?.length === 0; @@ -104,6 +140,7 @@ export default function PluginFilterSelect(props: PluginFilterSelectProps) { {(data || []).map(row => { - const option = `${groupby.map(col => row[col])[0]}`; + const [value] = groupby.map(col => row[col]); return ( - ); })} diff --git a/superset-frontend/src/filters/components/Select/buildQuery.ts b/superset-frontend/src/filters/components/Select/buildQuery.ts index d86931eb21a34..ad3cbb6658af0 100644 --- a/superset-frontend/src/filters/components/Select/buildQuery.ts +++ b/superset-frontend/src/filters/components/Select/buildQuery.ts @@ -21,14 +21,18 @@ import { DEFAULT_FORM_DATA, PluginFilterSelectQueryFormData } from './types'; export default function buildQuery(formData: PluginFilterSelectQueryFormData) { const { sortAscending } = { ...DEFAULT_FORM_DATA, ...formData }; - return buildQueryContext(formData, baseQueryObject => [ - { - ...baseQueryObject, - apply_fetch_values_predicate: true, - groupby: baseQueryObject.columns, - orderby: sortAscending - ? baseQueryObject.columns.map(column => [column, true]) - : [], - }, - ]); + return buildQueryContext(formData, baseQueryObject => { + const { columns, filters = [] } = baseQueryObject; + return [ + { + ...baseQueryObject, + apply_fetch_values_predicate: true, + groupby: columns, + filters: filters.concat( + columns.map(column => ({ col: column, op: 'IS NOT NULL' })), + ), + orderby: sortAscending ? columns.map(column => [column, true]) : [], + }, + ]; + }); } diff --git a/superset-frontend/src/filters/components/Select/transformProps.ts b/superset-frontend/src/filters/components/Select/transformProps.ts index 74c590e104f13..59262b3eed529 100644 --- a/superset-frontend/src/filters/components/Select/transformProps.ts +++ b/superset-frontend/src/filters/components/Select/transformProps.ts @@ -16,17 +16,24 @@ * specific language governing permissions and limitations * under the License. */ -import { ChartProps } from '@superset-ui/core'; -import { DEFAULT_FORM_DATA } from './types'; +import { GenericDataType } from '@superset-ui/core'; +import { DEFAULT_FORM_DATA, PluginFilterSelectChartProps } from './types'; -export default function transformProps(chartProps: ChartProps) { +export default function transformProps( + chartProps: PluginFilterSelectChartProps, +) { const { formData, height, hooks, queriesData, width, behaviors } = chartProps; const newFormData = { ...DEFAULT_FORM_DATA, ...formData }; const { setDataMask = () => {} } = hooks; - - const { data } = queriesData[0]; + const [queryData] = queriesData; + const { colnames = [], coltypes = [], data } = queryData || []; + const coltypeMap: Record = colnames.reduce( + (accumulator, item, index) => ({ ...accumulator, [item]: coltypes[index] }), + {}, + ); return { + coltypeMap, width, behaviors, height, diff --git a/superset-frontend/src/filters/components/Select/types.ts b/superset-frontend/src/filters/components/Select/types.ts index 7ab08b5c3cbb8..ddd2c60e83e9e 100644 --- a/superset-frontend/src/filters/components/Select/types.ts +++ b/superset-frontend/src/filters/components/Select/types.ts @@ -17,10 +17,13 @@ * under the License. */ import { + ChartProps, Behavior, DataRecord, + GenericDataType, QueryFormData, SetDataMaskHook, + ChartDataResponseResult, } from '@superset-ui/core'; import { RefObject } from 'react'; import { PluginFilterStylesProps } from '../types'; @@ -39,7 +42,12 @@ export type PluginFilterSelectQueryFormData = QueryFormData & PluginFilterStylesProps & PluginFilterSelectCustomizeProps; +export interface PluginFilterSelectChartProps extends ChartProps { + queriesData: ChartDataResponseResult[]; +} + export type PluginFilterSelectProps = PluginFilterStylesProps & { + coltypeMap: Record; data: DataRecord[]; setDataMask: SetDataMaskHook; behaviors: Behavior[]; diff --git a/superset-frontend/src/filters/utils.ts b/superset-frontend/src/filters/utils.ts index 005a15169358d..186cd8aec8171 100644 --- a/superset-frontend/src/filters/utils.ts +++ b/superset-frontend/src/filters/utils.ts @@ -16,7 +16,14 @@ * specific language governing permissions and limitations * under the License. */ -import { QueryObjectFilterClause } from '@superset-ui/core'; +import { + DataRecordValue, + GenericDataType, + NumberFormatter, + QueryObjectFilterClause, + TimeFormatter, +} from '@superset-ui/core'; +import { FALSE_STRING, NULL_STRING, TRUE_STRING } from 'src/utils/common'; export const getSelectExtraFormData = ( col: string, @@ -67,3 +74,47 @@ export const getRangeExtraFormData = ( }, }; }; + +export interface DataRecordValueFormatter { + (value: DataRecordValue, dtype: GenericDataType): string; +} + +export function getDataRecordFormatter({ + timeFormatter, + numberFormatter, +}: { + timeFormatter?: TimeFormatter; + numberFormatter?: NumberFormatter; +} = {}): DataRecordValueFormatter { + return (value, dtype) => { + if (value === null || value === undefined) { + return NULL_STRING; + } + if (typeof value === 'boolean') { + return value ? TRUE_STRING : FALSE_STRING; + } + if (dtype === GenericDataType.BOOLEAN) { + try { + return JSON.parse(String(value).toLowerCase()) + ? TRUE_STRING + : FALSE_STRING; + } catch { + return FALSE_STRING; + } + } + if (typeof value === 'string') { + return value; + } + if (timeFormatter && dtype === GenericDataType.TEMPORAL) { + return timeFormatter(value); + } + if ( + numberFormatter && + typeof value === 'number' && + dtype === GenericDataType.NUMERIC + ) { + return numberFormatter(value); + } + return String(value); + }; +} diff --git a/superset-frontend/src/utils/common.js b/superset-frontend/src/utils/common.js index ea452adc53d2d..974268c58e32c 100644 --- a/superset-frontend/src/utils/common.js +++ b/superset-frontend/src/utils/common.js @@ -25,6 +25,8 @@ import { // ATTENTION: If you change any constants, make sure to also change constants.py export const NULL_STRING = ''; +export const TRUE_STRING = 'TRUE'; +export const FALSE_STRING = 'FALSE'; // moment time format strings export const SHORT_DATE = 'MMM D, YYYY'; diff --git a/superset/connectors/base/models.py b/superset/connectors/base/models.py index bda2501477ac3..99075bf8d5e0d 100644 --- a/superset/connectors/base/models.py +++ b/superset/connectors/base/models.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. import json +from datetime import datetime from enum import Enum from typing import Any, Dict, Hashable, List, Optional, Type, Union @@ -334,7 +335,7 @@ def data_for_slices(self, slices: List[Slice]) -> Dict[str, Any]: @staticmethod def filter_values_handler( values: Optional[FilterValues], - target_column_is_numeric: bool = False, + target_column_type: utils.GenericDataType, is_list_target: bool = False, ) -> Optional[FilterValues]: if values is None: @@ -342,12 +343,18 @@ def filter_values_handler( def handle_single_value(value: Optional[FilterValue]) -> Optional[FilterValue]: # backward compatibility with previous