Skip to content

Commit

Permalink
[ML] Add support for percentiles aggregation to Transform wizard (#60763
Browse files Browse the repository at this point in the history 
)

* [ML] Add support for percentiles aggregation to Transform wizard

* [ML] Fix type error and comments from review

* [ML] Remove unused function

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
  • Loading branch information
@peteharverson @elasticmachine
peteharverson and elasticmachine authored Mar 24, 2020
1 parent a93efed commit 57f9a9f
Show file tree
Hide file tree
Showing 6 changed files with 217 additions and 8 deletions.
2 changes: 2 additions & 0 deletions x-pack/plugins/transform/public/app/common/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ export { GetTransformsResponse, PreviewData, PreviewMappings } from './pivot_pre
export {
getEsAggFromAggConfig,
isPivotAggsConfigWithUiSupport,
isPivotAggsConfigPercentiles,
PERCENTILES_AGG_DEFAULT_PERCENTS,
PivotAgg,
PivotAggDict,
PivotAggsConfig,
Expand Down
22 changes: 21 additions & 1 deletion x-pack/plugins/transform/public/app/common/pivot_aggs.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,13 @@ export enum PIVOT_SUPPORTED_AGGS {
CARDINALITY = 'cardinality',
MAX = 'max',
MIN = 'min',
PERCENTILES = 'percentiles',
SUM = 'sum',
VALUE_COUNT = 'value_count',
}

export const PERCENTILES_AGG_DEFAULT_PERCENTS = [1, 5, 25, 50, 75, 95, 99];

export const pivotAggsFieldSupport = {
[KBN_FIELD_TYPES.ATTACHMENT]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
[KBN_FIELD_TYPES.BOOLEAN]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
Expand All @@ -36,6 +39,7 @@ export const pivotAggsFieldSupport = {
PIVOT_SUPPORTED_AGGS.CARDINALITY,
PIVOT_SUPPORTED_AGGS.MAX,
PIVOT_SUPPORTED_AGGS.MIN,
PIVOT_SUPPORTED_AGGS.PERCENTILES,
PIVOT_SUPPORTED_AGGS.SUM,
PIVOT_SUPPORTED_AGGS.VALUE_COUNT,
],
Expand All @@ -60,10 +64,17 @@ export interface PivotAggsConfigBase {
dropDownName: string;
}

export interface PivotAggsConfigWithUiSupport extends PivotAggsConfigBase {
interface PivotAggsConfigWithUiBase extends PivotAggsConfigBase {
field: EsFieldName;
}

interface PivotAggsConfigPercentiles extends PivotAggsConfigWithUiBase {
agg: PIVOT_SUPPORTED_AGGS.PERCENTILES;
percents: number[];
}

export type PivotAggsConfigWithUiSupport = PivotAggsConfigWithUiBase | PivotAggsConfigPercentiles;

export function isPivotAggsConfigWithUiSupport(arg: any): arg is PivotAggsConfigWithUiSupport {
return (
arg.hasOwnProperty('agg') &&
Expand All @@ -74,6 +85,15 @@ export function isPivotAggsConfigWithUiSupport(arg: any): arg is PivotAggsConfig
);
}

export function isPivotAggsConfigPercentiles(arg: any): arg is PivotAggsConfigPercentiles {
return (
arg.hasOwnProperty('agg') &&
arg.hasOwnProperty('field') &&
arg.hasOwnProperty('percents') &&
arg.agg === PIVOT_SUPPORTED_AGGS.PERCENTILES
);
}

export type PivotAggsConfig = PivotAggsConfigBase | PivotAggsConfigWithUiSupport;

export type PivotAggsConfigWithUiSupportDict = Dictionary<PivotAggsConfigWithUiSupport>;
Expand Down
104 changes: 98 additions & 6 deletions ...ansform/public/app/sections/create_transform/components/aggregation_list/popover_form.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,10 @@ import { dictionaryToArray } from '../../../../../../common/types/common';
import {
AggName,
isAggName,
isPivotAggsConfigPercentiles,
isPivotAggsConfigWithUiSupport,
getEsAggFromAggConfig,
PERCENTILES_AGG_DEFAULT_PERCENTS,
PivotAggsConfig,
PivotAggsConfigWithUiSupportDict,
PIVOT_SUPPORTED_AGGS,
Expand All @@ -40,6 +42,33 @@ interface Props {
onChange(d: PivotAggsConfig): void;
}

function getDefaultPercents(defaultData: PivotAggsConfig): number[] | undefined {
if (isPivotAggsConfigPercentiles(defaultData)) {
return defaultData.percents;
}
}

function parsePercentsInput(inputValue: string | undefined) {
if (inputValue !== undefined) {
const strVals: string[] = inputValue.split(',');
const percents: number[] = [];
for (const str of strVals) {
if (str.trim().length > 0 && isNaN(str as any) === false) {
const val = Number(str);
if (val >= 0 && val <= 100) {
percents.push(val);
} else {
return [];
}
}
}

return percents;
}

return [];
}

export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onChange, options }) => {
const isUnsupportedAgg = !isPivotAggsConfigWithUiSupport(defaultData);

Expand All @@ -48,10 +77,45 @@ export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onCha
const [field, setField] = useState(
isPivotAggsConfigWithUiSupport(defaultData) ? defaultData.field : ''
);
const [percents, setPercents] = useState(getDefaultPercents(defaultData));

const availableFields: SelectOption[] = [];
const availableAggs: SelectOption[] = [];

function updateAgg(aggVal: PIVOT_SUPPORTED_AGGS) {
setAgg(aggVal);
if (aggVal === PIVOT_SUPPORTED_AGGS.PERCENTILES && percents === undefined) {
setPercents(PERCENTILES_AGG_DEFAULT_PERCENTS);
}
}

function updatePercents(inputValue: string) {
setPercents(parsePercentsInput(inputValue));
}

function getUpdatedItem(): PivotAggsConfig {
let updatedItem: PivotAggsConfig;

if (agg !== PIVOT_SUPPORTED_AGGS.PERCENTILES) {
updatedItem = {
agg,
aggName,
field,
dropDownName: defaultData.dropDownName,
};
} else {
updatedItem = {
agg,
aggName,
field,
dropDownName: defaultData.dropDownName,
percents,
};
}

return updatedItem;
}

if (!isUnsupportedAgg) {
const optionsArr = dictionaryToArray(options);
optionsArr
Expand Down Expand Up @@ -83,7 +147,18 @@ export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onCha
});
}

const formValid = validAggName;
let percentsText;
if (percents !== undefined) {
percentsText = percents.toString();
}

const validPercents =
agg === PIVOT_SUPPORTED_AGGS.PERCENTILES && parsePercentsInput(percentsText).length > 0;

let formValid = validAggName;
if (formValid && agg === PIVOT_SUPPORTED_AGGS.PERCENTILES) {
formValid = validPercents;
}

return (
<EuiForm style={{ width: '300px' }}>
Expand Down Expand Up @@ -117,7 +192,7 @@ export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onCha
<EuiSelect
options={availableAggs}
value={agg}
onChange={e => setAgg(e.target.value as PIVOT_SUPPORTED_AGGS)}
onChange={e => updateAgg(e.target.value as PIVOT_SUPPORTED_AGGS)}
/>
</EuiFormRow>
)}
Expand All @@ -134,6 +209,26 @@ export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onCha
/>
</EuiFormRow>
)}
{agg === PIVOT_SUPPORTED_AGGS.PERCENTILES && (
<EuiFormRow
label={i18n.translate('xpack.transform.agg.popoverForm.percentsLabel', {
defaultMessage: 'Percents',
})}
error={
!validPercents && [
i18n.translate('xpack.transform.groupBy.popoverForm.intervalPercents', {
defaultMessage: 'Enter a comma-separated list of percentiles',
}),
]
}
isInvalid={!validPercents}
>
<EuiFieldText
defaultValue={percentsText}
onChange={e => updatePercents(e.target.value)}
/>
</EuiFormRow>
)}
{isUnsupportedAgg && (
<EuiCodeEditor
mode="json"
Expand All @@ -147,10 +242,7 @@ export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onCha
/>
)}
<EuiFormRow hasEmptyLabelSpace>
<EuiButton
isDisabled={!formValid}
onClick={() => onChange({ ...defaultData, aggName, agg, field })}
>
<EuiButton isDisabled={!formValid} onClick={() => onChange(getUpdatedItem())}>
{i18n.translate('xpack.transform.agg.popoverForm.submitButtonLabel', {
defaultMessage: 'Apply',
})}
Expand Down
8 changes: 8 additions & 0 deletions ...gins/transform/public/app/sections/create_transform/components/step_define/common.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ describe('Transform: Define Pivot Common', () => {
{ label: 'cardinality( the-f[i]e>ld )' },
{ label: 'max( the-f[i]e>ld )' },
{ label: 'min( the-f[i]e>ld )' },
{ label: 'percentiles( the-f[i]e>ld )' },
{ label: 'sum( the-f[i]e>ld )' },
{ label: 'value_count( the-f[i]e>ld )' },
],
Expand Down Expand Up @@ -67,6 +68,13 @@ describe('Transform: Define Pivot Common', () => {
aggName: 'the-field.min',
dropDownName: 'min( the-f[i]e>ld )',
},
'percentiles( the-f[i]e>ld )': {
agg: 'percentiles',
field: ' the-f[i]e>ld ',
aggName: 'the-field.percentiles',
dropDownName: 'percentiles( the-f[i]e>ld )',
percents: [1, 5, 25, 50, 75, 95, 99],
},
'sum( the-f[i]e>ld )': {
agg: 'sum',
field: ' the-f[i]e>ld ',
Expand Down
35 changes: 34 additions & 1 deletion ...k/plugins/transform/public/app/sections/create_transform/components/step_define/common.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,13 @@ import {
DropDownOption,
EsFieldName,
GroupByConfigWithUiSupport,
PERCENTILES_AGG_DEFAULT_PERCENTS,
PivotAggsConfigWithUiSupport,
PivotAggsConfigWithUiSupportDict,
pivotAggsFieldSupport,
PivotGroupByConfigWithUiSupportDict,
pivotGroupByFieldSupport,
PIVOT_SUPPORTED_AGGS,
PIVOT_SUPPORTED_GROUP_BY_AGGS,
} from '../../../../common';

Expand Down Expand Up @@ -57,6 +60,31 @@ function getDefaultGroupByConfig(
}
}

function getDefaultAggregationConfig(
aggName: string,
dropDownName: string,
fieldName: EsFieldName,
agg: PIVOT_SUPPORTED_AGGS
): PivotAggsConfigWithUiSupport {
switch (agg) {
case PIVOT_SUPPORTED_AGGS.PERCENTILES:
return {
agg,
aggName,
dropDownName,
field: fieldName,
percents: PERCENTILES_AGG_DEFAULT_PERCENTS,
};
default:
return {
agg,
aggName,
dropDownName,
field: fieldName,
};
}
}

const illegalEsAggNameChars = /[[\]>]/g;

export function getPivotDropdownOptions(indexPattern: IndexPattern) {
Expand Down Expand Up @@ -105,7 +133,12 @@ export function getPivotDropdownOptions(indexPattern: IndexPattern) {
// Option name in the dropdown for the aggregation is in the form of `sum(fieldname)`.
const dropDownName = `${agg}(${field.name})`;
aggOption.options.push({ label: dropDownName });
aggOptionsData[dropDownName] = { agg, field: field.name, aggName, dropDownName };
aggOptionsData[dropDownName] = getDefaultAggregationConfig(
aggName,
dropDownName,
field.name,
agg
);
});
}
aggOptions.push(aggOption);
Expand Down
54 changes: 54 additions & 0 deletions x-pack/test/functional/apps/transform/creation_index_pattern.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,60 @@ export default function({ getService }: FtrProviderContext) {
},
},
},
{
suiteTitle: 'batch transform with terms group and percentiles agg',
source: 'ecommerce',
groupByEntries: [
{
identifier: 'terms(geoip.country_iso_code)',
label: 'geoip.country_iso_code',
} as GroupByEntry,
],
aggregationEntries: [
{
identifier: 'percentiles(products.base_price)',
label: 'products.base_price.percentiles',
},
],
transformId: `ec_2_${Date.now()}`,
transformDescription:
'ecommerce batch transform with group by terms(geoip.country_iso_code) and aggregation percentiles(products.base_price)',
get destinationIndex(): string {
return `user-${this.transformId}`;
},
expected: {
pivotAdvancedEditorValue: {
group_by: {
'geoip.country_iso_code': {
terms: {
field: 'geoip.country_iso_code',
},
},
},
aggregations: {
'products.base_price.percentiles': {
percentiles: {
field: 'products.base_price',
percents: [1, 5, 25, 50, 75, 95, 99],
},
},
},
},
pivotPreview: {
column: 0,
values: ['AE', 'CO', 'EG', 'FR', 'GB'],
},
row: {
status: 'stopped',
mode: 'batch',
progress: '100',
},
sourcePreview: {
columns: 45,
rows: 5,
},
},
},
];

for (const testData of testDataList) {
Expand Down

0 comments on commit 57f9a9f

Please sign in to comment.