Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Modify the import for timeline visualization to includes data source name in MDS scenario #6954

Merged
merged 1 commit into from
Jun 7, 2024
Merged

Modify the import for timeline visualization to includes data source name in MDS scenario #6954

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 114 additions & 0 deletions src/core/server/saved_objects/import/check_conflict_for_data_source.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,120 @@ describe('#checkConflictsForDataSource', () => {
);
});

/*
* Timeline test cases
zhyuanqi marked this conversation as resolved.
Show resolved Hide resolved
*/
it('will not change timeline expression when importing from datasource to different datasource', async () => {
const timelineSavedObject = createObject('visualization', 'old-datasource-id_some-object-id');
// @ts-expect-error
timelineSavedObject.attributes.visState =
'{"title":"(Timeline) Avg bytes over time","type":"timelion","aggs":[],"params":{"expression":".opensearch(opensearch_dashboards_sample_data_logs, metric=avg:bytes, timefield=@timestamp, data_source_name=newDataSource).lines(show=true).points(show=true).yaxis(label=\\"Average bytes\\")","interval":"auto"}}';
const params = setupParams({
objects: [timelineSavedObject],
ignoreRegularConflicts: true,
dataSourceId: 'some-datasource-id',
savedObjectsClient: getSavedObjectClient(),
});
const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params);

expect(checkConflictsForDataSourceResult).toEqual(
expect.objectContaining({
filteredObjects: [
{
...timelineSavedObject,
attributes: {
title: 'some-title',
visState:
'{"title":"(Timeline) Avg bytes over time","type":"timelion","aggs":[],"params":{"expression":".opensearch(opensearch_dashboards_sample_data_logs, metric=avg:bytes, timefield=@timestamp, data_source_name=newDataSource).lines(show=true).points(show=true).yaxis(label=\\"Average bytes\\")","interval":"auto"}}',
},
id: 'some-datasource-id_some-object-id',
},
],
errors: [],
importIdMap: new Map([
[
`visualization:old-datasource-id_some-object-id`,
{ id: 'some-datasource-id_some-object-id', omitOriginId: true },
],
]),
})
);
});

it('will change timeline expression when importing expression does not have a datasource name', async () => {
const timelineSavedObject = createObject('visualization', 'old-datasource-id_some-object-id');
// @ts-expect-error
timelineSavedObject.attributes.visState =
'{"title":"(Timeline) Avg bytes over time","type":"timelion","aggs":[],"params":{"expression":".opensearch(opensearch_dashboards_sample_data_logs, metric=avg:bytes, timefield=@timestamp).lines(show=true).points(show=true).yaxis(label=\\"Average bytes\\")","interval":"auto"}}';
const params = setupParams({
objects: [timelineSavedObject],
ignoreRegularConflicts: true,
dataSourceId: 'some-datasource-id',
savedObjectsClient: getSavedObjectClient(),
});
const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params);

expect(checkConflictsForDataSourceResult).toEqual(
expect.objectContaining({
filteredObjects: [
{
...timelineSavedObject,
attributes: {
title: 'some-title',
visState:
'{"title":"(Timeline) Avg bytes over time","type":"timelion","aggs":[],"params":{"expression":".opensearch(opensearch_dashboards_sample_data_logs, metric=avg:bytes, timefield=@timestamp, data_source_name=\\"some-datasource-title\\").lines(show=true).points(show=true).yaxis(label=\\"Average bytes\\")","interval":"auto"}}',
},
id: 'some-datasource-id_some-object-id',
},
],
errors: [],
importIdMap: new Map([
[
`visualization:old-datasource-id_some-object-id`,
{ id: 'some-datasource-id_some-object-id', omitOriginId: true },
],
]),
})
);
});

it('When there are multiple opensearch queries in the expression, it would go through each query and add data source name if it does not have any.', async () => {
const timelineSavedObject = createObject('visualization', 'old-datasource-id_some-object-id');
// @ts-expect-error
timelineSavedObject.attributes.visState =
'{"title":"some-other-title","type":"timelion","params":{"expression":".es(index=old-datasource-title, timefield=@timestamp, data_source_name=\\"aos 211\\"), .elasticsearch(index=old-datasource-title, timefield=@timestamp)"},"aggs":[]}';
const params = setupParams({
objects: [timelineSavedObject],
ignoreRegularConflicts: true,
dataSourceId: 'some-datasource-id',
savedObjectsClient: getSavedObjectClient(),
});
const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params);

expect(checkConflictsForDataSourceResult).toEqual(
expect.objectContaining({
filteredObjects: [
{
...timelineSavedObject,
attributes: {
title: 'some-title',
visState:
'{"title":"some-other-title","type":"timelion","params":{"expression":".es(index=old-datasource-title, timefield=@timestamp, data_source_name=\\"aos 211\\"), .elasticsearch(index=old-datasource-title, timefield=@timestamp, data_source_name=\\"some-datasource-title\\")"},"aggs":[]}',
},
id: 'some-datasource-id_some-object-id',
},
],
errors: [],
importIdMap: new Map([
[
`visualization:old-datasource-id_some-object-id`,
{ id: 'some-datasource-id_some-object-id', omitOriginId: true },
],
]),
})
);
});

/**
* TSVB test cases
*/
Expand Down
18 changes: 18 additions & 0 deletions src/core/server/saved_objects/import/check_conflict_for_data_source.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ import {
getDataSourceTitleFromId,
getUpdatedTSVBVisState,
updateDataSourceNameInVegaSpec,
extractTimelineExpression,
updateDataSourceNameInTimeline,
} from './utils';

export interface ConflictsForDataSourceParams {
Expand Down Expand Up @@ -120,6 +122,22 @@ export async function checkConflictsForDataSource({
}
}

// For timeline visualizations, update the data source name in the timeline expression
const timelineExpression = extractTimelineExpression(object);
if (!!timelineExpression && !!dataSourceTitle) {
// Get the timeline expression with the updated data source name
const modifiedExpression = updateDataSourceNameInTimeline(
timelineExpression,
dataSourceTitle
);

// @ts-expect-error
const timelineStateObject = JSON.parse(object.attributes?.visState);
timelineStateObject.params.expression = modifiedExpression;
// @ts-expect-error
object.attributes.visState = JSON.stringify(timelineStateObject);
}

if (!!dataSourceId) {
const visualizationObject = object as VisualizationObject;
const { visState, references } = getUpdatedTSVBVisState(
Expand Down
114 changes: 111 additions & 3 deletions src/core/server/saved_objects/import/create_saved_objects.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,39 @@ const getVegaMDSVisualizationObj = (id: string, dataSourceId: string) => ({
},
],
});

const getTimelineVisualizationObj = (id: string, dataSourceId: string) => ({
type: 'visualization',
id: dataSourceId ? `${dataSourceId}_${id}` : id,
attributes: {
title: 'some-other-title',
visState:
'{"title":"some-other-title","type":"timelion","params":{"expression":".es(index=old-datasource-title, timefield=@timestamp)"},"aggs":[]}',
},
references: [],
});

const getTimelineVisualizationObjWithMultipleQueries = (id: string, dataSourceId: string) => ({
type: 'visualization',
id: dataSourceId ? `${dataSourceId}_${id}` : id,
attributes: {
title: 'some-other-title',
visState:
'{"title":"some-other-title","type":"timelion","params":{"expression":".es(index=old-datasource-title, timefield=@timestamp, data_source_name=\\"aos 211\\"), .elasticsearch(index=old-datasource-title, timefield=@timestamp)"},"aggs":[]}',
},
references: [],
});

const getTimelineVisualizationObjWithDataSourceName = (id: string, dataSourceId: string) => ({
type: 'visualization',
id: dataSourceId ? `${dataSourceId}_${id}` : id,
attributes: {
title: 'some-other-title',
visState:
'{"title":"some-other-title","type":"timelion","params":{"expression":".es(index=old-datasource-title, timefield=@timestamp, data_source_name=ds1)"},"aggs":[]}',
},
references: [],
});
// non-multi-namespace types shouldn't have origin IDs, but we include test cases to ensure it's handled gracefully
// non-multi-namespace types by definition cannot result in an unresolvable conflict, so we don't include test cases for those
const importId3 = 'id-foo';
Expand Down Expand Up @@ -571,7 +604,7 @@ describe('#createSavedObjects', () => {
expect(results).toEqual(expectedResultsWithDataSource);
};

const testVegaVisualizationsWithDataSources = async (params: {
const testVegaTimelineVisualizationsWithDataSources = async (params: {
objects: SavedObject[];
expectedFilteredObjects: Array<Record<string, unknown>>;
dataSourceId?: string;
Expand Down Expand Up @@ -673,7 +706,7 @@ describe('#createSavedObjects', () => {
],
},
];
await testVegaVisualizationsWithDataSources({
await testVegaTimelineVisualizationsWithDataSources({
objects,
expectedFilteredObjects,
dataSourceId: 'some-datasource-id',
Expand All @@ -699,7 +732,82 @@ describe('#createSavedObjects', () => {
},
},
];
await testVegaVisualizationsWithDataSources({
await testVegaTimelineVisualizationsWithDataSources({
objects,
expectedFilteredObjects,
dataSourceId: 'some-datasource-id',
dataSourceTitle: 'dataSourceName',
});
});
});

describe('with a data source for timeline saved objects', () => {
test('can attach a data source name to the timeline expression', async () => {
zhyuanqi marked this conversation as resolved.
Show resolved Hide resolved
const objects = [getTimelineVisualizationObj('some-timeline-id', 'some-datasource-id')];
const expectedObject = getTimelineVisualizationObj('some-timeline-id', 'some-datasource-id');
const expectedFilteredObjects = [
{
...expectedObject,
attributes: {
title: 'some-other-title_dataSourceName',
visState:
'{"title":"some-other-title","type":"timelion","params":{"expression":".es(index=old-datasource-title, timefield=@timestamp, data_source_name=\\"dataSourceName\\")"},"aggs":[]}',
},
},
];
await testVegaTimelineVisualizationsWithDataSources({
objects,
expectedFilteredObjects,
dataSourceId: 'some-datasource-id',
dataSourceTitle: 'dataSourceName',
});
});

test('will not update the data source name in the timeline expression if no local cluster queries', async () => {
const objects = [
getTimelineVisualizationObjWithDataSourceName('some-timeline-id', 'old-datasource-id'),
];
const expectedObject = getTimelineVisualizationObjWithDataSourceName(
'some-timeline-id',
'old-datasource-id'
);
const expectedFilteredObjects = [
{
...expectedObject,
attributes: {
title: 'some-other-title_dataSourceName',
visState:
'{"title":"some-other-title","type":"timelion","params":{"expression":".es(index=old-datasource-title, timefield=@timestamp, data_source_name=ds1)"},"aggs":[]}',
},
},
];
await testVegaTimelineVisualizationsWithDataSources({
objects,
expectedFilteredObjects,
dataSourceId: 'some-datasource-id',
dataSourceTitle: 'dataSourceName',
});
});

test('When muliple opensearch query exists in expression, we can add data source name to the queries that missing data source name.', async () => {
const objects = [
getTimelineVisualizationObjWithMultipleQueries('some-timeline-id', 'some-datasource-id'),
];
const expectedObject = getTimelineVisualizationObjWithMultipleQueries(
'some-timeline-id',
'some-datasource-id'
);
const expectedFilteredObjects = [
{
...expectedObject,
attributes: {
title: 'some-other-title_dataSourceName',
visState:
'{"title":"some-other-title","type":"timelion","params":{"expression":".es(index=old-datasource-title, timefield=@timestamp, data_source_name=\\"aos 211\\"), .elasticsearch(index=old-datasource-title, timefield=@timestamp, data_source_name=\\"dataSourceName\\")"},"aggs":[]}',
},
},
];
await testVegaTimelineVisualizationsWithDataSources({
objects,
expectedFilteredObjects,
dataSourceId: 'some-datasource-id',
Expand Down
18 changes: 18 additions & 0 deletions src/core/server/saved_objects/import/create_saved_objects.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ import {
extractVegaSpecFromSavedObject,
getUpdatedTSVBVisState,
updateDataSourceNameInVegaSpec,
extractTimelineExpression,
updateDataSourceNameInTimeline,
} from './utils';

interface CreateSavedObjectsParams<T> {
Expand Down Expand Up @@ -130,6 +132,22 @@ export const createSavedObjects = async <T>({
});
}

// Some visualization types will need special modifications, like TSVB visualizations
const timelineExpression = extractTimelineExpression(object);
if (!!timelineExpression && !!dataSourceTitle) {
// Get the timeline expression with the updated data source name
const modifiedExpression = updateDataSourceNameInTimeline(
timelineExpression,
dataSourceTitle
);

// @ts-expect-error
const timelineStateObject = JSON.parse(object.attributes?.visState);
timelineStateObject.params.expression = modifiedExpression;
// @ts-expect-error
object.attributes.visState = JSON.stringify(timelineStateObject);
}

zhyuanqi marked this conversation as resolved.
Show resolved Hide resolved
const visualizationObject = object as VisualizationObject;
const { visState, references } = getUpdatedTSVBVisState(
visualizationObject,
Expand Down
3 changes: 2 additions & 1 deletion src/core/server/saved_objects/import/import_saved_objects.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,8 @@ export async function importSavedObjectsFromStream({
supportedTypes,
dataSourceId,
});
// if not enable data_source, throw error early
// if dataSource is not enabled, but object type is data-source, or saved object id contains datasource id
// return unsupported type error
if (!dataSourceEnabled) {
const notSupportedErrors: SavedObjectsImportError[] = collectSavedObjectsResult.collectedObjects.reduce(
(errors: SavedObjectsImportError[], obj) => {
Expand Down
Loading