diff --git a/CHANGELOG.md b/CHANGELOG.md index eecee0c09503..a870385db189 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -346,6 +346,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - [Dashboard De-Angular] Add more unit tests for utils folder ([#4641](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4641)) - [Dashboard De-Angular] Add unit tests for dashboard_listing and dashboard_top_nav ([#4640](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4640)) - Optimize `augment-vis` saved obj searching by adding arg to saved obj client ([#4595](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4595)) +- Change SavedObjects' Import API to allow selecting a data source when uploading files ([#5777](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5777)) ### 🐛 Bug Fixes diff --git a/config/opensearch_dashboards.yml b/config/opensearch_dashboards.yml index 189344ff1481..e1adad20eb33 100644 --- a/config/opensearch_dashboards.yml +++ b/config/opensearch_dashboards.yml @@ -274,4 +274,4 @@ # opensearchDashboards.survey.url: "https://survey.opensearch.org" # Set the value of this setting to true to enable plugin augmentation on Dashboard -# vis_augmenter.pluginAugmentationEnabled: true +# vis_augmenter.pluginAugmentationEnabled: true \ No newline at end of file diff --git a/src/core/server/saved_objects/import/check_conflict_for_data_source.test.ts b/src/core/server/saved_objects/import/check_conflict_for_data_source.test.ts new file mode 100644 index 000000000000..4bf3a8110454 --- /dev/null +++ b/src/core/server/saved_objects/import/check_conflict_for_data_source.test.ts @@ -0,0 +1,150 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { mockUuidv4 } from './__mocks__'; +import { SavedObjectReference, SavedObjectsImportRetry } from 'opensearch-dashboards/public'; +import { SavedObject } from '../types'; +import { SavedObjectsErrorHelpers } from '..'; +import { + checkConflictsForDataSource, + ConflictsForDataSourceParams, +} from './check_conflict_for_data_source'; + +type SavedObjectType = SavedObject<{ title?: string }>; + +/** + * Function to create a realistic-looking import object given a type and ID + */ +const createObject = (type: string, id: string): SavedObjectType => ({ + type, + id, + attributes: { title: 'some-title' }, + references: (Symbol() as unknown) as SavedObjectReference[], +}); + +const getResultMock = { + conflict: (type: string, id: string) => { + const error = SavedObjectsErrorHelpers.createConflictError(type, id).output.payload; + return { type, id, error }; + }, + unresolvableConflict: (type: string, id: string) => { + const conflictMock = getResultMock.conflict(type, id); + const metadata = { isNotOverwritable: true }; + return { ...conflictMock, error: { ...conflictMock.error, metadata } }; + }, + invalidType: (type: string, id: string) => { + const error = SavedObjectsErrorHelpers.createUnsupportedTypeError(type).output.payload; + return { type, id, error }; + }, +}; + +/** + * Create a variety of different objects to exercise different import / result scenarios + */ +const dataSourceObj = createObject('data-source', 'data-source-id-1'); // -> data-source type, no need to add in the filteredObjects +const dataSourceObj1 = createObject('type-1', 'ds_id-1'); // -> object with data source id +const dataSourceObj2 = createObject('type-2', 'ds_id-2'); // -> object with data source id +const objectsWithDataSource = [dataSourceObj, dataSourceObj1, dataSourceObj2]; +const dataSourceObj1Error = getResultMock.conflict(dataSourceObj1.type, dataSourceObj1.id); + +describe('#checkConflictsForDataSource', () => { + const setupParams = (partial: { + objects: SavedObjectType[]; + ignoreRegularConflicts?: boolean; + retries?: SavedObjectsImportRetry[]; + createNewCopies?: boolean; + dataSourceId?: string; + }): ConflictsForDataSourceParams => { + return { ...partial }; + }; + + beforeEach(() => { + mockUuidv4.mockReset(); + mockUuidv4.mockReturnValueOnce(`new-object-id`); + }); + + it('exits early if there are no objects to check', async () => { + const params = setupParams({ objects: [] }); + const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params); + expect(checkConflictsForDataSourceResult).toEqual({ + filteredObjects: [], + errors: [], + importIdMap: new Map(), + pendingOverwrites: new Set(), + }); + }); + + it('return obj if it is not data source obj and there is no conflict of the data source id', async () => { + const params = setupParams({ objects: objectsWithDataSource, dataSourceId: 'ds' }); + const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params); + expect(checkConflictsForDataSourceResult).toEqual({ + filteredObjects: [dataSourceObj1, dataSourceObj2], + errors: [], + importIdMap: new Map(), + pendingOverwrites: new Set(), + }); + }); + + it('can resolve the data source id conflict when the ds it not match when ignoreRegularConflicts=true', async () => { + const params = setupParams({ + objects: objectsWithDataSource, + ignoreRegularConflicts: true, + dataSourceId: 'currentDsId', + }); + const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params); + + expect(checkConflictsForDataSourceResult).toEqual( + expect.objectContaining({ + filteredObjects: [ + { + ...dataSourceObj1, + id: 'currentDsId_id-1', + }, + { + ...dataSourceObj2, + id: 'currentDsId_id-2', + }, + ], + errors: [], + importIdMap: new Map([ + [ + `${dataSourceObj1.type}:${dataSourceObj1.id}`, + { id: 'currentDsId_id-1', omitOriginId: true }, + ], + [ + `${dataSourceObj2.type}:${dataSourceObj2.id}`, + { id: 'currentDsId_id-2', omitOriginId: true }, + ], + ]), + pendingOverwrites: new Set([ + `${dataSourceObj1.type}:${dataSourceObj1.id}`, + `${dataSourceObj2.type}:${dataSourceObj2.id}`, + ]), + }) + ); + }); + + it('can push error when do not override with data source conflict', async () => { + const params = setupParams({ + objects: [dataSourceObj1], + ignoreRegularConflicts: false, + dataSourceId: 'currentDs', + }); + const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params); + expect(checkConflictsForDataSourceResult).toEqual({ + filteredObjects: [], + errors: [ + { + ...dataSourceObj1Error, + title: dataSourceObj1.attributes.title, + meta: { title: dataSourceObj1.attributes.title }, + error: { type: 'conflict' }, + }, + ], + importIdMap: new Map(), + pendingOverwrites: new Set(), + }); + }); +}); diff --git a/src/core/server/saved_objects/import/check_conflict_for_data_source.ts b/src/core/server/saved_objects/import/check_conflict_for_data_source.ts new file mode 100644 index 000000000000..50005777aba0 --- /dev/null +++ b/src/core/server/saved_objects/import/check_conflict_for_data_source.ts @@ -0,0 +1,85 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { SavedObject, SavedObjectsImportError, SavedObjectsImportRetry } from '../types'; + +export interface ConflictsForDataSourceParams { + objects: Array>; + ignoreRegularConflicts?: boolean; + retries?: SavedObjectsImportRetry[]; + dataSourceId?: string; +} + +interface ImportIdMapEntry { + id?: string; + omitOriginId?: boolean; +} + +/** + * function to check the conflict when multiple data sources are enabled. + * the purpose of this function is to check the conflict of the imported saved objects and data source + * @param objects, this the array of saved objects to be verified whether contains the data source conflict + * @param ignoreRegularConflicts whether to override + * @param retries import operations list + * @param dataSourceId the id to identify the data source + * @returns {filteredObjects, errors, importIdMap, pendingOverwrites } + */ +export async function checkConflictsForDataSource({ + objects, + ignoreRegularConflicts, + retries = [], + dataSourceId, +}: ConflictsForDataSourceParams) { + const filteredObjects: Array> = []; + const errors: SavedObjectsImportError[] = []; + const importIdMap = new Map(); + const pendingOverwrites = new Set(); + + // exit early if there are no objects to check + if (objects.length === 0) { + return { filteredObjects, errors, importIdMap, pendingOverwrites }; + } + const retryMap = retries.reduce( + (acc, cur) => acc.set(`${cur.type}:${cur.id}`, cur), + new Map() + ); + objects.forEach((object) => { + const { + type, + id, + attributes: { title }, + } = object; + const { destinationId } = retryMap.get(`${type}:${id}`) || {}; + + if (object.type !== 'data-source') { + const parts = id.split('_'); // this is the array to host the split results of the id + const previoudDataSourceId = parts.length > 1 ? parts[0] : undefined; + const rawId = previoudDataSourceId ? parts[1] : parts[0]; + + /** + * for import saved object from osd exported + * when the imported saved objects with the different dataSourceId comparing to the current dataSourceId + */ + + if ( + previoudDataSourceId && + previoudDataSourceId !== dataSourceId && + !ignoreRegularConflicts + ) { + const error = { type: 'conflict' as 'conflict', ...(destinationId && { destinationId }) }; + errors.push({ type, id, title, meta: { title }, error }); + } else if (previoudDataSourceId && previoudDataSourceId === dataSourceId) { + filteredObjects.push(object); + } else { + const omitOriginId = ignoreRegularConflicts; + importIdMap.set(`${type}:${id}`, { id: `${dataSourceId}_${rawId}`, omitOriginId }); + pendingOverwrites.add(`${type}:${id}`); + filteredObjects.push({ ...object, id: `${dataSourceId}_${rawId}` }); + } + } + }); + + return { filteredObjects, errors, importIdMap, pendingOverwrites }; +} diff --git a/src/core/server/saved_objects/import/check_origin_conflicts.ts b/src/core/server/saved_objects/import/check_origin_conflicts.ts index 7ef98982e149..5010c04f337a 100644 --- a/src/core/server/saved_objects/import/check_origin_conflicts.ts +++ b/src/core/server/saved_objects/import/check_origin_conflicts.ts @@ -45,6 +45,7 @@ interface CheckOriginConflictsParams { namespace?: string; ignoreRegularConflicts?: boolean; importIdMap: Map; + dataSourceId?: string; } type CheckOriginConflictParams = Omit & { diff --git a/src/core/server/saved_objects/import/collect_saved_objects.ts b/src/core/server/saved_objects/import/collect_saved_objects.ts index 8fd015c44991..747c384862e0 100644 --- a/src/core/server/saved_objects/import/collect_saved_objects.ts +++ b/src/core/server/saved_objects/import/collect_saved_objects.ts @@ -46,6 +46,7 @@ interface CollectSavedObjectsOptions { objectLimit: number; filter?: (obj: SavedObject) => boolean; supportedTypes: string[]; + dataSourceId?: string; } export async function collectSavedObjects({ @@ -53,6 +54,7 @@ export async function collectSavedObjects({ objectLimit, filter, supportedTypes, + dataSourceId, }: CollectSavedObjectsOptions) { const errors: SavedObjectsImportError[] = []; const entries: Array<{ type: string; id: string }> = []; diff --git a/src/core/server/saved_objects/import/create_saved_objects.test.ts b/src/core/server/saved_objects/import/create_saved_objects.test.ts index 9c40124c34cd..f1118842c967 100644 --- a/src/core/server/saved_objects/import/create_saved_objects.test.ts +++ b/src/core/server/saved_objects/import/create_saved_objects.test.ts @@ -53,6 +53,8 @@ const createObject = (type: string, id: string, originId?: string): SavedObject const MULTI_NS_TYPE = 'multi'; const OTHER_TYPE = 'other'; +const DATA_SOURCE = 'data-source'; + /** * Create a variety of different objects to exercise different import / result scenarios */ @@ -69,11 +71,56 @@ const obj10 = createObject(OTHER_TYPE, 'id-10', 'originId-f'); // -> success const obj11 = createObject(OTHER_TYPE, 'id-11', 'originId-g'); // -> conflict const obj12 = createObject(OTHER_TYPE, 'id-12'); // -> success const obj13 = createObject(OTHER_TYPE, 'id-13'); // -> conflict +// data source object +const dataSourceObj1 = createObject(DATA_SOURCE, 'ds-id1'); // -> success +const dataSourceObj2 = createObject(DATA_SOURCE, 'ds-id2'); // -> conflict +const dashboardObjWithDataSource = createObject('dashboard', 'ds_dashboard-id1'); // -> success +const visualizationObjWithDataSource = createObject('visualization', 'ds_visualization-id1'); // -> success +const searchObjWithDataSource = createObject('search', 'ds_search-id1'); // -> success + +// objs without data source id, used to test can get saved object with data source id +const searchObj = { + id: '6aea5700-ac94-11e8-a651-614b2788174a', + type: 'search', + attributes: { + title: 'some-title', + }, + references: [], + source: { + title: 'mysavedsearch', + kibanaSavedObjectMeta: { + searchSourceJSON: + '{"index":"4c3f3c30-ac94-11e8-a651-614b2788174a","highlightAll":true,"version":true,"query":{"query":"","language":"lucene"},"filter":[]}', + }, + }, +}; // -> success + +const visualizationObj = { + id: '8411daa0-ac94-11e8-a651-614b2788174a', + type: 'visualization', + attributes: { + title: 'visualization-title', + }, + references: [], + source: { + title: 'mysavedviz', + visState: + '{"title":"mysavedviz","type":"pie","params":{"type":"pie","addTooltip":true,"addLegend":true,"legendPosition":"right","isDonut":true,"labels":{"show":false,"values":true,"last_level":true,"truncate":100}},"aggs":[{"id":"1","enabled":true,"type":"count","schema":"metric","params":{}}]}', + uiStateJSON: '{}', + description: '', + savedSearchId: '6aea5700-ac94-11e8-a651-614b2788174a', + version: 1, + kibanaSavedObjectMeta: { + searchSourceJSON: '{"query":{"query":"","language":"lucene"},"filter":[]}', + }, + }, +}; // non-multi-namespace types shouldn't have origin IDs, but we include test cases to ensure it's handled gracefully // non-multi-namespace types by definition cannot result in an unresolvable conflict, so we don't include test cases for those const importId3 = 'id-foo'; const importId4 = 'id-bar'; const importId8 = 'id-baz'; + const importIdMap = new Map([ [`${obj3.type}:${obj3.id}`, { id: importId3, omitOriginId: true }], [`${obj4.type}:${obj4.id}`, { id: importId4 }], @@ -93,6 +140,8 @@ describe('#createSavedObjects', () => { accumulatedErrors?: SavedObjectsImportError[]; namespace?: string; overwrite?: boolean; + dataSourceId?: string; + dataSourceTitle?: string; }): CreateSavedObjectsParams => { savedObjectsClient = savedObjectsClientMock.create(); bulkCreate = savedObjectsClient.bulkCreate; @@ -177,6 +226,15 @@ describe('#createSavedObjects', () => { expect(createSavedObjectsResult).toEqual({ createdObjects: [], errors: [] }); }); + test('filters out objects that have errors present with data source', async () => { + const error = { type: dataSourceObj1.type, id: dataSourceObj1.id } as SavedObjectsImportError; + const options = setupParams({ objects: [dataSourceObj1], accumulatedErrors: [error] }); + + const createSavedObjectsResult = await createSavedObjects(options); + expect(bulkCreate).not.toHaveBeenCalled(); + expect(createSavedObjectsResult).toEqual({ createdObjects: [], errors: [] }); + }); + test('exits early if there are no objects to create', async () => { const options = setupParams({ objects: [] }); @@ -186,6 +244,13 @@ describe('#createSavedObjects', () => { }); const objs = [obj1, obj2, obj3, obj4, obj5, obj6, obj7, obj8, obj9, obj10, obj11, obj12, obj13]; + const dataSourceObjs = [ + dataSourceObj1, + dataSourceObj2, + dashboardObjWithDataSource, + visualizationObjWithDataSource, + searchObjWithDataSource, + ]; const setupMockResults = (options: CreateSavedObjectsParams) => { bulkCreate.mockResolvedValue({ @@ -207,6 +272,26 @@ describe('#createSavedObjects', () => { }); }; + const setupMockResultsWithDataSource = (options: CreateSavedObjectsParams) => { + bulkCreate.mockResolvedValue({ + saved_objects: [ + getResultMock.conflict(dataSourceObj1.type, dataSourceObj1.id), + getResultMock.success(dataSourceObj2, options), + getResultMock.success(dashboardObjWithDataSource, options), + getResultMock.success(visualizationObjWithDataSource, options), + getResultMock.success(searchObjWithDataSource, options), + ], + }); + }; + const setupMockResultsToConstructDataSource = (options: CreateSavedObjectsParams) => { + bulkCreate.mockResolvedValue({ + saved_objects: [ + getResultMock.success(searchObj, options), + getResultMock.success(visualizationObj, options), + ], + }); + }; + describe('handles accumulated errors as expected', () => { const resolvableErrors: SavedObjectsImportError[] = [ { type: 'foo', id: 'foo-id', error: { type: 'conflict' } } as SavedObjectsImportError, @@ -234,6 +319,14 @@ describe('#createSavedObjects', () => { } }); + test('does not call bulkCreate when resolvable errors are present with data source objects', async () => { + for (const error of resolvableErrors) { + const options = setupParams({ objects: dataSourceObjs, accumulatedErrors: [error] }); + await createSavedObjects(options); + expect(bulkCreate).not.toHaveBeenCalled(); + } + }); + test('calls bulkCreate when unresolvable errors or no errors are present', async () => { for (const error of unresolvableErrors) { const options = setupParams({ objects: objs, accumulatedErrors: [error] }); @@ -247,6 +340,20 @@ describe('#createSavedObjects', () => { await createSavedObjects(options); expect(bulkCreate).toHaveBeenCalledTimes(1); }); + + test('calls bulkCreate when unresolvable errors or no errors are present with data source', async () => { + for (const error of unresolvableErrors) { + const options = setupParams({ objects: dataSourceObjs, accumulatedErrors: [error] }); + setupMockResultsWithDataSource(options); + await createSavedObjects(options); + expect(bulkCreate).toHaveBeenCalledTimes(1); + bulkCreate.mockClear(); + } + const options = setupParams({ objects: dataSourceObjs }); + setupMockResultsWithDataSource(options); + await createSavedObjects(options); + expect(bulkCreate).toHaveBeenCalledTimes(1); + }); }); it('filters out version from objects before create', async () => { @@ -270,6 +377,52 @@ describe('#createSavedObjects', () => { const argObjs = [obj1, obj2, x3, x4, obj5, obj6, obj7, x8, obj9, obj10, obj11, obj12, obj13]; expectBulkCreateArgs.objects(1, argObjs); }; + + const testBulkCreateObjectsWithDataSource = async ( + namespace?: string, + dataSourceId?: string, + dataSourceTitle?: string + ) => { + const options = setupParams({ + objects: dataSourceObjs, + namespace, + dataSourceId, + dataSourceTitle, + }); + setupMockResultsWithDataSource(options); + + await createSavedObjects(options); + expect(bulkCreate).toHaveBeenCalledTimes(1); + const argObjs = [ + dataSourceObj1, + dataSourceObj2, + dashboardObjWithDataSource, + visualizationObjWithDataSource, + searchObjWithDataSource, + ]; + expectBulkCreateArgs.objects(1, argObjs); + }; + + // testBulkCreateObjectsToAddDataSourceTitle + const testBulkCreateObjectsToAddDataSourceTitle = async ( + namespace?: string, + dataSourceId?: string, + dataSourceTitle?: string + ) => { + const options = setupParams({ + objects: [searchObj, visualizationObj], + namespace, + dataSourceId, + dataSourceTitle, + }); + setupMockResultsToConstructDataSource(options); + const result = (await createSavedObjects(options)).createdObjects; + expect(bulkCreate).toHaveBeenCalledTimes(1); + result.map((resultObj) => + expect(JSON.stringify(resultObj.attributes)).toContain('some-data-source-title') + ); + }; + const testBulkCreateOptions = async (namespace?: string) => { const overwrite = (Symbol() as unknown) as boolean; const options = setupParams({ objects: objs, namespace, overwrite }); @@ -279,6 +432,26 @@ describe('#createSavedObjects', () => { expect(bulkCreate).toHaveBeenCalledTimes(1); expectBulkCreateArgs.options(1, options); }; + + const testBulkCreateOptionsWithDataSource = async ( + namespace?: string, + dataSourceId?: string, + dataSourceTitle?: string + ) => { + const overwrite = (Symbol() as unknown) as boolean; + const options = setupParams({ + objects: dataSourceObjs, + namespace, + overwrite, + dataSourceId, + dataSourceTitle, + }); + setupMockResultsWithDataSource(options); + + await createSavedObjects(options); + expect(bulkCreate).toHaveBeenCalledTimes(1); + expectBulkCreateArgs.options(1, options); + }; const testReturnValue = async (namespace?: string) => { const options = setupParams({ objects: objs, namespace }); setupMockResults(options); @@ -293,6 +466,30 @@ describe('#createSavedObjects', () => { expect(results).toEqual(expectedResults); }; + const testReturnValueWithDataSource = async ( + namespace?: string, + dataSourceId?: string, + dataSourceTitle?: string + ) => { + const options = setupParams({ + objects: dataSourceObjs, + namespace, + dataSourceId, + dataSourceTitle, + }); + setupMockResultsWithDataSource(options); + + const results = await createSavedObjects(options); + const resultSavedObjectsWithDataSource = (await bulkCreate.mock.results[0].value).saved_objects; + const [dsr1, dsr2, dsr3, dsr4, dsr5] = resultSavedObjectsWithDataSource; + const transformedResultsWithDataSource = [dsr1, dsr2, dsr3, dsr4, dsr5]; + const expectedResultsWithDataSource = getExpectedResults( + transformedResultsWithDataSource, + dataSourceObjs + ); + expect(results).toEqual(expectedResultsWithDataSource); + }; + describe('with an undefined namespace', () => { test('calls bulkCreate once with input objects', async () => { await testBulkCreateObjects(); @@ -317,4 +514,36 @@ describe('#createSavedObjects', () => { await testReturnValue(namespace); }); }); + + describe('with a data source', () => { + test('calls bulkCreate once with input objects with data source id', async () => { + await testBulkCreateObjectsWithDataSource( + 'some-namespace', + 'some-datasource-id', + 'some-data-source-title' + ); + }); + test('calls bulkCreate once with input options with data source id', async () => { + await testBulkCreateOptionsWithDataSource( + 'some-namespace', + 'some-datasource-id', + 'some-data-source-title' + ); + }); + test('returns bulkCreate results that are remapped to IDs of imported objects with data source id', async () => { + await testReturnValueWithDataSource( + 'some-namespace', + 'some-datasource-id', + 'some-data-source-title' + ); + }); + + test('can correct attach datasource id to a search object', async () => { + await testBulkCreateObjectsToAddDataSourceTitle( + 'some-namespace', + 'some-datasource-id', + 'some-data-source-title' + ); + }); + }); }); diff --git a/src/core/server/saved_objects/import/create_saved_objects.ts b/src/core/server/saved_objects/import/create_saved_objects.ts index a3a1eebbd2ab..6fd08520281e 100644 --- a/src/core/server/saved_objects/import/create_saved_objects.ts +++ b/src/core/server/saved_objects/import/create_saved_objects.ts @@ -39,6 +39,8 @@ interface CreateSavedObjectsParams { importIdMap: Map; namespace?: string; overwrite?: boolean; + dataSourceId?: string; + dataSourceTitle?: string; } interface CreateSavedObjectsResult { createdObjects: Array>; @@ -56,6 +58,8 @@ export const createSavedObjects = async ({ importIdMap, namespace, overwrite, + dataSourceId, + dataSourceTitle, }: CreateSavedObjectsParams): Promise> => { // filter out any objects that resulted in errors const errorSet = accumulatedErrors.reduce( @@ -76,7 +80,70 @@ export const createSavedObjects = async ({ ); // filter out the 'version' field of each object, if it exists + const objectsToCreate = filteredObjects.map(({ version, ...object }) => { + if (dataSourceId) { + // @ts-expect-error + if (dataSourceTitle && object.attributes.title) { + if ( + object.type === 'dashboard' || + object.type === 'visualization' || + object.type === 'search' + ) { + // @ts-expect-error + object.attributes.title = object.attributes.title + `_${dataSourceTitle}`; + } + } + + if (object.type === 'index-pattern') { + object.references = [ + { + id: `${dataSourceId}`, + type: 'data-source', + name: 'dataSource', + }, + ]; + } + + if (object.type === 'visualization' || object.type === 'search') { + // @ts-expect-error + const searchSourceString = object.attributes?.kibanaSavedObjectMeta?.searchSourceJSON; + // @ts-expect-error + const visStateString = object.attributes?.visState; + + if (searchSourceString) { + const searchSource = JSON.parse(searchSourceString); + if (searchSource.index) { + const searchSourceIndex = searchSource.index.includes('_') + ? searchSource.index.split('_')[searchSource.index.split('_').length - 1] + : searchSource.index; + searchSource.index = `${dataSourceId}_` + searchSourceIndex; + + // @ts-expect-error + object.attributes.kibanaSavedObjectMeta.searchSourceJSON = JSON.stringify(searchSource); + } + } + + if (visStateString) { + const visState = JSON.parse(visStateString); + const controlList = visState.params?.controls; + if (controlList) { + // @ts-expect-error + controlList.map((control) => { + if (control.indexPattern) { + const controlIndexPattern = control.indexPattern.includes('_') + ? control.indexPattern.split('_')[control.indexPattern.split('_').length - 1] + : control.indexPattern; + control.indexPattern = `${dataSourceId}_` + controlIndexPattern; + } + }); + } + // @ts-expect-error + object.attributes.visState = JSON.stringify(visState); + } + } + } + // use the import ID map to ensure that each reference is being created with the correct ID const references = object.references?.map((reference) => { const { type, id } = reference; @@ -96,7 +163,6 @@ export const createSavedObjects = async ({ } return { ...object, ...(references && { references }) }; }); - const resolvableErrors = ['conflict', 'ambiguous_conflict', 'missing_references']; let expectedResults = objectsToCreate; if (!accumulatedErrors.some(({ error: { type } }) => resolvableErrors.includes(type))) { diff --git a/src/core/server/saved_objects/import/import_saved_objects.test.ts b/src/core/server/saved_objects/import/import_saved_objects.test.ts index de8fb34dfbed..2a9c4b0ff5ed 100644 --- a/src/core/server/saved_objects/import/import_saved_objects.test.ts +++ b/src/core/server/saved_objects/import/import_saved_objects.test.ts @@ -47,6 +47,7 @@ import { validateReferences } from './validate_references'; import { checkConflicts } from './check_conflicts'; import { checkOriginConflicts } from './check_origin_conflicts'; import { createSavedObjects } from './create_saved_objects'; +import { checkConflictsForDataSource } from './check_conflict_for_data_source'; jest.mock('./collect_saved_objects'); jest.mock('./regenerate_ids'); @@ -54,6 +55,7 @@ jest.mock('./validate_references'); jest.mock('./check_conflicts'); jest.mock('./check_origin_conflicts'); jest.mock('./create_saved_objects'); +jest.mock('./check_conflict_for_data_source'); const getMockFn = any, U>(fn: (...args: Parameters) => U) => fn as jest.MockedFunction<(...args: Parameters) => U>; @@ -80,6 +82,12 @@ describe('#importSavedObjectsFromStream', () => { importIdMap: new Map(), pendingOverwrites: new Set(), }); + getMockFn(checkConflictsForDataSource).mockResolvedValue({ + errors: [], + filteredObjects: [], + importIdMap: new Map(), + pendingOverwrites: new Set(), + }); getMockFn(createSavedObjects).mockResolvedValue({ errors: [], createdObjects: [] }); }); @@ -89,8 +97,12 @@ describe('#importSavedObjectsFromStream', () => { let savedObjectsClient: jest.Mocked; let typeRegistry: jest.Mocked; const namespace = 'some-namespace'; + const testDataSourceId = 'some-datasource'; - const setupOptions = (createNewCopies: boolean = false): SavedObjectsImportOptions => { + const setupOptions = ( + createNewCopies: boolean = false, + dataSourceId: string | undefined = undefined + ): SavedObjectsImportOptions => { readStream = new Readable(); savedObjectsClient = savedObjectsClientMock.create(); typeRegistry = typeRegistryMock.create(); @@ -109,14 +121,17 @@ describe('#importSavedObjectsFromStream', () => { typeRegistry, namespace, createNewCopies, + dataSourceId, }; }; - const createObject = (): SavedObject<{ + const createObject = ( + dataSourceId: string | undefined = undefined + ): SavedObject<{ title: string; }> => { return { type: 'foo-type', - id: uuidv4(), + id: dataSourceId ? `${dataSourceId}_${uuidv4()}` : uuidv4(), references: [], attributes: { title: 'some-title' }, }; @@ -225,6 +240,24 @@ describe('#importSavedObjectsFromStream', () => { expect(checkOriginConflicts).toHaveBeenCalledWith(checkOriginConflictsParams); }); + test('checks data source conflicts', async () => { + const options = setupOptions(false, testDataSourceId); + const collectedObjects = [createObject()]; + getMockFn(collectSavedObjects).mockResolvedValue({ + errors: [], + collectedObjects, + importIdMap: new Map(), + }); + + await importSavedObjectsFromStream(options); + const checkConflictsForDataSourceParams = { + objects: collectedObjects, + ignoreRegularConflicts: overwrite, + dataSourceId: testDataSourceId, + }; + expect(checkConflictsForDataSource).toHaveBeenCalledWith(checkConflictsForDataSourceParams); + }); + test('creates saved objects', async () => { const options = setupOptions(); const collectedObjects = [createObject()]; @@ -281,16 +314,18 @@ describe('#importSavedObjectsFromStream', () => { }); await importSavedObjectsFromStream(options); - expect(regenerateIds).toHaveBeenCalledWith(collectedObjects); + expect(regenerateIds).toHaveBeenCalledWith(collectedObjects, undefined); }); - test('does not check conflicts or check origin conflicts', async () => { + test('does not check conflicts or check origin conflicts or check data source conflict', async () => { const options = setupOptions(true); + getMockFn(validateReferences).mockResolvedValue([]); await importSavedObjectsFromStream(options); expect(checkConflicts).not.toHaveBeenCalled(); expect(checkOriginConflicts).not.toHaveBeenCalled(); + expect(checkConflictsForDataSource).not.toHaveBeenCalled(); }); test('creates saved objects', async () => { @@ -348,10 +383,20 @@ describe('#importSavedObjectsFromStream', () => { const obj1 = createObject(); const tmp = createObject(); const obj2 = { ...tmp, destinationId: 'some-destinationId', originId: tmp.id }; - const obj3 = { ...createObject(), destinationId: 'another-destinationId' }; // empty originId + const obj3 = { ...createObject(undefined), destinationId: 'another-destinationId' }; // empty originId const createdObjects = [obj1, obj2, obj3]; const error1 = createError(); const error2 = createError(); + // add some objects with data source id + const dataSourceObj1 = createObject(testDataSourceId); + const tmp2 = createObject(testDataSourceId); + const dataSourceObj2 = { ...tmp2, destinationId: 'some-destinationId', originId: tmp.id }; + const dataSourceObj3 = { + ...createObject(testDataSourceId), + destinationId: 'another-destinationId', + }; + const createdDsObjects = [dataSourceObj1, dataSourceObj2, dataSourceObj3]; + // results const success1 = { type: obj1.type, @@ -372,8 +417,28 @@ describe('#importSavedObjectsFromStream', () => { }; const errors = [error1, error2]; + const dsSuccess1 = { + type: dataSourceObj1.type, + id: dataSourceObj1.id, + meta: { title: dataSourceObj1.attributes.title, icon: `${dataSourceObj1.type}-icon` }, + }; + + const dsSuccess2 = { + type: dataSourceObj2.type, + id: dataSourceObj2.id, + meta: { title: dataSourceObj2.attributes.title, icon: `${dataSourceObj2.type}-icon` }, + destinationId: dataSourceObj2.destinationId, + }; + + const dsSuccess3 = { + type: dataSourceObj3.type, + id: dataSourceObj3.id, + meta: { title: dataSourceObj3.attributes.title, icon: `${dataSourceObj3.type}-icon` }, + destinationId: dataSourceObj3.destinationId, + }; + test('with createNewCopies disabled', async () => { - const options = setupOptions(); + const options = setupOptions(false, undefined); getMockFn(checkConflicts).mockResolvedValue({ errors: [], filteredObjects: [], @@ -410,7 +475,7 @@ describe('#importSavedObjectsFromStream', () => { test('with createNewCopies enabled', async () => { // however, we include it here for posterity - const options = setupOptions(true); + const options = setupOptions(true, undefined); getMockFn(createSavedObjects).mockResolvedValue({ errors, createdObjects }); const result = await importSavedObjectsFromStream(options); @@ -428,10 +493,73 @@ describe('#importSavedObjectsFromStream', () => { errors: errorResults, }); }); + + test('with createNewCopies disabled and with data source id', async () => { + const options = setupOptions(false, testDataSourceId); + getMockFn(checkConflictsForDataSource).mockResolvedValue({ + errors: [], + filteredObjects: [], + importIdMap: new Map(), + pendingOverwrites: new Set([`${dsSuccess2.type}:${dsSuccess2.id}`]), + }); + getMockFn(checkConflicts).mockResolvedValue({ + errors: [], + filteredObjects: [], + importIdMap: new Map(), + pendingOverwrites: new Set([ + `${dsSuccess2.type}:${dsSuccess2.id}`, // the dsSuccess2 object was overwritten + `${error2.type}:${error2.id}`, // an attempt was made to overwrite the error2 object + ]), + }); + getMockFn(createSavedObjects).mockResolvedValue({ + errors, + createdObjects: createdDsObjects, + }); + + const result = await importSavedObjectsFromStream(options); + // successResults only includes the imported object's type, id, and destinationId (if a new one was generated) + const successResults = [ + dsSuccess1, + { ...dsSuccess2, overwrite: true }, + { ...dsSuccess3, createNewCopy: true }, + ]; + const errorResults = [ + { ...error1, meta: { ...error1.meta, icon: `${error1.type}-icon` } }, + { ...error2, meta: { ...error2.meta, icon: `${error2.type}-icon` }, overwrite: true }, + ]; + expect(result).toEqual({ + success: false, + successCount: 3, + successResults, + errors: errorResults, + }); + }); + + test('with createNewCopies enabled and with data source id', async () => { + const options = setupOptions(true, testDataSourceId); + getMockFn(createSavedObjects).mockResolvedValue({ + errors, + createdObjects: createdDsObjects, + }); + + const result = await importSavedObjectsFromStream(options); + const successDsResults = [dsSuccess1, dsSuccess2, dsSuccess3]; + + const errorResults = [ + { ...error1, meta: { ...error1.meta, icon: `${error1.type}-icon` } }, + { ...error2, meta: { ...error2.meta, icon: `${error2.type}-icon` } }, + ]; + expect(result).toEqual({ + success: false, + successCount: 3, + successResults: successDsResults, + errors: errorResults, + }); + }); }); test('accumulates multiple errors', async () => { - const options = setupOptions(); + const options = setupOptions(false, undefined); const errors = [createError(), createError(), createError(), createError(), createError()]; getMockFn(collectSavedObjects).mockResolvedValue({ errors: [errors[0]], diff --git a/src/core/server/saved_objects/import/import_saved_objects.ts b/src/core/server/saved_objects/import/import_saved_objects.ts index cd250fc5f65f..6cbb1059fdc3 100644 --- a/src/core/server/saved_objects/import/import_saved_objects.ts +++ b/src/core/server/saved_objects/import/import_saved_objects.ts @@ -39,6 +39,7 @@ import { checkOriginConflicts } from './check_origin_conflicts'; import { createSavedObjects } from './create_saved_objects'; import { checkConflicts } from './check_conflicts'; import { regenerateIds } from './regenerate_ids'; +import { checkConflictsForDataSource } from './check_conflict_for_data_source'; /** * Import saved objects from given stream. See the {@link SavedObjectsImportOptions | options} for more @@ -54,6 +55,8 @@ export async function importSavedObjectsFromStream({ savedObjectsClient, typeRegistry, namespace, + dataSourceId, + dataSourceTitle, }: SavedObjectsImportOptions): Promise { let errorAccumulator: SavedObjectsImportError[] = []; const supportedTypes = typeRegistry.getImportableAndExportableTypes().map((type) => type.name); @@ -63,6 +66,7 @@ export async function importSavedObjectsFromStream({ readStream, objectLimit, supportedTypes, + dataSourceId, }); errorAccumulator = [...errorAccumulator, ...collectSavedObjectsResult.errors]; /** Map of all IDs for objects that we are attempting to import; each value is empty by default */ @@ -78,8 +82,10 @@ export async function importSavedObjectsFromStream({ errorAccumulator = [...errorAccumulator, ...validateReferencesResult]; if (createNewCopies) { - importIdMap = regenerateIds(collectSavedObjectsResult.collectedObjects); + // randomly generated id + importIdMap = regenerateIds(collectSavedObjectsResult.collectedObjects, dataSourceId); } else { + // in check conclict and override mode // Check single-namespace objects for conflicts in this namespace, and check multi-namespace objects for conflicts across all namespaces const checkConflictsParams = { objects: collectSavedObjectsResult.collectedObjects, @@ -87,10 +93,27 @@ export async function importSavedObjectsFromStream({ namespace, ignoreRegularConflicts: overwrite, }; + + // resolve when data source exist, pass the filtered objects to next check conflict + if (dataSourceId) { + const checkConflictsForDataSourceResult = await checkConflictsForDataSource({ + objects: collectSavedObjectsResult.collectedObjects, + ignoreRegularConflicts: overwrite, + dataSourceId, + }); + + checkConflictsParams.objects = checkConflictsForDataSourceResult.filteredObjects; + + pendingOverwrites = new Set([ + ...pendingOverwrites, + ...checkConflictsForDataSourceResult.pendingOverwrites, + ]); + } + const checkConflictsResult = await checkConflicts(checkConflictsParams); errorAccumulator = [...errorAccumulator, ...checkConflictsResult.errors]; importIdMap = new Map([...importIdMap, ...checkConflictsResult.importIdMap]); - pendingOverwrites = checkConflictsResult.pendingOverwrites; + pendingOverwrites = new Set([...pendingOverwrites, ...checkConflictsResult.pendingOverwrites]); // Check multi-namespace object types for origin conflicts in this namespace const checkOriginConflictsParams = { @@ -112,12 +135,16 @@ export async function importSavedObjectsFromStream({ // Create objects in bulk const createSavedObjectsParams = { - objects: collectSavedObjectsResult.collectedObjects, + objects: dataSourceId + ? collectSavedObjectsResult.collectedObjects.filter((object) => object.type !== 'data-source') + : collectSavedObjectsResult.collectedObjects, accumulatedErrors: errorAccumulator, savedObjectsClient, importIdMap, overwrite, namespace, + dataSourceId, + dataSourceTitle, }; const createSavedObjectsResult = await createSavedObjects(createSavedObjectsParams); errorAccumulator = [...errorAccumulator, ...createSavedObjectsResult.errors]; @@ -136,6 +163,7 @@ export async function importSavedObjectsFromStream({ }; } ); + const errorResults = errorAccumulator.map((error) => { const icon = typeRegistry.getType(error.type)?.management?.icon; const attemptedOverwrite = pendingOverwrites.has(`${error.type}:${error.id}`); diff --git a/src/core/server/saved_objects/import/regenerate_ids.test.ts b/src/core/server/saved_objects/import/regenerate_ids.test.ts index 11556c8a21c1..c7dbfb8b50bc 100644 --- a/src/core/server/saved_objects/import/regenerate_ids.test.ts +++ b/src/core/server/saved_objects/import/regenerate_ids.test.ts @@ -39,12 +39,18 @@ describe('#regenerateIds', () => { { type: 'baz', id: '3' }, ] as any) as SavedObject[]; + const dataSourceObjects = ([{ type: 'data-source', id: '1' }] as any) as SavedObject[]; + + test('can filter out data source object', () => { + expect(regenerateIds(dataSourceObjects, '').size).toBe(0); + }); + test('returns expected values', () => { mockUuidv4 .mockReturnValueOnce('uuidv4 #1') .mockReturnValueOnce('uuidv4 #2') .mockReturnValueOnce('uuidv4 #3'); - expect(regenerateIds(objects)).toMatchInlineSnapshot(` + expect(regenerateIds(objects, '')).toMatchInlineSnapshot(` Map { "foo:1" => Object { "id": "uuidv4 #1", diff --git a/src/core/server/saved_objects/import/regenerate_ids.ts b/src/core/server/saved_objects/import/regenerate_ids.ts index 672a8f030620..f1092bed7f55 100644 --- a/src/core/server/saved_objects/import/regenerate_ids.ts +++ b/src/core/server/saved_objects/import/regenerate_ids.ts @@ -36,9 +36,14 @@ import { SavedObject } from '../types'; * * @param objects The saved objects to generate new IDs for. */ -export const regenerateIds = (objects: SavedObject[]) => { +export const regenerateIds = (objects: SavedObject[], dataSourceId: string | undefined) => { const importIdMap = objects.reduce((acc, object) => { - return acc.set(`${object.type}:${object.id}`, { id: uuidv4(), omitOriginId: true }); + return object.type === 'data-source' + ? acc + : acc.set(`${object.type}:${object.id}`, { + id: dataSourceId ? `${dataSourceId}_${uuidv4()}` : uuidv4(), + omitOriginId: true, + }); }, new Map()); return importIdMap; }; diff --git a/src/core/server/saved_objects/import/resolve_import_errors.test.ts b/src/core/server/saved_objects/import/resolve_import_errors.test.ts index bd0ba1afdc9d..ef22155f046b 100644 --- a/src/core/server/saved_objects/import/resolve_import_errors.test.ts +++ b/src/core/server/saved_objects/import/resolve_import_errors.test.ts @@ -369,7 +369,7 @@ describe('#importSavedObjectsFromStream', () => { }); await resolveSavedObjectsImportErrors(options); - expect(regenerateIds).toHaveBeenCalledWith(collectedObjects); + expect(regenerateIds).toHaveBeenCalledWith(collectedObjects, undefined); }); test('creates saved objects', async () => { diff --git a/src/core/server/saved_objects/import/resolve_import_errors.ts b/src/core/server/saved_objects/import/resolve_import_errors.ts index 162410c4ce9b..09207c893043 100644 --- a/src/core/server/saved_objects/import/resolve_import_errors.ts +++ b/src/core/server/saved_objects/import/resolve_import_errors.ts @@ -59,6 +59,8 @@ export async function resolveSavedObjectsImportErrors({ typeRegistry, namespace, createNewCopies, + dataSourceId, + dataSourceTitle, }: SavedObjectsResolveImportErrorsOptions): Promise { // throw a BadRequest error if we see invalid retries validateRetries(retries); @@ -76,6 +78,7 @@ export async function resolveSavedObjectsImportErrors({ objectLimit, filter, supportedTypes, + dataSourceId, } ); errorAccumulator = [...errorAccumulator, ...collectorErrors]; @@ -116,7 +119,7 @@ export async function resolveSavedObjectsImportErrors({ if (createNewCopies) { // In case any missing reference errors were resolved, ensure that we regenerate those object IDs as well // This is because a retry to resolve a missing reference error may not necessarily specify a destinationId - importIdMap = regenerateIds(objectsToResolve); + importIdMap = regenerateIds(objectsToResolve, dataSourceId); } // Check single-namespace objects for conflicts in this namespace, and check multi-namespace objects for conflicts across all namespaces @@ -126,6 +129,7 @@ export async function resolveSavedObjectsImportErrors({ namespace, retries, createNewCopies, + dataSourceId, }; const checkConflictsResult = await checkConflicts(checkConflictsParams); errorAccumulator = [...errorAccumulator, ...checkConflictsResult.errors]; @@ -157,6 +161,8 @@ export async function resolveSavedObjectsImportErrors({ importIdMap, namespace, overwrite, + dataSourceId, + dataSourceTitle, }; const { createdObjects, errors: bulkCreateErrors } = await createSavedObjects( createSavedObjectsParams diff --git a/src/core/server/saved_objects/import/types.ts b/src/core/server/saved_objects/import/types.ts index 88beacb9d2fd..73bc548b1f24 100644 --- a/src/core/server/saved_objects/import/types.ts +++ b/src/core/server/saved_objects/import/types.ts @@ -187,6 +187,8 @@ export interface SavedObjectsImportOptions { namespace?: string; /** If true, will create new copies of import objects, each with a random `id` and undefined `originId`. */ createNewCopies: boolean; + dataSourceId?: string; + dataSourceTitle?: string; } /** @@ -208,6 +210,8 @@ export interface SavedObjectsResolveImportErrorsOptions { namespace?: string; /** If true, will create new copies of import objects, each with a random `id` and undefined `originId`. */ createNewCopies: boolean; + dataSourceId?: string; + dataSourceTitle?: string; } export type CreatedObject = SavedObject & { destinationId?: string }; diff --git a/src/core/server/saved_objects/routes/import.ts b/src/core/server/saved_objects/routes/import.ts index b157feb0860e..259551298748 100644 --- a/src/core/server/saved_objects/routes/import.ts +++ b/src/core/server/saved_objects/routes/import.ts @@ -60,6 +60,7 @@ export const registerImportRoute = (router: IRouter, config: SavedObjectConfig) { overwrite: schema.boolean({ defaultValue: false }), createNewCopies: schema.boolean({ defaultValue: false }), + dataSourceId: schema.maybe(schema.string({ defaultValue: '' })), }, { validate: (object) => { @@ -75,13 +76,29 @@ export const registerImportRoute = (router: IRouter, config: SavedObjectConfig) }, }, router.handleLegacyErrors(async (context, req, res) => { - const { overwrite, createNewCopies } = req.query; + const { overwrite, createNewCopies, dataSourceId } = req.query; const file = req.body.file as FileStream; const fileExtension = extname(file.hapi.filename).toLowerCase(); if (fileExtension !== '.ndjson') { return res.badRequest({ body: `Invalid file extension ${fileExtension}` }); } + // get datasource from saved object service + // dataSource is '' when there is no dataSource pass in the url + const dataSource = dataSourceId + ? await context.core.savedObjects.client + .get('data-source', dataSourceId) + .then((response) => { + const attributes: any = response?.attributes || {}; + return { + id: response.id, + title: attributes.title, + }; + }) + : ''; + + const dataSourceTitle = dataSource ? dataSource.title : ''; + let readStream: Readable; try { readStream = await createSavedObjectsStreamFromNdJson(file); @@ -98,6 +115,8 @@ export const registerImportRoute = (router: IRouter, config: SavedObjectConfig) objectLimit: maxImportExportSize, overwrite, createNewCopies, + dataSourceId, + dataSourceTitle, }); return res.ok({ body: result }); diff --git a/src/core/server/saved_objects/routes/resolve_import_errors.ts b/src/core/server/saved_objects/routes/resolve_import_errors.ts index 5e07125671f1..8e2113af6378 100644 --- a/src/core/server/saved_objects/routes/resolve_import_errors.ts +++ b/src/core/server/saved_objects/routes/resolve_import_errors.ts @@ -58,6 +58,7 @@ export const registerResolveImportErrorsRoute = (router: IRouter, config: SavedO validate: { query: schema.object({ createNewCopies: schema.boolean({ defaultValue: false }), + dataSourceId: schema.maybe(schema.string({ defaultValue: '' })), }), body: schema.object({ file: schema.stream(), @@ -89,6 +90,24 @@ export const registerResolveImportErrorsRoute = (router: IRouter, config: SavedO return res.badRequest({ body: `Invalid file extension ${fileExtension}` }); } + const dataSourceId = req.query.dataSourceId; + + // get datasource from saved object service + + const dataSource = dataSourceId + ? await context.core.savedObjects.client + .get('data-source', dataSourceId) + .then((response) => { + const attributes: any = response?.attributes || {}; + return { + id: response.id, + title: attributes.title, + }; + }) + : ''; + + const dataSourceTitle = dataSource ? dataSource.title : ''; + let readStream: Readable; try { readStream = await createSavedObjectsStreamFromNdJson(file); @@ -105,6 +124,8 @@ export const registerResolveImportErrorsRoute = (router: IRouter, config: SavedO retries: req.body.retries, objectLimit: maxImportExportSize, createNewCopies: req.query.createNewCopies, + dataSourceId, + dataSourceTitle, }); return res.ok({ body: result });