Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Optimize package installation performance, phase 2 #131627

Merged
merged 5 commits into from
May 6, 2022
Merged

Optimize package installation performance, phase 2 #131627

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
1cc06e2
Remove refresh from SO import
joshdover May 2, 2022
1bc1f23
Install index templates and ingest pipelines in parallel
joshdover May 2, 2022
3526374
Fix
joshdover May 5, 2022
0372755
Add refresh back to createInstallation
joshdover May 6, 2022
b14e3e8
Refactor template and pipeline installation
joshdover May 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,6 @@
* 2.0.
*/

export { installPipelines, isTopLevelPipeline } from './install';
export { prepareToInstallPipelines, isTopLevelPipeline } from './install';

export { deletePreviousPipelines, deletePipeline } from './remove';
87 changes: 43 additions & 44 deletions x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,12 @@
*/

import type { TransportRequestOptions } from '@elastic/elasticsearch';
import type { ElasticsearchClient, Logger, SavedObjectsClientContract } from '@kbn/core/server';
import type { ElasticsearchClient, Logger } from '@kbn/core/server';

import { ElasticsearchAssetType } from '../../../../types';
import type { EsAssetReference, RegistryDataStream, InstallablePackage } from '../../../../types';
import { getAsset, getPathParts } from '../../archive';
import type { ArchiveEntry } from '../../archive';
import { updateEsAssetReferences } from '../../packages/install';
import {
FLEET_FINAL_PIPELINE_CONTENT,
FLEET_FINAL_PIPELINE_ID,
Expand All @@ -36,23 +35,23 @@ export const isTopLevelPipeline = (path: string) => {
);
};

export const installPipelines = async (
export const prepareToInstallPipelines = (
installablePackage: InstallablePackage,
paths: string[],
esClient: ElasticsearchClient,
savedObjectsClient: SavedObjectsClientContract,
logger: Logger,
esReferences: EsAssetReference[]
) => {
paths: string[]
): {
assetsToAdd: EsAssetReference[];
install: (esClient: ElasticsearchClient, logger: Logger) => Promise<void>;
} => {
// unlike other ES assets, pipeline names are versioned so after a template is updated
// it can be created pointing to the new template, without removing the old one and effecting data
// so do not remove the currently installed pipelines here
const dataStreams = installablePackage.data_streams;
const { name: pkgName, version: pkgVersion } = installablePackage;
const { version: pkgVersion } = installablePackage;
const pipelinePaths = paths.filter((path) => isPipeline(path));
const topLevelPipelinePaths = paths.filter((path) => isTopLevelPipeline(path));

if (!dataStreams?.length && topLevelPipelinePaths.length === 0) return [];
if (!dataStreams?.length && topLevelPipelinePaths.length === 0)
return { assetsToAdd: [], install: () => Promise.resolve() };

// get and save pipeline refs before installing pipelines
let pipelineRefs = dataStreams
Expand Down Expand Up @@ -85,41 +84,41 @@ export const installPipelines = async (

pipelineRefs = [...pipelineRefs, ...topLevelPipelineRefs];

esReferences = await updateEsAssetReferences(savedObjectsClient, pkgName, esReferences, {
return {
assetsToAdd: pipelineRefs,
});

const pipelines = dataStreams
? dataStreams.reduce<Array<Promise<EsAssetReference[]>>>((acc, dataStream) => {
if (dataStream.ingest_pipeline) {
acc.push(
installAllPipelines({
dataStream,
esClient,
logger,
paths: pipelinePaths,
installablePackage,
})
);
}
return acc;
}, [])
: [];

if (topLevelPipelinePaths) {
pipelines.push(
installAllPipelines({
dataStream: undefined,
esClient,
logger,
paths: topLevelPipelinePaths,
installablePackage,
})
);
}
install: async (esClient, logger) => {
const pipelines = dataStreams
? dataStreams.reduce<Array<Promise<EsAssetReference[]>>>((acc, dataStream) => {
if (dataStream.ingest_pipeline) {
acc.push(
installAllPipelines({
dataStream,
esClient,
logger,
paths: pipelinePaths,
installablePackage,
})
);
}
return acc;
}, [])
: [];

if (topLevelPipelinePaths) {
pipelines.push(
installAllPipelines({
dataStream: undefined,
esClient,
logger,
paths: topLevelPipelinePaths,
installablePackage,
})
);
}

await Promise.all(pipelines);
return esReferences;
await Promise.all(pipelines);
},
};
};

export function rewriteIngestPipeline(
Expand Down
140 changes: 20 additions & 120 deletions x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,30 +4,19 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { elasticsearchServiceMock } from '@kbn/core/server/mocks';
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { loggerMock } from '@kbn/logging-mocks';

import { createAppContextStartContractMock } from '../../../../mocks';
import { appContextService } from '../../..';

import type { RegistryDataStream } from '../../../../types';
import type { Field } from '../../fields/field';

import { installTemplate } from './install';
import { prepareTemplate } from './install';

describe('EPM install', () => {
describe('EPM index template install', () => {
beforeEach(async () => {
appContextService.start(createAppContextStartContractMock());
});

it('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix not set', async () => {
const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
esClient.indices.getIndexTemplate.mockImplementation(() =>
elasticsearchServiceMock.createSuccessTransportRequestPromise({ index_templates: [] })
);

const fields: Field[] = [];
it('tests prepareTemplate to use correct priority and index_patterns for data stream with dataset_is_prefix not set', async () => {
const dataStreamDatasetIsPrefixUnset = {
type: 'metrics',
dataset: 'package.dataset',
Expand All @@ -43,29 +32,14 @@ describe('EPM install', () => {
};
const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*';
const templatePriorityDatasetIsPrefixUnset = 200;
await installTemplate({
esClient,
logger: loggerMock.create(),
fields,
dataStream: dataStreamDatasetIsPrefixUnset,
packageVersion: pkg.version,
packageName: pkg.name,
});

const sentTemplate = (
esClient.indices.putIndexTemplate.mock.calls[0][0] as estypes.IndicesPutIndexTemplateRequest
).body;

expect(sentTemplate).toBeDefined();
expect(sentTemplate?.priority).toBe(templatePriorityDatasetIsPrefixUnset);
expect(sentTemplate?.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]);
const {
indexTemplate: { indexTemplate },
} = prepareTemplate({ pkg, dataStream: dataStreamDatasetIsPrefixUnset });
expect(indexTemplate.priority).toBe(templatePriorityDatasetIsPrefixUnset);
expect(indexTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]);
});

it('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to false', async () => {
const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
esClient.indices.getIndexTemplate.mockResponse({ index_templates: [] });

const fields: Field[] = [];
it('tests prepareTemplate to use correct priority and index_patterns for data stream with dataset_is_prefix set to false', async () => {
const dataStreamDatasetIsPrefixFalse = {
type: 'metrics',
dataset: 'package.dataset',
Expand All @@ -82,29 +56,15 @@ describe('EPM install', () => {
};
const templateIndexPatternDatasetIsPrefixFalse = 'metrics-package.dataset-*';
const templatePriorityDatasetIsPrefixFalse = 200;
await installTemplate({
esClient,
logger: loggerMock.create(),
fields,
dataStream: dataStreamDatasetIsPrefixFalse,
packageVersion: pkg.version,
packageName: pkg.name,
});

const sentTemplate = (
esClient.indices.putIndexTemplate.mock.calls[0][0] as estypes.IndicesPutIndexTemplateRequest
).body;
const {
indexTemplate: { indexTemplate },
} = prepareTemplate({ pkg, dataStream: dataStreamDatasetIsPrefixFalse });

expect(sentTemplate).toBeDefined();
expect(sentTemplate?.priority).toBe(templatePriorityDatasetIsPrefixFalse);
expect(sentTemplate?.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixFalse]);
expect(indexTemplate.priority).toBe(templatePriorityDatasetIsPrefixFalse);
expect(indexTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixFalse]);
});

it('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to true', async () => {
const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
esClient.indices.getIndexTemplate.mockResponse({ index_templates: [] });

const fields: Field[] = [];
it('tests prepareTemplate to use correct priority and index_patterns for data stream with dataset_is_prefix set to true', async () => {
const dataStreamDatasetIsPrefixTrue = {
type: 'metrics',
dataset: 'package.dataset',
Expand All @@ -121,71 +81,11 @@ describe('EPM install', () => {
};
const templateIndexPatternDatasetIsPrefixTrue = 'metrics-package.dataset.*-*';
const templatePriorityDatasetIsPrefixTrue = 150;
await installTemplate({
esClient,
logger: loggerMock.create(),
fields,
dataStream: dataStreamDatasetIsPrefixTrue,
packageVersion: pkg.version,
packageName: pkg.name,
});

const sentTemplate = (
esClient.indices.putIndexTemplate.mock.calls[0][0] as estypes.IndicesPutIndexTemplateRequest
).body;

expect(sentTemplate).toBeDefined();
expect(sentTemplate?.priority).toBe(templatePriorityDatasetIsPrefixTrue);
expect(sentTemplate?.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixTrue]);
});

it('tests installPackage remove the aliases property if the property existed', async () => {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I removed this whole test because this behavior is no longer necessary and was removed in the previous PR. This test just happens to still pass.

const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;

esClient.indices.getIndexTemplate.mockResponse({
index_templates: [
{
name: 'metrics-package.dataset',
// @ts-expect-error not full interface
index_template: {
index_patterns: ['metrics-package.dataset-*'],
template: { aliases: {} },
},
},
],
});

const fields: Field[] = [];
const dataStreamDatasetIsPrefixUnset = {
type: 'metrics',
dataset: 'package.dataset',
title: 'test data stream',
release: 'experimental',
package: 'package',
path: 'path',
ingest_pipeline: 'default',
} as RegistryDataStream;
const pkg = {
name: 'package',
version: '0.0.1',
};
const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*';
const templatePriorityDatasetIsPrefixUnset = 200;
await installTemplate({
esClient,
logger: loggerMock.create(),
fields,
dataStream: dataStreamDatasetIsPrefixUnset,
packageVersion: pkg.version,
packageName: pkg.name,
});
const {
indexTemplate: { indexTemplate },
} = prepareTemplate({ pkg, dataStream: dataStreamDatasetIsPrefixTrue });

const sentTemplate = (
esClient.indices.putIndexTemplate.mock.calls[0][0] as estypes.IndicesPutIndexTemplateRequest
).body;
expect(sentTemplate).toBeDefined();
expect(sentTemplate?.template?.aliases).not.toBeDefined();
expect(sentTemplate?.priority).toBe(templatePriorityDatasetIsPrefixUnset);
expect(sentTemplate?.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]);
expect(indexTemplate.priority).toBe(templatePriorityDatasetIsPrefixTrue);
expect(indexTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixTrue]);
});
});
Loading