From a0ce7b5aa887d34a7a892553c66f25d72e38d827 Mon Sep 17 00:00:00 2001 From: Spencer Date: Mon, 8 Feb 2021 09:47:55 -0800 Subject: [PATCH] [kbn/optimizer][ci-stats] ship metrics separate from build (#90482) Co-authored-by: spalger Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- package.json | 2 + .../src/ci_stats_reporter/index.ts | 1 + .../ci_stats_reporter/ship_ci_stats_cli.ts | 48 +++++++ packages/kbn-optimizer/src/cli.ts | 17 +-- .../kbn-optimizer/src/common/bundle.test.ts | 2 + packages/kbn-optimizer/src/common/bundle.ts | 16 ++- .../src/common/bundle_cache.test.ts | 14 +- .../kbn-optimizer/src/common/bundle_cache.ts | 22 ++- packages/kbn-optimizer/src/index.ts | 1 - .../basic_optimization.test.ts.snap | 35 ++++- .../basic_optimization.test.ts | 15 +- packages/kbn-optimizer/src/limits.ts | 21 ++- .../src/optimizer/get_output_stats.ts | 118 ---------------- .../src/optimizer/get_plugin_bundles.test.ts | 10 +- .../src/optimizer/get_plugin_bundles.ts | 5 +- packages/kbn-optimizer/src/optimizer/index.ts | 1 - .../src/optimizer/optimizer_config.test.ts | 8 +- .../src/optimizer/optimizer_config.ts | 10 +- .../src/report_optimizer_stats.ts | 46 ------ .../src/worker/bundle_metrics_plugin.ts | 108 ++++++++++++++ .../src/worker/emit_stats_plugin.ts | 34 +++++ .../worker/populate_bundle_cache_plugin.ts | 132 ++++++++++++++++++ .../kbn-optimizer/src/worker/run_compilers.ts | 122 +--------------- .../src/worker/webpack.config.ts | 6 + .../src/integration_tests/build.test.ts | 3 +- .../kbn-plugin-helpers/src/tasks/optimize.ts | 8 +- scripts/ship_ci_stats.js | 10 ++ .../tasks/build_kibana_platform_plugins.ts | 39 ++++-- test/scripts/jenkins_baseline.sh | 4 + test/scripts/jenkins_build_kibana.sh | 3 + test/scripts/jenkins_xpack_baseline.sh | 4 + test/scripts/jenkins_xpack_build_kibana.sh | 4 + yarn.lock | 2 +- 33 files changed, 518 insertions(+), 353 deletions(-) create mode 100644 packages/kbn-dev-utils/src/ci_stats_reporter/ship_ci_stats_cli.ts delete mode 100644 packages/kbn-optimizer/src/optimizer/get_output_stats.ts delete mode 100644 packages/kbn-optimizer/src/report_optimizer_stats.ts create mode 100644 packages/kbn-optimizer/src/worker/bundle_metrics_plugin.ts create mode 100644 packages/kbn-optimizer/src/worker/emit_stats_plugin.ts create mode 100644 packages/kbn-optimizer/src/worker/populate_bundle_cache_plugin.ts create mode 100644 scripts/ship_ci_stats.js diff --git a/package.json b/package.json index b224f0c1ae0d5..7144745f2ae35 100644 --- a/package.json +++ b/package.json @@ -558,6 +558,7 @@ "@types/webpack": "^4.41.3", "@types/webpack-env": "^1.15.3", "@types/webpack-merge": "^4.1.5", + "@types/webpack-sources": "^0.1.4", "@types/write-pkg": "^3.1.0", "@types/xml-crypto": "^1.4.1", "@types/xml2js": "^0.4.5", @@ -843,6 +844,7 @@ "webpack-cli": "^3.3.12", "webpack-dev-server": "^3.11.0", "webpack-merge": "^4.2.2", + "webpack-sources": "^1.4.1", "write-pkg": "^4.0.0", "xml-crypto": "^2.0.0", "xmlbuilder": "13.0.2", diff --git a/packages/kbn-dev-utils/src/ci_stats_reporter/index.ts b/packages/kbn-dev-utils/src/ci_stats_reporter/index.ts index 165239cbebb89..d99217c38b410 100644 --- a/packages/kbn-dev-utils/src/ci_stats_reporter/index.ts +++ b/packages/kbn-dev-utils/src/ci_stats_reporter/index.ts @@ -7,3 +7,4 @@ */ export * from './ci_stats_reporter'; +export * from './ship_ci_stats_cli'; diff --git a/packages/kbn-dev-utils/src/ci_stats_reporter/ship_ci_stats_cli.ts b/packages/kbn-dev-utils/src/ci_stats_reporter/ship_ci_stats_cli.ts new file mode 100644 index 0000000000000..244af7b657418 --- /dev/null +++ b/packages/kbn-dev-utils/src/ci_stats_reporter/ship_ci_stats_cli.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import Fs from 'fs'; + +import { CiStatsReporter } from './ci_stats_reporter'; +import { run, createFlagError } from '../run'; + +export function shipCiStatsCli() { + run( + async ({ log, flags }) => { + let metricPaths = flags.metrics; + if (typeof metricPaths === 'string') { + metricPaths = [metricPaths]; + } else if (!Array.isArray(metricPaths) || !metricPaths.every((p) => typeof p === 'string')) { + throw createFlagError('expected --metrics to be a string'); + } + + const reporter = CiStatsReporter.fromEnv(log); + for (const path of metricPaths) { + // resolve path from CLI relative to CWD + const abs = Path.resolve(path); + const json = Fs.readFileSync(abs, 'utf8'); + await reporter.metrics(JSON.parse(json)); + log.success('shipped metrics from', path); + } + }, + { + description: 'ship ci-stats which have been written to files', + usage: `node scripts/ship_ci_stats`, + log: { + defaultLevel: 'debug', + }, + flags: { + string: ['metrics'], + help: ` + --metrics [path] A path to a JSON file that includes metrics which should be sent. Multiple instances supported + `, + }, + } + ); +} diff --git a/packages/kbn-optimizer/src/cli.ts b/packages/kbn-optimizer/src/cli.ts index 3021982b8ed6a..8fb906aa4603e 100644 --- a/packages/kbn-optimizer/src/cli.ts +++ b/packages/kbn-optimizer/src/cli.ts @@ -12,11 +12,10 @@ import Path from 'path'; import { REPO_ROOT } from '@kbn/utils'; import { lastValueFrom } from '@kbn/std'; -import { run, createFlagError, CiStatsReporter } from '@kbn/dev-utils'; +import { run, createFlagError } from '@kbn/dev-utils'; import { logOptimizerState } from './log_optimizer_state'; import { OptimizerConfig } from './optimizer'; -import { reportOptimizerStats } from './report_optimizer_stats'; import { runOptimizer } from './run_optimizer'; import { validateLimitsForAllBundles, updateBundleLimits } from './limits'; @@ -120,17 +119,7 @@ run( return; } - let update$ = runOptimizer(config); - - if (reportStats) { - const reporter = CiStatsReporter.fromEnv(log); - - if (!reporter.isEnabled()) { - log.warning('Unable to initialize CiStatsReporter from env'); - } - - update$ = update$.pipe(reportOptimizerStats(reporter, config, log)); - } + const update$ = runOptimizer(config); await lastValueFrom(update$.pipe(logOptimizerState(log, config))); @@ -153,7 +142,6 @@ run( 'cache', 'profile', 'inspect-workers', - 'report-stats', 'validate-limits', 'update-limits', ], @@ -179,7 +167,6 @@ run( --dist create bundles that are suitable for inclusion in the Kibana distributable, enabled when running with --update-limits --scan-dir add a directory to the list of directories scanned for plugins (specify as many times as necessary) --no-inspect-workers when inspecting the parent process, don't inspect the workers - --report-stats attempt to report stats about this execution of the build to the kibana-ci-stats service using this name --validate-limits validate the limits.yml config to ensure that there are limits defined for every bundle --update-limits run a build and rewrite the limits file to include the current bundle sizes +5kb `, diff --git a/packages/kbn-optimizer/src/common/bundle.test.ts b/packages/kbn-optimizer/src/common/bundle.test.ts index b6d25f69e58b4..ff9aa6fd90628 100644 --- a/packages/kbn-optimizer/src/common/bundle.test.ts +++ b/packages/kbn-optimizer/src/common/bundle.test.ts @@ -42,6 +42,7 @@ it('creates cache keys', () => { "id": "bar", "manifestPath": undefined, "outputDir": "/foo/bar/target", + "pageLoadAssetSizeLimit": undefined, "publicDirNames": Array [ "public", ], @@ -79,6 +80,7 @@ it('parses bundles from JSON specs', () => { "id": "bar", "manifestPath": undefined, "outputDir": "/foo/bar/target", + "pageLoadAssetSizeLimit": undefined, "publicDirNames": Array [ "public", ], diff --git a/packages/kbn-optimizer/src/common/bundle.ts b/packages/kbn-optimizer/src/common/bundle.ts index cb6096759739b..64b44de0dd1b3 100644 --- a/packages/kbn-optimizer/src/common/bundle.ts +++ b/packages/kbn-optimizer/src/common/bundle.ts @@ -36,6 +36,8 @@ export interface BundleSpec { readonly banner?: string; /** Absolute path to a kibana.json manifest file, if omitted we assume there are not dependenices */ readonly manifestPath?: string; + /** Maximum allowed page load asset size for the bundles page load asset */ + readonly pageLoadAssetSizeLimit?: number; } export class Bundle { @@ -63,6 +65,8 @@ export class Bundle { * Every bundle mentioned in the `requiredBundles` must be built together. */ public readonly manifestPath: BundleSpec['manifestPath']; + /** Maximum allowed page load asset size for the bundles page load asset */ + public readonly pageLoadAssetSizeLimit: BundleSpec['pageLoadAssetSizeLimit']; public readonly cache: BundleCache; @@ -75,8 +79,9 @@ export class Bundle { this.outputDir = spec.outputDir; this.manifestPath = spec.manifestPath; this.banner = spec.banner; + this.pageLoadAssetSizeLimit = spec.pageLoadAssetSizeLimit; - this.cache = new BundleCache(Path.resolve(this.outputDir, '.kbn-optimizer-cache')); + this.cache = new BundleCache(this.outputDir); } /** @@ -107,6 +112,7 @@ export class Bundle { outputDir: this.outputDir, manifestPath: this.manifestPath, banner: this.banner, + pageLoadAssetSizeLimit: this.pageLoadAssetSizeLimit, }; } @@ -222,6 +228,13 @@ export function parseBundles(json: string) { } } + const { pageLoadAssetSizeLimit } = spec; + if (pageLoadAssetSizeLimit !== undefined) { + if (!(typeof pageLoadAssetSizeLimit === 'number')) { + throw new Error('`bundles[]` must have a numeric `pageLoadAssetSizeLimit` property'); + } + } + return new Bundle({ type, id, @@ -231,6 +244,7 @@ export function parseBundles(json: string) { outputDir, banner, manifestPath, + pageLoadAssetSizeLimit, }); } ); diff --git a/packages/kbn-optimizer/src/common/bundle_cache.test.ts b/packages/kbn-optimizer/src/common/bundle_cache.test.ts index 82a8c0debb83c..e903a687908b9 100644 --- a/packages/kbn-optimizer/src/common/bundle_cache.test.ts +++ b/packages/kbn-optimizer/src/common/bundle_cache.test.ts @@ -25,12 +25,12 @@ beforeEach(() => { }); it(`doesn't complain if files are not on disk`, () => { - const cache = new BundleCache('/foo/bar.json'); + const cache = new BundleCache('/foo'); expect(cache.get()).toEqual({}); }); it(`updates files on disk when calling set()`, () => { - const cache = new BundleCache('/foo/bar.json'); + const cache = new BundleCache('/foo'); cache.set(SOME_STATE); expect(mockReadFileSync).not.toHaveBeenCalled(); expect(mockMkdirSync.mock.calls).toMatchInlineSnapshot(` @@ -46,7 +46,7 @@ it(`updates files on disk when calling set()`, () => { expect(mockWriteFileSync.mock.calls).toMatchInlineSnapshot(` Array [ Array [ - "/foo/bar.json", + "/foo/.kbn-optimizer-cache", "{ \\"cacheKey\\": \\"abc\\", \\"files\\": [ @@ -61,7 +61,7 @@ it(`updates files on disk when calling set()`, () => { }); it(`serves updated state from memory`, () => { - const cache = new BundleCache('/foo/bar.json'); + const cache = new BundleCache('/foo'); cache.set(SOME_STATE); jest.clearAllMocks(); @@ -72,7 +72,7 @@ it(`serves updated state from memory`, () => { }); it('reads state from disk on get() after refresh()', () => { - const cache = new BundleCache('/foo/bar.json'); + const cache = new BundleCache('/foo'); cache.set(SOME_STATE); cache.refresh(); jest.clearAllMocks(); @@ -83,7 +83,7 @@ it('reads state from disk on get() after refresh()', () => { expect(mockReadFileSync.mock.calls).toMatchInlineSnapshot(` Array [ Array [ - "/foo/bar.json", + "/foo/.kbn-optimizer-cache", "utf8", ], ] @@ -91,7 +91,7 @@ it('reads state from disk on get() after refresh()', () => { }); it('provides accessors to specific state properties', () => { - const cache = new BundleCache('/foo/bar.json'); + const cache = new BundleCache('/foo'); expect(cache.getModuleCount()).toBe(undefined); expect(cache.getReferencedFiles()).toEqual(undefined); diff --git a/packages/kbn-optimizer/src/common/bundle_cache.ts b/packages/kbn-optimizer/src/common/bundle_cache.ts index 39b52095c819a..7c0770caa2623 100644 --- a/packages/kbn-optimizer/src/common/bundle_cache.ts +++ b/packages/kbn-optimizer/src/common/bundle_cache.ts @@ -9,6 +9,9 @@ import Fs from 'fs'; import Path from 'path'; +import webpack from 'webpack'; +import { RawSource } from 'webpack-sources'; + export interface State { optimizerCacheKey?: unknown; cacheKey?: unknown; @@ -20,13 +23,17 @@ export interface State { const DEFAULT_STATE: State = {}; const DEFAULT_STATE_JSON = JSON.stringify(DEFAULT_STATE); +const CACHE_FILENAME = '.kbn-optimizer-cache'; /** * Helper to read and update metadata for bundles. */ export class BundleCache { private state: State | undefined = undefined; - constructor(private readonly path: string | false) {} + private readonly path: string | false; + constructor(outputDir: string | false) { + this.path = outputDir === false ? false : Path.resolve(outputDir, CACHE_FILENAME); + } refresh() { this.state = undefined; @@ -63,6 +70,7 @@ export class BundleCache { set(updated: State) { this.state = updated; + if (this.path) { const directory = Path.dirname(this.path); Fs.mkdirSync(directory, { recursive: true }); @@ -107,4 +115,16 @@ export class BundleCache { } } } + + public writeWebpackAsset(compilation: webpack.compilation.Compilation) { + if (!this.path) { + return; + } + + const source = new RawSource(JSON.stringify(this.state, null, 2)); + + // see https://github.com/jantimon/html-webpack-plugin/blob/33d69f49e6e9787796402715d1b9cd59f80b628f/index.js#L266 + // @ts-expect-error undocumented, used to add assets to the output + compilation.emitAsset(CACHE_FILENAME, source); + } } diff --git a/packages/kbn-optimizer/src/index.ts b/packages/kbn-optimizer/src/index.ts index a74679bfff536..551d2ffacfcfb 100644 --- a/packages/kbn-optimizer/src/index.ts +++ b/packages/kbn-optimizer/src/index.ts @@ -9,6 +9,5 @@ export { OptimizerConfig } from './optimizer'; export * from './run_optimizer'; export * from './log_optimizer_state'; -export * from './report_optimizer_stats'; export * from './node'; export * from './limits'; diff --git a/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap b/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap index 1ed1b92f9c2d9..9e9e8960da21b 100644 --- a/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap +++ b/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap @@ -13,6 +13,7 @@ OptimizerConfig { "id": "bar", "manifestPath": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/kibana.json, "outputDir": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/target/public, + "pageLoadAssetSizeLimit": undefined, "publicDirNames": Array [ "public", ], @@ -29,6 +30,7 @@ OptimizerConfig { "id": "foo", "manifestPath": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/kibana.json, "outputDir": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/target/public, + "pageLoadAssetSizeLimit": undefined, "publicDirNames": Array [ "public", ], @@ -47,6 +49,7 @@ OptimizerConfig { "id": "baz", "manifestPath": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/x-pack/baz/kibana.json, "outputDir": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/x-pack/baz/target/public, + "pageLoadAssetSizeLimit": undefined, "publicDirNames": Array [ "public", ], @@ -57,7 +60,6 @@ OptimizerConfig { "cache": true, "dist": false, "inspectWorkers": false, - "limits": "", "maxWorkerCount": 1, "plugins": Array [ Object { @@ -109,3 +111,34 @@ exports[`prepares assets for distribution: baz bundle 1`] = ` exports[`prepares assets for distribution: foo async bundle 1`] = `"(window[\\"foo_bundle_jsonpfunction\\"]=window[\\"foo_bundle_jsonpfunction\\"]||[]).push([[1],{3:function(module,__webpack_exports__,__webpack_require__){\\"use strict\\";__webpack_require__.r(__webpack_exports__);__webpack_require__.d(__webpack_exports__,\\"foo\\",(function(){return foo}));function foo(){}}}]);"`; exports[`prepares assets for distribution: foo bundle 1`] = `"(function(modules){function webpackJsonpCallback(data){var chunkIds=data[0];var moreModules=data[1];var moduleId,chunkId,i=0,resolves=[];for(;i { dist: false, }); - expect(config.limits).toEqual(readLimits()); - (config as any).limits = ''; - expect(config).toMatchSnapshot('OptimizerConfig'); const msgs = await allValuesFrom( @@ -235,6 +226,10 @@ it('prepares assets for distribution', async () => { await allValuesFrom(runOptimizer(config).pipe(logOptimizerState(log, config))); + expect( + Fs.readFileSync(Path.resolve(MOCK_REPO_DIR, 'plugins/foo/target/public/metrics.json'), 'utf8') + ).toMatchSnapshot('metrics.json'); + expectFileMatchesSnapshotWithCompression('plugins/foo/target/public/foo.plugin.js', 'foo bundle'); expectFileMatchesSnapshotWithCompression( 'plugins/foo/target/public/foo.chunk.1.js', diff --git a/packages/kbn-optimizer/src/limits.ts b/packages/kbn-optimizer/src/limits.ts index fcfd36664c1f4..292314a4608e4 100644 --- a/packages/kbn-optimizer/src/limits.ts +++ b/packages/kbn-optimizer/src/limits.ts @@ -7,12 +7,13 @@ */ import Fs from 'fs'; +import Path from 'path'; import dedent from 'dedent'; import Yaml from 'js-yaml'; -import { createFailError, ToolingLog } from '@kbn/dev-utils'; +import { createFailError, ToolingLog, CiStatsMetrics } from '@kbn/dev-utils'; -import { OptimizerConfig, getMetrics, Limits } from './optimizer'; +import { OptimizerConfig, Limits } from './optimizer'; const LIMITS_PATH = require.resolve('../limits.yml'); const DEFAULT_BUDGET = 15000; @@ -33,7 +34,7 @@ export function readLimits(): Limits { } export function validateLimitsForAllBundles(log: ToolingLog, config: OptimizerConfig) { - const limitBundleIds = Object.keys(config.limits.pageLoadAssetSize || {}); + const limitBundleIds = Object.keys(readLimits().pageLoadAssetSize || {}); const configBundleIds = config.bundles.map((b) => b.id); const missingBundleIds = diff(configBundleIds, limitBundleIds); @@ -75,15 +76,21 @@ interface UpdateBundleLimitsOptions { } export function updateBundleLimits({ log, config, dropMissing }: UpdateBundleLimitsOptions) { - const metrics = getMetrics(log, config); + const limits = readLimits(); + const metrics: CiStatsMetrics = config.bundles + .map((bundle) => + JSON.parse(Fs.readFileSync(Path.resolve(bundle.outputDir, 'metrics.json'), 'utf-8')) + ) + .flat() + .sort((a, b) => a.id.localeCompare(b.id)); const pageLoadAssetSize: NonNullable = dropMissing ? {} - : config.limits.pageLoadAssetSize ?? {}; + : limits.pageLoadAssetSize ?? {}; - for (const metric of metrics.sort((a, b) => a.id.localeCompare(b.id))) { + for (const metric of metrics) { if (metric.group === 'page load bundle size') { - const existingLimit = config.limits.pageLoadAssetSize?.[metric.id]; + const existingLimit = limits.pageLoadAssetSize?.[metric.id]; pageLoadAssetSize[metric.id] = existingLimit != null && existingLimit >= metric.value ? existingLimit diff --git a/packages/kbn-optimizer/src/optimizer/get_output_stats.ts b/packages/kbn-optimizer/src/optimizer/get_output_stats.ts deleted file mode 100644 index e7059c4d6799c..0000000000000 --- a/packages/kbn-optimizer/src/optimizer/get_output_stats.ts +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import Fs from 'fs'; -import Path from 'path'; - -import { ToolingLog, CiStatsMetrics } from '@kbn/dev-utils'; -import { OptimizerConfig } from './optimizer_config'; - -const flatten = (arr: Array): T[] => - arr.reduce((acc: T[], item) => acc.concat(item), []); - -interface Entry { - relPath: string; - stats: Fs.Stats; -} - -const IGNORED_EXTNAME = ['.map', '.br', '.gz']; - -const getFiles = (dir: string, parent?: string) => - flatten( - Fs.readdirSync(dir).map((name): Entry | Entry[] => { - const absPath = Path.join(dir, name); - const relPath = parent ? Path.join(parent, name) : name; - const stats = Fs.statSync(absPath); - - if (stats.isDirectory()) { - return getFiles(absPath, relPath); - } - - return { - relPath, - stats, - }; - }) - ).filter((file) => { - const filename = Path.basename(file.relPath); - if (filename.startsWith('.')) { - return false; - } - - const ext = Path.extname(filename); - if (IGNORED_EXTNAME.includes(ext)) { - return false; - } - - return true; - }); - -export function getMetrics(log: ToolingLog, config: OptimizerConfig) { - return flatten( - config.bundles.map((bundle) => { - // make the cache read from the cache file since it was likely updated by the worker - bundle.cache.refresh(); - - const outputFiles = getFiles(bundle.outputDir); - const entryName = `${bundle.id}.${bundle.type}.js`; - const entry = outputFiles.find((f) => f.relPath === entryName); - if (!entry) { - throw new Error( - `Unable to find bundle entry named [${entryName}] in [${bundle.outputDir}]` - ); - } - - const chunkPrefix = `${bundle.id}.chunk.`; - const asyncChunks = outputFiles.filter((f) => f.relPath.startsWith(chunkPrefix)); - const miscFiles = outputFiles.filter((f) => f !== entry && !asyncChunks.includes(f)); - - if (asyncChunks.length) { - log.verbose(bundle.id, 'async chunks', asyncChunks); - } - if (miscFiles.length) { - log.verbose(bundle.id, 'misc files', asyncChunks); - } - - const sumSize = (files: Entry[]) => files.reduce((acc: number, f) => acc + f.stats!.size, 0); - - const bundleMetrics: CiStatsMetrics = [ - { - group: `@kbn/optimizer bundle module count`, - id: bundle.id, - value: bundle.cache.getModuleCount() || 0, - }, - { - group: `page load bundle size`, - id: bundle.id, - value: entry.stats!.size, - limit: config.limits.pageLoadAssetSize?.[bundle.id], - limitConfigPath: `packages/kbn-optimizer/limits.yml`, - }, - { - group: `async chunks size`, - id: bundle.id, - value: sumSize(asyncChunks), - }, - { - group: `async chunk count`, - id: bundle.id, - value: asyncChunks.length, - }, - { - group: `miscellaneous assets size`, - id: bundle.id, - value: sumSize(miscFiles), - }, - ]; - - log.debug(bundle.id, 'metrics', bundleMetrics); - - return bundleMetrics; - }) - ); -} diff --git a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts index d921d5e5cca31..e4cdddbf56dcb 100644 --- a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts +++ b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts @@ -48,7 +48,12 @@ it('returns a bundle for core and each plugin', () => { }, ], '/repo', - '/output' + '/output', + { + pageLoadAssetSize: { + box: 123, + }, + } ).map((b) => b.toSpec()) ).toMatchInlineSnapshot(` Array [ @@ -58,6 +63,7 @@ it('returns a bundle for core and each plugin', () => { "id": "foo", "manifestPath": /plugins/foo/kibana.json, "outputDir": /plugins/foo/target/public, + "pageLoadAssetSizeLimit": undefined, "publicDirNames": Array [ "public", ], @@ -70,6 +76,7 @@ it('returns a bundle for core and each plugin', () => { "id": "baz", "manifestPath": /plugins/baz/kibana.json, "outputDir": /plugins/baz/target/public, + "pageLoadAssetSizeLimit": undefined, "publicDirNames": Array [ "public", ], @@ -84,6 +91,7 @@ it('returns a bundle for core and each plugin', () => { "id": "box", "manifestPath": /x-pack/plugins/box/kibana.json, "outputDir": /x-pack/plugins/box/target/public, + "pageLoadAssetSizeLimit": 123, "publicDirNames": Array [ "public", ], diff --git a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts index 76a0d51edac82..8134707561bc0 100644 --- a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts +++ b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts @@ -9,13 +9,15 @@ import Path from 'path'; import { Bundle } from '../common'; +import { Limits } from './optimizer_config'; import { KibanaPlatformPlugin } from './kibana_platform_plugins'; export function getPluginBundles( plugins: KibanaPlatformPlugin[], repoRoot: string, - outputRoot: string + outputRoot: string, + limits: Limits ) { const xpackDirSlash = Path.resolve(repoRoot, 'x-pack') + Path.sep; @@ -39,6 +41,7 @@ export function getPluginBundles( ? `/*! Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one or more contributor license agreements. \n` + ` * Licensed under the Elastic License 2.0; you may not use this file except in compliance with the Elastic License 2.0. */\n` : undefined, + pageLoadAssetSizeLimit: limits.pageLoadAssetSize?.[p.id], }) ); } diff --git a/packages/kbn-optimizer/src/optimizer/index.ts b/packages/kbn-optimizer/src/optimizer/index.ts index ced61463d5edd..28d206488b0a4 100644 --- a/packages/kbn-optimizer/src/optimizer/index.ts +++ b/packages/kbn-optimizer/src/optimizer/index.ts @@ -14,4 +14,3 @@ export * from './watch_bundles_for_changes'; export * from './run_workers'; export * from './bundle_cache'; export * from './handle_optimizer_completion'; -export * from './get_output_stats'; diff --git a/packages/kbn-optimizer/src/optimizer/optimizer_config.test.ts b/packages/kbn-optimizer/src/optimizer/optimizer_config.test.ts index 5677719628b6a..c60d6719cdea7 100644 --- a/packages/kbn-optimizer/src/optimizer/optimizer_config.test.ts +++ b/packages/kbn-optimizer/src/optimizer/optimizer_config.test.ts @@ -435,7 +435,6 @@ describe('OptimizerConfig::create()', () => { "cache": Symbol(parsed cache), "dist": Symbol(parsed dist), "inspectWorkers": Symbol(parsed inspect workers), - "limits": Symbol(limits), "maxWorkerCount": Symbol(parsed max worker count), "plugins": Symbol(new platform plugins), "profileWebpack": Symbol(parsed profile webpack), @@ -457,7 +456,7 @@ describe('OptimizerConfig::create()', () => { [Window], ], "invocationCallOrder": Array [ - 21, + 22, ], "results": Array [ Object { @@ -480,7 +479,7 @@ describe('OptimizerConfig::create()', () => { [Window], ], "invocationCallOrder": Array [ - 24, + 25, ], "results": Array [ Object { @@ -498,13 +497,14 @@ describe('OptimizerConfig::create()', () => { Symbol(new platform plugins), Symbol(parsed repo root), Symbol(parsed output root), + Symbol(limits), ], ], "instances": Array [ [Window], ], "invocationCallOrder": Array [ - 22, + 23, ], "results": Array [ Object { diff --git a/packages/kbn-optimizer/src/optimizer/optimizer_config.ts b/packages/kbn-optimizer/src/optimizer/optimizer_config.ts index b93d7a753c9ac..ed521d32a0a29 100644 --- a/packages/kbn-optimizer/src/optimizer/optimizer_config.ts +++ b/packages/kbn-optimizer/src/optimizer/optimizer_config.ts @@ -211,6 +211,7 @@ export class OptimizerConfig { } static create(inputOptions: Options) { + const limits = readLimits(); const options = OptimizerConfig.parseOptions(inputOptions); const plugins = findKibanaPlatformPlugins(options.pluginScanDirs, options.pluginPaths); const bundles = [ @@ -223,10 +224,11 @@ export class OptimizerConfig { sourceRoot: options.repoRoot, contextDir: Path.resolve(options.repoRoot, 'src/core'), outputDir: Path.resolve(options.outputRoot, 'src/core/target/public'), + pageLoadAssetSizeLimit: limits.pageLoadAssetSize?.core, }), ] : []), - ...getPluginBundles(plugins, options.repoRoot, options.outputRoot), + ...getPluginBundles(plugins, options.repoRoot, options.outputRoot, limits), ]; return new OptimizerConfig( @@ -239,8 +241,7 @@ export class OptimizerConfig { options.maxWorkerCount, options.dist, options.profileWebpack, - options.themeTags, - readLimits() + options.themeTags ); } @@ -254,8 +255,7 @@ export class OptimizerConfig { public readonly maxWorkerCount: number, public readonly dist: boolean, public readonly profileWebpack: boolean, - public readonly themeTags: ThemeTags, - public readonly limits: Limits + public readonly themeTags: ThemeTags ) {} getWorkerConfig(optimizerCacheKey: unknown): WorkerConfig { diff --git a/packages/kbn-optimizer/src/report_optimizer_stats.ts b/packages/kbn-optimizer/src/report_optimizer_stats.ts deleted file mode 100644 index eeed2fb1b156c..0000000000000 --- a/packages/kbn-optimizer/src/report_optimizer_stats.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { materialize, mergeMap, dematerialize } from 'rxjs/operators'; -import { CiStatsReporter, ToolingLog } from '@kbn/dev-utils'; - -import { OptimizerUpdate$ } from './run_optimizer'; -import { OptimizerConfig, getMetrics } from './optimizer'; -import { pipeClosure } from './common'; - -export function reportOptimizerStats( - reporter: CiStatsReporter, - config: OptimizerConfig, - log: ToolingLog -) { - return pipeClosure((update$: OptimizerUpdate$) => - update$.pipe( - materialize(), - mergeMap(async (n) => { - if (n.kind === 'C') { - const metrics = getMetrics(log, config); - - await reporter.metrics(metrics); - - for (const metric of metrics) { - if (metric.limit != null && metric.value > metric.limit) { - const value = metric.value.toLocaleString(); - const limit = metric.limit.toLocaleString(); - log.warning( - `Metric [${metric.group}] for [${metric.id}] of [${value}] over the limit of [${limit}]` - ); - } - } - } - - return n; - }), - dematerialize() - ) - ); -} diff --git a/packages/kbn-optimizer/src/worker/bundle_metrics_plugin.ts b/packages/kbn-optimizer/src/worker/bundle_metrics_plugin.ts new file mode 100644 index 0000000000000..909a97a3e11c7 --- /dev/null +++ b/packages/kbn-optimizer/src/worker/bundle_metrics_plugin.ts @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; + +import webpack from 'webpack'; +import { RawSource } from 'webpack-sources'; +import { CiStatsMetrics } from '@kbn/dev-utils'; + +import { Bundle } from '../common'; + +interface Asset { + name: string; + size: number; +} + +const IGNORED_EXTNAME = ['.map', '.br', '.gz']; + +export class BundleMetricsPlugin { + constructor(private readonly bundle: Bundle) {} + + public apply(compiler: webpack.Compiler) { + const { bundle } = this; + + compiler.hooks.emit.tap('BundleMetricsPlugin', (compilation) => { + const assets = Object.entries(compilation.assets) + .map( + ([name, source]: [string, any]): Asset => ({ + name, + size: source.size(), + }) + ) + .filter((asset) => { + const filename = Path.basename(asset.name); + if (filename.startsWith('.')) { + return false; + } + + const ext = Path.extname(filename); + if (IGNORED_EXTNAME.includes(ext)) { + return false; + } + + return true; + }); + + const entryName = `${bundle.id}.${bundle.type}.js`; + const entry = assets.find((a) => a.name === entryName); + if (!entry) { + throw new Error( + `Unable to find bundle entry named [${entryName}] in [${bundle.outputDir}]` + ); + } + + const chunkPrefix = `${bundle.id}.chunk.`; + const asyncChunks = assets.filter((a) => a.name.startsWith(chunkPrefix)); + const miscFiles = assets.filter((a) => a !== entry && !asyncChunks.includes(a)); + + const sumSize = (files: Asset[]) => files.reduce((acc: number, a) => acc + a.size, 0); + + const moduleCount = bundle.cache.getModuleCount(); + if (moduleCount === undefined) { + throw new Error(`moduleCount wasn't populated by PopulateBundleCachePlugin`); + } + + const bundleMetrics: CiStatsMetrics = [ + { + group: `@kbn/optimizer bundle module count`, + id: bundle.id, + value: moduleCount, + }, + { + group: `page load bundle size`, + id: bundle.id, + value: entry.size, + limit: bundle.pageLoadAssetSizeLimit, + limitConfigPath: `packages/kbn-optimizer/limits.yml`, + }, + { + group: `async chunks size`, + id: bundle.id, + value: sumSize(asyncChunks), + }, + { + group: `async chunk count`, + id: bundle.id, + value: asyncChunks.length, + }, + { + group: `miscellaneous assets size`, + id: bundle.id, + value: sumSize(miscFiles), + }, + ]; + + const metricsSource = new RawSource(JSON.stringify(bundleMetrics, null, 2)); + + // see https://github.com/jantimon/html-webpack-plugin/blob/33d69f49e6e9787796402715d1b9cd59f80b628f/index.js#L266 + // @ts-expect-error undocumented, used to add assets to the output + compilation.emitAsset('metrics.json', metricsSource); + }); + } +} diff --git a/packages/kbn-optimizer/src/worker/emit_stats_plugin.ts b/packages/kbn-optimizer/src/worker/emit_stats_plugin.ts new file mode 100644 index 0000000000000..c964219e1fed6 --- /dev/null +++ b/packages/kbn-optimizer/src/worker/emit_stats_plugin.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Fs from 'fs'; +import Path from 'path'; + +import webpack from 'webpack'; + +import { Bundle } from '../common'; + +export class EmitStatsPlugin { + constructor(private readonly bundle: Bundle) {} + + public apply(compiler: webpack.Compiler) { + compiler.hooks.done.tap( + { + name: 'EmitStatsPlugin', + // run at the very end, ensure that it's after clean-webpack-plugin + stage: 10, + }, + (stats) => { + Fs.writeFileSync( + Path.resolve(this.bundle.outputDir, 'stats.json'), + JSON.stringify(stats.toJson()) + ); + } + ); + } +} diff --git a/packages/kbn-optimizer/src/worker/populate_bundle_cache_plugin.ts b/packages/kbn-optimizer/src/worker/populate_bundle_cache_plugin.ts new file mode 100644 index 0000000000000..6d296b9be089c --- /dev/null +++ b/packages/kbn-optimizer/src/worker/populate_bundle_cache_plugin.ts @@ -0,0 +1,132 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import webpack from 'webpack'; + +import Path from 'path'; +import { inspect } from 'util'; + +import { Bundle, WorkerConfig, ascending, parseFilePath } from '../common'; +import { BundleRefModule } from './bundle_ref_module'; +import { + isExternalModule, + isNormalModule, + isIgnoredModule, + isConcatenatedModule, + getModulePath, +} from './webpack_helpers'; + +/** + * sass-loader creates about a 40% overhead on the overall optimizer runtime, and + * so this constant is used to indicate to assignBundlesToWorkers() that there is + * extra work done in a bundle that has a lot of scss imports. The value is + * arbitrary and just intended to weigh the bundles so that they are distributed + * across mulitple workers on machines with lots of cores. + */ +const EXTRA_SCSS_WORK_UNITS = 100; + +export class PopulateBundleCachePlugin { + constructor(private readonly workerConfig: WorkerConfig, private readonly bundle: Bundle) {} + + public apply(compiler: webpack.Compiler) { + const { bundle, workerConfig } = this; + + compiler.hooks.emit.tap( + { + name: 'PopulateBundleCachePlugin', + before: ['BundleMetricsPlugin'], + }, + (compilation) => { + const bundleRefExportIds: string[] = []; + const referencedFiles = new Set(); + let moduleCount = 0; + let workUnits = compilation.fileDependencies.size; + + if (bundle.manifestPath) { + referencedFiles.add(bundle.manifestPath); + } + + for (const module of compilation.modules) { + if (isNormalModule(module)) { + moduleCount += 1; + const path = getModulePath(module); + const parsedPath = parseFilePath(path); + + if (!parsedPath.dirs.includes('node_modules')) { + referencedFiles.add(path); + + if (path.endsWith('.scss')) { + workUnits += EXTRA_SCSS_WORK_UNITS; + + for (const depPath of module.buildInfo.fileDependencies) { + referencedFiles.add(depPath); + } + } + + continue; + } + + const nmIndex = parsedPath.dirs.lastIndexOf('node_modules'); + const isScoped = parsedPath.dirs[nmIndex + 1].startsWith('@'); + referencedFiles.add( + Path.join( + parsedPath.root, + ...parsedPath.dirs.slice(0, nmIndex + 1 + (isScoped ? 2 : 1)), + 'package.json' + ) + ); + continue; + } + + if (module instanceof BundleRefModule) { + bundleRefExportIds.push(module.ref.exportId); + continue; + } + + if (isConcatenatedModule(module)) { + moduleCount += module.modules.length; + continue; + } + + if (isExternalModule(module) || isIgnoredModule(module)) { + continue; + } + + throw new Error(`Unexpected module type: ${inspect(module)}`); + } + + const files = Array.from(referencedFiles).sort(ascending((p) => p)); + const mtimes = new Map( + files.map((path): [string, number | undefined] => { + try { + return [path, compiler.inputFileSystem.statSync(path)?.mtimeMs]; + } catch (error) { + if (error?.code === 'ENOENT') { + return [path, undefined]; + } + + throw error; + } + }) + ); + + bundle.cache.set({ + bundleRefExportIds: bundleRefExportIds.sort(ascending((p) => p)), + optimizerCacheKey: workerConfig.optimizerCacheKey, + cacheKey: bundle.createCacheKey(files, mtimes), + moduleCount, + workUnits, + files, + }); + + // write the cache to the compilation so that it isn't cleaned by clean-webpack-plugin + bundle.cache.writeWebpackAsset(compilation); + } + ); + } +} diff --git a/packages/kbn-optimizer/src/worker/run_compilers.ts b/packages/kbn-optimizer/src/worker/run_compilers.ts index 61f9c243a4def..4f5bb23c3550d 100644 --- a/packages/kbn-optimizer/src/worker/run_compilers.ts +++ b/packages/kbn-optimizer/src/worker/run_compilers.ts @@ -8,46 +8,16 @@ import 'source-map-support/register'; -import Fs from 'fs'; -import Path from 'path'; -import { inspect } from 'util'; - import webpack, { Stats } from 'webpack'; import * as Rx from 'rxjs'; import { mergeMap, map, mapTo, takeUntil } from 'rxjs/operators'; -import { - CompilerMsgs, - CompilerMsg, - maybeMap, - Bundle, - WorkerConfig, - ascending, - parseFilePath, - BundleRefs, -} from '../common'; -import { BundleRefModule } from './bundle_ref_module'; +import { CompilerMsgs, CompilerMsg, maybeMap, Bundle, WorkerConfig, BundleRefs } from '../common'; import { getWebpackConfig } from './webpack.config'; import { isFailureStats, failedStatsToErrorMessage } from './webpack_helpers'; -import { - isExternalModule, - isNormalModule, - isIgnoredModule, - isConcatenatedModule, - getModulePath, -} from './webpack_helpers'; const PLUGIN_NAME = '@kbn/optimizer'; -/** - * sass-loader creates about a 40% overhead on the overall optimizer runtime, and - * so this constant is used to indicate to assignBundlesToWorkers() that there is - * extra work done in a bundle that has a lot of scss imports. The value is - * arbitrary and just intended to weigh the bundles so that they are distributed - * across mulitple workers on machines with lots of cores. - */ -const EXTRA_SCSS_WORK_UNITS = 100; - /** * Create an Observable for a specific child compiler + bundle */ @@ -80,13 +50,6 @@ const observeCompiler = ( return undefined; } - if (workerConfig.profileWebpack) { - Fs.writeFileSync( - Path.resolve(bundle.outputDir, 'stats.json'), - JSON.stringify(stats.toJson()) - ); - } - if (!workerConfig.watch) { process.nextTick(() => done$.next()); } @@ -97,88 +60,11 @@ const observeCompiler = ( }); } - const bundleRefExportIds: string[] = []; - const referencedFiles = new Set(); - let moduleCount = 0; - let workUnits = stats.compilation.fileDependencies.size; - - if (bundle.manifestPath) { - referencedFiles.add(bundle.manifestPath); - } - - for (const module of stats.compilation.modules) { - if (isNormalModule(module)) { - moduleCount += 1; - const path = getModulePath(module); - const parsedPath = parseFilePath(path); - - if (!parsedPath.dirs.includes('node_modules')) { - referencedFiles.add(path); - - if (path.endsWith('.scss')) { - workUnits += EXTRA_SCSS_WORK_UNITS; - - for (const depPath of module.buildInfo.fileDependencies) { - referencedFiles.add(depPath); - } - } - - continue; - } - - const nmIndex = parsedPath.dirs.lastIndexOf('node_modules'); - const isScoped = parsedPath.dirs[nmIndex + 1].startsWith('@'); - referencedFiles.add( - Path.join( - parsedPath.root, - ...parsedPath.dirs.slice(0, nmIndex + 1 + (isScoped ? 2 : 1)), - 'package.json' - ) - ); - continue; - } - - if (module instanceof BundleRefModule) { - bundleRefExportIds.push(module.ref.exportId); - continue; - } - - if (isConcatenatedModule(module)) { - moduleCount += module.modules.length; - continue; - } - - if (isExternalModule(module) || isIgnoredModule(module)) { - continue; - } - - throw new Error(`Unexpected module type: ${inspect(module)}`); + const moduleCount = bundle.cache.getModuleCount(); + if (moduleCount === undefined) { + throw new Error(`moduleCount wasn't populated by PopulateBundleCachePlugin`); } - const files = Array.from(referencedFiles).sort(ascending((p) => p)); - const mtimes = new Map( - files.map((path): [string, number | undefined] => { - try { - return [path, compiler.inputFileSystem.statSync(path)?.mtimeMs]; - } catch (error) { - if (error?.code === 'ENOENT') { - return [path, undefined]; - } - - throw error; - } - }) - ); - - bundle.cache.set({ - bundleRefExportIds: bundleRefExportIds.sort(ascending((p) => p)), - optimizerCacheKey: workerConfig.optimizerCacheKey, - cacheKey: bundle.createCacheKey(files, mtimes), - moduleCount, - workUnits, - files, - }); - return compilerMsgs.compilerSuccess({ moduleCount, }); diff --git a/packages/kbn-optimizer/src/worker/webpack.config.ts b/packages/kbn-optimizer/src/worker/webpack.config.ts index 331fbde6ea0ba..c4beb959284cc 100644 --- a/packages/kbn-optimizer/src/worker/webpack.config.ts +++ b/packages/kbn-optimizer/src/worker/webpack.config.ts @@ -19,6 +19,9 @@ import * as UiSharedDeps from '@kbn/ui-shared-deps'; import { Bundle, BundleRefs, WorkerConfig } from '../common'; import { BundleRefsPlugin } from './bundle_refs_plugin'; +import { BundleMetricsPlugin } from './bundle_metrics_plugin'; +import { EmitStatsPlugin } from './emit_stats_plugin'; +import { PopulateBundleCachePlugin } from './populate_bundle_cache_plugin'; const IS_CODE_COVERAGE = !!process.env.CODE_COVERAGE; const ISTANBUL_PRESET_PATH = require.resolve('@kbn/babel-preset/istanbul_preset'); @@ -67,6 +70,9 @@ export function getWebpackConfig(bundle: Bundle, bundleRefs: BundleRefs, worker: plugins: [ new CleanWebpackPlugin(), new BundleRefsPlugin(bundle, bundleRefs), + new PopulateBundleCachePlugin(worker, bundle), + new BundleMetricsPlugin(bundle), + ...(worker.profileWebpack ? [new EmitStatsPlugin(bundle)] : []), ...(bundle.banner ? [new webpack.BannerPlugin({ banner: bundle.banner, raw: true })] : []), ], diff --git a/packages/kbn-plugin-helpers/src/integration_tests/build.test.ts b/packages/kbn-plugin-helpers/src/integration_tests/build.test.ts index 559d9da35c320..9723c0107cf8e 100644 --- a/packages/kbn-plugin-helpers/src/integration_tests/build.test.ts +++ b/packages/kbn-plugin-helpers/src/integration_tests/build.test.ts @@ -74,13 +74,14 @@ it('builds a generated plugin into a viable archive', async () => { await extract(PLUGIN_ARCHIVE, { dir: TMP_DIR }); - const files = await globby(['**/*'], { cwd: TMP_DIR }); + const files = await globby(['**/*'], { cwd: TMP_DIR, dot: true }); files.sort((a, b) => a.localeCompare(b)); expect(files).toMatchInlineSnapshot(` Array [ "kibana/fooTestPlugin/common/index.js", "kibana/fooTestPlugin/kibana.json", + "kibana/fooTestPlugin/node_modules/.yarn-integrity", "kibana/fooTestPlugin/package.json", "kibana/fooTestPlugin/server/index.js", "kibana/fooTestPlugin/server/plugin.js", diff --git a/packages/kbn-plugin-helpers/src/tasks/optimize.ts b/packages/kbn-plugin-helpers/src/tasks/optimize.ts index 0f0ac93086c9e..2478947e79f18 100644 --- a/packages/kbn-plugin-helpers/src/tasks/optimize.ts +++ b/packages/kbn-plugin-helpers/src/tasks/optimize.ts @@ -34,9 +34,15 @@ export async function optimize({ log, plugin, sourceDir, buildDir }: BuildContex pluginScanDirs: [], }); + const target = Path.resolve(sourceDir, 'target'); + await runOptimizer(config).pipe(logOptimizerState(log, config)).toPromise(); + // clean up unnecessary files + Fs.unlinkSync(Path.resolve(target, 'public/metrics.json')); + Fs.unlinkSync(Path.resolve(target, 'public/.kbn-optimizer-cache')); + // move target into buildDir - await asyncRename(Path.resolve(sourceDir, 'target'), Path.resolve(buildDir, 'target')); + await asyncRename(target, Path.resolve(buildDir, 'target')); log.indent(-2); } diff --git a/scripts/ship_ci_stats.js b/scripts/ship_ci_stats.js new file mode 100644 index 0000000000000..5aed9fc446240 --- /dev/null +++ b/scripts/ship_ci_stats.js @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +require('../src/setup_node_env/no_transpilation'); +require('@kbn/dev-utils').shipCiStatsCli(); diff --git a/src/dev/build/tasks/build_kibana_platform_plugins.ts b/src/dev/build/tasks/build_kibana_platform_plugins.ts index 91fad2ca52617..d2d2d3275270b 100644 --- a/src/dev/build/tasks/build_kibana_platform_plugins.ts +++ b/src/dev/build/tasks/build_kibana_platform_plugins.ts @@ -6,20 +6,18 @@ * Side Public License, v 1. */ +import Path from 'path'; + import { REPO_ROOT } from '@kbn/utils'; -import { CiStatsReporter } from '@kbn/dev-utils'; -import { - runOptimizer, - OptimizerConfig, - logOptimizerState, - reportOptimizerStats, -} from '@kbn/optimizer'; +import { lastValueFrom } from '@kbn/std'; +import { CiStatsMetrics } from '@kbn/dev-utils'; +import { runOptimizer, OptimizerConfig, logOptimizerState } from '@kbn/optimizer'; -import { Task } from '../lib'; +import { Task, deleteAll, write, read } from '../lib'; export const BuildKibanaPlatformPlugins: Task = { description: 'Building distributable versions of Kibana platform plugins', - async run(_, log, build) { + async run(buildConfig, log, build) { const config = OptimizerConfig.create({ repoRoot: REPO_ROOT, outputRoot: build.resolvePath(), @@ -31,12 +29,27 @@ export const BuildKibanaPlatformPlugins: Task = { includeCoreBundle: true, }); - const reporter = CiStatsReporter.fromEnv(log); + await lastValueFrom(runOptimizer(config).pipe(logOptimizerState(log, config))); + + const combinedMetrics: CiStatsMetrics = []; + const metricFilePaths: string[] = []; + for (const bundle of config.bundles) { + const path = Path.resolve(bundle.outputDir, 'metrics.json'); + const metrics: CiStatsMetrics = JSON.parse(await read(path)); + combinedMetrics.push(...metrics); + metricFilePaths.push(path); + } + + // write combined metrics to target + await write( + buildConfig.resolveFromTarget('optimizer_bundle_metrics.json'), + JSON.stringify(combinedMetrics, null, 2) + ); - await runOptimizer(config) - .pipe(reportOptimizerStats(reporter, config, log), logOptimizerState(log, config)) - .toPromise(); + // delete all metric files + await deleteAll(metricFilePaths, log); + // delete all bundle cache files await Promise.all(config.bundles.map((b) => b.cache.clear())); }, }; diff --git a/test/scripts/jenkins_baseline.sh b/test/scripts/jenkins_baseline.sh index e679ac7f31bd1..60926238576c7 100755 --- a/test/scripts/jenkins_baseline.sh +++ b/test/scripts/jenkins_baseline.sh @@ -5,6 +5,10 @@ source "$KIBANA_DIR/src/dev/ci_setup/setup_percy.sh" echo " -> building and extracting OSS Kibana distributable for use in functional tests" node scripts/build --debug --oss + +echo " -> shipping metrics from build to ci-stats" +node scripts/ship_ci_stats --metrics target/optimizer_bundle_metrics.json + linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')" installDir="$PARENT_DIR/install/kibana" mkdir -p "$installDir" diff --git a/test/scripts/jenkins_build_kibana.sh b/test/scripts/jenkins_build_kibana.sh index 6184708ea3fc6..5819a3ce6765e 100755 --- a/test/scripts/jenkins_build_kibana.sh +++ b/test/scripts/jenkins_build_kibana.sh @@ -17,6 +17,9 @@ if [[ -z "$CODE_COVERAGE" ]] ; then echo " -> building and extracting OSS Kibana distributable for use in functional tests" node scripts/build --debug --oss + echo " -> shipping metrics from build to ci-stats" + node scripts/ship_ci_stats --metrics target/optimizer_bundle_metrics.json + mkdir -p "$WORKSPACE/kibana-build-oss" cp -pR build/oss/kibana-*-SNAPSHOT-linux-x86_64/. $WORKSPACE/kibana-build-oss/ fi diff --git a/test/scripts/jenkins_xpack_baseline.sh b/test/scripts/jenkins_xpack_baseline.sh index 7577b6927d166..aaacdd4ea3aae 100755 --- a/test/scripts/jenkins_xpack_baseline.sh +++ b/test/scripts/jenkins_xpack_baseline.sh @@ -6,6 +6,10 @@ source "$KIBANA_DIR/src/dev/ci_setup/setup_percy.sh" echo " -> building and extracting default Kibana distributable" cd "$KIBANA_DIR" node scripts/build --debug --no-oss + +echo " -> shipping metrics from build to ci-stats" +node scripts/ship_ci_stats --metrics target/optimizer_bundle_metrics.json + linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')" installDir="$KIBANA_DIR/install/kibana" mkdir -p "$installDir" diff --git a/test/scripts/jenkins_xpack_build_kibana.sh b/test/scripts/jenkins_xpack_build_kibana.sh index a9e603f63bd42..36865ce7c4967 100755 --- a/test/scripts/jenkins_xpack_build_kibana.sh +++ b/test/scripts/jenkins_xpack_build_kibana.sh @@ -32,6 +32,10 @@ if [[ -z "$CODE_COVERAGE" ]] ; then echo " -> building and extracting default Kibana distributable for use in functional tests" cd "$KIBANA_DIR" node scripts/build --debug --no-oss + + echo " -> shipping metrics from build to ci-stats" + node scripts/ship_ci_stats --metrics target/optimizer_bundle_metrics.json + linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')" installDir="$KIBANA_DIR/install/kibana" mkdir -p "$installDir" diff --git a/yarn.lock b/yarn.lock index 6df258e9715b7..ec6cf338a43da 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6803,7 +6803,7 @@ dependencies: "@types/webpack" "*" -"@types/webpack-sources@*": +"@types/webpack-sources@*", "@types/webpack-sources@^0.1.4": version "0.1.5" resolved "https://registry.yarnpkg.com/@types/webpack-sources/-/webpack-sources-0.1.5.tgz#be47c10f783d3d6efe1471ff7f042611bd464a92" integrity sha512-zfvjpp7jiafSmrzJ2/i3LqOyTYTuJ7u1KOXlKgDlvsj9Rr0x7ZiYu5lZbXwobL7lmsRNtPXlBfmaUD8eU2Hu8w==