From 7febd5a5f112ed84bcc3bf2540a901f0fdfc701e Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Fri, 4 Sep 2020 10:53:03 +0200 Subject: [PATCH] [7.x] es-archiver: Drop invalid settings, --query flag (#76522) (#76742) * ESArchiver: Drop invalid index settings, support --query flag for saving archive * Describe query flag --- packages/kbn-es-archiver/src/actions/save.ts | 4 +++- packages/kbn-es-archiver/src/cli.ts | 14 +++++++++++++- packages/kbn-es-archiver/src/es_archiver.ts | 7 ++++++- .../docs/__tests__/generate_doc_records_stream.ts | 6 +++--- .../src/lib/docs/generate_doc_records_stream.ts | 15 ++++++++++++++- .../lib/indices/generate_index_records_stream.ts | 4 ++++ 6 files changed, 43 insertions(+), 7 deletions(-) diff --git a/packages/kbn-es-archiver/src/actions/save.ts b/packages/kbn-es-archiver/src/actions/save.ts index 2f87cabadee6c..84a0ce09936d0 100644 --- a/packages/kbn-es-archiver/src/actions/save.ts +++ b/packages/kbn-es-archiver/src/actions/save.ts @@ -39,6 +39,7 @@ export async function saveAction({ dataDir, log, raw, + query, }: { name: string; indices: string | string[]; @@ -46,6 +47,7 @@ export async function saveAction({ dataDir: string; log: ToolingLog; raw: boolean; + query?: Record; }) { const outputDir = resolve(dataDir, name); const stats = createStats(name, log); @@ -69,7 +71,7 @@ export async function saveAction({ // export all documents from matching indexes into data.json.gz createPromiseFromStreams([ createListStream(indices), - createGenerateDocRecordsStream(client, stats, progress), + createGenerateDocRecordsStream({ client, stats, progress, query }), ...createFormatArchiveStreams({ gzip: !raw }), createWriteStream(resolve(outputDir, `data.json${raw ? '' : '.gz'}`)), ] as [Readable, ...Writable[]]), diff --git a/packages/kbn-es-archiver/src/cli.ts b/packages/kbn-es-archiver/src/cli.ts index 1745bd862b434..41abe83c148cd 100644 --- a/packages/kbn-es-archiver/src/cli.ts +++ b/packages/kbn-es-archiver/src/cli.ts @@ -122,8 +122,10 @@ export function runCli() { `, flags: { boolean: ['raw'], + string: ['query'], help: ` --raw don't gzip the archives + --query query object to limit the documents being archived, needs to be properly escaped JSON `, }, async run({ flags, esArchiver }) { @@ -140,7 +142,17 @@ export function runCli() { throw createFlagError('--raw does not take a value'); } - await esArchiver.save(name, indices, { raw }); + const query = flags.query; + let parsedQuery; + if (typeof query === 'string') { + try { + parsedQuery = JSON.parse(query); + } catch (err) { + throw createFlagError('--query should be valid JSON'); + } + } + + await esArchiver.save(name, indices, { raw, query: parsedQuery }); }, }) .command({ diff --git a/packages/kbn-es-archiver/src/es_archiver.ts b/packages/kbn-es-archiver/src/es_archiver.ts index e335652195b86..d61e7d2a422e8 100644 --- a/packages/kbn-es-archiver/src/es_archiver.ts +++ b/packages/kbn-es-archiver/src/es_archiver.ts @@ -62,7 +62,11 @@ export class EsArchiver { * @property {Boolean} options.raw - should the archive be raw (unzipped) or not * @return Promise */ - async save(name: string, indices: string | string[], { raw = false }: { raw?: boolean } = {}) { + async save( + name: string, + indices: string | string[], + { raw = false, query }: { raw?: boolean; query?: Record } = {} + ) { return await saveAction({ name, indices, @@ -70,6 +74,7 @@ export class EsArchiver { client: this.client, dataDir: this.dataDir, log: this.log, + query, }); } diff --git a/packages/kbn-es-archiver/src/lib/docs/__tests__/generate_doc_records_stream.ts b/packages/kbn-es-archiver/src/lib/docs/__tests__/generate_doc_records_stream.ts index 2214f7ae9f2ea..3c5fc742a6e10 100644 --- a/packages/kbn-es-archiver/src/lib/docs/__tests__/generate_doc_records_stream.ts +++ b/packages/kbn-es-archiver/src/lib/docs/__tests__/generate_doc_records_stream.ts @@ -47,7 +47,7 @@ describe('esArchiver: createGenerateDocRecordsStream()', () => { const progress = new Progress(); await createPromiseFromStreams([ createListStream(['logstash-*']), - createGenerateDocRecordsStream(client, stats, progress), + createGenerateDocRecordsStream({ client, stats, progress }), ]); expect(progress.getTotal()).to.be(0); @@ -74,7 +74,7 @@ describe('esArchiver: createGenerateDocRecordsStream()', () => { const progress = new Progress(); await createPromiseFromStreams([ createListStream(['logstash-*']), - createGenerateDocRecordsStream(client, stats, progress), + createGenerateDocRecordsStream({ client, stats, progress }), ]); expect(progress.getTotal()).to.be(0); @@ -115,7 +115,7 @@ describe('esArchiver: createGenerateDocRecordsStream()', () => { const progress = new Progress(); const docRecords = await createPromiseFromStreams([ createListStream(['index1', 'index2']), - createGenerateDocRecordsStream(client, stats, progress), + createGenerateDocRecordsStream({ client, stats, progress }), createConcatStream([]), ]); diff --git a/packages/kbn-es-archiver/src/lib/docs/generate_doc_records_stream.ts b/packages/kbn-es-archiver/src/lib/docs/generate_doc_records_stream.ts index e255a0abc36c5..87c166fe275cc 100644 --- a/packages/kbn-es-archiver/src/lib/docs/generate_doc_records_stream.ts +++ b/packages/kbn-es-archiver/src/lib/docs/generate_doc_records_stream.ts @@ -25,7 +25,17 @@ import { Progress } from '../progress'; const SCROLL_SIZE = 1000; const SCROLL_TIMEOUT = '1m'; -export function createGenerateDocRecordsStream(client: Client, stats: Stats, progress: Progress) { +export function createGenerateDocRecordsStream({ + client, + stats, + progress, + query, +}: { + client: Client; + stats: Stats; + progress: Progress; + query?: Record; +}) { return new Transform({ writableObjectMode: true, readableObjectMode: true, @@ -41,6 +51,9 @@ export function createGenerateDocRecordsStream(client: Client, stats: Stats, pro scroll: SCROLL_TIMEOUT, size: SCROLL_SIZE, _source: true, + body: { + query, + }, rest_total_hits_as_int: true, // not declared on SearchParams type } as SearchParams); remainingHits = resp.hits.total; diff --git a/packages/kbn-es-archiver/src/lib/indices/generate_index_records_stream.ts b/packages/kbn-es-archiver/src/lib/indices/generate_index_records_stream.ts index b4b98f8ae262c..07ee1420741c9 100644 --- a/packages/kbn-es-archiver/src/lib/indices/generate_index_records_stream.ts +++ b/packages/kbn-es-archiver/src/lib/indices/generate_index_records_stream.ts @@ -37,6 +37,10 @@ export function createGenerateIndexRecordsStream(client: Client, stats: Stats) { '-*.settings.index.uuid', '-*.settings.index.version', '-*.settings.index.provided_name', + '-*.settings.index.frozen', + '-*.settings.index.search.throttled', + '-*.settings.index.query', + '-*.settings.index.routing', ], })) as Record;