es-archiver: Drop invalid index settings, support --query flag (#76522)

* ESArchiver: Drop invalid index settings, support --query flag for saving archive * Describe query flag
elastic · Sep 3, 2020 · 53c7414 · 53c7414
1 parent 9fe4f2d
commit 53c7414
Show file tree

Hide file tree

Showing 6 changed files with 43 additions and 7 deletions.
diff --git a/packages/kbn-es-archiver/src/actions/save.ts b/packages/kbn-es-archiver/src/actions/save.ts
@@ -39,13 +39,15 @@ export async function saveAction({
   dataDir,
   log,
   raw,
+  query,
 }: {
   name: string;
   indices: string | string[];
   client: Client;
   dataDir: string;
   log: ToolingLog;
   raw: boolean;
+  query?: Record<string, any>;
 }) {
   const outputDir = resolve(dataDir, name);
   const stats = createStats(name, log);
@@ -69,7 +71,7 @@ export async function saveAction({
     // export all documents from matching indexes into data.json.gz
     createPromiseFromStreams([
       createListStream(indices),
-      createGenerateDocRecordsStream(client, stats, progress),
+      createGenerateDocRecordsStream({ client, stats, progress, query }),
       ...createFormatArchiveStreams({ gzip: !raw }),
       createWriteStream(resolve(outputDir, `data.json${raw ? '' : '.gz'}`)),
     ] as [Readable, ...Writable[]]),

diff --git a/packages/kbn-es-archiver/src/cli.ts b/packages/kbn-es-archiver/src/cli.ts
@@ -122,8 +122,10 @@ export function runCli() {
       `,
       flags: {
         boolean: ['raw'],
+        string: ['query'],
         help: `
           --raw              don't gzip the archives
+          --query            query object to limit the documents being archived, needs to be properly escaped JSON
         `,
       },
       async run({ flags, esArchiver }) {
@@ -140,7 +142,17 @@ export function runCli() {
           throw createFlagError('--raw does not take a value');
         }
 
-        await esArchiver.save(name, indices, { raw });
+        const query = flags.query;
+        let parsedQuery;
+        if (typeof query === 'string') {
+          try {
+            parsedQuery = JSON.parse(query);
+          } catch (err) {
+            throw createFlagError('--query should be valid JSON');
+          }
+        }
+
+        await esArchiver.save(name, indices, { raw, query: parsedQuery });
       },
     })
     .command({

diff --git a/packages/kbn-es-archiver/src/es_archiver.ts b/packages/kbn-es-archiver/src/es_archiver.ts
@@ -62,14 +62,19 @@ export class EsArchiver {
    *  @property {Boolean} options.raw - should the archive be raw (unzipped) or not
    *  @return Promise<Stats>
    */
-  async save(name: string, indices: string | string[], { raw = false }: { raw?: boolean } = {}) {
+  async save(
+    name: string,
+    indices: string | string[],
+    { raw = false, query }: { raw?: boolean; query?: Record<string, any> } = {}
+  ) {
     return await saveAction({
       name,
       indices,
       raw,
       client: this.client,
       dataDir: this.dataDir,
       log: this.log,
+      query,
     });
   }
 

diff --git a/packages/kbn-es-archiver/src/lib/docs/__tests__/generate_doc_records_stream.ts b/packages/kbn-es-archiver/src/lib/docs/__tests__/generate_doc_records_stream.ts
@@ -47,7 +47,7 @@ describe('esArchiver: createGenerateDocRecordsStream()', () => {
     const progress = new Progress();
     await createPromiseFromStreams([
       createListStream(['logstash-*']),
-      createGenerateDocRecordsStream(client, stats, progress),
+      createGenerateDocRecordsStream({ client, stats, progress }),
     ]);
 
     expect(progress.getTotal()).to.be(0);
@@ -74,7 +74,7 @@ describe('esArchiver: createGenerateDocRecordsStream()', () => {
     const progress = new Progress();
     await createPromiseFromStreams([
       createListStream(['logstash-*']),
-      createGenerateDocRecordsStream(client, stats, progress),
+      createGenerateDocRecordsStream({ client, stats, progress }),
     ]);
 
     expect(progress.getTotal()).to.be(0);
@@ -115,7 +115,7 @@ describe('esArchiver: createGenerateDocRecordsStream()', () => {
     const progress = new Progress();
     const docRecords = await createPromiseFromStreams([
       createListStream(['index1', 'index2']),
-      createGenerateDocRecordsStream(client, stats, progress),
+      createGenerateDocRecordsStream({ client, stats, progress }),
       createConcatStream([]),
     ]);
 

diff --git a/packages/kbn-es-archiver/src/lib/docs/generate_doc_records_stream.ts b/packages/kbn-es-archiver/src/lib/docs/generate_doc_records_stream.ts
@@ -25,7 +25,17 @@ import { Progress } from '../progress';
 const SCROLL_SIZE = 1000;
 const SCROLL_TIMEOUT = '1m';
 
-export function createGenerateDocRecordsStream(client: Client, stats: Stats, progress: Progress) {
+export function createGenerateDocRecordsStream({
+  client,
+  stats,
+  progress,
+  query,
+}: {
+  client: Client;
+  stats: Stats;
+  progress: Progress;
+  query?: Record<string, any>;
+}) {
   return new Transform({
     writableObjectMode: true,
     readableObjectMode: true,
@@ -41,6 +51,9 @@ export function createGenerateDocRecordsStream(client: Client, stats: Stats, pro
               scroll: SCROLL_TIMEOUT,
               size: SCROLL_SIZE,
               _source: true,
+              body: {
+                query,
+              },
               rest_total_hits_as_int: true, // not declared on SearchParams type
             } as SearchParams);
             remainingHits = resp.hits.total;

diff --git a/packages/kbn-es-archiver/src/lib/indices/generate_index_records_stream.ts b/packages/kbn-es-archiver/src/lib/indices/generate_index_records_stream.ts
@@ -37,6 +37,10 @@ export function createGenerateIndexRecordsStream(client: Client, stats: Stats) {
             '-*.settings.index.uuid',
             '-*.settings.index.version',
             '-*.settings.index.provided_name',
+            '-*.settings.index.frozen',
+            '-*.settings.index.search.throttled',
+            '-*.settings.index.query',
+            '-*.settings.index.routing',
           ],
         })) as Record<string, any>;