diff --git a/package.json b/package.json index b73ceac736130..7e9ee509109c4 100644 --- a/package.json +++ b/package.json @@ -296,6 +296,7 @@ "@elastic/makelogs": "^6.0.0", "@kbn/dev-utils": "1.0.0", "@kbn/es": "1.0.0", + "@kbn/es-archiver": "1.0.0", "@kbn/eslint-import-resolver-kibana": "2.0.0", "@kbn/eslint-plugin-eslint": "1.0.0", "@kbn/expect": "1.0.0", diff --git a/packages/kbn-dev-utils/src/run/run_with_commands.test.ts b/packages/kbn-dev-utils/src/run/run_with_commands.test.ts index eb7708998751c..f6759b218bf32 100644 --- a/packages/kbn-dev-utils/src/run/run_with_commands.test.ts +++ b/packages/kbn-dev-utils/src/run/run_with_commands.test.ts @@ -61,9 +61,7 @@ it('extends the context using extendContext()', async () => { expect(context.flags).toMatchInlineSnapshot(` Object { - "_": Array [ - "foo", - ], + "_": Array [], "debug": false, "help": false, "quiet": false, diff --git a/packages/kbn-dev-utils/src/run/run_with_commands.ts b/packages/kbn-dev-utils/src/run/run_with_commands.ts index 9fb069e4b2d35..ca56a17b545a7 100644 --- a/packages/kbn-dev-utils/src/run/run_with_commands.ts +++ b/packages/kbn-dev-utils/src/run/run_with_commands.ts @@ -91,6 +91,13 @@ export class RunWithCommands { const commandFlagOptions = mergeFlagOptions(this.options.globalFlags, command.flags); const commandFlags = getFlags(process.argv.slice(2), commandFlagOptions); + // strip command name plus "help" if we're actually executing the fake "help" command + if (isHelpCommand) { + commandFlags._.splice(0, 2); + } else { + commandFlags._.splice(0, 1); + } + const commandHelp = getCommandLevelHelp({ usage: this.options.usage, globalFlagHelp: this.options.globalFlags?.help, @@ -115,7 +122,7 @@ export class RunWithCommands { log, flags: commandFlags, procRunner, - addCleanupTask: cleanup.add, + addCleanupTask: cleanup.add.bind(cleanup), }; const extendedContext = { diff --git a/packages/kbn-es-archiver/package.json b/packages/kbn-es-archiver/package.json new file mode 100644 index 0000000000000..13b5662519b19 --- /dev/null +++ b/packages/kbn-es-archiver/package.json @@ -0,0 +1,17 @@ +{ + "name": "@kbn/es-archiver", + "version": "1.0.0", + "license": "Apache-2.0", + "main": "target/index.js", + "scripts": { + "kbn:bootstrap": "tsc", + "kbn:watch": "tsc --watch" + }, + "dependencies": { + "@kbn/dev-utils": "1.0.0", + "elasticsearch": "^16.7.0" + }, + "devDependencies": { + "@types/elasticsearch": "^5.0.33" + } +} \ No newline at end of file diff --git a/src/es_archiver/actions/edit.ts b/packages/kbn-es-archiver/src/actions/edit.ts similarity index 97% rename from src/es_archiver/actions/edit.ts rename to packages/kbn-es-archiver/src/actions/edit.ts index afa51a3b96477..1194637b1ff89 100644 --- a/src/es_archiver/actions/edit.ts +++ b/packages/kbn-es-archiver/src/actions/edit.ts @@ -24,7 +24,7 @@ import { promisify } from 'util'; import globby from 'globby'; import { ToolingLog } from '@kbn/dev-utils'; -import { createPromiseFromStreams } from '../../legacy/utils'; +import { createPromiseFromStreams } from '../lib/streams'; const unlinkAsync = promisify(Fs.unlink); diff --git a/src/es_archiver/actions/empty_kibana_index.ts b/packages/kbn-es-archiver/src/actions/empty_kibana_index.ts similarity index 100% rename from src/es_archiver/actions/empty_kibana_index.ts rename to packages/kbn-es-archiver/src/actions/empty_kibana_index.ts diff --git a/src/es_archiver/actions/index.ts b/packages/kbn-es-archiver/src/actions/index.ts similarity index 100% rename from src/es_archiver/actions/index.ts rename to packages/kbn-es-archiver/src/actions/index.ts diff --git a/src/es_archiver/actions/load.ts b/packages/kbn-es-archiver/src/actions/load.ts similarity index 99% rename from src/es_archiver/actions/load.ts rename to packages/kbn-es-archiver/src/actions/load.ts index 03de8f39a7c04..efb1fe9f9ea54 100644 --- a/src/es_archiver/actions/load.ts +++ b/packages/kbn-es-archiver/src/actions/load.ts @@ -23,7 +23,7 @@ import { Readable } from 'stream'; import { ToolingLog, KbnClient } from '@kbn/dev-utils'; import { Client } from 'elasticsearch'; -import { createPromiseFromStreams, concatStreamProviders } from '../../legacy/utils'; +import { createPromiseFromStreams, concatStreamProviders } from '../lib/streams'; import { isGzip, diff --git a/src/es_archiver/actions/rebuild_all.ts b/packages/kbn-es-archiver/src/actions/rebuild_all.ts similarity index 97% rename from src/es_archiver/actions/rebuild_all.ts rename to packages/kbn-es-archiver/src/actions/rebuild_all.ts index dfbd51300e04d..470a566a6eef0 100644 --- a/src/es_archiver/actions/rebuild_all.ts +++ b/packages/kbn-es-archiver/src/actions/rebuild_all.ts @@ -23,7 +23,7 @@ import { Readable, Writable } from 'stream'; import { fromNode } from 'bluebird'; import { ToolingLog } from '@kbn/dev-utils'; -import { createPromiseFromStreams } from '../../legacy/utils'; +import { createPromiseFromStreams } from '../lib/streams'; import { prioritizeMappings, readDirectory, diff --git a/src/es_archiver/actions/save.ts b/packages/kbn-es-archiver/src/actions/save.ts similarity index 96% rename from src/es_archiver/actions/save.ts rename to packages/kbn-es-archiver/src/actions/save.ts index 7a3a9dd97c0ab..2f87cabadee6c 100644 --- a/src/es_archiver/actions/save.ts +++ b/packages/kbn-es-archiver/src/actions/save.ts @@ -23,7 +23,7 @@ import { Readable, Writable } from 'stream'; import { Client } from 'elasticsearch'; import { ToolingLog } from '@kbn/dev-utils'; -import { createListStream, createPromiseFromStreams } from '../../legacy/utils'; +import { createListStream, createPromiseFromStreams } from '../lib/streams'; import { createStats, createGenerateIndexRecordsStream, diff --git a/src/es_archiver/actions/unload.ts b/packages/kbn-es-archiver/src/actions/unload.ts similarity index 97% rename from src/es_archiver/actions/unload.ts rename to packages/kbn-es-archiver/src/actions/unload.ts index 130a6b542b218..ae23ef21fb79f 100644 --- a/src/es_archiver/actions/unload.ts +++ b/packages/kbn-es-archiver/src/actions/unload.ts @@ -23,7 +23,7 @@ import { Readable, Writable } from 'stream'; import { Client } from 'elasticsearch'; import { ToolingLog, KbnClient } from '@kbn/dev-utils'; -import { createPromiseFromStreams } from '../../legacy/utils'; +import { createPromiseFromStreams } from '../lib/streams'; import { isGzip, createStats, diff --git a/packages/kbn-es-archiver/src/cli.ts b/packages/kbn-es-archiver/src/cli.ts new file mode 100644 index 0000000000000..1745bd862b434 --- /dev/null +++ b/packages/kbn-es-archiver/src/cli.ts @@ -0,0 +1,244 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** *********************************************************** + * + * Run `node scripts/es_archiver --help` for usage information + * + *************************************************************/ + +import Path from 'path'; +import Url from 'url'; +import readline from 'readline'; + +import { RunWithCommands, createFlagError } from '@kbn/dev-utils'; +import { readConfigFile } from '@kbn/test'; +import legacyElasticsearch from 'elasticsearch'; + +import { EsArchiver } from './es_archiver'; + +const resolveConfigPath = (v: string) => Path.resolve(process.cwd(), v); +const defaultConfigPath = resolveConfigPath('test/functional/config.js'); + +export function runCli() { + new RunWithCommands({ + description: 'CLI to manage archiving/restoring data in elasticsearch', + globalFlags: { + string: ['es-url', 'kibana-url', 'dir', 'config'], + help: ` + --config path to an FTR config file that sets --es-url, --kibana-url, and --dir + default: ${defaultConfigPath} + --es-url url for Elasticsearch, prefer the --config flag + --kibana-url url for Kibana, prefer the --config flag + --dir where arechives are stored, prefer the --config flag + `, + }, + async extendContext({ log, flags, addCleanupTask }) { + const configPath = flags.config || defaultConfigPath; + if (typeof configPath !== 'string') { + throw createFlagError('--config must be a string'); + } + const config = await readConfigFile(log, Path.resolve(configPath)); + + let esUrl = flags['es-url']; + if (esUrl && typeof esUrl !== 'string') { + throw createFlagError('--es-url must be a string'); + } + if (!esUrl && config) { + esUrl = Url.format(config.get('servers.elasticsearch')); + } + if (!esUrl) { + throw createFlagError('--es-url or --config must be defined'); + } + + let kibanaUrl = flags['kibana-url']; + if (kibanaUrl && typeof kibanaUrl !== 'string') { + throw createFlagError('--kibana-url must be a string'); + } + if (!kibanaUrl && config) { + kibanaUrl = Url.format(config.get('servers.kibana')); + } + if (!kibanaUrl) { + throw createFlagError('--kibana-url or --config must be defined'); + } + + let dir = flags.dir; + if (dir && typeof dir !== 'string') { + throw createFlagError('--dir must be a string'); + } + if (!dir && config) { + dir = Path.resolve(config.get('esArchiver.directory')); + } + if (!dir) { + throw createFlagError('--dir or --config must be defined'); + } + + const client = new legacyElasticsearch.Client({ + host: esUrl, + log: flags.verbose ? 'trace' : [], + }); + addCleanupTask(() => client.close()); + + const esArchiver = new EsArchiver({ + log, + client, + dataDir: dir, + kibanaUrl, + }); + + return { + esArchiver, + }; + }, + }) + .command({ + name: 'save', + usage: 'save [name] [...indices]', + description: ` + archive the [indices ...] into the --dir with [name] + + Example: + Save all [logstash-*] indices from http://localhost:9200 to [snapshots/my_test_data] directory + + WARNING: If the [my_test_data] snapshot exists it will be deleted! + + $ node scripts/es_archiver save my_test_data logstash-* --dir snapshots + `, + flags: { + boolean: ['raw'], + help: ` + --raw don't gzip the archives + `, + }, + async run({ flags, esArchiver }) { + const [name, ...indices] = flags._; + if (!name) { + throw createFlagError('missing [name] argument'); + } + if (!indices.length) { + throw createFlagError('missing [...indices] arguments'); + } + + const raw = flags.raw; + if (typeof raw !== 'boolean') { + throw createFlagError('--raw does not take a value'); + } + + await esArchiver.save(name, indices, { raw }); + }, + }) + .command({ + name: 'load', + usage: 'load [name]', + description: ` + load the archive in --dir with [name] + + Example: + Load the [my_test_data] snapshot from the archive directory and elasticsearch instance defined + in the [test/functional/config.js] config file + + WARNING: If the indices exist already they will be deleted! + + $ node scripts/es_archiver load my_test_data --config test/functional/config.js + `, + flags: { + boolean: ['use-create'], + help: ` + --use-create use create instead of index for loading documents + `, + }, + async run({ flags, esArchiver }) { + const [name] = flags._; + if (!name) { + throw createFlagError('missing [name] argument'); + } + if (flags._.length > 1) { + throw createFlagError(`unknown extra arguments: [${flags._.slice(1).join(', ')}]`); + } + + const useCreate = flags['use-create']; + if (typeof useCreate !== 'boolean') { + throw createFlagError('--use-create does not take a value'); + } + + await esArchiver.load(name, { useCreate }); + }, + }) + .command({ + name: 'unload', + usage: 'unload [name]', + description: 'remove indices created by the archive in --dir with [name]', + async run({ flags, esArchiver }) { + const [name] = flags._; + if (!name) { + throw createFlagError('missing [name] argument'); + } + if (flags._.length > 1) { + throw createFlagError(`unknown extra arguments: [${flags._.slice(1).join(', ')}]`); + } + + await esArchiver.unload(name); + }, + }) + .command({ + name: 'edit', + usage: 'edit [prefix]', + description: + 'extract the archives under the prefix, wait for edits to be completed, and then recompress the archives', + async run({ flags, esArchiver }) { + const [prefix] = flags._; + if (!prefix) { + throw createFlagError('missing [prefix] argument'); + } + if (flags._.length > 1) { + throw createFlagError(`unknown extra arguments: [${flags._.slice(1).join(', ')}]`); + } + + await esArchiver.edit(prefix, async () => { + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); + + await new Promise((resolveInput) => { + rl.question(`Press enter when you're done`, () => { + rl.close(); + resolveInput(); + }); + }); + }); + }, + }) + .command({ + name: 'empty-kibana-index', + description: + '[internal] Delete any Kibana indices, and initialize the Kibana index as Kibana would do on startup.', + async run({ esArchiver }) { + await esArchiver.emptyKibanaIndex(); + }, + }) + .command({ + name: 'rebuild-all', + description: '[internal] read and write all archives in --dir to remove any inconsistencies', + async run({ esArchiver }) { + await esArchiver.rebuildAll(); + }, + }) + .execute(); +} diff --git a/src/es_archiver/es_archiver.ts b/packages/kbn-es-archiver/src/es_archiver.ts similarity index 100% rename from src/es_archiver/es_archiver.ts rename to packages/kbn-es-archiver/src/es_archiver.ts diff --git a/src/es_archiver/index.ts b/packages/kbn-es-archiver/src/index.ts similarity index 97% rename from src/es_archiver/index.ts rename to packages/kbn-es-archiver/src/index.ts index f7a579a98a42d..c00d457c939ce 100644 --- a/src/es_archiver/index.ts +++ b/packages/kbn-es-archiver/src/index.ts @@ -18,3 +18,4 @@ */ export { EsArchiver } from './es_archiver'; +export * from './cli'; diff --git a/src/es_archiver/lib/__tests__/stats.ts b/packages/kbn-es-archiver/src/lib/__tests__/stats.ts similarity index 100% rename from src/es_archiver/lib/__tests__/stats.ts rename to packages/kbn-es-archiver/src/lib/__tests__/stats.ts diff --git a/src/es_archiver/lib/archives/__tests__/format.ts b/packages/kbn-es-archiver/src/lib/archives/__tests__/format.ts similarity index 96% rename from src/es_archiver/lib/archives/__tests__/format.ts rename to packages/kbn-es-archiver/src/lib/archives/__tests__/format.ts index f3829273ea808..044a0e82d9df2 100644 --- a/src/es_archiver/lib/archives/__tests__/format.ts +++ b/packages/kbn-es-archiver/src/lib/archives/__tests__/format.ts @@ -22,11 +22,7 @@ import { createGunzip } from 'zlib'; import expect from '@kbn/expect'; -import { - createListStream, - createPromiseFromStreams, - createConcatStream, -} from '../../../../legacy/utils'; +import { createListStream, createPromiseFromStreams, createConcatStream } from '../../streams'; import { createFormatArchiveStreams } from '../format'; diff --git a/src/es_archiver/lib/archives/__tests__/parse.ts b/packages/kbn-es-archiver/src/lib/archives/__tests__/parse.ts similarity index 97% rename from src/es_archiver/lib/archives/__tests__/parse.ts rename to packages/kbn-es-archiver/src/lib/archives/__tests__/parse.ts index 50cbdfe06f361..25b8fe46a81fc 100644 --- a/src/es_archiver/lib/archives/__tests__/parse.ts +++ b/packages/kbn-es-archiver/src/lib/archives/__tests__/parse.ts @@ -22,11 +22,7 @@ import { createGzip } from 'zlib'; import expect from '@kbn/expect'; -import { - createConcatStream, - createListStream, - createPromiseFromStreams, -} from '../../../../legacy/utils'; +import { createConcatStream, createListStream, createPromiseFromStreams } from '../../streams'; import { createParseArchiveStreams } from '../parse'; @@ -109,7 +105,7 @@ describe('esArchiver createParseArchiveStreams', () => { Buffer.from('{"a": 2}\n\n'), ]), ...createParseArchiveStreams({ gzip: false }), - createConcatStream(), + createConcatStream([]), ] as [Readable, ...Writable[]]); throw new Error('should have failed'); } catch (err) { @@ -172,7 +168,7 @@ describe('esArchiver createParseArchiveStreams', () => { await createPromiseFromStreams([ createListStream([Buffer.from('{"a": 1}')]), ...createParseArchiveStreams({ gzip: true }), - createConcatStream(), + createConcatStream([]), ] as [Readable, ...Writable[]]); throw new Error('should have failed'); } catch (err) { diff --git a/src/es_archiver/lib/archives/constants.ts b/packages/kbn-es-archiver/src/lib/archives/constants.ts similarity index 100% rename from src/es_archiver/lib/archives/constants.ts rename to packages/kbn-es-archiver/src/lib/archives/constants.ts diff --git a/src/es_archiver/lib/archives/filenames.ts b/packages/kbn-es-archiver/src/lib/archives/filenames.ts similarity index 100% rename from src/es_archiver/lib/archives/filenames.ts rename to packages/kbn-es-archiver/src/lib/archives/filenames.ts diff --git a/src/es_archiver/lib/archives/format.ts b/packages/kbn-es-archiver/src/lib/archives/format.ts similarity index 94% rename from src/es_archiver/lib/archives/format.ts rename to packages/kbn-es-archiver/src/lib/archives/format.ts index ac18147ad6948..3cd698c3f82c3 100644 --- a/src/es_archiver/lib/archives/format.ts +++ b/packages/kbn-es-archiver/src/lib/archives/format.ts @@ -21,7 +21,7 @@ import { createGzip, Z_BEST_COMPRESSION } from 'zlib'; import { PassThrough } from 'stream'; import stringify from 'json-stable-stringify'; -import { createMapStream, createIntersperseStream } from '../../../legacy/utils'; +import { createMapStream, createIntersperseStream } from '../streams'; import { RECORD_SEPARATOR } from './constants'; export function createFormatArchiveStreams({ gzip = false }: { gzip?: boolean } = {}) { diff --git a/src/es_archiver/lib/archives/index.ts b/packages/kbn-es-archiver/src/lib/archives/index.ts similarity index 100% rename from src/es_archiver/lib/archives/index.ts rename to packages/kbn-es-archiver/src/lib/archives/index.ts diff --git a/src/es_archiver/lib/archives/parse.ts b/packages/kbn-es-archiver/src/lib/archives/parse.ts similarity index 87% rename from src/es_archiver/lib/archives/parse.ts rename to packages/kbn-es-archiver/src/lib/archives/parse.ts index 1d650815f9358..9236a618aa01a 100644 --- a/src/es_archiver/lib/archives/parse.ts +++ b/packages/kbn-es-archiver/src/lib/archives/parse.ts @@ -19,8 +19,12 @@ import { createGunzip } from 'zlib'; import { PassThrough } from 'stream'; -import { createFilterStream } from '../../../legacy/utils/streams/filter_stream'; -import { createSplitStream, createReplaceStream, createMapStream } from '../../../legacy/utils'; +import { + createFilterStream, + createSplitStream, + createReplaceStream, + createMapStream, +} from '../streams'; import { RECORD_SEPARATOR } from './constants'; diff --git a/src/es_archiver/lib/directory.ts b/packages/kbn-es-archiver/src/lib/directory.ts similarity index 100% rename from src/es_archiver/lib/directory.ts rename to packages/kbn-es-archiver/src/lib/directory.ts diff --git a/src/es_archiver/lib/docs/__tests__/generate_doc_records_stream.ts b/packages/kbn-es-archiver/src/lib/docs/__tests__/generate_doc_records_stream.ts similarity index 97% rename from src/es_archiver/lib/docs/__tests__/generate_doc_records_stream.ts rename to packages/kbn-es-archiver/src/lib/docs/__tests__/generate_doc_records_stream.ts index 03599cdc9fbcf..2214f7ae9f2ea 100644 --- a/src/es_archiver/lib/docs/__tests__/generate_doc_records_stream.ts +++ b/packages/kbn-es-archiver/src/lib/docs/__tests__/generate_doc_records_stream.ts @@ -21,11 +21,7 @@ import sinon from 'sinon'; import expect from '@kbn/expect'; import { delay } from 'bluebird'; -import { - createListStream, - createPromiseFromStreams, - createConcatStream, -} from '../../../../legacy/utils'; +import { createListStream, createPromiseFromStreams, createConcatStream } from '../../streams'; import { createGenerateDocRecordsStream } from '../generate_doc_records_stream'; import { Progress } from '../../progress'; diff --git a/src/es_archiver/lib/docs/__tests__/index_doc_records_stream.ts b/packages/kbn-es-archiver/src/lib/docs/__tests__/index_doc_records_stream.ts similarity index 99% rename from src/es_archiver/lib/docs/__tests__/index_doc_records_stream.ts rename to packages/kbn-es-archiver/src/lib/docs/__tests__/index_doc_records_stream.ts index 178f0a6a7e5ca..3205549bf97c7 100644 --- a/src/es_archiver/lib/docs/__tests__/index_doc_records_stream.ts +++ b/packages/kbn-es-archiver/src/lib/docs/__tests__/index_doc_records_stream.ts @@ -20,7 +20,7 @@ import expect from '@kbn/expect'; import { delay } from 'bluebird'; -import { createListStream, createPromiseFromStreams } from '../../../../legacy/utils'; +import { createListStream, createPromiseFromStreams } from '../../streams'; import { Progress } from '../../progress'; import { createIndexDocRecordsStream } from '../index_doc_records_stream'; diff --git a/src/es_archiver/lib/docs/__tests__/stubs.ts b/packages/kbn-es-archiver/src/lib/docs/__tests__/stubs.ts similarity index 100% rename from src/es_archiver/lib/docs/__tests__/stubs.ts rename to packages/kbn-es-archiver/src/lib/docs/__tests__/stubs.ts diff --git a/src/es_archiver/lib/docs/generate_doc_records_stream.ts b/packages/kbn-es-archiver/src/lib/docs/generate_doc_records_stream.ts similarity index 100% rename from src/es_archiver/lib/docs/generate_doc_records_stream.ts rename to packages/kbn-es-archiver/src/lib/docs/generate_doc_records_stream.ts diff --git a/src/es_archiver/lib/docs/index.ts b/packages/kbn-es-archiver/src/lib/docs/index.ts similarity index 100% rename from src/es_archiver/lib/docs/index.ts rename to packages/kbn-es-archiver/src/lib/docs/index.ts diff --git a/src/es_archiver/lib/docs/index_doc_records_stream.ts b/packages/kbn-es-archiver/src/lib/docs/index_doc_records_stream.ts similarity index 100% rename from src/es_archiver/lib/docs/index_doc_records_stream.ts rename to packages/kbn-es-archiver/src/lib/docs/index_doc_records_stream.ts diff --git a/src/es_archiver/lib/index.ts b/packages/kbn-es-archiver/src/lib/index.ts similarity index 100% rename from src/es_archiver/lib/index.ts rename to packages/kbn-es-archiver/src/lib/index.ts diff --git a/src/es_archiver/lib/indices/__tests__/create_index_stream.ts b/packages/kbn-es-archiver/src/lib/indices/__tests__/create_index_stream.ts similarity index 98% rename from src/es_archiver/lib/indices/__tests__/create_index_stream.ts rename to packages/kbn-es-archiver/src/lib/indices/__tests__/create_index_stream.ts index 3ae5387e10c60..8f79cac6daaea 100644 --- a/src/es_archiver/lib/indices/__tests__/create_index_stream.ts +++ b/packages/kbn-es-archiver/src/lib/indices/__tests__/create_index_stream.ts @@ -21,11 +21,7 @@ import expect from '@kbn/expect'; import sinon from 'sinon'; import Chance from 'chance'; -import { - createPromiseFromStreams, - createConcatStream, - createListStream, -} from '../../../../legacy/utils'; +import { createPromiseFromStreams, createConcatStream, createListStream } from '../../streams'; import { createCreateIndexStream } from '../create_index_stream'; diff --git a/src/es_archiver/lib/indices/__tests__/delete_index_stream.ts b/packages/kbn-es-archiver/src/lib/indices/__tests__/delete_index_stream.ts similarity index 99% rename from src/es_archiver/lib/indices/__tests__/delete_index_stream.ts rename to packages/kbn-es-archiver/src/lib/indices/__tests__/delete_index_stream.ts index 1c989ba158a29..551b744415c83 100644 --- a/src/es_archiver/lib/indices/__tests__/delete_index_stream.ts +++ b/packages/kbn-es-archiver/src/lib/indices/__tests__/delete_index_stream.ts @@ -19,7 +19,7 @@ import sinon from 'sinon'; -import { createListStream, createPromiseFromStreams } from '../../../../legacy/utils'; +import { createListStream, createPromiseFromStreams } from '../../streams'; import { createDeleteIndexStream } from '../delete_index_stream'; diff --git a/src/es_archiver/lib/indices/__tests__/generate_index_records_stream.ts b/packages/kbn-es-archiver/src/lib/indices/__tests__/generate_index_records_stream.ts similarity index 97% rename from src/es_archiver/lib/indices/__tests__/generate_index_records_stream.ts rename to packages/kbn-es-archiver/src/lib/indices/__tests__/generate_index_records_stream.ts index fe927483da7b0..cb3746c015dad 100644 --- a/src/es_archiver/lib/indices/__tests__/generate_index_records_stream.ts +++ b/packages/kbn-es-archiver/src/lib/indices/__tests__/generate_index_records_stream.ts @@ -20,11 +20,7 @@ import sinon from 'sinon'; import expect from '@kbn/expect'; -import { - createListStream, - createPromiseFromStreams, - createConcatStream, -} from '../../../../legacy/utils'; +import { createListStream, createPromiseFromStreams, createConcatStream } from '../../streams'; import { createStubClient, createStubStats } from './stubs'; diff --git a/src/es_archiver/lib/indices/__tests__/stubs.ts b/packages/kbn-es-archiver/src/lib/indices/__tests__/stubs.ts similarity index 100% rename from src/es_archiver/lib/indices/__tests__/stubs.ts rename to packages/kbn-es-archiver/src/lib/indices/__tests__/stubs.ts diff --git a/src/es_archiver/lib/indices/create_index_stream.ts b/packages/kbn-es-archiver/src/lib/indices/create_index_stream.ts similarity index 100% rename from src/es_archiver/lib/indices/create_index_stream.ts rename to packages/kbn-es-archiver/src/lib/indices/create_index_stream.ts diff --git a/src/es_archiver/lib/indices/delete_index.ts b/packages/kbn-es-archiver/src/lib/indices/delete_index.ts similarity index 100% rename from src/es_archiver/lib/indices/delete_index.ts rename to packages/kbn-es-archiver/src/lib/indices/delete_index.ts diff --git a/src/es_archiver/lib/indices/delete_index_stream.ts b/packages/kbn-es-archiver/src/lib/indices/delete_index_stream.ts similarity index 100% rename from src/es_archiver/lib/indices/delete_index_stream.ts rename to packages/kbn-es-archiver/src/lib/indices/delete_index_stream.ts diff --git a/src/es_archiver/lib/indices/generate_index_records_stream.ts b/packages/kbn-es-archiver/src/lib/indices/generate_index_records_stream.ts similarity index 100% rename from src/es_archiver/lib/indices/generate_index_records_stream.ts rename to packages/kbn-es-archiver/src/lib/indices/generate_index_records_stream.ts diff --git a/src/es_archiver/lib/indices/index.ts b/packages/kbn-es-archiver/src/lib/indices/index.ts similarity index 100% rename from src/es_archiver/lib/indices/index.ts rename to packages/kbn-es-archiver/src/lib/indices/index.ts diff --git a/src/es_archiver/lib/indices/kibana_index.ts b/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts similarity index 98% rename from src/es_archiver/lib/indices/kibana_index.ts rename to packages/kbn-es-archiver/src/lib/indices/kibana_index.ts index ed436ad7773e1..a30cf3865a108 100644 --- a/src/es_archiver/lib/indices/kibana_index.ts +++ b/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts @@ -75,7 +75,7 @@ export async function migrateKibanaIndex({ }, } as any); - return await kbnClient.savedObjects.migrate(); + await kbnClient.savedObjects.migrate(); } /** diff --git a/src/es_archiver/lib/progress.ts b/packages/kbn-es-archiver/src/lib/progress.ts similarity index 100% rename from src/es_archiver/lib/progress.ts rename to packages/kbn-es-archiver/src/lib/progress.ts diff --git a/src/es_archiver/lib/records/__tests__/filter_records_stream.ts b/packages/kbn-es-archiver/src/lib/records/__tests__/filter_records_stream.ts similarity index 95% rename from src/es_archiver/lib/records/__tests__/filter_records_stream.ts rename to packages/kbn-es-archiver/src/lib/records/__tests__/filter_records_stream.ts index f4f9f32e239ea..b23ff2e4e52ac 100644 --- a/src/es_archiver/lib/records/__tests__/filter_records_stream.ts +++ b/packages/kbn-es-archiver/src/lib/records/__tests__/filter_records_stream.ts @@ -20,11 +20,7 @@ import Chance from 'chance'; import expect from '@kbn/expect'; -import { - createListStream, - createPromiseFromStreams, - createConcatStream, -} from '../../../../legacy/utils'; +import { createListStream, createPromiseFromStreams, createConcatStream } from '../../streams'; import { createFilterRecordsStream } from '../filter_records_stream'; diff --git a/src/es_archiver/lib/records/filter_records_stream.ts b/packages/kbn-es-archiver/src/lib/records/filter_records_stream.ts similarity index 100% rename from src/es_archiver/lib/records/filter_records_stream.ts rename to packages/kbn-es-archiver/src/lib/records/filter_records_stream.ts diff --git a/src/es_archiver/lib/records/index.ts b/packages/kbn-es-archiver/src/lib/records/index.ts similarity index 100% rename from src/es_archiver/lib/records/index.ts rename to packages/kbn-es-archiver/src/lib/records/index.ts diff --git a/src/es_archiver/lib/stats.ts b/packages/kbn-es-archiver/src/lib/stats.ts similarity index 100% rename from src/es_archiver/lib/stats.ts rename to packages/kbn-es-archiver/src/lib/stats.ts diff --git a/packages/kbn-es-archiver/src/lib/streams.ts b/packages/kbn-es-archiver/src/lib/streams.ts new file mode 100644 index 0000000000000..a90afbe0c4d25 --- /dev/null +++ b/packages/kbn-es-archiver/src/lib/streams.ts @@ -0,0 +1,20 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export * from '../../../../src/legacy/utils/streams'; diff --git a/packages/kbn-es-archiver/tsconfig.json b/packages/kbn-es-archiver/tsconfig.json new file mode 100644 index 0000000000000..6ffa64d91fba0 --- /dev/null +++ b/packages/kbn-es-archiver/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "./target", + "declaration": true, + "sourceMap": true, + "target": "ES2019" + }, + "include": [ + "src/**/*" + ] +} diff --git a/packages/kbn-es-archiver/yarn.lock b/packages/kbn-es-archiver/yarn.lock new file mode 120000 index 0000000000000..3f82ebc9cdbae --- /dev/null +++ b/packages/kbn-es-archiver/yarn.lock @@ -0,0 +1 @@ +../../yarn.lock \ No newline at end of file diff --git a/packages/kbn-test/src/functional_test_runner/index.ts b/packages/kbn-test/src/functional_test_runner/index.ts index cf65ceb51df8e..b13c311350ff6 100644 --- a/packages/kbn-test/src/functional_test_runner/index.ts +++ b/packages/kbn-test/src/functional_test_runner/index.ts @@ -18,6 +18,6 @@ */ export { FunctionalTestRunner } from './functional_test_runner'; -export { readConfigFile } from './lib'; +export { readConfigFile, Config } from './lib'; export { runFtrCli } from './cli'; export * from './lib/docker_servers'; diff --git a/scripts/es_archiver.js b/scripts/es_archiver.js index faa8d9131240d..88674ce9eafb3 100755 --- a/scripts/es_archiver.js +++ b/scripts/es_archiver.js @@ -18,4 +18,4 @@ */ require('../src/setup_node_env'); -require('../src/es_archiver/cli'); +require('@kbn/es-archiver').runCli(); diff --git a/src/dev/build/tasks/copy_source_task.js b/src/dev/build/tasks/copy_source_task.js index e34f05bd6cfff..52809449ba338 100644 --- a/src/dev/build/tasks/copy_source_task.js +++ b/src/dev/build/tasks/copy_source_task.js @@ -37,7 +37,6 @@ export const CopySourceTask = { '!src/legacy/core_plugins/console/public/tests/**', '!src/cli/cluster/**', '!src/cli/repl/**', - '!src/es_archiver/**', '!src/functional_test_runner/**', '!src/dev/**', 'typings/**', diff --git a/src/es_archiver/cli.ts b/src/es_archiver/cli.ts deleted file mode 100644 index 85e10b31a87ee..0000000000000 --- a/src/es_archiver/cli.ts +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/** *********************************************************** - * - * Run `node scripts/es_archiver --help` for usage information - * - *************************************************************/ - -import { resolve } from 'path'; -import { readFileSync } from 'fs'; -import { format as formatUrl } from 'url'; -import readline from 'readline'; -import { Command } from 'commander'; -import * as legacyElasticsearch from 'elasticsearch'; - -import { ToolingLog } from '@kbn/dev-utils'; -import { readConfigFile } from '@kbn/test'; - -import { EsArchiver } from './es_archiver'; - -const cmd = new Command('node scripts/es_archiver'); - -const resolveConfigPath = (v: string) => resolve(process.cwd(), v); -const defaultConfigPath = resolveConfigPath('test/functional/config.js'); - -cmd - .description(`CLI to manage archiving/restoring data in elasticsearch`) - .option('--es-url [url]', 'url for elasticsearch') - .option( - '--kibana-url [url]', - 'url for kibana (only necessary if using "load" or "unload" methods)' - ) - .option(`--dir [path]`, 'where archives are stored') - .option('--verbose', 'turn on verbose logging') - .option( - '--config [path]', - 'path to a functional test config file to use for default values', - resolveConfigPath, - defaultConfigPath - ) - .on('--help', () => { - // eslint-disable-next-line no-console - console.log(readFileSync(resolve(__dirname, './cli_help.txt'), 'utf8')); - }); - -cmd - .option('--raw', `don't gzip the archive`) - .command('save ') - .description('archive the into the --dir with ') - .action((name, indices) => execute((archiver, { raw }) => archiver.save(name, indices, { raw }))); - -cmd - .option('--use-create', 'use create instead of index for loading documents') - .command('load ') - .description('load the archive in --dir with ') - .action((name) => execute((archiver, { useCreate }) => archiver.load(name, { useCreate }))); - -cmd - .command('unload ') - .description('remove indices created by the archive in --dir with ') - .action((name) => execute((archiver) => archiver.unload(name))); - -cmd - .command('empty-kibana-index') - .description( - '[internal] Delete any Kibana indices, and initialize the Kibana index as Kibana would do on startup.' - ) - .action(() => execute((archiver) => archiver.emptyKibanaIndex())); - -cmd - .command('edit [prefix]') - .description( - 'extract the archives under the prefix, wait for edits to be completed, and then recompress the archives' - ) - .action((prefix) => - execute((archiver) => - archiver.edit(prefix, async () => { - const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout, - }); - - await new Promise((resolveInput) => { - rl.question(`Press enter when you're done`, () => { - rl.close(); - resolveInput(); - }); - }); - }) - ) - ); - -cmd - .command('rebuild-all') - .description('[internal] read and write all archives in --dir to remove any inconsistencies') - .action(() => execute((archiver) => archiver.rebuildAll())); - -cmd.parse(process.argv); - -const missingCommand = cmd.args.every((a) => !((a as any) instanceof Command)); -if (missingCommand) { - execute(); -} - -async function execute(fn?: (esArchiver: EsArchiver, command: Command) => void): Promise { - try { - const log = new ToolingLog({ - level: cmd.verbose ? 'debug' : 'info', - writeTo: process.stdout, - }); - - if (cmd.config) { - // load default values from the specified config file - const config = await readConfigFile(log, resolve(cmd.config)); - if (!cmd.esUrl) cmd.esUrl = formatUrl(config.get('servers.elasticsearch')); - if (!cmd.kibanaUrl) cmd.kibanaUrl = formatUrl(config.get('servers.kibana')); - if (!cmd.dir) cmd.dir = config.get('esArchiver.directory'); - } - - // log and count all validation errors - let errorCount = 0; - const error = (msg: string) => { - errorCount++; - log.error(msg); - }; - - if (!fn) { - error(`Unknown command "${cmd.args[0]}"`); - } - - if (!cmd.esUrl) { - error('You must specify either --es-url or --config flags'); - } - - if (!cmd.dir) { - error('You must specify either --dir or --config flags'); - } - - // if there was a validation error display the help - if (errorCount) { - cmd.help(); - return; - } - - // run! - const client = new legacyElasticsearch.Client({ - host: cmd.esUrl, - log: cmd.verbose ? 'trace' : [], - }); - - try { - const esArchiver = new EsArchiver({ - log, - client, - dataDir: resolve(cmd.dir), - kibanaUrl: cmd.kibanaUrl, - }); - await fn!(esArchiver, cmd); - } finally { - await client.close(); - } - } catch (err) { - // eslint-disable-next-line no-console - console.log('FATAL ERROR', err.stack); - } -} diff --git a/src/es_archiver/cli_help.txt b/src/es_archiver/cli_help.txt deleted file mode 100644 index 1e2f8e40824ba..0000000000000 --- a/src/es_archiver/cli_help.txt +++ /dev/null @@ -1,15 +0,0 @@ - Examples: - Dump an index to disk: - Save all `logstash-*` indices from http://localhost:9200 to `snapshots/my_test_data` directory - - WARNING: If the `my_test_data` snapshot exists it will be deleted! - - $ node scripts/es_archiver save my_test_data logstash-* --dir snapshots - - Load an index from disk - Load the `my_test_data` snapshot from the archive directory and elasticsearch instance defined - in the `test/functional/config.js` config file - - WARNING: If the indices exist already they will be deleted! - - $ node scripts/es_archiver load my_test_data --config test/functional/config.js diff --git a/test/common/services/es_archiver.ts b/test/common/services/es_archiver.ts index cfe0610414b4f..9c99445fa4827 100644 --- a/test/common/services/es_archiver.ts +++ b/test/common/services/es_archiver.ts @@ -18,9 +18,9 @@ */ import { format as formatUrl } from 'url'; +import { EsArchiver } from '@kbn/es-archiver'; import { FtrProviderContext } from '../ftr_provider_context'; -import { EsArchiver } from '../../../src/es_archiver'; // @ts-ignore not TS yet import * as KibanaServer from './kibana_server'; diff --git a/x-pack/test/functional_enterprise_search/apps/enterprise_search/with_host_configured/app_search/engines.ts b/x-pack/test/functional_enterprise_search/apps/enterprise_search/with_host_configured/app_search/engines.ts index e4ebd61c0692a..1742ed443984b 100644 --- a/x-pack/test/functional_enterprise_search/apps/enterprise_search/with_host_configured/app_search/engines.ts +++ b/x-pack/test/functional_enterprise_search/apps/enterprise_search/with_host_configured/app_search/engines.ts @@ -5,7 +5,7 @@ */ import expect from '@kbn/expect'; -import { EsArchiver } from 'src/es_archiver'; +import { EsArchiver } from '@kbn/es-archiver'; import { AppSearchService, IEngine } from '../../../../services/app_search_service'; import { Browser } from '../../../../../../../test/functional/services/common'; import { FtrProviderContext } from '../../../../ftr_provider_context'; diff --git a/x-pack/test/spaces_api_integration/common/suites/copy_to_space.ts b/x-pack/test/spaces_api_integration/common/suites/copy_to_space.ts index ebec70793e8fd..2dd4484ffcde8 100644 --- a/x-pack/test/spaces_api_integration/common/suites/copy_to_space.ts +++ b/x-pack/test/spaces_api_integration/common/suites/copy_to_space.ts @@ -6,7 +6,7 @@ import expect from '@kbn/expect'; import { SuperTest } from 'supertest'; -import { EsArchiver } from 'src/es_archiver'; +import { EsArchiver } from '@kbn/es-archiver'; import { DEFAULT_SPACE_ID } from '../../../../plugins/spaces/common/constants'; import { CopyResponse } from '../../../../plugins/spaces/server/lib/copy_to_spaces'; import { getUrlPrefix } from '../lib/space_test_utils'; diff --git a/x-pack/test/spaces_api_integration/common/suites/resolve_copy_to_space_conflicts.ts b/x-pack/test/spaces_api_integration/common/suites/resolve_copy_to_space_conflicts.ts index 3529d8f3ae9c9..6d80688b7a703 100644 --- a/x-pack/test/spaces_api_integration/common/suites/resolve_copy_to_space_conflicts.ts +++ b/x-pack/test/spaces_api_integration/common/suites/resolve_copy_to_space_conflicts.ts @@ -6,7 +6,7 @@ import expect from '@kbn/expect'; import { SuperTest } from 'supertest'; -import { EsArchiver } from 'src/es_archiver'; +import { EsArchiver } from '@kbn/es-archiver'; import { SavedObject } from 'src/core/server'; import { DEFAULT_SPACE_ID } from '../../../../plugins/spaces/common/constants'; import { CopyResponse } from '../../../../plugins/spaces/server/lib/copy_to_spaces';