diff --git a/packages/kbn-es/package.json b/packages/kbn-es/package.json index 64f67d37e4119..1edb90fbb6aae 100644 --- a/packages/kbn-es/package.json +++ b/packages/kbn-es/package.json @@ -14,6 +14,7 @@ "node-fetch": "^2.0.0", "simple-git": "^1.91.0", "tar-fs": "^1.16.0", + "yauzl": "^2.10.0", "zlib": "^1.0.5" } } diff --git a/packages/kbn-es/src/install/archive.js b/packages/kbn-es/src/install/archive.js index 048bb467e37ec..2851a3e8451f8 100644 --- a/packages/kbn-es/src/install/archive.js +++ b/packages/kbn-es/src/install/archive.js @@ -21,7 +21,7 @@ const fs = require('fs'); const path = require('path'); const chalk = require('chalk'); const execa = require('execa'); -const { log: defaultLog, extractTarball } = require('../utils'); +const { log: defaultLog, decompress } = require('../utils'); const { BASE_PATH, ES_CONFIG, ES_KEYSTORE_BIN } = require('../paths'); /** @@ -49,7 +49,7 @@ exports.installArchive = async function installArchive(archive, options = {}) { } log.info('extracting %s', chalk.bold(archive)); - await extractTarball(archive, installPath); + await decompress(archive, installPath); log.info('extracted to %s', chalk.bold(installPath)); if (license !== 'oss') { diff --git a/packages/kbn-es/src/install/snapshot.js b/packages/kbn-es/src/install/snapshot.js index d568f6a207a71..dac535acb5aa7 100644 --- a/packages/kbn-es/src/install/snapshot.js +++ b/packages/kbn-es/src/install/snapshot.js @@ -19,6 +19,7 @@ const fetch = require('node-fetch'); const fs = require('fs'); +const os = require('os'); const mkdirp = require('mkdirp'); const chalk = require('chalk'); const path = require('path'); @@ -116,9 +117,10 @@ function downloadFile(url, dest, log) { } function getFilename(license, version) { + const extension = os.platform().startsWith('win') ? 'zip' : 'tar.gz'; const basename = `elasticsearch${ license === 'oss' ? '-oss-' : '-' }${version}`; - return `${basename}-SNAPSHOT.tar.gz`; + return `${basename}-SNAPSHOT.${extension}`; } diff --git a/packages/kbn-es/src/utils/__fixtures__/snapshot.tar.gz b/packages/kbn-es/src/utils/__fixtures__/snapshot.tar.gz new file mode 100644 index 0000000000000..0218324a23da6 Binary files /dev/null and b/packages/kbn-es/src/utils/__fixtures__/snapshot.tar.gz differ diff --git a/packages/kbn-es/src/utils/__fixtures__/snapshot.zip b/packages/kbn-es/src/utils/__fixtures__/snapshot.zip new file mode 100644 index 0000000000000..ace84883270dc Binary files /dev/null and b/packages/kbn-es/src/utils/__fixtures__/snapshot.zip differ diff --git a/packages/kbn-es/src/utils/decompress.js b/packages/kbn-es/src/utils/decompress.js new file mode 100644 index 0000000000000..223eb56d47000 --- /dev/null +++ b/packages/kbn-es/src/utils/decompress.js @@ -0,0 +1,102 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +const fs = require('fs'); +const path = require('path'); + +const yauzl = require('yauzl'); +const mkdirp = require('mkdirp'); +const zlib = require('zlib'); +const tarFs = require('tar-fs'); + +function decompressTarball(archive, dirPath) { + return new Promise((resolve, reject) => { + fs + .createReadStream(archive) + .on('error', reject) + .pipe(zlib.createGunzip()) + .on('error', reject) + .pipe(tarFs.extract(dirPath, { strip: true })) + .on('error', reject) + .on('finish', resolve); + }); +} + +function decompressZip(input, output) { + return new Promise((resolve, reject) => { + yauzl.open(input, { lazyEntries: true }, (err, zipfile) => { + if (err) { + reject(err); + } + + zipfile.readEntry(); + + zipfile.on('close', () => { + resolve(); + }); + + zipfile.on('error', err => { + reject(err); + }); + + zipfile.on('entry', entry => { + const zipPath = entry.fileName + .split(/\/|\\/) + .slice(1) + .join(path.sep); + const fileName = path.resolve(output, zipPath); + + if (/\/$/.test(entry.fileName)) { + zipfile.readEntry(); + } else { + // file entry + zipfile.openReadStream(entry, (err, readStream) => { + if (err) { + reject(err); + } + + readStream.on('end', () => { + zipfile.readEntry(); + }); + + mkdirp(path.dirname(fileName), () => { + readStream.pipe(fs.createWriteStream(fileName)); + }); + }); + } + }); + }); + }); +} + +exports.decompress = async function(input, output) { + const ext = path.extname(input); + + switch (path.extname(input)) { + case '.zip': + await decompressZip(input, output); + break; + case '.tar': + case '.gz': + await decompressTarball(input, output); + break; + default: + throw new Error(`unknown extension "${ext}"`); + } +}; diff --git a/packages/kbn-es/src/utils/tarball.js b/packages/kbn-es/src/utils/decompress.test.js similarity index 52% rename from packages/kbn-es/src/utils/tarball.js rename to packages/kbn-es/src/utils/decompress.test.js index f4d396343c817..fe5d802ccb668 100644 --- a/packages/kbn-es/src/utils/tarball.js +++ b/packages/kbn-es/src/utils/decompress.test.js @@ -17,32 +17,35 @@ * under the License. */ +const { decompress } = require('./decompress'); +const mockFs = require('mock-fs'); const fs = require('fs'); -const zlib = require('zlib'); const path = require('path'); -const tarFs = require('tar-fs'); -/** - * @param {String} archive - * @param {String} dirPath - */ -exports.extractTarball = function extractTarball(archive, dirPath) { - const stripOne = header => { - header.name = header.name - .split(/\/|\\/) - .slice(1) - .join(path.sep); - return header; - }; - - return new Promise((resolve, reject) => { - fs - .createReadStream(archive) - .on('error', reject) - .pipe(zlib.createGunzip()) - .on('error', reject) - .pipe(tarFs.extract(dirPath, { map: stripOne })) - .on('error', reject) - .on('finish', resolve); +beforeEach(() => { + mockFs({ + '/data': { + 'snapshot.zip': fs.readFileSync( + path.resolve(__dirname, '__fixtures__/snapshot.zip') + ), + 'snapshot.tar.gz': fs.readFileSync( + path.resolve(__dirname, '__fixtures__/snapshot.tar.gz') + ), + }, + '/.es': {}, }); -}; +}); + +afterEach(() => { + mockFs.restore(); +}); + +test('zip strips root directory', async () => { + await decompress('/data/snapshot.zip', '/.es/foo'); + expect(fs.readdirSync('/.es/foo/bin')).toContain('elasticsearch.bat'); +}); + +test('tar strips root directory', async () => { + await decompress('/data/snapshot.tar.gz', '/.es/foo'); + expect(fs.readdirSync('/.es/foo/bin')).toContain('elasticsearch'); +}); diff --git a/packages/kbn-es/src/utils/index.js b/packages/kbn-es/src/utils/index.js index cc06bbbbfe024..74c50387d3e08 100644 --- a/packages/kbn-es/src/utils/index.js +++ b/packages/kbn-es/src/utils/index.js @@ -20,6 +20,6 @@ exports.cache = require('./cache'); exports.log = require('./log').log; exports.parseEsLog = require('./parse_es_log').parseEsLog; -exports.extractTarball = require('./tarball').extractTarball; exports.findMostRecentlyChanged = require('./find_most_recently_changed').findMostRecentlyChanged; exports.extractConfigFiles = require('./extract_config_files').extractConfigFiles; +exports.decompress = require('./decompress').decompress; diff --git a/packages/kbn-es/yarn.lock b/packages/kbn-es/yarn.lock index ab15faef4bcdd..2543d8798364b 100644 --- a/packages/kbn-es/yarn.lock +++ b/packages/kbn-es/yarn.lock @@ -32,6 +32,10 @@ bl@^1.0.0: dependencies: readable-stream "^2.0.5" +buffer-crc32@~0.2.3: + version "0.2.13" + resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242" + chalk@^1.0.0: version "1.1.3" resolved "https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98" @@ -121,6 +125,12 @@ execa@^0.10.0: signal-exit "^3.0.0" strip-eof "^1.0.0" +fd-slicer@~1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/fd-slicer/-/fd-slicer-1.1.0.tgz#25c7c89cb1f9077f8891bbe61d8f390eae256f1e" + dependencies: + pend "~1.2.0" + get-stream@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-3.0.0.tgz#8e943d1358dc37555054ecbe2edb05aa174ede14" @@ -223,6 +233,10 @@ path-key@^2.0.0, path-key@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/path-key/-/path-key-2.0.1.tgz#411cadb574c5a140d3a4b1910d40d80cc9f40b40" +pend@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50" + process-nextick-args@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.0.tgz#a37d732f4271b4ab1ad070d35508e8290788ffaa" @@ -350,6 +364,13 @@ xtend@^4.0.0: version "4.0.1" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af" +yauzl@^2.10.0: + version "2.10.0" + resolved "https://registry.yarnpkg.com/yauzl/-/yauzl-2.10.0.tgz#c7eb17c93e112cb1086fa6d8e51fb0667b79a5f9" + dependencies: + buffer-crc32 "~0.2.3" + fd-slicer "~1.1.0" + zlib@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/zlib/-/zlib-1.0.5.tgz#6e7c972fc371c645a6afb03ab14769def114fcc0"