diff --git a/lib/util.js b/lib/util.js index 5988ae2..ce45a12 100644 --- a/lib/util.js +++ b/lib/util.js @@ -1,13 +1,16 @@ const { readFile } = require('fs'); const { basename, extname, join } = require('path'); const { promisify } = require('util'); -const hcl = require('gopher-hcl'); const tar = require('tar'); const { Duplex } = require('stream'); const { URLSearchParams } = require('url'); const recursive = require('recursive-readdir'); const _ = require('lodash'); const tmp = require('tmp'); +const debug = require('debug')('citizen:server'); +const util = require('util'); + +const hcl = require('@evops/hcl-terraform-parser'); const readFileProm = promisify(readFile); @@ -20,22 +23,37 @@ const makeUrl = (req, search) => { }; const ignore = (file, stats) => { - if ((stats.isDirectory() || extname(file) === '.tf') && !basename(file).startsWith('._')) { + if (stats.isDirectory()) { + return false; + } + + if (extname(file) === '.tf') { return false; } + + if (!basename(file).startsWith('._')) { + return false; + } + return true; }; const hclToJson = async (filePath) => { const content = await readFileProm(filePath); - const json = hcl.parse(content.toString()); - - return json; + return hcl.parse(content.toString(), filePath); }; const extractDefinition = async (files, targetPath) => { const tfFiles = files.filter((f) => { + // Need to constraint lookup to files within target path + // otherwise we are exposing ourselves to FS based security + // attacks + if (f.indexOf(targetPath) !== 0) { + return false; + } + const relativePath = f.replace(targetPath, ''); + if (relativePath.lastIndexOf('/') > 0) { return false; } @@ -51,36 +69,54 @@ const extractDefinition = async (files, targetPath) => { return _.reduce(list, (l, accum) => _.merge(accum, l), {}); }; +// As module information is stored directly +// in the database, we need to normalise the +// object keys to ensure no invalid characters +// in the names e.g. full stop (.) +const normalizeKeyNamesForDbStorage = (obj) => { + if (!obj) { + return obj; + } + + const result = {}; + Object.keys(obj).forEach((key) => { + const newKey = key.replace(/[^\w\d_]/g, '__'); + result[newKey] = obj[key]; + }); + return result; +}; + const nomarlizeModule = (module) => ({ path: '', name: module.name || '', readme: '', empty: !module, - inputs: module.variable - ? Object.keys(module.variable).map((name) => ({ name, ...module.variable[name] })) - : [], - outputs: module.output - ? Object.keys(module.output).map((name) => ({ name, ...module.output[name] })) - : [], + inputs: normalizeKeyNamesForDbStorage(module.inputs), + outputs: normalizeKeyNamesForDbStorage(module.outputs), dependencies: [], - resources: module.resource - ? Object.keys(module.resource).map((type) => ({ name: Object.keys(module.resource)[0], type })) - : [], + module_calls: normalizeKeyNamesForDbStorage(module.module_calls), + resources: normalizeKeyNamesForDbStorage(module.managed_resources), }); const extractSubmodules = async (definition, files, targetPath) => { - let pathes = []; - if (definition.module) { - const submodules = Object.keys(definition.module).map((key) => definition.module[key].source); - pathes = _.uniq(submodules); + let submodulePaths = []; + if (definition.module_calls) { + const submodules = Object.keys(definition.module_calls) + .map((k) => definition.module_calls[k].source); + + submodulePaths = _.uniq(submodules); } - const promises = pathes.map(async (p) => { + const promises = submodulePaths.map(async (p) => { const data = await extractDefinition(files, join(targetPath, p)); + // Submodule was not found in the archive, ignore + if (Object.keys(data).length === 0) { + return []; + } data.name = p.substr(p.lastIndexOf('/') + 1); let result = [data]; - if (data.module) { + if (data.module_calls) { const m = await extractSubmodules(data, files, join(targetPath, p)); result = result.concat(m); } @@ -104,11 +140,15 @@ const parseHcl = (moduleName, compressedModule) => new Promise((resolve, reject) try { const files = await recursive(tempDir, [ignore]); + debug('Files found in the archive: %s', files); + // make a root module definition const rootData = await extractDefinition(files, tempDir); rootData.name = moduleName; const rootDefinition = nomarlizeModule(rootData); + debug('Module definition: %s', util.inspect(rootDefinition, false, 15)); + // make submodules definition const submodulesData = await extractSubmodules(rootData, files, tempDir); const submodulesDefinition = submodulesData.map((s) => nomarlizeModule(s)); diff --git a/lib/util.spec.js b/lib/util.spec.js index fb3d57a..b1edab2 100644 --- a/lib/util.spec.js +++ b/lib/util.spec.js @@ -42,7 +42,7 @@ describe('util\'s', () => { tarball = await readFile(tarballPath); }); - it('should make JSON from HCL in a compressed module file ', async () => { + it('should make JSON from HCL in a compressed module file', async () => { const result = await parseHcl('citizen', tarball); expect(result).to.have.property('root'); diff --git a/package-lock.json b/package-lock.json index ce84bad..6e74c0d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -102,6 +102,11 @@ } } }, + "@evops/hcl-terraform-parser": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@evops/hcl-terraform-parser/-/hcl-terraform-parser-1.0.0.tgz", + "integrity": "sha512-LBAjvfkslFUuWqOOIf6a3u/DqMNpHitfyUAOBb5KNFbZHMvDG/BjkMYS4yNRIQgzATfEayxw9NZs6wNdak7aCg==" + }, "@nodelib/fs.scandir": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.3.tgz", @@ -2165,14 +2170,6 @@ "slash": "^3.0.0" } }, - "gopher-hcl": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/gopher-hcl/-/gopher-hcl-0.2.0.tgz", - "integrity": "sha512-myBMpbVvoqC8X8P4PUJ2s34yHgl4X/S+DyXlFxU1a290PvKcGawVNzGOjzd6MxAGh41d4ySwOxsYy5XArysNKA==", - "requires": { - "mixin-deep": "^2.0.1" - } - }, "got": { "version": "9.6.0", "resolved": "https://registry.npmjs.org/got/-/got-9.6.0.tgz", @@ -2993,11 +2990,6 @@ "yallist": "^4.0.0" } }, - "mixin-deep": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-2.0.1.tgz", - "integrity": "sha512-imbHQNRglyaplMmjBLL3V5R6Bfq5oM+ivds3SKgc6oRtzErEnBUUc5No11Z2pilkUvl42gJvi285xTNswcKCMA==" - }, "mkdirp": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", diff --git a/package.json b/package.json index 596e5d4..b9a7a34 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "build": "pkg . --out-path dist --targets node10-linux-x64,node10-macos-x64,node10-win-x64" }, "dependencies": { + "@evops/hcl-terraform-parser": "^1.0.0", "aws-sdk": "^2.787.0", "body-parser": "^1.19.0", "colors": "^1.4.0", @@ -23,7 +24,6 @@ "express": "^4.17.1", "glob-gitignore": "^1.0.14", "globby": "^11.0.1", - "gopher-hcl": "^0.2.0", "helmet": "^4.2.0", "jten": "^0.2.0", "listr": "^0.14.2", diff --git a/routes/modules.spec.js b/routes/modules.spec.js index fc5466e..99c9422 100644 --- a/routes/modules.spec.js +++ b/routes/modules.spec.js @@ -13,6 +13,31 @@ const { db, save } = require('../lib/store'); const writeFile = promisify(fs.writeFile); const readFile = promisify(fs.readFile); +describe('Terraform 0.12 support', () => { + describe('POST /v1/modules/:namespace/:name/:provider/:version', () => { + let modulePath; + + beforeEach(async () => { + modulePath = `hashicorp/consul/aws/${(new Date()).getTime()}`; + }); + + afterEach(async () => { + await deleteDbAll(db); + await rimraf(process.env.CITIZEN_STORAGE_PATH); + }); + + it('should register new v12 module', () => request(app) + .post(`/v1/modules/${modulePath}`) + .attach('module', 'test/fixture/module.v12.tar.gz') + .expect('Content-Type', /application\/json/) + .expect(201) + .then((res) => { + expect(res.body).to.have.property('modules').to.be.an('array'); + expect(res.body.modules[0]).to.have.property('id').to.equal(modulePath); + })); + }); +}); + describe('POST /v1/modules/:namespace/:name/:provider/:version', () => { let moduleBuf; let modulePath; diff --git a/test/fixture/module.v12.tar.gz b/test/fixture/module.v12.tar.gz new file mode 100644 index 0000000..27d3383 Binary files /dev/null and b/test/fixture/module.v12.tar.gz differ