Adding HCL v2 (Terraform v12) language support

lib/util.js

@@ -1,13 +1,16 @@
 const { readFile } = require('fs');
 const { basename, extname, join } = require('path');
 const { promisify } = require('util');
-const hcl = require('gopher-hcl');
 const tar = require('tar');
 const { Duplex } = require('stream');
 const { URLSearchParams } = require('url');
 const recursive = require('recursive-readdir');
 const _ = require('lodash');
 const tmp = require('tmp');
+const debug = require('debug')('citizen:server');
+const util = require('util');
+
+const hcl = require('@evops/hcl-terraform-parser');
 
 const readFileProm = promisify(readFile);
 
@@ -20,22 +23,37 @@ const makeUrl = (req, search) => {
 };
 
 const ignore = (file, stats) => {
-  if ((stats.isDirectory() || extname(file) === '.tf') && !basename(file).startsWith('._')) {
+  if (stats.isDirectory()) {
+    return false;
+  }
+
+  if (extname(file) === '.tf') {
     return false;
   }
+
+  if (!basename(file).startsWith('._')) {
+    return false;
+  }
+
   return true;
 };
 
 const hclToJson = async (filePath) => {
   const content = await readFileProm(filePath);
-  const json = hcl.parse(content.toString());
-
-  return json;
+  return hcl.parse(content.toString(), filePath);
 };
 
 const extractDefinition = async (files, targetPath) => {
   const tfFiles = files.filter((f) => {
+    // Need to constraint lookup to files within target path
+    // otherwise we are exposing ourselves to FS based security
+    // attacks
+    if (f.indexOf(targetPath) !== 0) {
+      return false;
+    }
+
     const relativePath = f.replace(targetPath, '');
+
     if (relativePath.lastIndexOf('/') > 0) {
       return false;
     }
@@ -51,36 +69,54 @@ const extractDefinition = async (files, targetPath) => {
   return _.reduce(list, (l, accum) => _.merge(accum, l), {});
 };
 
+// As module information is stored directly
+// in the database, we need to normalise the
+// object keys to ensure no invalid characters
+// in the names e.g. full stop (.)
+const normalizeKeyNamesForDbStorage = (obj) => {
+  if (!obj) {
+    return obj;
+  }
+
+  const result = {};
+  Object.keys(obj).forEach((key) => {
+    const newKey = key.replace(/[^\w\d_]/g, '__');
+    result[newKey] = obj[key];
+  });
+  return result;
+};
+
 const nomarlizeModule = (module) => ({
   path: '',
   name: module.name || '',
   readme: '',
   empty: !module,
-  inputs: module.variable
-    ? Object.keys(module.variable).map((name) => ({ name, ...module.variable[name] }))
-    : [],
-  outputs: module.output
-    ? Object.keys(module.output).map((name) => ({ name, ...module.output[name] }))
-    : [],
+  inputs: normalizeKeyNamesForDbStorage(module.inputs),
+  outputs: normalizeKeyNamesForDbStorage(module.outputs),
   dependencies: [],
-  resources: module.resource
-    ? Object.keys(module.resource).map((type) => ({ name: Object.keys(module.resource)[0], type }))
-    : [],
+  module_calls: normalizeKeyNamesForDbStorage(module.module_calls),
+  resources: normalizeKeyNamesForDbStorage(module.managed_resources),
 });
 
 const extractSubmodules = async (definition, files, targetPath) => {
-  let pathes = [];
-  if (definition.module) {
-    const submodules = Object.keys(definition.module).map((key) => definition.module[key].source);
-    pathes = _.uniq(submodules);
+  let submodulePaths = [];
+  if (definition.module_calls) {
+    const submodules = Object.keys(definition.module_calls)
+      .map((k) => definition.module_calls[k].source);
+
+    submodulePaths = _.uniq(submodules);
   }
 
-  const promises = pathes.map(async (p) => {
+  const promises = submodulePaths.map(async (p) => {
     const data = await extractDefinition(files, join(targetPath, p));
+    // Submodule was not found in the archive, ignore
+    if (Object.keys(data).length === 0) {
+      return [];
+    }
     data.name = p.substr(p.lastIndexOf('/') + 1);
 
     let result = [data];
-    if (data.module) {
+    if (data.module_calls) {
       const m = await extractSubmodules(data, files, join(targetPath, p));
       result = result.concat(m);
     }
@@ -104,11 +140,15 @@ const parseHcl = (moduleName, compressedModule) => new Promise((resolve, reject)
         try {
           const files = await recursive(tempDir, [ignore]);
 
+          debug('Files found in the archive: %s', files);
+
           // make a root module definition
           const rootData = await extractDefinition(files, tempDir);
           rootData.name = moduleName;
           const rootDefinition = nomarlizeModule(rootData);
 
+          debug('Module definition: %s', util.inspect(rootDefinition, false, 15));
+
           // make submodules definition
           const submodulesData = await extractSubmodules(rootData, files, tempDir);
           const submodulesDefinition = submodulesData.map((s) => nomarlizeModule(s));

lib/util.spec.js

@@ -42,7 +42,7 @@ describe('util\'s', () => {
       tarball = await readFile(tarballPath);
     });
 
-    it('should make JSON from HCL in a compressed module file ', async () => {
+    it('should make JSON from HCL in a compressed module file', async () => {
       const result = await parseHcl('citizen', tarball);
 
       expect(result).to.have.property('root');

package.json

@@ -15,6 +15,7 @@
     "build": "pkg . --out-path dist --targets node10-linux-x64,node10-macos-x64,node10-win-x64"
   },
   "dependencies": {
+    "@evops/hcl-terraform-parser": "^1.0.0",
     "aws-sdk": "^2.787.0",
     "body-parser": "^1.19.0",
     "colors": "^1.4.0",
@@ -23,7 +24,6 @@
     "express": "^4.17.1",
     "glob-gitignore": "^1.0.14",
     "globby": "^11.0.1",
-    "gopher-hcl": "^0.2.0",
     "helmet": "^4.2.0",
     "jten": "^0.2.0",
     "listr": "^0.14.2",

routes/modules.spec.js

@@ -13,6 +13,31 @@ const { db, save } = require('../lib/store');
 const writeFile = promisify(fs.writeFile);
 const readFile = promisify(fs.readFile);
 
+describe('Terraform 0.12 support', () => {
+  describe('POST /v1/modules/:namespace/:name/:provider/:version', () => {
+    let modulePath;
+
+    beforeEach(async () => {
+      modulePath = `hashicorp/consul/aws/${(new Date()).getTime()}`;
+    });
+
+    afterEach(async () => {
+      await deleteDbAll(db);
+      await rimraf(process.env.CITIZEN_STORAGE_PATH);
+    });
+
+    it('should register new v12 module', () => request(app)
+      .post(`/v1/modules/${modulePath}`)
+      .attach('module', 'test/fixture/module.v12.tar.gz')
+      .expect('Content-Type', /application\/json/)
+      .expect(201)
+      .then((res) => {
+        expect(res.body).to.have.property('modules').to.be.an('array');
+        expect(res.body.modules[0]).to.have.property('id').to.equal(modulePath);
+      }));
+  });
+});
+
 describe('POST /v1/modules/:namespace/:name/:provider/:version', () => {
   let moduleBuf;
   let modulePath;