Implement support for doctesting ESM files

This commit adds a new choice to the 'module' option: 'esm'. When
running doctest with Node version 9 or up available, setting 'module'
to 'esm' allows for the documentation comments to be embedded in
ECMAScript modules, and use 'import' to load dependencies.

The approach included in this commit has the following consequences:

1. The CLI transparently switches between ESM and non-ESM enabled
   based on the Node version running it. This means the CLI is
   fully backwards compatible.
2. The programmatic version also transparently switches between
   ESM and non-ESM depending on whether it's loaded via import
   or via require.
3. The programmatic version has a breaking change, in that its
   primary function returns a Promise now.

Co-Authored-By: David Chambers <dc@davidchambers.me>

.eslintrc.json

@@ -2,13 +2,21 @@
   "root": true,
   "extends": ["./node_modules/sanctuary-style/eslint-es3.json"],
   "env": {"node": true},
+  "globals": {
+    "Promise": "readonly"
+  },
   "overrides": [
     {
       "files": ["lib/doctest.js"],
       "rules": {
         "no-multiple-empty-lines": ["error", {"max": 2, "maxEOF": 0}],
         "spaced-comment": ["error", "always", {"markers": ["/"]}]
       }
+    },
+    {
+      "files": ["*.mjs"],
+      "env": {"es6": true},
+      "parser": "babel-eslint"
     }
   ]
 }

.travis.yml

@@ -3,6 +3,7 @@ language: node_js
 node_js:
   - "6"
   - "8"
+  - "9"
   - "10"
 before_install:
   - git fetch origin refs/heads/master:refs/heads/master

README.md

@@ -46,16 +46,13 @@ $ doctest lib/temperature.js
 
 The exit code is 0 if all tests pass, 1 otherwise.
 
-### AMD and CommonJS modules
+### Supported module systems
 
-doctest supports CommonJS modules and partially supports AMD modules:
-
-| Module system               | Node.js |
-| --------------------------- |:-------:|
-| AMD                         |    ✔︎    |
-| AMD w/ dependencies         |    ✘    |
-| CommonJS                    |    ✔︎    |
-| CommonJS w/ dependencies    |    ✔︎    |
+| Module system         | Option        | Node.js       | Dependencies  |
+| --------------------- | ------------- |:-------------:|:-------------:|
+| AMD                   | `amd`         |       ✔︎       |       ✘       |
+| CommonJS              | `commonjs`    |       ✔︎       |       ✔︎       |
+| ECMAScript modules    | `esm`         |       ✔︎       |       ✔︎       |
 
 Specify module system via JavaScript API:
 

bin/doctest

@@ -2,18 +2,21 @@
 
 'use strict';
 
-// Based on:
-// https://github.com/jashkenas/coffeescript/blob/1.9.1/src/command.coffee#L431-L441
-var args = process.argv.slice (1);
+var path = require ('path');
+
+var esmSupported = Number ((process.versions.node.split ('.'))[0]) >= 9;
+var args = process.argv.slice (2);
 var idx = args.indexOf ('--nodejs');
-if (idx < 0 || idx === args.length - 1) {
-  require ('../lib/command');
-} else {
-  require ('child_process')
-  .spawn (process.execPath,
-          args[idx + 1]
-            .split (/\s+/)
-            .concat (args.slice (0, idx), args.slice (idx + 2)),
-          {cwd: process.cwd (), env: process.env, stdio: [0, 1, 2]})
-  .on ('exit', process.exit);
-}
+var flags = idx >= 0 && idx < args.length - 1;
+
+require ('child_process')
+.spawn (
+  process.execPath,
+  [].concat (esmSupported ? ['--experimental-modules'] : [])
+    .concat (flags ? args[idx + 1].split (/\s+/) : [])
+    .concat (['--', path.resolve (__dirname, '..', 'lib', 'command')])
+    .concat (flags ? (args.slice (0, idx)).concat (args.slice (idx + 2))
+                   : args),
+  {cwd: process.cwd (), env: process.env, stdio: [0, 1, 2]}
+)
+.on ('exit', process.exit);

lib/command.js

@@ -1,62 +1,19 @@
 'use strict';
 
-var program = require ('commander');
-
+var common = require ('./common');
+var program = require ('./program');
 var doctest = require ('..');
-var pkg = require ('../package.json');
-
-
-program
-.version (pkg.version)
-.usage ('[options] path/to/js/or/coffee/module')
-.option ('-m, --module <type>',
-         'specify module system ("amd" or "commonjs")')
-.option ('    --nodejs <options>',
-         'pass options directly to the "node" binary')
-.option ('    --prefix <prefix>',
-         'specify Transcribe-style prefix (e.g. ".")')
-.option ('    --opening-delimiter <delimiter>',
-         'specify line preceding doctest block (e.g. "```javascript")')
-.option ('    --closing-delimiter <delimiter>',
-         'specify line following doctest block (e.g. "```")')
-.option ('-p, --print',
-         'output the rewritten source without running tests')
-.option ('-s, --silent',
-         'suppress output')
-.option ('-t, --type <type>',
-         'specify file type ("coffee" or "js")')
-.parse (process.argv);
 
-//  formatErrors :: Array String -> String
-function formatErrors(errors) {
-  return (errors.map (function(s) { return 'error: ' + s + '\n'; })).join ('');
-}
 
-var errors = [];
-if (program.module != null &&
-    program.module !== 'amd' &&
-    program.module !== 'commonjs') {
-  errors.push ('Invalid module `' + program.module + "'");
-}
-if (program.type != null &&
-    program.type !== 'coffee' &&
-    program.type !== 'js') {
-  errors.push ('Invalid type `' + program.type + "'");
-}
-if (errors.length > 0) {
-  process.stderr.write (formatErrors (errors));
+if (program.module === 'esm') {
+  process.stderr.write (
+    common.formatErrors ([
+      'Node.js v' +
+      process.versions.node +
+      ' does not support ECMAScript modules (supported since v9.0.0)'
+    ])
+  );
   process.exit (1);
 }
 
-process.exit (program.args.reduce (function(status, path) {
-  var results;
-  try {
-    results = doctest (path, program);
-  } catch (err) {
-    process.stderr.write (formatErrors ([err.message]));
-    process.exit (1);
-  }
-  return results.reduce (function(status, tuple) {
-    return tuple[0] ? status : 1;
-  }, status);
-}, 0));
+common.runDoctests (doctest, program);

lib/command.mjs

@@ -0,0 +1,5 @@
+import common from './common';
+import program from './program';
+import doctest from '..';
+
+common.runDoctests (doctest, program);

lib/common.js

@@ -0,0 +1,38 @@
+'use strict';
+
+
+//  formatErrors :: Array String -> String
+exports.formatErrors = function(errors) {
+  return (errors.map (function(s) { return 'error: ' + s + '\n'; })).join ('');
+};
+
+//  sanitizeFileContents :: String -> String
+exports.sanitizeFileContents = function(contents) {
+  return contents.replace (/\r\n?/g, '\n').replace (/^#!.*/, '');
+};
+
+//  unlines :: Array String -> String
+exports.unlines = function(lines) {
+  return lines.reduce (function(s, line) { return s + line + '\n'; }, '');
+};
+
+exports.runDoctests = function(doctest, program) {
+  if (program.args.length === 0) {
+    process.stderr.write (exports.formatErrors ([
+      'No files for doctesting provided'
+    ]));
+    process.exit (1);
+  }
+  Promise.all (program.args.map (function(path) {
+    return (doctest (path, program)).then (function(results) {
+      return results.reduce (function(status, tuple) {
+        return tuple[0] ? status : 1;
+      }, 0);
+    });
+  })).then (function(statuses) {
+    process.exit (statuses.every (function(s) { return s === 0; }) ? 0 : 1);
+  }, function(err) {
+    process.stderr.write (exports.formatErrors ([err.message]));
+    process.exit (1);
+  });
+};

lib/doctest.js

@@ -18,12 +18,14 @@ var esprima = require ('esprima');
 var _show = require ('sanctuary-show');
 var Z = require ('sanctuary-type-classes');
 
+var common = require ('./common');
+
 
 function inferType(path) {
   switch (pathlib.extname (path)) {
     case '.coffee': return 'coffee';
     case '.js':     return 'js';
-    default:        throw new Error ('Cannot infer type from extension');
+    default:        return null;
   }
 }
 
@@ -39,36 +41,46 @@ module.exports = function(path, options) {
   if (options.module != null &&
       options.module !== 'amd' &&
       options.module !== 'commonjs') {
-    throw new Error ('Invalid module `' + options.module + "'");
+    return Promise.reject (new Error (
+      'Invalid module `' + options.module + "'"
+    ));
   }
   if (options.type != null &&
       options.type !== 'coffee' &&
       options.type !== 'js') {
-    throw new Error ('Invalid type `' + options.type + "'");
+    return Promise.reject (new Error (
+      'Invalid type `' + options.type + "'"
+    ));
+  }
+
+  var type = options.type == null ? inferType (path) : options.type;
+  if (type == null) {
+    return Promise.reject (new Error (
+      'Cannot infer type from extension'
+    ));
   }
 
   var source = toModule (
-    rewriters[options.type == null ? inferType (path) : options.type] (
+    rewriters[type] (
       {prefix: options.prefix == null ? '' : options.prefix,
        openingDelimiter: options.openingDelimiter,
-       closingDelimiter: options.closingDelimiter},
-      fs.readFileSync (path, 'utf8')
-        .replace (/\r\n?/g, '\n')
-        .replace (/^#!.*/, '')
+       closingDelimiter: options.closingDelimiter,
+       sourceType: 'script'},
+      common.sanitizeFileContents (fs.readFileSync (path, 'utf8'))
     ),
     options.module
   );
 
   if (options.print) {
     console.log (source.replace (/\n$/, ''));
-    return [];
+    return Promise.resolve ([]);
   } else if (options.silent) {
-    return evaluate (options.module, source, path);
+    return Promise.resolve (evaluate (options.module, source, path));
   } else {
     console.log ('running doctests in ' + path + '...');
     var results = evaluate (options.module, source, path);
     log (results);
-    return results;
+    return Promise.resolve (results);
   }
 };
 
@@ -111,11 +123,6 @@ function stripLeading(n, c, s) {
   return s.slice (idx);
 }
 
-//  unlines :: Array String -> String
-function unlines(lines) {
-  return lines.reduce (function(s, line) { return s + line + '\n'; }, '');
-}
-
 //  iifeWrap :: String -> String
 function iifeWrap(s) {
   return 'void function() {\n' + indentN (2, s) + '}.call(this);';
@@ -125,7 +132,7 @@ function iifeWrap(s) {
 function toModule(source, moduleType) {
   switch (moduleType) {
     case 'amd':
-      return unlines ([
+      return common.unlines ([
         source,
         'function define() {',
         '  for (var idx = 0; idx < arguments.length; idx += 1) {',
@@ -137,7 +144,7 @@ function toModule(source, moduleType) {
         '}'
       ]);
     case 'commonjs':
-      return iifeWrap (unlines ([
+      return iifeWrap (common.unlines ([
         'var __doctest = {',
         '  require: require,',
         '  queue: [],',
@@ -308,9 +315,12 @@ function substring(input, start, end) {
   );
 }
 
-function wrap$js(test) {
-  var type = (esprima.parse (test[INPUT].value)).body[0].type;
-  return type === 'FunctionDeclaration' || type === 'VariableDeclaration' ?
+function wrap$js(test, sourceType) {
+  var ast = esprima.parse (test[INPUT].value, {sourceType: sourceType});
+  var type = ast.body[0].type;
+  return type === 'FunctionDeclaration' ||
+         type === 'ImportDeclaration' ||
+         type === 'VariableDeclaration' ?
     test[INPUT].value :
     [
       '__doctest.enqueue({',
@@ -378,7 +388,16 @@ function rewrite$js(options, input) {
   //     produced by step 6 (substituting "step 6" for "step 2").
 
   function getComments(input) {
-    return (esprima.parse (input, {comment: true, loc: true})).comments;
+    var ast = esprima.parse (input, {
+      comment: true,
+      loc: true,
+      sourceType: options.sourceType
+    });
+    return ast.comments;
+  }
+
+  function wrapCode(js) {
+    return wrap$js (js, options.sourceType);
   }
 
   //  comments :: { Block :: Array Comment, Line :: Array Comment }
@@ -401,7 +420,7 @@ function rewrite$js(options, input) {
 
   //  source :: String
   var source = lineTests
-    .map (wrap$js)
+    .map (wrapCode)
     .concat ([''])
     .reduce (function(accum, s, idx) { return accum + chunks[idx] + s; }, '');
 
@@ -413,7 +432,7 @@ function rewrite$js(options, input) {
         substring (source, accum.loc, comment.loc.start),
         blockTests
           .filter (function(test) { return test.commentIndex === idx; })
-          .map (wrap$js)
+          .map (wrapCode)
           .join ('\n')
       );
       accum.loc = comment.loc.end;
@@ -423,6 +442,8 @@ function rewrite$js(options, input) {
     .join ('');
 }
 
+module.exports.rewrite$js = rewrite$js;
+
 function rewrite$coffee(options, input) {
   var lines = input.match (/^.*(?=\n)/gm);
   var chunks = lines.reduce (function(accum, line, idx) {
@@ -548,6 +569,8 @@ function run(queue) {
   }, {results: [], thunk: null}).results;
 }
 
+module.exports.run = run;
+
 function log(results) {
   console.log (results.reduce (function(s, tuple) {
     return s + (tuple[0] ? '.' : 'x');
@@ -559,3 +582,5 @@ function log(results) {
     }
   });
 }
+
+module.exports.log = log;