tools: produce JSON documentation using unified/remark/rehype

.gitignore

@@ -106,6 +106,9 @@ deps/npm/node_modules/.bin/
 /*.pkg
 /SHASUMS*.txt*
 
+# api docs artifacts
+tools/doc/node_modules
+
 # test artifacts
 tools/remark-cli/node_modules
 tools/remark-preset-lint-node/node_modules

Makefile

@@ -329,7 +329,7 @@ ifeq ($(OSTYPE),aix)
 DOCBUILDSTAMP_PREREQS := $(DOCBUILDSTAMP_PREREQS) out/$(BUILDTYPE)/node.exp
 endif
 
-test/addons/.docbuildstamp: $(DOCBUILDSTAMP_PREREQS)
+test/addons/.docbuildstamp: $(DOCBUILDSTAMP_PREREQS) tools/doc/node_modules
 	$(RM) -r test/addons/??_*/
 	[ -x $(NODE) ] && $(NODE) $< || node $<
 	touch $@
@@ -650,15 +650,15 @@ available-node = \
 run-npm-install = $(PWD)/$(NPM) install --production --no-package-lock
 run-npm-ci = $(PWD)/$(NPM) ci
 
-gen-json = tools/doc/generate.js --format=json $< > $@
-gen-html = tools/doc/generate.js --node-version=$(FULLVERSION) --format=html \
-			--analytics=$(DOCS_ANALYTICS) $< > $@
+tools/doc/node_modules/js-yaml/package.json:
+	cd tools/doc && $(call available-node,$(run-npm-install))
 
-out/doc/api/%.json: doc/api/%.md tools/doc/generate.js tools/doc/json.js
-	$(call available-node, $(gen-json))
+gen-api = tools/doc/generate.js --node-version=$(FULLVERSION) \
+		--analytics=$(DOCS_ANALYTICS) $< --output-directory=out/doc/api
 
-out/doc/api/%.html: doc/api/%.md tools/doc/generate.js tools/doc/html.js
-	$(call available-node, $(gen-html))
+out/doc/api/%.json out/doc/api/%.html: doc/api/%.md tools/doc/generate.js \
+	tools/doc/html.js tools/doc/json.js
+	$(call available-node, $(gen-api))
 
 out/doc/api/all.html: $(apidocs_html) tools/doc/allhtml.js
 	$(call available-node, tools/doc/allhtml.js)

doc/api/https.md

@@ -123,7 +123,7 @@ changes:
     pr-url: https://github.com/nodejs/node/pull/10638
     description: The `options` parameter can be a WHATWG `URL` object.
 -->
-- `url` {string | URL}
+* `url` {string | URL}
 * `options` {Object | string | URL} Accepts the same `options` as
   [`https.request()`][], with the `method` always set to `GET`.
 * `callback` {Function}
@@ -174,7 +174,7 @@ changes:
     pr-url: https://github.com/nodejs/node/pull/10638
     description: The `options` parameter can be a WHATWG `URL` object.
 -->
-- `url` {string | URL}
+* `url` {string | URL}
 * `options` {Object | string | URL} Accepts all `options` from
   [`http.request()`][], with some differences in default values:
   - `protocol` **Default:** `'https:'`

test/doctool/test-doctool-html.js

@@ -11,7 +11,34 @@ try {
 const assert = require('assert');
 const { readFile } = require('fs');
 const fixtures = require('../common/fixtures');
-const toHTML = require('../../tools/doc/html.js');
+const html = require('../../tools/doc/html.js');
+const path = require('path');
+
+module.paths.unshift(
+  path.join(__dirname, '..', '..', 'tools', 'doc', 'node_modules'));
+const unified = require('unified');
+const markdown = require('remark-parse');
+const remark2rehype = require('remark-rehype');
+const raw = require('rehype-raw');
+const htmlStringify = require('rehype-stringify');
+
+function toHTML({ input, filename, nodeVersion, analytics }, cb) {
+  const content = unified()
+    .use(markdown)
+    .use(html.firstHeader)
+    .use(html.preprocessText)
+    .use(html.preprocessElements, { filename })
+    .use(html.buildToc, { filename })
+    .use(remark2rehype, { allowDangerousHTML: true })
+    .use(raw)
+    .use(htmlStringify)
+    .processSync(input);
+
+  html.toHTML(
+    { input, content, filename, nodeVersion, analytics },
+    cb
+  );
+}
 
 // Test data is a list of objects with two properties.
 // The file property is the file path.
@@ -80,7 +107,6 @@ testData.forEach(({ file, html, analyticsId }) => {
 
   readFile(file, 'utf8', common.mustCall((err, input) => {
     assert.ifError(err);
-
     toHTML(
       {
         input: input,

test/doctool/test-doctool-json.js

@@ -10,9 +10,27 @@ try {
 
 const assert = require('assert');
 const fs = require('fs');
+const path = require('path');
 const fixtures = require('../common/fixtures');
 const json = require('../../tools/doc/json.js');
 
+module.paths.unshift(
+  path.join(__dirname, '..', '..', 'tools', 'doc', 'node_modules'));
+const unified = require('unified');
+const markdown = require('remark-parse');
+
+function toJSON(input, filename, cb) {
+  function nullCompiler() {
+    this.Compiler = (tree) => tree;
+  }
+
+  unified()
+    .use(markdown)
+    .use(json.jsonAPI, { filename })
+    .use(nullCompiler)
+    .process(input, cb);
+}
+
 // Outputs valid json with the expected fields when given simple markdown
 // Test data is a list of objects with two properties.
 // The file property is the file path.
@@ -21,15 +39,16 @@ const testData = [
   {
     file: fixtures.path('sample_document.md'),
     json: {
+      type: 'module',
       source: 'foo',
       modules: [{
         textRaw: 'Sample Markdown',
         name: 'sample_markdown',
         modules: [{
           textRaw: 'Seussian Rhymes',
           name: 'seussian_rhymes',
-          desc: '<ol>\n<li>fish</li>\n<li><p>fish</p>\n</li>\n<li>' +
-                  '<p>Red fish</p>\n</li>\n<li>Blue fish</li>\n</ol>\n',
+          desc: '<ol>\n<li>fish</li>\n<li>fish</li>\n</ol>\n' +
+                  '<ul>\n<li>Red fish</li>\n<li>Blue fish</li>\n</ul>',
           type: 'module',
           displayName: 'Seussian Rhymes'
         }],
@@ -41,6 +60,7 @@ const testData = [
   {
     file: fixtures.path('order_of_end_tags_5873.md'),
     json: {
+      type: 'module',
       source: 'foo',
       modules: [{
         textRaw: 'Title',
@@ -55,15 +75,10 @@ const testData = [
             signatures: [
               {
                 params: [{
-                  textRaw: '`array` {Array} ',
+                  textRaw: '`array` {Array}',
                   name: 'array',
                   type: 'Array'
                 }]
-              },
-              {
-                params: [{
-                  name: 'array'
-                }]
               }
             ]
           }],
@@ -78,6 +93,7 @@ const testData = [
   {
     file: fixtures.path('doc_with_yaml.md'),
     json: {
+      type: 'module',
       source: 'foo',
       modules: [
         {
@@ -92,7 +108,7 @@ const testData = [
                 changes: []
               },
               desc: '<p>Describe <code>Foobar</code> in more detail ' +
-                'here.</p>\n',
+                'here.</p>',
               type: 'module',
               displayName: 'Foobar'
             },
@@ -110,7 +126,7 @@ const testData = [
                 ]
               },
               desc: '<p>Describe <code>Foobar II</code> in more detail ' +
-                'here. fg(1)</p>\n',
+                'here. fg(1)</p>',
               type: 'module',
               displayName: 'Foobar II'
             },
@@ -123,15 +139,15 @@ const testData = [
                 changes: []
               },
               desc: '<p>Describe <code>Deprecated thingy</code> in more ' +
-                'detail here. fg(1p)</p>\n',
+                'detail here. fg(1p)</p>',
               type: 'module',
               displayName: 'Deprecated thingy'
             },
             {
               textRaw: 'Something',
               name: 'something',
               desc: '<!-- This is not a metadata comment -->\n<p>' +
-                'Describe <code>Something</code> in more detail here.</p>\n',
+                'Describe <code>Something</code> in more detail here.</p>',
               type: 'module',
               displayName: 'Something'
             }
@@ -147,9 +163,9 @@ const testData = [
 testData.forEach((item) => {
   fs.readFile(item.file, 'utf8', common.mustCall((err, input) => {
     assert.ifError(err);
-    json(input, 'foo', common.mustCall((err, output) => {
+    toJSON(input, 'foo', common.mustCall((err, output) => {
       assert.ifError(err);
-      assert.deepStrictEqual(output, item.json);
+      assert.deepStrictEqual(output.json, item.json);
     }));
   }));
 });

tools/doc/generate.js

@@ -22,53 +22,72 @@
 'use strict';
 
 const fs = require('fs');
+const path = require('path');
+const unified = require('unified');
+const markdown = require('remark-parse');
+const remark2rehype = require('remark-rehype');
+const raw = require('rehype-raw');
+const htmlStringify = require('rehype-stringify');
+
+const html = require('./html');
+const json = require('./json');
 
 // Parse the args.
 // Don't use nopt or whatever for this. It's simple enough.
 
 const args = process.argv.slice(2);
-let format = 'json';
 let filename = null;
 let nodeVersion = null;
 let analytics = null;
+let outputDir = null;
 
 args.forEach(function(arg) {
   if (!arg.startsWith('--')) {
     filename = arg;
-  } else if (arg.startsWith('--format=')) {
-    format = arg.replace(/^--format=/, '');
   } else if (arg.startsWith('--node-version=')) {
     nodeVersion = arg.replace(/^--node-version=/, '');
   } else if (arg.startsWith('--analytics=')) {
     analytics = arg.replace(/^--analytics=/, '');
+  } else if (arg.startsWith('--output-directory=')) {
+    outputDir = arg.replace(/^--output-directory=/, '');
   }
 });
 
 nodeVersion = nodeVersion || process.version;
 
 if (!filename) {
   throw new Error('No input file specified');
+} else if (!outputDir) {
+  throw new Error('No output directory specified');
 }
 
+
 fs.readFile(filename, 'utf8', (er, input) => {
   if (er) throw er;
-  switch (format) {
-    case 'json':
-      require('./json.js')(input, filename, (er, obj) => {
-        if (er) throw er;
-        console.log(JSON.stringify(obj, null, 2));
-      });
-      break;
 
-    case 'html':
-      require('./html')({ input, filename, nodeVersion, analytics },
-                        (err, html) => {
-                          if (err) throw err;
-                          console.log(html);
-                        });
-      break;
+  const content = unified()
+    .use(markdown)
+    .use(json.jsonAPI, { filename })
+    .use(html.firstHeader)
+    .use(html.preprocessText)
+    .use(html.preprocessElements, { filename })
+    .use(html.buildToc, { filename })
+    .use(remark2rehype, { allowDangerousHTML: true })
+    .use(raw)
+    .use(htmlStringify)
+    .processSync(input);
 
-    default:
-      throw new Error(`Invalid format: ${format}`);
-  }
+  const basename = path.basename(filename, '.md');
+
+  html.toHTML(
+    { input, content, filename, nodeVersion, analytics },
+    (err, html) => {
+      const target = path.join(outputDir, `${basename}.html`);
+      if (err) throw err;
+      fs.writeFileSync(target, html);
+    }
+  );
+
+  const target = path.join(outputDir, `${basename}.json`);
+  fs.writeFileSync(target, JSON.stringify(content.json, null, 2));
 });

tools/doc/html.js

@@ -29,11 +29,13 @@ const visit = require('unist-util-visit');
 const markdown = require('remark-parse');
 const remark2rehype = require('remark-rehype');
 const raw = require('rehype-raw');
-const html = require('rehype-stringify');
+const htmlStringify = require('rehype-stringify');
 const path = require('path');
 const typeParser = require('./type-parser.js');
 
-module.exports = toHTML;
+module.exports = {
+  toHTML, firstHeader, preprocessText, preprocessElements, buildToc
+};
 
 const docPath = path.resolve(__dirname, '..', '..', 'doc');
 
@@ -54,26 +56,15 @@ const gtocHTML = unified()
   .use(remark2rehype, { allowDangerousHTML: true })
   .use(raw)
   .use(navClasses)
-  .use(html)
+  .use(htmlStringify)
   .processSync(gtocMD).toString();
 
 const templatePath = path.join(docPath, 'template.html');
 const template = fs.readFileSync(templatePath, 'utf8');
 
-function toHTML({ input, filename, nodeVersion, analytics }, cb) {
+function toHTML({ input, content, filename, nodeVersion, analytics }, cb) {
   filename = path.basename(filename, '.md');
 
-  const content = unified()
-    .use(markdown)
-    .use(firstHeader)
-    .use(preprocessText)
-    .use(preprocessElements, { filename })
-    .use(buildToc, { filename })
-    .use(remark2rehype, { allowDangerousHTML: true })
-    .use(raw)
-    .use(html)
-    .processSync(input);
-
   const id = filename.replace(/\W+/g, '-');
 
   let HTML = template.replace('__ID__', id)
@@ -280,7 +271,7 @@ function parseYAML(text) {
         .use(markdown)
         .use(remark2rehype, { allowDangerousHTML: true })
         .use(raw)
-        .use(html)
+        .use(htmlStringify)
         .processSync(change.description).toString();
 
       result += `<tr><td>${change.version}</td>\n` +
@@ -366,7 +357,7 @@ function buildToc({ filename }) {
       .use(markdown)
       .use(remark2rehype, { allowDangerousHTML: true })
       .use(raw)
-      .use(html)
+      .use(htmlStringify)
       .processSync(toc).toString();
   };
 }