feat(html): enable HAST parsing and serialization of HTML responses

HTML response bodies will now be parsed and then serialized before the final response is created. This creates the opportunity of changing the HTML AST for HTML-post-processing Fixes #285
adobe · Apr 26, 2019 · 224c665 · 224c665
1 parent aa2538f
commit 224c665
Show file tree

Hide file tree

Showing 12 changed files with 361 additions and 24 deletions.
diff --git a/docs/response.schema.json b/docs/response.schema.json
diff --git a/docs/response.schema.md b/docs/response.schema.md
diff --git a/package.json b/package.json
@@ -66,7 +66,6 @@
     "micromatch": "^4.0.0",
     "object-hash": "^1.3.1",
     "rehype-parse": "^6.0.0",
-    "rehype-stringify": "^5.0.0",
     "remark-parse": "^6.0.0",
     "remark-rehype": "^4.0.0",
     "request": "^2.87.0",

diff --git a/src/defaults/html.pipe.js b/src/defaults/html.pipe.js
@@ -32,6 +32,8 @@ const { cache, uncached } = require('../html/shared-cache');
 const embeds = require('../html/find-embeds');
 const parseFrontmatter = require('../html/parse-frontmatter');
 const rewriteLinks = require('../html/static-asset-links');
+const tohast = require('../html/html-to-hast');
+const tohtml = require('../html/stringify-hast');
 
 /* eslint no-param-reassign: off */
 /* eslint newline-per-chained-call: off */
@@ -63,7 +65,9 @@ const htmlpipe = (cont, payload, action) => {
     .after(cache).when(uncached)
     .after(key)
     .after(debug)
+    .after(tohast) // start HTML post-processing
     .after(rewriteLinks).when(production)
+    .after(tohtml) // end HTML post-processing
     .after(flag).expose('esi').when(esi) // flag ESI when there is ESI in the response
     .error(selectStatus(production()));
 

diff --git a/src/html/html-to-hast.js b/src/html/html-to-hast.js
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2019 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+const unified = require('unified');
+const parse = require('rehype-parse');
+
+function tohast({ response: { body } }) {
+  const fragment = !body.match(/<html/i);
+  const hast = unified().use(parse, { fragment }).parse(body);
+  return {
+    response: {
+      hast,
+    },
+  };
+}
+
+module.exports = tohast;
diff --git a/src/html/static-asset-links.js b/src/html/static-asset-links.js
@@ -10,9 +10,6 @@
  * governing permissions and limitations under the License.
  */
 
-const unified = require('unified');
-const parse = require('rehype-parse');
-const stringify = require('rehype-stringify');
 const map = require('unist-util-map');
 const Url = require('url-parse');
 
@@ -51,25 +48,14 @@ function links() {
   };
 }
 
-function rewrite({ response: { body, headers } }) {
-  if (headers && headers['Content-Type'] && headers['Content-Type'].match(/html/)) {
-    const doc = unified()
-      .use(parse, {
-        fragment: false,
-      })
-      .use(scripts)
-      .use(links)
-      .use(stringify, {
-        allowParseErrors: true,
-        allowDangerousHTML: true,
-        allowDangerousCharacters: true,
-        quoteSmart: true,
+function rewrite({ response: { hast, headers } }) {
+  if (headers && headers['Content-Type'] && headers['Content-Type'].match(/html/) && hast) {
+    links()(hast);
+    scripts()(hast);
 
-      })
-      .processSync(body);
     return {
       response: {
-        body: doc.contents,
+        hast,
       },
     };
   }

diff --git a/src/html/stringify-hast.js b/src/html/stringify-hast.js
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2019 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+const serialize = require('hast-util-to-html');
+
+function stringify({ response: { hast } }) {
+  const body = serialize(hast, {
+    allowParseErrors: true,
+    allowDangerousHTML: true,
+    allowDangerousCharacters: true,
+    quoteSmart: true,
+    entities: {
+      useNamedReferences: true,
+    },
+  });
+
+  return {
+    response: {
+      body,
+    },
+  };
+}
+
+module.exports = stringify;
diff --git a/src/schemas/response.schema.json b/src/schemas/response.schema.json
@@ -34,6 +34,10 @@
         }
       ]
     },
+    "hast": {
+      "type": "object",
+      "description": "The Hypertext AST of the reponse body"
+    },
     "headers": {
       "description": "The HTTP headers of the response",
       "additionalProperties": {

diff --git a/test/testEmbedHandler.js b/test/testEmbedHandler.js
@@ -115,6 +115,6 @@ https://www.youtube.com/watch?v=KOxbO0EI4MA
     assert.equal(result.response.body, `<p>Hello World
 Here comes an embed.</p>
 <esi:include src="https://example-embed-service.com/https://www.youtube.com/watch?v=KOxbO0EI4MA"></esi:include>
-<p><img src="easy.png" alt="Easy!" srcset="easy.png?width=480&amp;auto=webp 480w,easy.png?width=1384&amp;auto=webp 1384w,easy.png?width=2288&amp;auto=webp 2288w,easy.png?width=3192&amp;auto=webp 3192w,easy.png?width=4096&amp;auto=webp 4096w" sizes="100vw"></p>`);
+<p><img src="easy.png" alt="Easy!" srcset="easy.png?width=480&amp;auto=webp 480w, easy.png?width=1384&amp;auto=webp 1384w, easy.png?width=2288&amp;auto=webp 2288w, easy.png?width=3192&amp;auto=webp 3192w, easy.png?width=4096&amp;auto=webp 4096w" sizes="100vw"></p>`);
   });
 });
diff --git a/test/testRewriteStatic.js b/test/testRewriteStatic.js
@@ -13,23 +13,36 @@
 const assert = require('assert');
 const { Logger } = require('@adobe/helix-shared');
 const rewrite = require('../src/html/static-asset-links');
+const tohast = require('../src/html/html-to-hast');
+const stringify = require('../src/html/stringify-hast');
 const { pipe } = require('../src/defaults/html.pipe.js');
 
-
 const logger = Logger.getTestLogger({
   // tune this for debugging
   level: 'info',
 });
 
 function rw(content) {
-  return rewrite({
+  const hastcontext = tohast({
     response: {
       body: content,
       headers: {
         'Content-Type': 'text/html',
       },
     },
-  }).response.body;
+  });
+
+  const rewritecontext = rewrite({
+    response: {
+      body: content,
+      hast: hastcontext.response.hast,
+      headers: {
+        'Content-Type': 'text/html',
+      },
+    },
+  });
+
+  return stringify(rewritecontext).response.body;
 }
 
 describe('Integration Test Static Asset Rewriting', () => {

diff --git a/test/testStringifyHast.js b/test/testStringifyHast.js
@@ -0,0 +1,120 @@
+/*
+ * Copyright 2018 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+/* eslint-env mocha */
+const assert = require('assert');
+const stringify = require('../src/html/stringify-hast');
+
+describe('Testing stringify pipeline step', () => {
+  it('Simple HTML can be transformed', () => {
+    assert.deepEqual(
+      stringify({
+        response: {
+          hast: {
+            type: 'root',
+            children: [
+              {
+                type: 'element',
+                tagName: 'html',
+                properties: {},
+                children: [
+                  {
+                    type: 'element',
+                    tagName: 'head',
+                    properties: {},
+                    children: [
+                      {
+                        type: 'text',
+                        value: '\n    ',
+                        position: {
+                          start: { line: 2, column: 9, offset: 15 },
+                          end: { line: 3, column: 5, offset: 20 },
+                        },
+                      },
+                      {
+                        type: 'element',
+                        tagName: 'title',
+                        properties: {},
+                        children: [
+                          {
+                            type: 'text',
+                            value: 'Foo',
+                            position: {
+                              start: { line: 3, column: 12, offset: 27 },
+                              end: { line: 3, column: 15, offset: 30 },
+                            },
+                          },
+                        ],
+                        position: {
+                          start: { line: 3, column: 5, offset: 20 },
+                          end: { line: 3, column: 23, offset: 38 },
+                        },
+                      },
+                      {
+                        type: 'text',
+                        value: '\n  ',
+                        position: {
+                          start: { line: 3, column: 23, offset: 38 },
+                          end: { line: 4, column: 3, offset: 41 },
+                        },
+                      },
+                    ],
+                    position: {
+                      start: { line: 2, column: 3, offset: 9 },
+                      end: { line: 4, column: 10, offset: 48 },
+                    },
+                  },
+                  {
+                    type: 'text',
+                    value: '\n  ',
+                    position: {
+                      start: { line: 4, column: 10, offset: 48 },
+                      end: { line: 5, column: 3, offset: 51 },
+                    },
+                  },
+                  {
+                    type: 'element',
+                    tagName: 'body',
+                    properties: {},
+                    children: [
+                      {
+                        type: 'text',
+                        value: 'bar\n',
+                        position: {
+                          start: { line: 5, column: 10, offset: 58 },
+                          end: { line: 6, column: 1, offset: 69 },
+                        },
+                      },
+                    ],
+                  },
+                ],
+                position: {
+                  start: { line: 1, column: 1, offset: 0 },
+                  end: { line: 6, column: 8, offset: 76 },
+                },
+              },
+            ],
+            data: { quirksMode: true },
+            position: {
+              start: { line: 1, column: 1, offset: 0 },
+              end: { line: 6, column: 8, offset: 76 },
+            },
+          },
+        },
+      }).response.body,
+      `<html><head>
+    <title>Foo</title>
+  </head>
+  <body>bar
+</body></html>`,
+    );
+  });
+});