formatter: render HTML entities (#481)

filter-entities.js

@@ -0,0 +1,19 @@
+'use strict';
+
+// entities.json is from https://html.spec.whatwg.org/entities.json
+// which is built by https://github.com/whatwg/html-build/tree/main/entities
+// this processes it into a form more useful for us
+// no new entities will be added, so the output of this file is committed
+// rather than running it every time
+
+let fs = require('fs');
+let entities = require('./entities.json');
+
+let transformed = Object.fromEntries(Object.entries(entities).map(([k, v]) => {
+  // whitespace, default-ignorable, combining characters, control characters
+  if (v.characters === '&' || v.characters === '<' || /\p{White_Space}|\p{DI}|\p{gc=M}|\p{gc=C}/u.test(v.characters)) {
+    return [k, null];
+  }
+  return [k, v.characters];
+}));
+fs.writeFileSync('./entities-processed.json', JSON.stringify(transformed), 'utf8');

package.json

@@ -27,7 +27,8 @@
 		"/lib",
 		"/js",
 		"/css",
-		"/boilerplate"
+		"/boilerplate",
+		"/entities-processed.json"
 	],
 	"repository": "tc39/ecmarkup",
 	"keywords": [

src/formatter/text.ts

@@ -1,10 +1,22 @@
 import { LineBuilder } from './line-builder';
+const entities = require('../../entities-processed.json');
 
 export function printText(text: string, indent: number): LineBuilder {
   const output: LineBuilder = new LineBuilder(indent);
   if (text === '') {
     return output;
   }
+  text = text.replace(/&[a-zA-Z0-9]+;?/g, m => {
+    // entities[m] is null if the entity expands to '&', '<', or a string which has blank/control/etc characters
+    if ({}.hasOwnProperty.call(entities, m) && entities[m] !== null) {
+      return entities[m];
+    }
+    const lower = m.toLowerCase();
+    if (lower === '&le;' || lower === '&amp;') {
+      return lower;
+    }
+    return m;
+  });
 
   const leadingSpace = text[0] === ' ' || text[0] === '\t';
   const trailingSpace = text[text.length - 1] === ' ' || text[text.length - 1] === '\t';

test/formatter.js

@@ -546,32 +546,32 @@ describe('equation formatting', () => {
       `
       <emu-eqn id="eqn-DaysInYear" aoid="DaysInYear">
       DaysInYear(_y_)
-      = *365*<sub>𝔽</sub> if (ℝ(_y_) modulo 4) &ne; 0
-        = *366*<sub>𝔽</sub> if (ℝ(_y_) modulo 4) = 0 and (ℝ(_y_) modulo 100) &ne; 0
-    = *365*<sub>𝔽</sub> if (ℝ(_y_) modulo 100) = 0 and (ℝ(_y_) modulo 400) &ne; 0
+      = *365*<sub>𝔽</sub> if (ℝ(_y_) modulo 4) ≠ 0
+        = *366*<sub>𝔽</sub> if (ℝ(_y_) modulo 4) = 0 and (ℝ(_y_) modulo 100) ≠ 0
+    = *365*<sub>𝔽</sub> if (ℝ(_y_) modulo 100) = 0 and (ℝ(_y_) modulo 400) ≠ 0
       = *366*<sub>𝔽</sub> if (ℝ(_y_) modulo 400) = 0
       </emu-eqn>
 
       <emu-eqn id="eqn-DaysInYear" aoid="DaysInYear">DaysInYear(_y_)
-      = *365*<sub>𝔽</sub> if (ℝ(_y_) modulo 4) &ne; 0
-      = *366*<sub>𝔽</sub> if (ℝ(_y_) modulo 4) = 0 and (ℝ(_y_) modulo 100) &ne; 0
-      = *365*<sub>𝔽</sub> if (ℝ(_y_) modulo 100) = 0 and (ℝ(_y_) modulo 400) &ne; 0
+      = *365*<sub>𝔽</sub> if (ℝ(_y_) modulo 4) ≠ 0
+      = *366*<sub>𝔽</sub> if (ℝ(_y_) modulo 4) = 0 and (ℝ(_y_) modulo 100) ≠ 0
+      = *365*<sub>𝔽</sub> if (ℝ(_y_) modulo 100) = 0 and (ℝ(_y_) modulo 400) ≠ 0
       = *366*<sub>𝔽</sub> if (ℝ(_y_) modulo 400) = 0 </emu-eqn>
       `,
       dedentKeepingTrailingNewline`
       <emu-eqn id="eqn-DaysInYear" aoid="DaysInYear">
         DaysInYear(_y_)
-          = *365*<sub>𝔽</sub> if (ℝ(_y_) modulo 4) &ne; 0
-          = *366*<sub>𝔽</sub> if (ℝ(_y_) modulo 4) = 0 and (ℝ(_y_) modulo 100) &ne; 0
-          = *365*<sub>𝔽</sub> if (ℝ(_y_) modulo 100) = 0 and (ℝ(_y_) modulo 400) &ne; 0
+          = *365*<sub>𝔽</sub> if (ℝ(_y_) modulo 4) ≠ 0
+          = *366*<sub>𝔽</sub> if (ℝ(_y_) modulo 4) = 0 and (ℝ(_y_) modulo 100) ≠ 0
+          = *365*<sub>𝔽</sub> if (ℝ(_y_) modulo 100) = 0 and (ℝ(_y_) modulo 400) ≠ 0
           = *366*<sub>𝔽</sub> if (ℝ(_y_) modulo 400) = 0
       </emu-eqn>
 
       <emu-eqn id="eqn-DaysInYear" aoid="DaysInYear">
         DaysInYear(_y_)
-          = *365*<sub>𝔽</sub> if (ℝ(_y_) modulo 4) &ne; 0
-          = *366*<sub>𝔽</sub> if (ℝ(_y_) modulo 4) = 0 and (ℝ(_y_) modulo 100) &ne; 0
-          = *365*<sub>𝔽</sub> if (ℝ(_y_) modulo 100) = 0 and (ℝ(_y_) modulo 400) &ne; 0
+          = *365*<sub>𝔽</sub> if (ℝ(_y_) modulo 4) ≠ 0
+          = *366*<sub>𝔽</sub> if (ℝ(_y_) modulo 4) = 0 and (ℝ(_y_) modulo 100) ≠ 0
+          = *365*<sub>𝔽</sub> if (ℝ(_y_) modulo 100) = 0 and (ℝ(_y_) modulo 400) ≠ 0
           = *366*<sub>𝔽</sub> if (ℝ(_y_) modulo 400) = 0
       </emu-eqn>
       `
@@ -652,6 +652,31 @@ describe('structured header formatting', () => {
   });
 });
 
+describe('entities', () => {
+  it('entities are transformed or not as appropriate', async () => {
+    await assertDocFormatsAs(
+      `
+      <div>
+        some entities are transformed: &AMP; &frac12; &frac12 &fjlig; &CapitalDifferentialD;
+        others are preserved: &amp; &lt; &nbsp; &nbsp &NotAnEntity;
+      </div>
+      <emu-alg>
+        1. This also works in algorithms, as in &laquo; 0, 1 &raquo;.
+      </emu-alg>
+      `,
+      dedentKeepingTrailingNewline`
+      <div>
+        some entities are transformed: &amp; ½ ½ fj ⅅ
+        others are preserved: &amp; &lt; &nbsp; &nbsp &NotAnEntity;
+      </div>
+      <emu-alg>
+        1. This also works in algorithms, as in « 0, 1 ».
+      </emu-alg>
+      `
+    );
+  });
+});
+
 async function assertRoundTrips(src) {
   await assertDocFormatsAs(src, src);
 }