feat: add support for templates

docs/documentation.html

@@ -36,6 +36,7 @@ <h2 id="table-of-contents">Table of Contents</h2>
 <a href="#grammar-syntax-and-semantics">Grammar Syntax and Semantics</a>
 <ul>
 <li><a href="#grammar-syntax-and-semantics-parsing-expression-types">Parsing Expression Types</a></li>
+<li><a href="#grammar-syntax-and-semantics-templates">Templated rules</a></li>
 <li><a href="#action-execution-environment">Action Execution Environment</a></li>
 <li><a href="#parsing-lists">Parsing Lists</a></li>
 </ul>
@@ -597,6 +598,38 @@ <h2 id="grammar-syntax-and-semantics">Grammar Syntax and Semantics</h2>
 of strings containing digits, as its parameter. It joins the digits together to
 form a number and converts it to a JavaScript <code>number</code> object.</p>
 
+<h3 id="grammar-syntax-and-semantics-templates">Templates</h3>
+<p>
+  Rules can become <em>templated</em>, meaning you can define a rule as a template,
+  and create an instance of this template. <br/>
+
+  To define a rule as a template, 
+  just decide of parameters and add them like this : <br/>
+  <code>
+    TemplatedRule&lt;Parameter&gt; = /* an expression that can depend of Parameter */ 
+  </code>
+  <br/>
+  Example:<br/>
+  <code>
+    SeparatedList&lt;Item,Separator&gt; = (head:Item tail:(Separator @Item)* )?
+  </code>
+  <br/>
+  Then you can use it by giving it arguments:<br/>
+  <code>
+    CSV = SeparatedList&lt;StringLiteral,CommaToken&gt;
+  </code>
+
+  <br/>
+
+  Note: some limitations : 
+  <ul>
+    <li>Templates arguments are required to be template references, and not expressions</li>
+    <li>Template do not "inline" for now when they instantiate, they create additional rules</li>
+    <li>No type constraints on template parameters, and no variadic parameters</li>
+  </ul>
+
+</p>
+
 <h3 id="grammar-syntax-and-semantics-parsing-expression-types">Parsing Expression Types</h3>
 
 <p>There are several types of parsing expressions, some of them containing

examples/generics.pegjs

@@ -0,0 +1,25 @@
+Entry 
+ = File / Directory
+
+File 
+ = "file"
+
+Directory 
+ = "directory" _
+   ":" _ 
+   Map<name,Entry>
+
+Map<Key,Value> 
+ = "{" _
+    head:(@MapEntry<Key,Value> _)?
+    tail:("," _ @MapEntry<Key,Value> _)*
+   "}"
+
+MapEntry<Key,Value>
+ = Key _ ":" _ Value
+
+name = [a-zA-Z0-9]+
+
+_ "whitespace"
+  = [ \t\n\r]*
+

lib/compiler/index.js

@@ -10,6 +10,8 @@ const reportInfiniteRecursion = require("./passes/report-infinite-recursion");
 const reportInfiniteRepetition = require("./passes/report-infinite-repetition");
 const reportUndefinedRules = require("./passes/report-undefined-rules");
 const reportIncorrectPlucking = require("./passes/report-incorrect-plucking");
+const instantiateTemplates = require("./passes/instantiate-templates");
+const removeNonInstantiatedTemplates =  require("./passes/remove-non-instantiated-templates");
 const Session = require("./session");
 const visitor = require("./visitor");
 const { base64 } = require("./utils");
@@ -41,6 +43,10 @@ const compiler = {
   // or modify it as needed. If the pass encounters a semantic error, it throws
   // |peg.GrammarError|.
   passes: {
+    templates: [
+      instantiateTemplates,
+      removeNonInstantiatedTemplates,
+    ],
     check: [
       reportUndefinedRules,
       reportDuplicateRules,

lib/compiler/passes/instantiate-templates.js

@@ -0,0 +1,137 @@
+"use strict";
+
+const GrammarError = require("../../grammar-error");
+const asts = require("../asts");
+const visitor = require("../visitor");
+
+
+/**
+ * This pass will instantiate referenced templates
+ * The instantiation is done by duplicating the templated rule, and replacing any parameters by whatever they are
+ */
+function instantiateTemplates(ast, options, session) {
+  function instantiateTemplate(template, templateArgs) {
+    // Javascript does not let us have variables names such as Map<Key,Value>
+    // Instead, it accepts MapᐊKeyͺValueᐅ
+    // See https://es5.github.io/x7.html#x7.6
+    const name =  `${template.name}ᐊ${templateArgs.map(a => a.name).join("ͺ")}ᐅ`;
+    const existing = asts.findRule(ast, name);
+    if (existing) {
+      return existing;
+    }
+
+    session.info("Instantiating ", template.name, " with ", templateArgs.map(x => x.name));
+
+    /**
+     * This will replace every usage of the template parameters by the args specified in instantiateTemplate
+     */
+    const instantiateVisitor = visitor.build({
+
+      /**
+       * Check if the rule we reference is one of the template parameter.
+       * If yes, we replace by the template arguments.
+       * Then, if the target rule is a template, we instantiate this template
+       */
+      rule_ref(node) {
+        const i = template.templateParams.declarations.indexOf(node.name);
+        if (i === -1 && !node.templateArgs) {
+          // We reference a rule that is not a template, and this rule has no parameters
+          return node;
+        }
+
+        // Find the target rule
+        const targetName = (i !== -1) ? templateArgs[i].name : node.name;
+
+        if (node.templateArgs) {
+          const targetRule = asts.findRule(ast, targetName);
+          if (!targetRule) {
+            throw new GrammarError(`Rule "${targetName}" is not defined`, node.location);
+          }
+
+          const expected = targetRule.templateParams;
+
+          if (!expected) {
+            throw new GrammarError(`Rule "${node.name}" is not a template `, node.location);
+          }
+          if (expected.declarations.length !== node.templateArgs.length) {
+            throw new GrammarError(`Template "${node.name}" expect ${expected.declarations.length} "
+            +" arguments, but ${node.templateArgs.length} were given`, node.location);
+          }
+          const params = node.templateArgs.map(arg => instantiateVisitor(arg));
+          // We deep clone to be able to mutate this without side effect
+          const clonedTargetRule = JSON.parse(JSON.stringify(targetRule));
+          const { name } = instantiateTemplate(clonedTargetRule, params);
+
+          node.name = name;
+        } else {
+          node.name = targetName;
+        }
+        delete node.templateArgs;
+        return node;
+      },
+
+      /**
+       * We replace the target template by its instance
+       */
+      rule(node) {
+        if (node !== template) {
+          throw new Error("We only support replacing one rule at once, call instantiateRule please ");
+        }
+
+        instantiateVisitor(node.expression);
+
+        delete node.templateParams;
+        return node;
+      },
+    });
+
+    const rule = instantiateVisitor(template);
+    rule.name = name;
+    ast.rules.push(rule);
+
+    return rule;
+  }
+
+  // Replace the first template calls: a non-template rule referencing a templated rule.
+  // For example the grammar A = B<u> , B<X>=C<X>, C<X>=X will be transformed first as
+  // 2. A = Bᐊuᐅ , B<X>=C<X>, C<X>=X , Bᐊuᐅ = Cᐊuᐅ, Cᐊuᐅ = u 
+  const replaceRootTemplateCalls = visitor.build({
+    rule_ref(node) {
+      if (!node.templateArgs) {
+        return node;
+      }
+
+      if (node.templateArgs.some(g => !asts.findRule(ast, g.name))) {
+        // This ref depends on parameters, so it is not a root template call
+        return node;
+      }
+
+      const targetRule = asts.findRule(ast, node.name);
+      if (!targetRule) {
+        throw new GrammarError(`Rule "${node.name}" is not defined`, node.location);
+      }
+
+      const expected = targetRule.templateParams;
+
+      if (!expected) {
+        throw new GrammarError(`Rule "${node.name}" is not a template`, node.location);
+      }
+      if (expected.declarations.length !== node.templateArgs.length) {
+        throw new GrammarError(`Template "${node.name}" expect ${expected.declarations.length} "
+            +" arguments, but ${node.templateArgs.length} were given`, node.location);
+      }
+
+      // We clone because it mutates
+      const clonedTargetRule = JSON.parse(JSON.stringify(targetRule));
+      const { name } = instantiateTemplate(clonedTargetRule, node.templateArgs);
+
+      // Finally override the redirection
+      node.name = name;
+      return node;
+    },
+  });
+
+  replaceRootTemplateCalls(ast);
+}
+
+module.exports = instantiateTemplates;

lib/compiler/passes/remove-non-instantiated-templates.js

@@ -0,0 +1,33 @@
+"use strict";
+
+const visitor = require("../visitor");
+
+/**
+ * This passes will remove any non-used templates
+ */
+function removeNonInstantiatedTemplates(ast, options, session) {
+  const rulesToRemove = [];
+  const danglingRemoveVisitor = visitor.build({
+
+    rule(node) {
+      if (!node.templateParams) {
+        return node;
+      }
+      rulesToRemove.push(node.name);
+
+      return null;
+    },
+  });
+
+  danglingRemoveVisitor(ast);
+
+  while (rulesToRemove.length) {
+    const name = rulesToRemove.pop();
+    session.info("Removing dangling generic rule", name);
+
+    const idx = ast.rules.findIndex(x => x.name === name);
+    ast.rules.splice(idx, 1);
+  }
+}
+
+module.exports = removeNonInstantiatedTemplates;

lib/compiler/visitor.js

@@ -11,6 +11,19 @@ const visitor = {
       // Do nothing.
     }
 
+    function visitRule(node, ...args) {
+      if (node.templateParams) {
+        visit(node.templateParams, ...args);
+      }
+      return visitExpression(node, ...args);
+    }
+
+    function visitRuleRef(node, ...args) {
+      if (node.templateArgs) {
+        node.templateArgs.forEach(child => visit(child, ...args));
+      }
+    }
+
     function visitExpression(node, ...args) {
       return visit(node.expression, ...args);
     }
@@ -41,7 +54,7 @@ const visitor = {
 
       top_level_initializer: visitNop,
       initializer: visitNop,
-      rule: visitExpression,
+      rule: visitRule,
       named: visitExpression,
       choice: visitChildren("alternatives"),
       action: visitExpression,
@@ -56,10 +69,11 @@ const visitor = {
       group: visitExpression,
       semantic_and: visitNop,
       semantic_not: visitNop,
-      rule_ref: visitNop,
+      rule_ref: visitRuleRef,
       literal: visitNop,
       class: visitNop,
       any: visitNop,
+      template_params: visitNop,
     };
 
     Object.keys(DEFAULT_FUNCTIONS).forEach(type => {