Traits (mdn#3280)

Co-authored-by: Schalk Neethling <sneethling@mozilla.com>

.npmignore

@@ -17,6 +17,7 @@ server/node_modules/
 /content/files/en-us/
 
 /deployer
+/traits
 kumascript/tests
 testing/
 import/

build/index.js

@@ -12,6 +12,7 @@ const {
   execGit,
 } = require("../content");
 const kumascript = require("../kumascript");
+const { normalizeMacroName } = require("../kumascript/src/render.js");
 
 const { FLAW_LEVELS } = require("./constants");
 const {
@@ -668,10 +669,129 @@ function renderContributorsTxt(wikiContributorNames = null, githubURL = null) {
   return txt;
 }
 
+function* fastKSParser(s, includeArgs = false) {
+  for (const match of s.matchAll(
+    /\{\{\s*(\w+[\w-.]*\w+)\s*(\((.*?)\)|)\s*\}\}/gms
+  )) {
+    const { index } = match;
+    if (s.charAt(index - 1) === "\\") {
+      continue;
+    }
+
+    const split = (match[3] || "").trim().split(",");
+    const found = { name: match[1] };
+    if (includeArgs) {
+      found.args = split
+        .map((s) => s.trim())
+        .map((s) => {
+          if (s.startsWith('"') && s.endsWith('"')) {
+            return s.slice(1, -1);
+          }
+          if (s.startsWith("'") && s.endsWith("'")) {
+            return s.slice(1, -1);
+          }
+          return s;
+        })
+        .filter((s, i) => {
+          if (!s) {
+            // Only return false if it's NOT first
+            if (i === 0) {
+              return Boolean(s);
+            }
+          }
+          return true;
+        });
+    }
+    yield found;
+  }
+}
+
+async function analyzeDocument(document) {
+  const { metadata } = document;
+
+  const doc = {
+    ...metadata,
+    contributors: metadata.contributors ? metadata.contributors.length : 0,
+    isArchive: !!document.isArchive,
+    isTranslated: !!document.isTranslated,
+  };
+
+  doc.normalizedMacrosCount = {};
+  for (const token of fastKSParser(document.rawBody)) {
+    const normalizedMacroName = normalizeMacroName(token.name);
+    if (!(normalizedMacroName in doc.normalizedMacrosCount)) {
+      doc.normalizedMacrosCount[normalizedMacroName] = 0;
+    }
+    doc.normalizedMacrosCount[normalizedMacroName]++;
+  }
+  doc.tags = document.metadata.tags || [];
+
+  doc.fileSize = fs.statSync(document.fileInfo.path).size;
+  doc.wordCount = document.rawBody
+    .replace(/(<([^>]+)>)/g, "")
+    .split(/\s+/).length;
+  const $ = cheerio.load(document.rawBody);
+  const imageCounts = countImages($);
+  doc.images = imageCounts.total;
+  doc.externalImages = imageCounts.external;
+  doc.internalImages = imageCounts.internal;
+  doc.h2s = $("h2").length;
+  doc.h3s = $("h3").length;
+  doc.pres = $("pre").length;
+  doc.title = metadata.title;
+  doc.mdn_url = document.url;
+  doc.depth = document.url.split("/").length - 3;
+
+  // If the document has a `.popularity` make sure don't bother with too
+  // many significant figures on it.
+  doc.popularity = metadata.popularity
+    ? Number(metadata.popularity.toFixed(4))
+    : 0.0;
+
+  doc.modified = metadata.modified || null;
+
+  const otherTranslations = document.translations || [];
+  if (!otherTranslations.length && metadata.translation_of) {
+    // If built just-in-time, we won't have a record of all the other translations
+    // available. But if the current document has a translation_of, we can
+    // at least use that.
+    otherTranslations.push({ locale: "en-US", slug: metadata.translation_of });
+  }
+
+  if (otherTranslations.length) {
+    doc.other_translations = otherTranslations;
+  }
+
+  return doc;
+}
+
+function countImages($) {
+  const counts = {
+    external: 0,
+    internal: 0,
+    total: 0,
+  };
+  $("img[src]").each((i, img) => {
+    const src = $(img).attr("src");
+    if (
+      src.includes("://") ||
+      src.startsWith("/@api/") ||
+      src.startsWith("/files")
+    ) {
+      counts.external++;
+    } else {
+      counts.internal++;
+    }
+  });
+  counts.total = counts.external + counts.internal;
+  return counts;
+}
+
 module.exports = {
   FLAW_LEVELS,
 
   buildDocument,
+  analyzeDocument,
 
   buildLiveSamplePageFromURL,
   renderContributorsTxt,

client/pwa/src/unpack-cache.ts

@@ -1,7 +1,6 @@
 /* eslint no-restricted-globals: ["off", "self"] */
 import * as zip from "@zip.js/zip.js";
 import { openContentCache } from "./caches";
-import { INTERACTIVE_EXAMPLES_URL } from "./service-worker";
 
 zip.configure({
   useWebWorkers: false,

client/src/app.tsx

@@ -25,6 +25,7 @@ import { ContributorSpotlight } from "./contributor-spotlight";
 import { Banner, hasActiveBanners } from "./banners";
 
 const AllFlaws = React.lazy(() => import("./flaws"));
+const AllTraits = React.lazy(() => import("./traits"));
 const Translations = React.lazy(() => import("./translations"));
 const DocumentEdit = React.lazy(() => import("./document/forms/edit"));
 const DocumentCreate = React.lazy(() => import("./document/forms/create"));
@@ -167,6 +168,14 @@ export function App(appProps) {
                     </StandardLayout>
                   }
                 />
+                <Route
+                  path="/_traits/*"
+                  element={
+                    <StandardLayout>
+                      <AllTraits />
+                    </StandardLayout>
+                  }
+                />
                 <Route
                   path="/_edit/*"
                   element={

client/src/setupProxy.js

@@ -10,7 +10,7 @@ module.exports = function (app) {
   });
   app.use("/api", proxy);
   app.use("/users", proxy);
-  app.use("/_+(flaws|translations|open|document)", proxy);
+  app.use("/_+(flaws|translations|open|document|traits)", proxy);
   // E.g. search-index.json or index.json
   app.use("**/*.json", proxy);
   // This has to match what we do in server/index.js in the catchall handler

client/src/traits/index.scss

@@ -0,0 +1,62 @@
+button.button-inline-small {
+  display: inline;
+  margin-right: 2px;
+  white-space: nowrap;
+  padding: 2px 4px;
+  font-weight: normal;
+  min-height: 30px;
+}
+button:disabled {
+  opacity: 0.6;
+}
+
+.all-traits {
+  .sub-nav {
+    margin: 100px;
+    li {
+      margin: 20px;
+    }
+  }
+  .loading {
+    text-align: center;
+    margin: 200px;
+
+    progress {
+      width: 600px;
+      height: 30px;
+      border-radius: 2px;
+    }
+  }
+
+  .all-macros-used {
+    th.filter {
+      min-width: 350px;
+      .filter-macros {
+        text-align: right;
+      }
+    }
+
+    td.mdn_url {
+      font-size: 80%;
+      text-align: right;
+      text-overflow: ellipsis;
+      overflow: hidden;
+    }
+    td.count,
+    th.macro {
+      max-width: 22px;
+      font-size: 90%;
+    }
+    td.count {
+      span.zero {
+        color: #a4a4a4;
+      }
+    }
+    th.macro {
+      text-align: center;
+      transform-origin: left;
+      transform: rotate(-90deg);
+      vertical-align: bottom;
+    }
+  }
+}