feat(translations/differences): visualize how many commits behind translations are (#8338)

.gitignore

@@ -103,6 +103,8 @@ testing/content/files/en-us/markdown/tool/m2h/index.html
 # eslintcache
 client/.eslintcache
 popularities.json
+source-commit.json
+source-commit-invalid-report.txt
 
 client/src/.linaria_cache
 ssr/.linaria-cache/

client/src/translations/differences/index.tsx

@@ -35,6 +35,8 @@ interface DocumentEdits {
   parentModified: string;
   commitURL: string;
   parentCommitURL: string;
+  sourceCommitsBehindCount?: number;
+  sourceCommitURL?: string;
 }
 
 interface Document {
@@ -574,6 +576,10 @@ function DocumentsTable({
         const a = A.mdn_url;
         const b = B.mdn_url;
         return reverse * a.localeCompare(b);
+      } else if (sort === "sourceCommit") {
+        const a = A.edits.sourceCommitsBehindCount ?? -1;
+        const b = B.edits.sourceCommitsBehindCount ?? -1;
+        return reverse * (b - a);
       } else {
         throw new Error(`Unrecognized sort '${sort}'`);
       }
@@ -606,6 +612,7 @@ function DocumentsTable({
             <TableHead id="popularity" title="Popularity" />
             <TableHead id="modified" title="Last modified" />
             <TableHead id="differences" title="Differences" />
+            <TableHead id="sourceCommit" title="Source Commit" />
           </tr>
         </thead>
         <tbody>
@@ -658,6 +665,14 @@ function DocumentsTable({
                     <LastModified edits={doc.edits} />
                   </td>
                   <td>{doc.differences.total.toLocaleString()}</td>
+                  <td>
+                    <L10nSourceCommitModified
+                      sourceCommitsBehindCount={
+                        doc.edits.sourceCommitsBehindCount
+                      }
+                      sourceCommitURL={doc.edits.sourceCommitURL}
+                    />
+                  </td>
                 </tr>
               );
             })}
@@ -680,6 +695,29 @@ function DocumentsTable({
   );
 }
 
+function L10nSourceCommitModified({
+  sourceCommitsBehindCount,
+  sourceCommitURL,
+}: Pick<DocumentEdits, "sourceCommitsBehindCount" | "sourceCommitURL">) {
+  if (
+    !sourceCommitURL ||
+    (!sourceCommitsBehindCount && sourceCommitsBehindCount !== 0)
+  ) {
+    return <>Metadata does not exist.</>;
+  }
+
+  const getImportanceColor = () => {
+    if (sourceCommitsBehindCount === 0) return "🟢";
+    return sourceCommitsBehindCount < 10 ? "🟠" : "🔴";
+  };
+
+  return (
+    <a
+      href={sourceCommitURL}
+    >{`${getImportanceColor()} ${sourceCommitsBehindCount} commits behind`}</a>
+  );
+}
+
 function LastModified({ edits }: { edits: DocumentEdits }) {
   const modified = dayjs(edits.modified);
   const parentModified = dayjs(edits.parentModified);

server/translations.ts

@@ -3,6 +3,7 @@ import path from "node:path";
 
 import express from "express";
 import { fdir } from "fdir";
+import { execSync } from "node:child_process";
 
 import { getPopularities, Document, Translation } from "../content/index.js";
 import {
@@ -60,7 +61,42 @@ function packageTranslationDifferences(translationDifferences) {
 }
 
 const _foundDocumentsCache = new Map();
+const sourceCommitCache = fs.existsSync("./source-commit.json")
+  ? new Map<string, number>(
+      Object.entries(
+        JSON.parse(fs.readFileSync("./source-commit.json", "utf8"))
+      )
+    )
+  : new Map<string, number>();
+const memCommitStore = new Map<string, string[]>();
+let memCommitStoreOldest = "HEAD";
 export async function findDocuments({ locale }) {
+  function checkCacheValidation(prevCache: Map<any, any>): void {
+    const contentHash = getRecentRepoHash(CONTENT_ROOT);
+    const translatedContentHash = getRecentRepoHash(CONTENT_TRANSLATED_ROOT);
+
+    function getRecentRepoHash(cwd: string): string {
+      return execSync("git rev-parse HEAD", { cwd }).toString().trimEnd();
+    }
+    function isValidCache(): boolean {
+      return (
+        prevCache.has(CONTENT_ROOT) &&
+        prevCache.has(CONTENT_TRANSLATED_ROOT) &&
+        prevCache.get(CONTENT_ROOT) === contentHash &&
+        prevCache.get(CONTENT_TRANSLATED_ROOT) === translatedContentHash
+      );
+    }
+
+    if (!isValidCache()) {
+      prevCache.clear();
+      prevCache.set(CONTENT_ROOT, contentHash);
+      prevCache.set(CONTENT_TRANSLATED_ROOT, translatedContentHash);
+      sourceCommitCache.clear();
+      memCommitStore.clear();
+      memCommitStoreOldest = "HEAD";
+    }
+  }
+
   const counts = {
     // Number of documents found that aren't skipped
     found: 0,
@@ -81,6 +117,7 @@ export async function findDocuments({ locale }) {
   });
   counts.total = documentsFound.count;
 
+  checkCacheValidation(_foundDocumentsCache);
   if (!_foundDocumentsCache.has(locale)) {
     _foundDocumentsCache.set(locale, new Map());
   }
@@ -114,6 +151,12 @@ export async function findDocuments({ locale }) {
     took,
   };
 
+  fs.writeFileSync(
+    "./source-commit.json",
+    JSON.stringify(Object.fromEntries(sourceCommitCache)),
+    "utf8"
+  );
+
   return {
     counts,
     times,
@@ -141,22 +184,96 @@ function getDocument(filePath) {
     };
   }
 
+  function recordInvalidSourceCommit(
+    isValid: boolean,
+    fileFolder: string,
+    commitHash: string
+  ) {
+    const filePath = "./source-commit-invalid-report.txt";
+    const errorMessage = `- ${commitHash} commit hash is invalid in ${fileFolder}`;
+    if (isValid) return;
+    if (!fs.existsSync(filePath)) {
+      fs.writeFileSync(filePath, "");
+    }
+
+    fs.appendFile(filePath, `${errorMessage}\n`, function (err) {
+      if (err) throw err;
+    });
+  }
+
+  function getCommitBehindFromLatest(
+    fileFolder: string,
+    parentFilePath: string,
+    commitHash: string
+  ): number {
+    try {
+      let count = 0;
+      if (!memCommitStore.has(commitHash)) {
+        execSync(
+          `git log --pretty=format:'%H' --name-only ${commitHash}..${memCommitStoreOldest}`,
+          {
+            cwd: CONTENT_ROOT,
+          }
+        )
+          .toString()
+          .trimEnd()
+          .split("\n\n")
+          .forEach((commit) => {
+            const [hash, ...files] = commit.split("\n");
+            memCommitStore.set(hash, files);
+          });
+        memCommitStoreOldest = commitHash;
+      }
+      for (const [hash, files] of memCommitStore.entries()) {
+        if (hash === commitHash) {
+          recordInvalidSourceCommit(
+            files.includes(parentFilePath),
+            fileFolder,
+            commitHash
+          );
+          break;
+        }
+        if (files.includes(parentFilePath)) count++;
+      }
+      sourceCommitCache.set(fileFolder, count);
+    } catch (err) {
+      recordInvalidSourceCommit(false, fileFolder, commitHash);
+    }
+
+    return sourceCommitCache.get(fileFolder);
+  }
+
   function packageEdits(document, parentDocument) {
-    const commitURL = getLastCommitURL(
-      document.fileInfo.root,
-      document.metadata.hash
-    );
-    const parentCommitURL = getLastCommitURL(
-      parentDocument.fileInfo.root,
-      parentDocument.metadata.hash
-    );
-    const modified = document.metadata.modified;
-    const parentModified = parentDocument.metadata.modified;
+    const {
+      fileInfo: { root: fileRoot, folder: fileFolder },
+      metadata: { hash: fileHash, modified, l10n },
+    } = document;
+    const {
+      fileInfo: { root: parentFileRoot, path: parentFilePath },
+      metadata: { hash: parentFileHash, parentModified },
+    } = parentDocument;
+
+    const commitURL = getLastCommitURL(fileRoot, fileHash);
+    const parentCommitURL = getLastCommitURL(parentFileRoot, parentFileHash);
+    let sourceCommitURL;
+    let sourceCommitsBehindCount;
+
+    if (l10n?.sourceCommit) {
+      sourceCommitURL = getLastCommitURL(CONTENT_ROOT, l10n.sourceCommit);
+      sourceCommitsBehindCount = getCommitBehindFromLatest(
+        fileFolder,
+        parentFilePath.replace(parentFileRoot, "files"),
+        l10n.sourceCommit
+      );
+    }
+
     return {
       commitURL,
       parentCommitURL,
       modified,
       parentModified,
+      sourceCommitURL,
+      sourceCommitsBehindCount,
     };
   }