Sync missing translation keys in app/locales/*.json files to a Google Sheet

.gitignore

@@ -6,6 +6,8 @@ yarn-debug.log*
 yarn-error.log*
 lerna-debug.log*
 
+*-credentials.json
+
 # Diagnostic reports (https://nodejs.org/api/report.html)
 report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
 

ops/lost-in-translation.js

@@ -0,0 +1,172 @@
+const { parse } = require('path');
+const { readdirSync } = require('fs');
+const { spawnSync } = require('child_process');
+const { GoogleSpreadsheet } = require('google-spreadsheet');
+
+/**
+ * This script finds all the english translation keys across
+ * all commits (including branches and pull requests) for each
+ * locale.json in app/locales. Afterwards it looks at each locale
+ * separately to see if it's missing any one the english keys.
+ *
+ * Important: You need to run it from the root directory.
+ *
+ * Usage:
+ *   node ops/lost-in-translation.js
+ */
+
+/**
+ * Calls git cli with arguments and returns stdout.
+ */
+function git(...args) {
+  const { stdout } = spawnSync('git', args, { encoding: 'utf-8'});
+  return stdout;
+}
+
+/**
+ * Returns the commit hashes for a file path across all local branches.
+ */
+function findCommitHashesForFile(filePath) {
+  const output = git('log', '--pretty=format:"%h"', '--all', '--', filePath);
+  const hashes = output.split('\n').map(line => line.replace(/"/g, ''));
+  return hashes;
+}
+
+/**
+ * Returns the JSON representation of the contents of a file at certain commit hash.
+ */
+function retrieveJSONForFileAtCommitHash(filePath, commitHash) {
+  const contentsAtCommitHash = git('show', `${commitHash}:${filePath}`);
+  try {
+    return JSON.parse(contentsAtCommitHash);
+  } catch (error) {
+    // Sometimes the file is not in a proper JSON format. Simply return nothing in that case.
+    return {};
+  }
+}
+
+/**
+ * Normalizes translation key like we do in app/server.ts.
+ */
+function normalizeTranslationKey(translationKey) {
+  return translationKey.replace(/[\s\n\t]+/g, ' ').trim();
+}
+
+/**
+ * Find all the locales (ie. en-IN, no, se) in the provided directory.
+ */
+function retrieveAllLocales(directoryPath) {
+  const filenames = readdirSync(directoryPath);
+  const locales = filenames.map(filename => parse(filename).name);
+  return locales;
+}
+
+/**
+ * Add rows to a Google Spreadsheet that are not already added.
+ */
+async function addUniqueRowsToGoogleSheet(sheet, rowsToAdd) {
+  const alreadyAddedRows = await sheet.getRows({ limit: 1000 });
+  const uniqueRowsToAdd = rowsToAdd.filter(rowToAdd =>
+    !alreadyAddedRows.find(alreadyAddedRow => alreadyAddedRow.key === rowToAdd.key));
+  await sheet.addRows(uniqueRowsToAdd);
+  return uniqueRowsToAdd;
+}
+
+/**
+ * Step 1: Find all the (english) translation keys across all branches and PRs.
+ *
+ * If a Dutch developer has made a feature in a branch, we expect that him/her added a key
+ * to one or many locale files (usually the english locale (en.json) or to their own locale
+ * file (nl.json), or both).
+ *
+ * But they probably haven't added translations to all the other locales, because they don't
+ * speak the other languages. And this is the problem, because now we need find people to help
+ * translate the added keys to all the other locales.
+ *
+ * So what we do here is simply to find *all* the english translation keys across the entire
+ * project. That means looking at all the english translations keys in all the locale files
+ * since the start of the project across all branches and PRs. We throw them into a set that
+ * we use in the next step.
+ */
+const allLocales = retrieveAllLocales('app/locales/');
+const allEnglishTranslationKeys = new Set([]);
+for (const locale of allLocales) {
+  const filePath = `app/locales/${locale}.json`;
+  for (const commitHash of findCommitHashesForFile(filePath)) {
+    const translation = retrieveJSONForFileAtCommitHash(filePath, commitHash);
+    for (const translationKey of Object.keys(translation)) {
+      allEnglishTranslationKeys.add(normalizeTranslationKey(translationKey));
+    }
+  }
+}
+
+/**
+ * Step 2: For each locale file check if there are any missing english translation keys
+ *         across all branches and PRs and all commits/changes.
+ *
+ * If there's a missing english translation key, we know that we're most likely missing
+ * a translation for that locale. So what we need to do is to translate it, or get help
+ * to translate it.
+ * 
+ * We throw it into a dictionary where the key is the locale and the value is a set of 
+ * missing translations from english to that locale.
+ */
+const translationKeysByLocale = {};
+for (const locale of allLocales) {
+  const filePath = `app/locales/${locale}.json`;
+  for (const commitHash of findCommitHashesForFile(filePath)) {
+    const translation = retrieveJSONForFileAtCommitHash(filePath, commitHash);
+    const translationKeys = Object.keys(translation).map(key => normalizeTranslationKey(key));
+    if (translationKeysByLocale[locale] === undefined) {
+      translationKeysByLocale[locale] = new Set([]);
+    }
+    translationKeysByLocale[locale] = new Set([...translationKeysByLocale[locale], ...translationKeys]);
+  }
+}
+
+/**
+ * Step 3: Add rows of missing translations to Google Spreadsheet.
+ * 
+ * https://docs.google.com/spreadsheets/d/1ILFfc1DX4ujMnLnf9UqhwQGM9Ke3s1cAWciy8VqMHZw
+ */
+(async () => {
+  const doc = new GoogleSpreadsheet('1ILFfc1DX4ujMnLnf9UqhwQGM9Ke3s1cAWciy8VqMHZw');
+  await doc.useServiceAccountAuth(require('./coronastatus-translation-486cef09736e-credentials.json'));
+  await doc.loadInfo();
+
+  const allLocales = retrieveAllLocales('app/locales/');
+  for (let sheetIndex = 0; sheetIndex < doc.sheetCount; sheetIndex++) {
+    const sheet = doc.sheetsByIndex[sheetIndex];
+    for (const locale of allLocales) {
+      if (sheet.title !== locale) {
+        continue;
+      }
+
+      try {
+        // Create a sheet if it doesn't already exist.
+        await doc.addSheet({ title: locale, headerValues: ['key', 'translation'] });
+        await new Promise(resolve => setTimeout(resolve, 10*1000));
+      } catch (error) {
+        // We don't do anything if the sheet for this locale exists.
+      }
+
+      // Add the missing rows by looking at the key column.
+      const rows = [];
+      for (const englishTranslationKey of allEnglishTranslationKeys) {
+        if (!translationKeysByLocale[locale].has(englishTranslationKey)) {
+          const row = { 'key': englishTranslationKey, translation: '' };
+          rows.push(row);
+        }
+      }
+
+      // Print out how many rows we added.
+      const addedRows = await addUniqueRowsToGoogleSheet(sheet, rows);
+      console.log(`Added ${addedRows.length} of ${rows.length} missing translations to the ${locale} sheet.`);
+
+      // Avoid getting rate limited by Google's API (max writes per 100 seconds).
+      console.log('Waiting before processing the next sheet.');
+      await new Promise(resolve => setTimeout(resolve, 20*1000));
+    }
+  }
+
+})();

package.json

@@ -24,6 +24,7 @@
     "ejs": "^3.0.1",
     "express": "^4.17.1",
     "express-rate-limit": "^5.1.1",
+    "google-spreadsheet": "^3.0.10",
     "i18n": "^0.8.6",
     "node-fetch": "^2.6.0",
     "pg": "^7.18.2",