From 7ec5413c0e71300814350164c69e6452136bd4cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9my=20Raes?= <raes.remy@gmail.com>
Date: Thu, 6 Apr 2023 00:28:09 +0200
Subject: [PATCH] Add CI logic for finding missing translations (#608)

* build: add script to get missing keys per language

* fix: typo

* fix: check arguments before launching script

* feat: script can be run with language argument

* docs: add readme

* fix: remove multiple spaces while searching for translations

* build: add translations check CI

* refactor: put translation job alongside encoding job

* feat: add unicode characters because why not

* build: exit with an error code if some translations are missing

* build: exit using process.exit

* feat: CI should continue running with missing translations

* build: use actions/checkout v3

* refactor: put build directory into .github directory
---
 .github/build/README.md                    | 28 +++++++++
 .github/build/find-missing-translations.js | 66 ++++++++++++++++++++++
 .github/workflows/encoding.yml             |  8 +++
 3 files changed, 102 insertions(+)
 create mode 100644 .github/build/README.md
 create mode 100644 .github/build/find-missing-translations.js

diff --git a/.github/build/README.md b/.github/build/README.md
new file mode 100644
index 000000000..84d479a32
--- /dev/null
+++ b/.github/build/README.md
@@ -0,0 +1,28 @@
+# Finding missing translations
+
+This package contains a script that detects missing translation keys in Titanfall2 translation files contained in this repository (in the `Northstar.Client/mod/resource` folder).
+
+It uses english translations file as reference.
+
+You have to launch it **from the repository root folder**:
+```shell
+node .github/build/find-missing-translations.js
+```
+The script will then list all missing translations for all supported languages.
+
+If you want to display missing keys for a given language, just add it as an argument:
+```shell
+node .github/build/find-missing-translations.js french
+```
+
+Here's the list of supported languages: 
+* english
+* french
+* german
+* italian
+* japanese
+* mspanish
+* portuguese
+* russian
+* spanish
+* tchinese
\ No newline at end of file
diff --git a/.github/build/find-missing-translations.js b/.github/build/find-missing-translations.js
new file mode 100644
index 000000000..3f6c6c995
--- /dev/null
+++ b/.github/build/find-missing-translations.js
@@ -0,0 +1,66 @@
+const fs = require('fs');
+const { exit } = require('process');
+const langPath = "Northstar.Client/mod/resource";
+const knownLanguages = ['english', 'french', 'german', 'italian', 'japanese', 'mspanish', 'portuguese', 'russian', 'spanish', 'tchinese'];
+
+
+// Proceed checks before launch
+if (![2,3].includes(process.argv.length)) {
+    console.error('Wrong number of arguments, either call this script with no argument, or with a language.');
+    return;
+}
+const inputLang = process.argv[2];
+if (process.argv.length === 3 && !knownLanguages.includes(inputLang)) {
+    console.error(`"${inputLang}" is not a valid language.\nValid languages are: ${knownLanguages}`);
+    return;
+}
+
+
+// Get language files names
+const langs = fs.readdirSync(langPath)
+    .filter(lang => lang.indexOf('northstar_client_localisation_') !== -1);
+
+
+function getLanguageKeys (lang) {
+    if (knownLanguages.indexOf(lang) === -1) return;
+    return fs.readFileSync(`${langPath}/northstar_client_localisation_${lang}.txt`, {encoding:'utf16le', flag:'r'})
+        .split('\n')
+        .filter(line => line.length !== 0)                  // remove empty lines
+        .map(line => line.replaceAll(/\s+/g, ' ').trim())   // remove multiple spaces
+        .map(line => line.replaceAll('\t', ''))             // remove tabs characters
+        
+        // keep only lines with translation keys
+        .filter(line => {
+            const words = line.split('" "');
+            return words.length === 2 && words[1] !== 'english"'
+        })
+        .map(line => line.split('" "')[0].substring(1)); // only keep translation keys (throw values)
+}
+
+// We use english keys as reference for other languages
+const englishKeys = getLanguageKeys('english');
+const inputLanguages = inputLang !== undefined ? ["", inputLang] : [...knownLanguages];
+inputLanguages.shift();
+
+// Check for each language if there are missing keys
+var missingKeysCount = 0;
+
+for (const language of inputLanguages) {
+    const languageKeys = getLanguageKeys(language);
+    const missingKeys = [...englishKeys] // clone
+        .filter(key => languageKeys.indexOf(key) === -1);
+    const missingKeysLength = missingKeys.length;
+    console.log(
+        missingKeysLength === 0 
+        ? `✔️ "${language}" doesn't have missing keys.`
+        : `❌ "${language}" has ${missingKeys.length} missing key${missingKeys.length === 1 ? '' : 's'}:`
+    );
+
+    if (missingKeysLength !== 0) {
+        console.log(missingKeys);
+        missingKeysCount += missingKeys.length;
+    }
+}
+
+process.exitCode = missingKeysCount;
+exit();
diff --git a/.github/workflows/encoding.yml b/.github/workflows/encoding.yml
index ac63e7844..bafdb15b7 100644
--- a/.github/workflows/encoding.yml
+++ b/.github/workflows/encoding.yml
@@ -11,3 +11,11 @@ jobs:
         run: |
           files=$(ls Northstar.Client/mod/resource/northstar_client_localisation_*.txt)
           IFS=$'\n'; files=($files); unset IFS; ! file --mime "${files[@]}" | grep -v "charset=utf-16le"
+  check-missing-translations:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Look out for missing translations
+        run: node .github/build/find-missing-translations.js
+        continue-on-error: true