Add translation feature from CSV (qmk#601)

* Add i18n feature

Adding loader

move to csv parse

translations cleanup

Add sync and chk for i18n

Doc addition

Legacy translation deletion

Rebase and sync

delete useless ignore

replace npm to yarn

fix lang label

* fix status

* merge fix

* fix missing trad

* missing translation

README.md

@@ -87,3 +87,7 @@ If for some reason you do need to build it yourself, you can use this command:
 This process will take a while. You may want to go make some tea or something. When it finishes you can run it with this command:
 
     docker run -p 8080:80 qmk_configurator
+
+## Internationalization Guide
+
+Please refer to [this document](internationalization_guide.md)

internationalization_guide.md

@@ -0,0 +1,66 @@
+# :globe_with_meridians: Internationalization Contribution Guide
+
+## How it works?
+
+All the translations are stored in the `CSV` files in `src/i18n`. Each line describe a key/value pair. Those files are parsed and integrated into the Front-End application during the build process. The source language for all keys is the English, stored in `src/i18n/en.csv`.
+
+The key maybe be simple or may describe a nested element with the use of namespaces. For example:
+```bash
+# this describe Loading translation as a global translation
+loading,Loading
+# output is : TranslationObject.loading
+```
+
+```bash
+# This describe the Reload translation in the Configuration section in the admin panel
+configuration:admin:reload,Reload
+# output is : TranslationObject.configuration.admin.reload
+```
+
+Using namespaces give us more readability of where the translation is, mainly for people not familiar with front-end development or javascript.
+
+If you want to dig into the code:
+
+Csv to json parsing: [scripts/i18n/csv-json.js](scripts/i18n/csv-json.js) (called with `loader.js`)
+
+Csv synchronization: [scripts/i18n/csv-sync.js](scripts/i18n/csv-sync.js) (called with `i18n:sync`)
+
+The CSV synchronization make sure the translations files are synchronized beetween each others and also reorder keys.
+
+Front-end integration: [src/i18n/index.js](src/i18n/index.js)
+
+## :suspect: How do i contribute?
+
+Just fork the project, make a branch for the completion of your translation and create a pull request.
+
+If you're not familiar with all those aspects GitHub offers a web based csv editor. Here how you proceed from the qmk_configurator repository.
+
+- go on [https://github.com/qmk/qmk_configurator](https://github.com/qmk/qmk_configurator)
+- Click on fork (you'll be redirected to your fork on your account)
+- Go on the CSV file you want to edit and click `Edit the file` with a pencil icon
+- Make your changes
+- Once you're done go to the bottom of the page and add a revelant commit message.
+- Click on `create a new branch for this commit and start a pull request`
+- go on [https://github.com/qmk/qmk_configurator](https://github.com/qmk/qmk_configurator)
+- You'll see `compare & pull request` button appear at the top of the page, Click on it.
+- Accept the pull request
+- Wait for the review.
+
+:raised_hands: TADA :raised_hands:
+
+__Note__: If you want to add a sentence with a comma ',' you'll have to use quotes " around the string to escape the sentence with the comma; otherwise the CSV format will not be respected.
+
+## :gift: I can provide a new language translation, how do i proceed?
+
+First you have to create a file in the `src/i18n/` folder with the [ISO 639-1 code](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) (2 letters one) of the language you want to provide. For example `zh.csv` for chinese. And insert the headers in it:
+```
+Key,Translation
+```
+
+Once it's done you can use the command:
+
+```
+yarn i18n:sync
+```
+
+And fill the document :smiley:

package.json

@@ -11,7 +11,9 @@
     "prettier:json:check": "prettier -c \"./public/**/*.json\"; if [ $? = 1 ]; then exit 0; else exit 1; fi",
     "test:cypress": "cypress run",
     "test:cypress:ci": "start-server-and-test serve http://localhost:8080 test:cypress",
-    "test:unit": "vue-cli-service test:unit"
+    "test:unit": "vue-cli-service test:unit",
+    "i18n:sync": "node scripts/i18n/csv-sync.js sync",
+    "i18n:chk": "node scripts/i18n/csv-sync.js chk"
   },
   "dependencies": {
     "@fortawesome/fontawesome-svg-core": "^1.2.12",
@@ -39,6 +41,8 @@
     "babel-core": "7.0.0-bridge.0",
     "babel-eslint": "^10.0.1",
     "babel-jest": "^24.0.0",
+    "csv-parse": "^4.8.2",
+    "csv-stringify": "^5.3.4",
     "cypress": "^3.4.1",
     "eslint": "^5.8.0",
     "eslint-plugin-cypress": "^2.6.1",

scripts/i18n/csv-json.js

@@ -0,0 +1,36 @@
+function namespaceNesting(o, k, v) {
+  let namespace = '';
+  const splittedKey = k.split(':');
+  let key = '';
+  if (splittedKey.length > 1) {
+    namespace = splittedKey[0];
+    key = splittedKey[1];
+  } else {
+    key = splittedKey[0];
+  }
+  if (!o[namespace]) {
+    o[namespace] = {};
+  }
+  if (splittedKey.length > 2) {
+    splittedKey.shift();
+    namespaceNesting(o[namespace], splittedKey.join(':'), v);
+  } else {
+    if (v && namespace) {
+      o[namespace][key] = v;
+    } else if (v) {
+      o[key] = v;
+    }
+  }
+}
+
+function genTsObj(csvEntries) {
+  const o = {};
+  for (const item of csvEntries) {
+    namespaceNesting(o, item.Key, item.Translation);
+  }
+  return o;
+}
+
+module.exports = {
+  genTsObj
+};

scripts/i18n/csv-sync.js

@@ -0,0 +1,148 @@
+const fs = require('fs');
+const args = process.argv.slice(2);
+const { PATH_CSV } = require('./env');
+const referenceFile = 'en.csv';
+const files = fs
+  .readdirSync(PATH_CSV)
+  .filter(f => f !== referenceFile && f.endsWith('csv'));
+const stringify = require('csv-stringify');
+const parse = require('csv-parse/lib/sync');
+
+function aggregateRecords(records) {
+  const common = records.filter(r => {
+    return r.Key.split(',')[0].indexOf(':') === -1;
+  });
+  const others = records.filter(r => {
+    return r.Key.split(',')[0].indexOf(':') !== -1;
+  });
+  return common.sort(sortKeys).concat(others.sort(sortKeys));
+}
+
+function sortKeys(a, b) {
+  if (a.Key < b.Key) {
+    return -1;
+  }
+  if (a.Key > b.Key) {
+    return 1;
+  }
+  return 0;
+}
+
+function readReferenceFile() {
+  return readTranslation(referenceFile);
+}
+
+function readTranslation(filename) {
+  const source = fs.readFileSync(`${PATH_CSV}${filename}`, 'utf8');
+  const records = parse(source, {
+    columns: true,
+    skip_empty_lines: true
+  });
+  records.sort(sortKeys);
+  return records;
+}
+
+function updateMissingTranslation(
+  referenceTranslation,
+  dialectTranslationArray
+) {
+  const sK = dialectTranslationArray.find(
+    s => referenceTranslation.Key === s.Key
+  );
+  if (!sK) {
+    dialectTranslationArray.push({
+      Key: referenceTranslation.Key,
+      Translation: ''
+    });
+  }
+}
+
+function deleteDeprecatedTranslations(dialectArray, referenceArray) {
+  const referenceKeys = referenceArray.map(x => x.Key);
+  dialectArray = dialectArray.filter(x => referenceKeys.includes(x.Key));
+  return dialectArray;
+}
+
+function syncTranslationFiles(referenceTranslations) {
+  files.forEach(f => {
+    let dialect = fs.readFileSync(`${PATH_CSV}${f}`, 'utf8');
+    let records = parse(dialect, {
+      columns: true,
+      skip_empty_lines: true
+    });
+    referenceTranslations.forEach(t => {
+      updateMissingTranslation(t, records);
+    });
+    records = deleteDeprecatedTranslations(records, referenceTranslations);
+    stringify(
+      [{ Key: 'Key', Translation: 'Translation' }].concat(
+        aggregateRecords(records)
+      ),
+      (err, csvOutput) => {
+        fs.writeFileSync(`${PATH_CSV}${f}`, csvOutput);
+      }
+    );
+  });
+  // Reordering the reference Translation
+  stringify(
+    [{ Key: 'Key', Translation: 'Translation' }].concat(
+      aggregateRecords(referenceTranslations)
+    ),
+    (err, csvOutput) => {
+      fs.writeFileSync(`${PATH_CSV}en.csv`, csvOutput);
+    }
+  );
+}
+
+function checkTranslationFiles(referenceTranslations) {
+  let errors = {};
+  files.forEach(f => {
+    const dialectTranslations = readTranslation(f);
+    const dialectKeys = dialectTranslations.map(s => s.Key);
+    const missingTranslations = referenceTranslations.filter(
+      t => !dialectKeys.includes(t.Key)
+    );
+    if (missingTranslations.length > 0) {
+      errors[f] = missingTranslations.map(s => s.Key);
+    }
+  });
+  if (Object.keys(errors).length > 0) {
+    console.log('Missing translations found');
+    console.log('');
+    Object.keys(errors).forEach(e => {
+      console.log(`Missing Keys in ${e}`);
+      console.log('');
+      console.log(errors[e].join('\n'));
+      console.log('');
+    });
+    console.log('Translation report:');
+    Object.keys(errors).forEach(e => {
+      console.log(`${e} : ${errors[e].length} entries missing`);
+    });
+    return false;
+  } else {
+    console.log('Everything is synchronized');
+    return true;
+  }
+}
+
+function main() {
+  const reference = readReferenceFile();
+  switch (args[0]) {
+    case 'sync':
+      syncTranslationFiles(reference);
+      break;
+    case 'chk':
+      if (checkTranslationFiles(reference)) {
+        process.exit(0);
+      } else {
+        process.exit(1);
+      }
+      break;
+    default:
+      console.log('Invalid argument. sync / ci are only available');
+      process.exit(1);
+  }
+}
+
+main();

scripts/i18n/env.js

@@ -0,0 +1,3 @@
+module.exports = {
+  PATH_CSV: './src/i18n/'
+};

scripts/i18n/loader.js

@@ -0,0 +1,11 @@
+const { genTsObj } = require('./csv-json');
+const parse = require('csv-parse/lib/sync');
+
+exports.default = function(source) {
+  const records = parse(source, {
+    columns: true,
+    skip_empty_lines: true
+  });
+  const output = genTsObj(records);
+  return `export default ${JSON.stringify(output)}`;
+};

src/App.vue

@@ -11,17 +11,17 @@
     </div>
     <slideout-panel></slideout-panel>
     <footer>
-      <p>{{ $t('message.maintain') }}</p>
-      <p>{{ $t('message.hostedOn') }}</p>
+      <p>{{ $t('maintain') }}</p>
+      <p>{{ $t('hostedOn') }}</p>
       <p style="font-size:10px">version: {{ revision }}</p>
     </footer>
     <div
       class="help"
       :class="helpClasses"
       @click="toggleTutorial"
-      :title="$t('message.help.label')"
+      :title="$t('help.label')"
       @mouseenter="
-        setMessage($t('message.help.label'));
+        setMessage($t('help.label'));
         hover = true;
       "
       @mouseleave="
@@ -151,8 +151,8 @@ export default {
     ]),
     ...mapActions('app', ['loadApplicationState']),
     randomPotatoFact() {
-      const len = size(this.$t('message.potato'));
-      this.potatoFact = this.$t('message.potato.' + random(1, len));
+      const len = size(this.$t('potato'));
+      this.potatoFact = this.$t('potato.' + random(1, len));
     },
     async appLoad() {
       await this.loadApplicationState();

src/components/BrowserWarn.vue

@@ -1,7 +1,7 @@
 <template>
   <div id="browser-warn" v-show="isNotSupported && !isDimissed">
     <a class="dismiss" title="dismiss" v-on:click="dismiss">X</a>
-    {{ $t('message.errors.unsupportedBrowser') }}
+    {{ $t('errors.unsupportedBrowser') }}
     <a href="https://www.google.com/intl/en_us/chrome/" target="_blank"
       >Google Chrome</a
     >