From b379cbc87a6fb8562df74a632eccdf1580553425 Mon Sep 17 00:00:00 2001 From: Michael McMillan Date: Sun, 29 Mar 2020 17:09:35 +0200 Subject: [PATCH 01/16] add script for finding missing translations across all locales --- ops/lost-in-translation.py | 80 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 ops/lost-in-translation.py diff --git a/ops/lost-in-translation.py b/ops/lost-in-translation.py new file mode 100644 index 00000000..00951c60 --- /dev/null +++ b/ops/lost-in-translation.py @@ -0,0 +1,80 @@ +from subprocess import call, check_output, STDOUT +from json import loads as decode_json, JSONDecodeError + +''' + This script finds all the english translation keys across + all commits (including branches and pull requests) for each + locale.json in app/locales. Afterwards it looks at each locale + separately to see if it's missing any one the english keys. + + Usage: python3 ops/lost-in-translation.py + Important: You need to run it from the root directory. +''' + +def git(*args): + '''Calls git cli with arguments.''' + stdout = check_output(['git', *args], stderr=STDOUT) + return stdout + +def find_commit_hashes_for_file(file_path): + '''Returns the commit hashes for a file path across all local branches.''' + lines = git('log', '--pretty=format:"%h"', '--all', '--', file_path) + hashes = [line.replace('"', '') for line in lines.decode('utf-8').split('\n')] + return hashes + +def retrieve_json_for_file_at_commit_hash(file_path, commit_hash): + '''Returns the JSON representation of the contents of a file at certain commit hash.''' + contents_at_commit_hash = git('show', '%s:%s' % (commit_hash, file_path)) + try: + json = decode_json(contents_at_commit_hash.decode('utf-8')) + return json + except JSONDecodeError: + # Sometimes the file is not in a proper JSON format. We simply return nothing in that case. + return {} + +if __name__ == '__main__': + all_locales = ('no', 'se', 'en-IN') + locale_to_check_for = 'no' + + # Step 1: Find all the (english) translation keys across all branches and PRs. + # + # If a Dutch developer has made a feature in a branch, we expect that him/her added a key + # to one or many locale files (usually the english locale (en.json) or to their own locale + # file (nl.json), or both). + # + # But they probably haven't added translations to all the other locales, because they don't + # speak the other languages. And this is the problem, because now we need find people to help + # translate the added keys to all the other locales. + # + # So what we do here is simply to find *all* the english translation keys across the entire + # project. That means looking at all the english translations keys in all the locale files + # since the start of the project across all branches and PRs. We throw them into a set that + # we use in the next step. + all_english_translation_keys = set([]) + for locale in all_locales: + file_path = 'app/locales/%s.json' % locale + for commit_hash in find_commit_hashes_for_file(file_path): + translation = retrieve_json_for_file_at_commit_hash(file_path, commit_hash) + all_english_translation_keys.update(translation.keys()) + + # Step 2: For each locale file check if there are any missing english translation keys + # across all branches and PRs and all commits/changes. + # + # If there's a missing english translation key, we know that we're most likely missing + # a translation for that locale. So what we need to do is to translate it, or get help + # to translate it. + # + # We throw it into a dictionary where the key is the locale and the value is a set of + # missing translations from english to that locale. + translation_keys_by_locale = {} + for locale in all_locales: + file_path = 'app/locales/%s.json' % locale + for commit_hash in find_commit_hashes_for_file(file_path): + translation = retrieve_json_for_file_at_commit_hash(file_path, commit_hash) + translation_keys_by_locale[locale] = translation_keys_by_locale.get(locale, set()).union(translation.keys()) + + # Step 3: Print out the missing keys for each locale. + for locale in all_locales: + for english_translation_key in all_english_translation_keys: + if not english_translation_key in translation_keys_by_locale[locale]: + print(locale, 'missing translation for:', english_translation_key) From 06d1ef0858985d281fd23b4b836843613f435240 Mon Sep 17 00:00:00 2001 From: Michael McMillan Date: Sun, 29 Mar 2020 17:12:52 +0200 Subject: [PATCH 02/16] remove unused var --- ops/lost-in-translation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ops/lost-in-translation.py b/ops/lost-in-translation.py index 00951c60..4d4b664c 100644 --- a/ops/lost-in-translation.py +++ b/ops/lost-in-translation.py @@ -34,7 +34,6 @@ def retrieve_json_for_file_at_commit_hash(file_path, commit_hash): if __name__ == '__main__': all_locales = ('no', 'se', 'en-IN') - locale_to_check_for = 'no' # Step 1: Find all the (english) translation keys across all branches and PRs. # From 5a50bc14d1123414f86b8164d6417919cef42a44 Mon Sep 17 00:00:00 2001 From: Michael McMillan Date: Sun, 29 Mar 2020 21:44:07 +0200 Subject: [PATCH 03/16] port lost-in-translation.py to javascript for consistency --- ops/lost-in-translation.js | 93 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 ops/lost-in-translation.js diff --git a/ops/lost-in-translation.js b/ops/lost-in-translation.js new file mode 100644 index 00000000..9366b1e5 --- /dev/null +++ b/ops/lost-in-translation.js @@ -0,0 +1,93 @@ +const { spawnSync } = require('child_process'); + +/** + * Calls git cli with arguments and returns stdout. + */ +function git(...args) { + const { stdout } = spawnSync('git', args, { encoding: 'utf-8'}); + return stdout; +} + +/** + * Returns the commit hashes for a file path across all local branches. + */ +function findCommitHashesForFile(filePath) { + const output = git('log', '--pretty=format:"%h"', '--all', '--', filePath); + const hashes = output.split('\n').map(line => line.replace(/"/g, '')); + return hashes; +} + +/** + * Returns the JSON representation of the contents of a file at certain commit hash. + */ +function retrieveJSONForFileAtCommitHash(filePath, commitHash) { + const contentsAtCommitHash = git('show', `${commitHash}:${filePath}`); + try { + return JSON.parse(contentsAtCommitHash); + } catch (error) { + // Sometimes the file is not in a proper JSON format. Simply return nothing in that case. + return {}; + } +} + +/** + * Step 1: Find all the (english) translation keys across all branches and PRs. + * + * If a Dutch developer has made a feature in a branch, we expect that him/her added a key + * to one or many locale files (usually the english locale (en.json) or to their own locale + * file (nl.json), or both). + * + * But they probably haven't added translations to all the other locales, because they don't + * speak the other languages. And this is the problem, because now we need find people to help + * translate the added keys to all the other locales. + * + * So what we do here is simply to find *all* the english translation keys across the entire + * project. That means looking at all the english translations keys in all the locale files + * since the start of the project across all branches and PRs. We throw them into a set that + * we use in the next step. + */ +allLocales = new Set(['no', 'se']); +allEnglishTranslationKeys = new Set([]); +for (const locale of allLocales) { + const filePath = `../app/locales/${locale}.json`; + for (const commitHash of findCommitHashesForFile(filePath)) { + const translation = retrieveJSONForFileAtCommitHash(filePath, commitHash); + for (const translationKey of Object.keys(translation)) { + allEnglishTranslationKeys.add(translationKey); + } + } +} + +/** + * Step 2: For each locale file check if there are any missing english translation keys + * across all branches and PRs and all commits/changes. + * + * If there's a missing english translation key, we know that we're most likely missing + * a translation for that locale. So what we need to do is to translate it, or get help + * to translate it. + * + * We throw it into a dictionary where the key is the locale and the value is a set of + * missing translations from english to that locale. + */ +const translationKeysByLocale = {}; +for (const locale of allLocales) { + const filePath = `../app/locales/${locale}.json`; + for (const commitHash of findCommitHashesForFile(filePath)) { + const translation = retrieveJSONForFileAtCommitHash(filePath, commitHash); + if (translationKeysByLocale[locale] === undefined) { + translationKeysByLocale[locale] = new Set([]); + } + translationKeysByLocale[locale] = new Set([...translationKeysByLocale[locale], ...Object.keys(translation)]); + } +} + +/** + * Step 3: Print out the missing keys for each locale. + */ +for (const locale of allLocales) { + for (const englishTranslationKey of allEnglishTranslationKeys) { + if (!translationKeysByLocale[locale].has(englishTranslationKey)) { + console.log(locale, 'missing translation for:', englishTranslationKey); + } + } +} From 004d5e059ba6da56dda36ea47b371bf44bb8c9d6 Mon Sep 17 00:00:00 2001 From: Michael McMillan Date: Sun, 29 Mar 2020 21:49:32 +0200 Subject: [PATCH 04/16] ported over script usage --- ops/lost-in-translation.js | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/ops/lost-in-translation.js b/ops/lost-in-translation.js index 9366b1e5..51bfd276 100644 --- a/ops/lost-in-translation.js +++ b/ops/lost-in-translation.js @@ -1,5 +1,14 @@ const { spawnSync } = require('child_process'); +/** + * This script finds all the english translation keys across + * all commits (including branches and pull requests) for each + * locale.json in app/locales. Afterwards it looks at each locale + * separately to see if it's missing any one the english keys. + * Usage: node ops/lost-in-translation.js + * Important: You need to run it from the root directory. + */ + /** * Calls git cli with arguments and returns stdout. */ @@ -49,7 +58,7 @@ function retrieveJSONForFileAtCommitHash(filePath, commitHash) { allLocales = new Set(['no', 'se']); allEnglishTranslationKeys = new Set([]); for (const locale of allLocales) { - const filePath = `../app/locales/${locale}.json`; + const filePath = `app/locales/${locale}.json`; for (const commitHash of findCommitHashesForFile(filePath)) { const translation = retrieveJSONForFileAtCommitHash(filePath, commitHash); for (const translationKey of Object.keys(translation)) { @@ -71,7 +80,7 @@ for (const locale of allLocales) { */ const translationKeysByLocale = {}; for (const locale of allLocales) { - const filePath = `../app/locales/${locale}.json`; + const filePath = `app/locales/${locale}.json`; for (const commitHash of findCommitHashesForFile(filePath)) { const translation = retrieveJSONForFileAtCommitHash(filePath, commitHash); if (translationKeysByLocale[locale] === undefined) { From 721769d4424468ff71284961041c58202d3f1d1a Mon Sep 17 00:00:00 2001 From: Michael McMillan Date: Sun, 29 Mar 2020 21:49:52 +0200 Subject: [PATCH 05/16] remove python script --- ops/lost-in-translation.py | 79 -------------------------------------- 1 file changed, 79 deletions(-) delete mode 100644 ops/lost-in-translation.py diff --git a/ops/lost-in-translation.py b/ops/lost-in-translation.py deleted file mode 100644 index 4d4b664c..00000000 --- a/ops/lost-in-translation.py +++ /dev/null @@ -1,79 +0,0 @@ -from subprocess import call, check_output, STDOUT -from json import loads as decode_json, JSONDecodeError - -''' - This script finds all the english translation keys across - all commits (including branches and pull requests) for each - locale.json in app/locales. Afterwards it looks at each locale - separately to see if it's missing any one the english keys. - - Usage: python3 ops/lost-in-translation.py - Important: You need to run it from the root directory. -''' - -def git(*args): - '''Calls git cli with arguments.''' - stdout = check_output(['git', *args], stderr=STDOUT) - return stdout - -def find_commit_hashes_for_file(file_path): - '''Returns the commit hashes for a file path across all local branches.''' - lines = git('log', '--pretty=format:"%h"', '--all', '--', file_path) - hashes = [line.replace('"', '') for line in lines.decode('utf-8').split('\n')] - return hashes - -def retrieve_json_for_file_at_commit_hash(file_path, commit_hash): - '''Returns the JSON representation of the contents of a file at certain commit hash.''' - contents_at_commit_hash = git('show', '%s:%s' % (commit_hash, file_path)) - try: - json = decode_json(contents_at_commit_hash.decode('utf-8')) - return json - except JSONDecodeError: - # Sometimes the file is not in a proper JSON format. We simply return nothing in that case. - return {} - -if __name__ == '__main__': - all_locales = ('no', 'se', 'en-IN') - - # Step 1: Find all the (english) translation keys across all branches and PRs. - # - # If a Dutch developer has made a feature in a branch, we expect that him/her added a key - # to one or many locale files (usually the english locale (en.json) or to their own locale - # file (nl.json), or both). - # - # But they probably haven't added translations to all the other locales, because they don't - # speak the other languages. And this is the problem, because now we need find people to help - # translate the added keys to all the other locales. - # - # So what we do here is simply to find *all* the english translation keys across the entire - # project. That means looking at all the english translations keys in all the locale files - # since the start of the project across all branches and PRs. We throw them into a set that - # we use in the next step. - all_english_translation_keys = set([]) - for locale in all_locales: - file_path = 'app/locales/%s.json' % locale - for commit_hash in find_commit_hashes_for_file(file_path): - translation = retrieve_json_for_file_at_commit_hash(file_path, commit_hash) - all_english_translation_keys.update(translation.keys()) - - # Step 2: For each locale file check if there are any missing english translation keys - # across all branches and PRs and all commits/changes. - # - # If there's a missing english translation key, we know that we're most likely missing - # a translation for that locale. So what we need to do is to translate it, or get help - # to translate it. - # - # We throw it into a dictionary where the key is the locale and the value is a set of - # missing translations from english to that locale. - translation_keys_by_locale = {} - for locale in all_locales: - file_path = 'app/locales/%s.json' % locale - for commit_hash in find_commit_hashes_for_file(file_path): - translation = retrieve_json_for_file_at_commit_hash(file_path, commit_hash) - translation_keys_by_locale[locale] = translation_keys_by_locale.get(locale, set()).union(translation.keys()) - - # Step 3: Print out the missing keys for each locale. - for locale in all_locales: - for english_translation_key in all_english_translation_keys: - if not english_translation_key in translation_keys_by_locale[locale]: - print(locale, 'missing translation for:', english_translation_key) From 432870dcdd8a260e46dee5fb9819c0eba54edcbc Mon Sep 17 00:00:00 2001 From: Michael McMillan Date: Sun, 29 Mar 2020 21:54:56 +0200 Subject: [PATCH 06/16] delimit language with --- --- ops/lost-in-translation.js | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/ops/lost-in-translation.js b/ops/lost-in-translation.js index 51bfd276..246c2103 100644 --- a/ops/lost-in-translation.js +++ b/ops/lost-in-translation.js @@ -39,6 +39,13 @@ function retrieveJSONForFileAtCommitHash(filePath, commitHash) { } } +/** + * Normalizes translation key like we do in app/server.ts. + */ +function normalizeTranslationKey(translationKey) { + return translationKey.replace(/[\s\n\t]+/g, ' ').trim(); +} + /** * Step 1: Find all the (english) translation keys across all branches and PRs. * @@ -62,7 +69,7 @@ for (const locale of allLocales) { for (const commitHash of findCommitHashesForFile(filePath)) { const translation = retrieveJSONForFileAtCommitHash(filePath, commitHash); for (const translationKey of Object.keys(translation)) { - allEnglishTranslationKeys.add(translationKey); + allEnglishTranslationKeys.add(normalizeTranslationKey(translationKey)); } } } @@ -83,10 +90,11 @@ for (const locale of allLocales) { const filePath = `app/locales/${locale}.json`; for (const commitHash of findCommitHashesForFile(filePath)) { const translation = retrieveJSONForFileAtCommitHash(filePath, commitHash); + const translationKeys = Object.keys(translation).map(key => normalizeTranslationKey(key)); if (translationKeysByLocale[locale] === undefined) { translationKeysByLocale[locale] = new Set([]); } - translationKeysByLocale[locale] = new Set([...translationKeysByLocale[locale], ...Object.keys(translation)]); + translationKeysByLocale[locale] = new Set([...translationKeysByLocale[locale], ...translationKeys]); } } @@ -94,6 +102,7 @@ for (const locale of allLocales) { * Step 3: Print out the missing keys for each locale. */ for (const locale of allLocales) { + console.log('---'); for (const englishTranslationKey of allEnglishTranslationKeys) { if (!translationKeysByLocale[locale].has(englishTranslationKey)) { console.log(locale, 'missing translation for:', englishTranslationKey); From f70d5a6583780b6ea644124dd8c18347abd862ee Mon Sep 17 00:00:00 2001 From: Michael McMillan Date: Sun, 29 Mar 2020 22:03:04 +0200 Subject: [PATCH 07/16] determine locales from dir --- ops/lost-in-translation.js | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/ops/lost-in-translation.js b/ops/lost-in-translation.js index 246c2103..5e35b745 100644 --- a/ops/lost-in-translation.js +++ b/ops/lost-in-translation.js @@ -1,3 +1,5 @@ +const { parse } = require('path'); +const { readdirSync } = require('fs'); const { spawnSync } = require('child_process'); /** @@ -46,6 +48,15 @@ function normalizeTranslationKey(translationKey) { return translationKey.replace(/[\s\n\t]+/g, ' ').trim(); } +/** + * Find all the locales (ie. en-IN, no, se) in the provided directory. + */ +function retrieveAllLocales(directoryPath) { + const filenames = readdirSync(directoryPath); + const locales = filenames.map(filename => parse(filename).name); + return locales; +} + /** * Step 1: Find all the (english) translation keys across all branches and PRs. * @@ -62,8 +73,8 @@ function normalizeTranslationKey(translationKey) { * since the start of the project across all branches and PRs. We throw them into a set that * we use in the next step. */ -allLocales = new Set(['no', 'se']); -allEnglishTranslationKeys = new Set([]); +const allLocales = retrieveAllLocales('app/locales/'); +const allEnglishTranslationKeys = new Set([]); for (const locale of allLocales) { const filePath = `app/locales/${locale}.json`; for (const commitHash of findCommitHashesForFile(filePath)) { From ae006803e2a7c9311b6e57a21c5150cdf479715b Mon Sep 17 00:00:00 2001 From: Michael McMillan Date: Sun, 29 Mar 2020 23:00:25 +0200 Subject: [PATCH 08/16] upload missing translations to a google sheet --- .gitignore | 2 + ops/lost-in-translation.js | 31 ++++++- package.json | 1 + yarn.lock | 184 ++++++++++++++++++++++++++++++++++++- 4 files changed, 209 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 9249a563..a8bcb26d 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,8 @@ yarn-debug.log* yarn-error.log* lerna-debug.log* +*-credentials.json + # Diagnostic reports (https://nodejs.org/api/report.html) report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json diff --git a/ops/lost-in-translation.js b/ops/lost-in-translation.js index 5e35b745..78e6230c 100644 --- a/ops/lost-in-translation.js +++ b/ops/lost-in-translation.js @@ -1,6 +1,7 @@ const { parse } = require('path'); const { readdirSync } = require('fs'); const { spawnSync } = require('child_process'); +const { GoogleSpreadsheet } = require('google-spreadsheet'); /** * This script finds all the english translation keys across @@ -57,6 +58,16 @@ function retrieveAllLocales(directoryPath) { return locales; } +/** + * Add rows to a Google Spreadsheet that are not already added. + */ +async function addUniqueRowsToGoogleSheet(sheet, rowsToAdd) { + const alreadyAddedRows = await sheet.getRows({ limit: 1000 }); + const uniqueRowsToAdd = rowsToAdd.filter(rowToAdd => + !alreadyAddedRows.find(alreadyAddedRow => alreadyAddedRow.key === rowToAdd.key)); + await sheet.addRows(uniqueRowsToAdd); +} + /** * Step 1: Find all the (english) translation keys across all branches and PRs. * @@ -110,13 +121,23 @@ for (const locale of allLocales) { } /** - * Step 3: Print out the missing keys for each locale. + * Step 3: Add rows of missing translations to Google Spreadsheet. + * + * https://docs.google.com/spreadsheets/d/1ILFfc1DX4ujMnLnf9UqhwQGM9Ke3s1cAWciy8VqMHZw */ -for (const locale of allLocales) { - console.log('---'); +(async () => { + const doc = new GoogleSpreadsheet('1ILFfc1DX4ujMnLnf9UqhwQGM9Ke3s1cAWciy8VqMHZw'); + await doc.useServiceAccountAuth(require('./coronastatus-translation-486cef09736e-credentials.json')); + await doc.loadInfo(); + const sheet = doc.sheetsByIndex[0]; + + const rows = []; + const locale = 'no'; for (const englishTranslationKey of allEnglishTranslationKeys) { if (!translationKeysByLocale[locale].has(englishTranslationKey)) { - console.log(locale, 'missing translation for:', englishTranslationKey); + const row = { 'key': englishTranslationKey, translation: '...' }; + rows.push(row); } } -} + await addUniqueRowsToGoogleSheet(sheet, rows); +})(); diff --git a/package.json b/package.json index cd16e7e5..cf16a192 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "ejs": "^3.0.1", "express": "^4.17.1", "express-rate-limit": "^5.1.1", + "google-spreadsheet": "^3.0.10", "i18n": "^0.8.6", "node-fetch": "^2.6.0", "pg": "^7.18.2", diff --git a/yarn.lock b/yarn.lock index 30a68a05..19a29541 100644 --- a/yarn.lock +++ b/yarn.lock @@ -672,6 +672,13 @@ abbrev@1: resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q== +abort-controller@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392" + integrity sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg== + dependencies: + event-target-shim "^5.0.0" + accepts@~1.3.7: version "1.3.7" resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.7.tgz#531bc726517a3b2b41f850021c6cc15eaab507cd" @@ -727,6 +734,13 @@ agent-base@5: resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-5.1.1.tgz#e8fb3f242959db44d63be665db7a8e739537a32c" integrity sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g== +agent-base@6: + version "6.0.0" + resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.0.tgz#5d0101f19bbfaed39980b22ae866de153b93f09a" + integrity sha512-j1Q7cSCqN+AwrmDd+pzgqc0/NpC655x2bUf5ZjRIO77DcNBFmh+OgRNzF6OKdCC9RSCb19fGd99+bhXFdkRNqw== + dependencies: + debug "4" + ajv@^6.10.0, ajv@^6.10.2, ajv@^6.5.5: version "6.12.0" resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.0.tgz#06d60b96d87b8454a5adaba86e7854da629db4b7" @@ -892,6 +906,11 @@ array.prototype.flat@^1.2.1: define-properties "^1.1.3" es-abstract "^1.17.0-next.1" +arrify@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/arrify/-/arrify-2.0.1.tgz#c9655e9331e0abcd588d2a7cad7e9956f66701fa" + integrity sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug== + asn1@~0.2.3: version "0.2.4" resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.4.tgz#8d2475dfab553bb33e77b54e59e880bb8ce23136" @@ -957,6 +976,13 @@ aws4@^1.8.0: resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.9.1.tgz#7e33d8f7d449b3f673cd72deb9abdc552dbe528e" integrity sha512-wMHVg2EOHaMRxbzgFJ9gtjOOCrI80OHLG14rxi28XwOW8ux6IiEbRCGGGqCtdAIg4FQCbW20k9RsT4y3gJlFug== +axios@^0.19.1: + version "0.19.2" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.19.2.tgz#3ea36c5d8818d0d5f8a8a97a6d36b86cdc00cb27" + integrity sha512-fjgm5MvRHLhx+osE2xoekY70AhARk3a6hkN+3Io1jc00jtquGvxYlKlsFUhmUET0V5te6CcZI7lcv2Ym61mjHA== + dependencies: + follow-redirects "1.5.10" + axobject-query@^2.0.2: version "2.1.2" resolved "https://registry.yarnpkg.com/axobject-query/-/axobject-query-2.1.2.tgz#2bdffc0371e643e5f03ba99065d5179b9ca79799" @@ -1019,6 +1045,11 @@ balanced-match@^1.0.0: resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c= +base64-js@^1.3.0: + version "1.3.1" + resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.3.1.tgz#58ece8cb75dd07e71ed08c736abc5fac4dbf8df1" + integrity sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g== + base@^0.11.1: version "0.11.2" resolved "https://registry.yarnpkg.com/base/-/base-0.11.2.tgz#7bde5ced145b6d551a90db87f83c558b4eb48a8f" @@ -1039,6 +1070,11 @@ bcrypt-pbkdf@^1.0.0: dependencies: tweetnacl "^0.14.3" +bignumber.js@^7.0.0: + version "7.2.1" + resolved "https://registry.yarnpkg.com/bignumber.js/-/bignumber.js-7.2.1.tgz#80c048759d826800807c4bfd521e50edbba57a5f" + integrity sha512-S4XzBk5sMB+Rcb/LNcpzXr57VRTxgAvaAEDAl1AwRx27j00hT84O6OkteE7u8UB3NuaaygCRrEpqox4uDOrbdQ== + binary-extensions@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.0.0.tgz#23c0df14f6a88077f5f986c0d167ec03c3d5537c" @@ -1150,6 +1186,11 @@ buffer-crc32@~0.2.3: resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242" integrity sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI= +buffer-equal-constant-time@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz#f8e71132f7ffe6e01a5c9697a4c6f3e48d5cc819" + integrity sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk= + buffer-from@1.x, buffer-from@^1.0.0: version "1.1.1" resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" @@ -1782,6 +1823,13 @@ debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.9: dependencies: ms "2.0.0" +debug@=3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" + integrity sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g== + dependencies: + ms "2.0.0" + debug@^3.2.6: version "3.2.6" resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.6.tgz#e83d17de16d8a7efb7717edbe5fb10135eee629b" @@ -1979,6 +2027,13 @@ ecc-jsbn@~0.1.1: jsbn "~0.1.0" safer-buffer "^2.1.0" +ecdsa-sig-formatter@1.0.11, ecdsa-sig-formatter@^1.0.11: + version "1.0.11" + resolved "https://registry.yarnpkg.com/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz#ae0f0fa2d85045ef14a817daa3ce9acd0489e5bf" + integrity sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ== + dependencies: + safe-buffer "^5.0.1" + ee-first@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" @@ -2274,6 +2329,11 @@ etag@~1.8.1: resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc= +event-target-shim@^5.0.0: + version "5.0.1" + resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789" + integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ== + exec-sh@^0.3.2: version "0.3.4" resolved "https://registry.yarnpkg.com/exec-sh/-/exec-sh-0.3.4.tgz#3a018ceb526cc6f6df2bb504b2bfe8e3a4934ec5" @@ -2407,7 +2467,7 @@ extend-shallow@^3.0.0, extend-shallow@^3.0.2: assign-symbols "^1.0.0" is-extendable "^1.0.1" -extend@~3.0.2: +extend@^3.0.2, extend@~3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== @@ -2487,6 +2547,11 @@ fast-levenshtein@~2.0.6: resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" integrity sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc= +fast-text-encoding@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/fast-text-encoding/-/fast-text-encoding-1.0.1.tgz#4a428566f74fc55ebdd447555b1eb4d9cf514455" + integrity sha512-x4FEgaz3zNRtJfLFqJmHWxkMDDvXVtaznj2V9jiP8ACUJrUgist4bP9FmDL2Vew2Y9mEQI/tG4GqabaitYp9CQ== + fastq@^1.6.0: version "1.6.1" resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.6.1.tgz#4570c74f2ded173e71cf0beb08ac70bb85826791" @@ -2588,6 +2653,13 @@ flatted@^2.0.0: resolved "https://registry.yarnpkg.com/flatted/-/flatted-2.0.1.tgz#69e57caa8f0eacbc281d2e2cb458d46fdb449e08" integrity sha512-a1hQMktqW9Nmqr5aktAux3JMNqaucxGcjtjWnZLHX7yyPCmlSV3M54nGYbqT8K+0GhF3NBgmJCc3ma+WOgX8Jg== +follow-redirects@1.5.10: + version "1.5.10" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.5.10.tgz#7b7a9f9aea2fdff36786a94ff643ed07f4ff5e2a" + integrity sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ== + dependencies: + debug "=3.1.0" + for-in@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80" @@ -2683,6 +2755,25 @@ gauge@~2.7.3: strip-ansi "^3.0.1" wide-align "^1.1.0" +gaxios@^2.1.0: + version "2.3.4" + resolved "https://registry.yarnpkg.com/gaxios/-/gaxios-2.3.4.tgz#eea99353f341c270c5f3c29fc46b8ead56f0a173" + integrity sha512-US8UMj8C5pRnao3Zykc4AAVr+cffoNKRTg9Rsf2GiuZCW69vgJj38VK2PzlPuQU73FZ/nTk9/Av6/JGcE1N9vA== + dependencies: + abort-controller "^3.0.0" + extend "^3.0.2" + https-proxy-agent "^5.0.0" + is-stream "^2.0.0" + node-fetch "^2.3.0" + +gcp-metadata@^3.4.0: + version "3.5.0" + resolved "https://registry.yarnpkg.com/gcp-metadata/-/gcp-metadata-3.5.0.tgz#6d28343f65a6bbf8449886a0c0e4a71c77577055" + integrity sha512-ZQf+DLZ5aKcRpLzYUyBS3yo3N0JSa82lNDO8rj3nMSlovLcz2riKFBsYgDzeXcv75oo5eqB2lx+B14UvPoCRnA== + dependencies: + gaxios "^2.1.0" + json-bigint "^0.3.0" + gensync@^1.0.0-beta.1: version "1.0.0-beta.1" resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.1.tgz#58f4361ff987e5ff6e1e7a210827aa371eaac269" @@ -2803,6 +2894,37 @@ globrex@^0.1.1: resolved "https://registry.yarnpkg.com/globrex/-/globrex-0.1.2.tgz#dd5d9ec826232730cd6793a5e33a9302985e6098" integrity sha512-uHJgbwAMwNFf5mLst7IWLNg14x1CkeqglJb/K3doi4dw6q2IvAAmM/Y81kevy83wP+Sst+nutFTYOGg3d1lsxg== +google-auth-library@^5.9.1: + version "5.10.1" + resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-5.10.1.tgz#504ec75487ad140e68dd577c21affa363c87ddff" + integrity sha512-rOlaok5vlpV9rSiUu5EpR0vVpc+PhN62oF4RyX/6++DG1VsaulAFEMlDYBLjJDDPI6OcNOCGAKy9UVB/3NIDXg== + dependencies: + arrify "^2.0.0" + base64-js "^1.3.0" + ecdsa-sig-formatter "^1.0.11" + fast-text-encoding "^1.0.0" + gaxios "^2.1.0" + gcp-metadata "^3.4.0" + gtoken "^4.1.0" + jws "^4.0.0" + lru-cache "^5.0.0" + +google-p12-pem@^2.0.0: + version "2.0.4" + resolved "https://registry.yarnpkg.com/google-p12-pem/-/google-p12-pem-2.0.4.tgz#036462394e266472632a78b685f0cc3df4ef337b" + integrity sha512-S4blHBQWZRnEW44OcR7TL9WR+QCqByRvhNDZ/uuQfpxywfupikf/miba8js1jZi6ZOGv5slgSuoshCWh6EMDzg== + dependencies: + node-forge "^0.9.0" + +google-spreadsheet@^3.0.10: + version "3.0.10" + resolved "https://registry.yarnpkg.com/google-spreadsheet/-/google-spreadsheet-3.0.10.tgz#3aac6fd2a766a3943a27f09fbb45ed501e54ddcd" + integrity sha512-FQNaXAUzE+W9AE3Sd3O9BMPnC4x1FZSjBt+X8G+s9x14nJ/xk0waIJiou44q+E0WKr9fi12UANEwVRAd5vvDEw== + dependencies: + axios "^0.19.1" + google-auth-library "^5.9.1" + lodash "^4.17.15" + got@^6.7.1: version "6.7.1" resolved "https://registry.yarnpkg.com/got/-/got-6.7.1.tgz#240cd05785a9a18e561dc1b44b41c763ef1e8db0" @@ -2830,6 +2952,16 @@ growly@^1.3.0: resolved "https://registry.yarnpkg.com/growly/-/growly-1.3.0.tgz#f10748cbe76af964b7c96c93c6bcc28af120c081" integrity sha1-8QdIy+dq+WS3yWyTxrzCivEgwIE= +gtoken@^4.1.0: + version "4.1.4" + resolved "https://registry.yarnpkg.com/gtoken/-/gtoken-4.1.4.tgz#925ff1e7df3aaada06611d30ea2d2abf60fcd6a7" + integrity sha512-VxirzD0SWoFUo5p8RDP8Jt2AGyOmyYcT/pOUgDKJCK+iSw0TMqwrVfY37RXTNmoKwrzmDHSk0GMT9FsgVmnVSA== + dependencies: + gaxios "^2.1.0" + google-p12-pem "^2.0.0" + jws "^4.0.0" + mime "^2.2.0" + har-schema@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92" @@ -2977,6 +3109,14 @@ https-proxy-agent@^4.0.0: agent-base "5" debug "4" +https-proxy-agent@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz#e2a90542abb68a762e0a0850f6c9edadfd8506b2" + integrity sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA== + dependencies: + agent-base "6" + debug "4" + human-signals@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-1.1.1.tgz#c5b1cd14f50aeae09ab6c59fe63ba3395fe4dfa3" @@ -3882,6 +4022,13 @@ jsesc@^2.5.1: resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4" integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA== +json-bigint@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/json-bigint/-/json-bigint-0.3.0.tgz#0ccd912c4b8270d05f056fbd13814b53d3825b1e" + integrity sha1-DM2RLEuCcNBfBW+9E4FLU9OCWx4= + dependencies: + bignumber.js "^7.0.0" + json-parse-better-errors@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz#bb867cfb3450e69107c131d1c514bab3dc8bcaa9" @@ -3946,6 +4093,23 @@ jsx-ast-utils@^2.2.1: array-includes "^3.0.3" object.assign "^4.1.0" +jwa@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/jwa/-/jwa-2.0.0.tgz#a7e9c3f29dae94027ebcaf49975c9345593410fc" + integrity sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA== + dependencies: + buffer-equal-constant-time "1.0.1" + ecdsa-sig-formatter "1.0.11" + safe-buffer "^5.0.1" + +jws@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/jws/-/jws-4.0.0.tgz#2d4e8cf6a318ffaa12615e9dec7e86e6c97310f4" + integrity sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg== + dependencies: + jwa "^2.0.0" + safe-buffer "^5.0.1" + kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.2.0: version "3.2.2" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" @@ -4080,6 +4244,13 @@ lru-cache@^4.0.1: pseudomap "^1.0.2" yallist "^2.1.2" +lru-cache@^5.0.0: + version "5.1.1" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-5.1.1.tgz#1da27e6710271947695daf6848e847f01d84b920" + integrity sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w== + dependencies: + yallist "^3.0.2" + make-dir@^1.0.0: version "1.3.0" resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-1.3.0.tgz#79c1033b80515bd6d24ec9933e860ca75ee27f0c" @@ -4228,7 +4399,7 @@ mime@1.6.0: resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== -mime@^2.0.3: +mime@^2.0.3, mime@^2.2.0: version "2.4.4" resolved "https://registry.yarnpkg.com/mime/-/mime-2.4.4.tgz#bd7b91135fc6b01cde3e9bae33d659b63d8857e5" integrity sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA== @@ -4370,11 +4541,16 @@ node-emoji@^1.8.1: dependencies: lodash.toarray "^4.4.0" -node-fetch@^2.6.0: +node-fetch@^2.3.0, node-fetch@^2.6.0: version "2.6.0" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd" integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA== +node-forge@^0.9.0: + version "0.9.1" + resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.9.1.tgz#775368e6846558ab6676858a4d8c6e8d16c677b5" + integrity sha512-G6RlQt5Sb4GMBzXvhfkeFmbqR6MzhtnT7VTHuLadjkii3rdYHNdw0m8zA4BTxVIh68FicCQ2NSUANpsqkr9jvQ== + node-int64@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/node-int64/-/node-int64-0.4.0.tgz#87a9065cdb355d3182d8f94ce11188b825c68a3b" @@ -6985,7 +7161,7 @@ yallist@^2.1.2: resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52" integrity sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI= -yallist@^3.0.0, yallist@^3.0.3: +yallist@^3.0.0, yallist@^3.0.2, yallist@^3.0.3: version "3.1.1" resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== From 490106c511f772dcb63c2551f81d734f41a75f03 Mon Sep 17 00:00:00 2001 From: Michael McMillan Date: Sun, 29 Mar 2020 23:03:23 +0200 Subject: [PATCH 09/16] empty string as missing --- ops/lost-in-translation.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops/lost-in-translation.js b/ops/lost-in-translation.js index 78e6230c..76653f11 100644 --- a/ops/lost-in-translation.js +++ b/ops/lost-in-translation.js @@ -135,7 +135,7 @@ for (const locale of allLocales) { const locale = 'no'; for (const englishTranslationKey of allEnglishTranslationKeys) { if (!translationKeysByLocale[locale].has(englishTranslationKey)) { - const row = { 'key': englishTranslationKey, translation: '...' }; + const row = { 'key': englishTranslationKey, translation: '' }; rows.push(row); } } From 6ecc6fabc6beb114add66c60c32f91f36d97001a Mon Sep 17 00:00:00 2001 From: Michael McMillan Date: Sun, 29 Mar 2020 23:13:56 +0200 Subject: [PATCH 10/16] sync all available sheets --- ops/lost-in-translation.js | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/ops/lost-in-translation.js b/ops/lost-in-translation.js index 76653f11..f6aa891c 100644 --- a/ops/lost-in-translation.js +++ b/ops/lost-in-translation.js @@ -66,6 +66,7 @@ async function addUniqueRowsToGoogleSheet(sheet, rowsToAdd) { const uniqueRowsToAdd = rowsToAdd.filter(rowToAdd => !alreadyAddedRows.find(alreadyAddedRow => alreadyAddedRow.key === rowToAdd.key)); await sheet.addRows(uniqueRowsToAdd); + return uniqueRowsToAdd; } /** @@ -129,15 +130,25 @@ for (const locale of allLocales) { const doc = new GoogleSpreadsheet('1ILFfc1DX4ujMnLnf9UqhwQGM9Ke3s1cAWciy8VqMHZw'); await doc.useServiceAccountAuth(require('./coronastatus-translation-486cef09736e-credentials.json')); await doc.loadInfo(); - const sheet = doc.sheetsByIndex[0]; - const rows = []; - const locale = 'no'; - for (const englishTranslationKey of allEnglishTranslationKeys) { - if (!translationKeysByLocale[locale].has(englishTranslationKey)) { - const row = { 'key': englishTranslationKey, translation: '' }; - rows.push(row); + const allLocales = retrieveAllLocales('app/locales/'); + for (let sheetIndex = 0; sheetIndex < doc.sheetCount; sheetIndex++) { + const sheet = doc.sheetsByIndex[sheetIndex]; + for (const locale of allLocales) { + if (sheet.title !== locale) { + continue; + } + + const rows = []; + for (const englishTranslationKey of allEnglishTranslationKeys) { + if (!translationKeysByLocale[locale].has(englishTranslationKey)) { + const row = { 'key': englishTranslationKey, translation: '' }; + rows.push(row); + } + } + const addedRows = await addUniqueRowsToGoogleSheet(sheet, rows); + console.log(`Added ${addedRows.length} of ${rows.length} missing translations to the ${locale} sheet.`); } } - await addUniqueRowsToGoogleSheet(sheet, rows); + })(); From 34e600a5584b64d0a46bbad75c057c91f515c8a6 Mon Sep 17 00:00:00 2001 From: Michael McMillan Date: Sun, 29 Mar 2020 23:24:01 +0200 Subject: [PATCH 11/16] add sheet if it doesnt already exist --- ops/lost-in-translation.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ops/lost-in-translation.js b/ops/lost-in-translation.js index f6aa891c..540d35a9 100644 --- a/ops/lost-in-translation.js +++ b/ops/lost-in-translation.js @@ -136,6 +136,11 @@ for (const locale of allLocales) { const sheet = doc.sheetsByIndex[sheetIndex]; for (const locale of allLocales) { if (sheet.title !== locale) { + try { + await doc.addSheet({ title: locale, headerValues: ['key', 'translation'] }); + } catch (error) { + // We don't do anything if the sheet for this locale exists. + } continue; } From e2def1ed76a3329d639d3e538b44b9aae07645e5 Mon Sep 17 00:00:00 2001 From: Michael McMillan Date: Sun, 29 Mar 2020 23:28:36 +0200 Subject: [PATCH 12/16] avoid rate limit --- ops/lost-in-translation.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ops/lost-in-translation.js b/ops/lost-in-translation.js index 540d35a9..796d5a25 100644 --- a/ops/lost-in-translation.js +++ b/ops/lost-in-translation.js @@ -154,6 +154,10 @@ for (const locale of allLocales) { const addedRows = await addUniqueRowsToGoogleSheet(sheet, rows); console.log(`Added ${addedRows.length} of ${rows.length} missing translations to the ${locale} sheet.`); } + + // Avoid getting rate limited by Google's API by waiting 100 seconds between each sheet. + console.log('Waiting a 100 seconds before processing the next sheet.'); + await new Promise(resolve => setTimeout(resolve, 100*1000)); } })(); From 1fe146f88dba2279f1e9aaddfa816983bd7212a6 Mon Sep 17 00:00:00 2001 From: Michael McMillan Date: Sun, 29 Mar 2020 23:43:22 +0200 Subject: [PATCH 13/16] rate limit circumvension --- ops/lost-in-translation.js | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/ops/lost-in-translation.js b/ops/lost-in-translation.js index 796d5a25..0ff3236a 100644 --- a/ops/lost-in-translation.js +++ b/ops/lost-in-translation.js @@ -136,14 +136,18 @@ for (const locale of allLocales) { const sheet = doc.sheetsByIndex[sheetIndex]; for (const locale of allLocales) { if (sheet.title !== locale) { - try { - await doc.addSheet({ title: locale, headerValues: ['key', 'translation'] }); - } catch (error) { - // We don't do anything if the sheet for this locale exists. - } continue; } + try { + // Create a sheet if it doesn't already exist. + await doc.addSheet({ title: locale, headerValues: ['key', 'translation'] }); + await new Promise(resolve => setTimeout(resolve, 10*1000)); + } catch (error) { + // We don't do anything if the sheet for this locale exists. + } + + // Add the missing rows by looking at the key column. const rows = []; for (const englishTranslationKey of allEnglishTranslationKeys) { if (!translationKeysByLocale[locale].has(englishTranslationKey)) { @@ -151,13 +155,15 @@ for (const locale of allLocales) { rows.push(row); } } + + // Print out how many rows we added. const addedRows = await addUniqueRowsToGoogleSheet(sheet, rows); console.log(`Added ${addedRows.length} of ${rows.length} missing translations to the ${locale} sheet.`); - } - // Avoid getting rate limited by Google's API by waiting 100 seconds between each sheet. - console.log('Waiting a 100 seconds before processing the next sheet.'); - await new Promise(resolve => setTimeout(resolve, 100*1000)); + // Avoid getting rate limited by Google's API (max writes per 100 seconds). + console.log('Waiting before processing the next sheet.'); + await new Promise(resolve => setTimeout(resolve, 20*1000)); + } } })(); From 99f1c80a4a7d3e6d0316660afc963c0a5704a20d Mon Sep 17 00:00:00 2001 From: Michael McMillan Date: Mon, 30 Mar 2020 00:07:34 +0200 Subject: [PATCH 14/16] prettify comment --- ops/lost-in-translation.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ops/lost-in-translation.js b/ops/lost-in-translation.js index 0ff3236a..aa83428f 100644 --- a/ops/lost-in-translation.js +++ b/ops/lost-in-translation.js @@ -8,8 +8,11 @@ const { GoogleSpreadsheet } = require('google-spreadsheet'); * all commits (including branches and pull requests) for each * locale.json in app/locales. Afterwards it looks at each locale * separately to see if it's missing any one the english keys. - * Usage: node ops/lost-in-translation.js + * * Important: You need to run it from the root directory. + * + * Usage: + * node ops/lost-in-translation.js */ /** From d355fbf0b403826527d8731f6c961fdaaf8d74d3 Mon Sep 17 00:00:00 2001 From: Michael McMillan Date: Mon, 30 Mar 2020 00:31:31 +0200 Subject: [PATCH 15/16] refactored code and made it less complex after feedback from @fossecode --- ops/lost-in-translation.js | 61 +++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/ops/lost-in-translation.js b/ops/lost-in-translation.js index aa83428f..24b98e7c 100644 --- a/ops/lost-in-translation.js +++ b/ops/lost-in-translation.js @@ -72,6 +72,18 @@ async function addUniqueRowsToGoogleSheet(sheet, rowsToAdd) { return uniqueRowsToAdd; } +/** + * Retrieve all the sheets in a Google Spreadsheet document. + */ +async function retrieveSheetsInDocument(doc) { + const sheets = []; + for (let sheetIndex = 0; sheetIndex < doc.sheetCount; sheetIndex++) { + const sheet = await doc.sheetsByIndex[sheetIndex]; + sheets.push(sheet); + } + return sheets; +} + /** * Step 1: Find all the (english) translation keys across all branches and PRs. * @@ -134,39 +146,34 @@ for (const locale of allLocales) { await doc.useServiceAccountAuth(require('./coronastatus-translation-486cef09736e-credentials.json')); await doc.loadInfo(); + const sheets = await retrieveSheetsInDocument(doc); const allLocales = retrieveAllLocales('app/locales/'); - for (let sheetIndex = 0; sheetIndex < doc.sheetCount; sheetIndex++) { - const sheet = doc.sheetsByIndex[sheetIndex]; - for (const locale of allLocales) { - if (sheet.title !== locale) { - continue; - } - try { - // Create a sheet if it doesn't already exist. - await doc.addSheet({ title: locale, headerValues: ['key', 'translation'] }); - await new Promise(resolve => setTimeout(resolve, 10*1000)); - } catch (error) { - // We don't do anything if the sheet for this locale exists. - } + for (const locale of allLocales) { + + // Create sheet for this locale if it doesn't already exist. + let matchingSheet = sheets.find(sheet => sheet.title === locale); + if (!matchingSheet) { + matchingSheet = await doc.addSheet({ title: locale, headerValues: ['key', 'translation'] }); + } - // Add the missing rows by looking at the key column. - const rows = []; - for (const englishTranslationKey of allEnglishTranslationKeys) { - if (!translationKeysByLocale[locale].has(englishTranslationKey)) { - const row = { 'key': englishTranslationKey, translation: '' }; - rows.push(row); - } + // Find rows that don't already exist by looking at the key column. + const rows = []; + for (const englishTranslationKey of allEnglishTranslationKeys) { + if (!translationKeysByLocale[locale].has(englishTranslationKey)) { + const row = { 'key': englishTranslationKey, translation: '' }; + rows.push(row); } + } - // Print out how many rows we added. - const addedRows = await addUniqueRowsToGoogleSheet(sheet, rows); - console.log(`Added ${addedRows.length} of ${rows.length} missing translations to the ${locale} sheet.`); + // Add and print out how many rows we added. + const addedRows = await addUniqueRowsToGoogleSheet(matchingSheet, rows); + console.log(`Added ${addedRows.length} of ${rows.length} missing translations to the ${locale} sheet.`); + + // Avoid getting rate limited by Google's API (max writes per 100 seconds). + console.log('Waiting before processing the next sheet.'); + await new Promise(resolve => setTimeout(resolve, 20*1000)); - // Avoid getting rate limited by Google's API (max writes per 100 seconds). - console.log('Waiting before processing the next sheet.'); - await new Promise(resolve => setTimeout(resolve, 20*1000)); - } } })(); From be1a7e180936f47624fd6b9374a94cc725a83c5d Mon Sep 17 00:00:00 2001 From: Michael McMillan Date: Mon, 30 Mar 2020 00:41:26 +0200 Subject: [PATCH 16/16] linted --- ops/lost-in-translation.js | 54 ++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/ops/lost-in-translation.js b/ops/lost-in-translation.js index 24b98e7c..089d8d14 100644 --- a/ops/lost-in-translation.js +++ b/ops/lost-in-translation.js @@ -1,3 +1,4 @@ +/* eslint-disable no-await-in-loop, global-require */ const { parse } = require('path'); const { readdirSync } = require('fs'); const { spawnSync } = require('child_process'); @@ -19,7 +20,7 @@ const { GoogleSpreadsheet } = require('google-spreadsheet'); * Calls git cli with arguments and returns stdout. */ function git(...args) { - const { stdout } = spawnSync('git', args, { encoding: 'utf-8'}); + const { stdout } = spawnSync('git', args, { encoding: 'utf-8' }); return stdout; } @@ -66,8 +67,12 @@ function retrieveAllLocales(directoryPath) { */ async function addUniqueRowsToGoogleSheet(sheet, rowsToAdd) { const alreadyAddedRows = await sheet.getRows({ limit: 1000 }); - const uniqueRowsToAdd = rowsToAdd.filter(rowToAdd => - !alreadyAddedRows.find(alreadyAddedRow => alreadyAddedRow.key === rowToAdd.key)); + const uniqueRowsToAdd = rowsToAdd.filter( + rowToAdd => + !alreadyAddedRows.find( + alreadyAddedRow => alreadyAddedRow.key === rowToAdd.key + ) + ); await sheet.addRows(uniqueRowsToAdd); return uniqueRowsToAdd; } @@ -77,7 +82,7 @@ async function addUniqueRowsToGoogleSheet(sheet, rowsToAdd) { */ async function retrieveSheetsInDocument(doc) { const sheets = []; - for (let sheetIndex = 0; sheetIndex < doc.sheetCount; sheetIndex++) { + for (let sheetIndex = 0; sheetIndex < doc.sheetCount; sheetIndex += 1) { const sheet = await doc.sheetsByIndex[sheetIndex]; sheets.push(sheet); } @@ -119,8 +124,8 @@ for (const locale of allLocales) { * If there's a missing english translation key, we know that we're most likely missing * a translation for that locale. So what we need to do is to translate it, or get help * to translate it. - * - * We throw it into a dictionary where the key is the locale and the value is a set of + * + * We throw it into a dictionary where the key is the locale and the value is a set of * missing translations from english to that locale. */ const translationKeysByLocale = {}; @@ -128,52 +133,61 @@ for (const locale of allLocales) { const filePath = `app/locales/${locale}.json`; for (const commitHash of findCommitHashesForFile(filePath)) { const translation = retrieveJSONForFileAtCommitHash(filePath, commitHash); - const translationKeys = Object.keys(translation).map(key => normalizeTranslationKey(key)); + const translationKeys = Object.keys(translation).map(key => + normalizeTranslationKey(key) + ); if (translationKeysByLocale[locale] === undefined) { translationKeysByLocale[locale] = new Set([]); } - translationKeysByLocale[locale] = new Set([...translationKeysByLocale[locale], ...translationKeys]); + translationKeysByLocale[locale] = new Set([ + ...translationKeysByLocale[locale], + ...translationKeys + ]); } } /** * Step 3: Add rows of missing translations to Google Spreadsheet. - * + * * https://docs.google.com/spreadsheets/d/1ILFfc1DX4ujMnLnf9UqhwQGM9Ke3s1cAWciy8VqMHZw */ (async () => { - const doc = new GoogleSpreadsheet('1ILFfc1DX4ujMnLnf9UqhwQGM9Ke3s1cAWciy8VqMHZw'); - await doc.useServiceAccountAuth(require('./coronastatus-translation-486cef09736e-credentials.json')); + const doc = new GoogleSpreadsheet( + '1ILFfc1DX4ujMnLnf9UqhwQGM9Ke3s1cAWciy8VqMHZw' + ); + await doc.useServiceAccountAuth( + require('./coronastatus-translation-486cef09736e-credentials.json') + ); await doc.loadInfo(); - const sheets = await retrieveSheetsInDocument(doc); - const allLocales = retrieveAllLocales('app/locales/'); for (const locale of allLocales) { - // Create sheet for this locale if it doesn't already exist. let matchingSheet = sheets.find(sheet => sheet.title === locale); if (!matchingSheet) { - matchingSheet = await doc.addSheet({ title: locale, headerValues: ['key', 'translation'] }); + matchingSheet = await doc.addSheet({ + title: locale, + headerValues: ['key', 'translation'] + }); } // Find rows that don't already exist by looking at the key column. const rows = []; for (const englishTranslationKey of allEnglishTranslationKeys) { if (!translationKeysByLocale[locale].has(englishTranslationKey)) { - const row = { 'key': englishTranslationKey, translation: '' }; + const row = { key: englishTranslationKey, translation: '' }; rows.push(row); } } // Add and print out how many rows we added. const addedRows = await addUniqueRowsToGoogleSheet(matchingSheet, rows); - console.log(`Added ${addedRows.length} of ${rows.length} missing translations to the ${locale} sheet.`); + console.log( + `Added ${addedRows.length} of ${rows.length} missing translations to the ${locale} sheet.` + ); // Avoid getting rate limited by Google's API (max writes per 100 seconds). console.log('Waiting before processing the next sheet.'); - await new Promise(resolve => setTimeout(resolve, 20*1000)); - + await new Promise(resolve => setTimeout(resolve, 20 * 1000)); } - })();