From 9a01da76e1b51566f2a5b4d92157a3706022e42b Mon Sep 17 00:00:00 2001 From: EYHN Date: Thu, 19 Sep 2024 05:30:17 +0000 Subject: [PATCH] feat: sync i18n with crowdin (#8293) --- .github/workflows/languages-sync.yml | 35 --- .github/workflows/sync-i18n.yml | 38 ++++ packages/frontend/i18n/crowdin.yml | 12 + packages/frontend/i18n/package.json | 5 +- packages/frontend/i18n/src/scripts/api.ts | 212 ------------------ .../frontend/i18n/src/scripts/download.ts | 155 ------------- packages/frontend/i18n/src/scripts/request.ts | 52 ----- packages/frontend/i18n/src/scripts/sync.ts | 153 ------------- packages/frontend/i18n/src/scripts/utils.ts | 35 --- packages/frontend/i18n/tsconfig.json | 3 - packages/frontend/i18n/tsconfig.node.json | 8 - 11 files changed, 51 insertions(+), 657 deletions(-) delete mode 100644 .github/workflows/languages-sync.yml create mode 100644 .github/workflows/sync-i18n.yml create mode 100644 packages/frontend/i18n/crowdin.yml delete mode 100644 packages/frontend/i18n/src/scripts/api.ts delete mode 100644 packages/frontend/i18n/src/scripts/download.ts delete mode 100644 packages/frontend/i18n/src/scripts/request.ts delete mode 100644 packages/frontend/i18n/src/scripts/sync.ts delete mode 100644 packages/frontend/i18n/src/scripts/utils.ts delete mode 100644 packages/frontend/i18n/tsconfig.node.json diff --git a/.github/workflows/languages-sync.yml b/.github/workflows/languages-sync.yml deleted file mode 100644 index 073cf208130da..0000000000000 --- a/.github/workflows/languages-sync.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Languages Sync - -on: - push: - branches: ['canary'] - paths: - - 'packages/frontend/i18n/**' - - '.github/workflows/languages-sync.yml' - - '!.github/actions/setup-node/action.yml' - pull_request_target: - branches: ['canary'] - paths: - - 'packages/frontend/i18n/**' - - '.github/workflows/languages-sync.yml' - - '!.github/actions/setup-node/action.yml' - workflow_dispatch: - -jobs: - main: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Setup Node.js - uses: ./.github/actions/setup-node - - name: Check Language Key - if: github.ref != 'refs/heads/canary' - run: yarn workspace @affine/i18n run sync-languages:check - env: - TOLGEE_API_KEY: ${{ secrets.TOLGEE_API_KEY }} - - - name: Sync Languages - if: github.ref == 'refs/heads/canary' - run: yarn workspace @affine/i18n run sync-languages - env: - TOLGEE_API_KEY: ${{ secrets.TOLGEE_API_KEY }} diff --git a/.github/workflows/sync-i18n.yml b/.github/workflows/sync-i18n.yml new file mode 100644 index 0000000000000..09d0f4181a8d9 --- /dev/null +++ b/.github/workflows/sync-i18n.yml @@ -0,0 +1,38 @@ +name: Sync I18n with Crowdin + +on: + push: + workflow_dispatch: + +jobs: + synchronize-with-crowdin: + runs-on: ubuntu-latest + + permissions: + contents: write + pull-requests: write + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: crowdin action + uses: crowdin/github-action@v2 + with: + upload_sources: true + upload_translations: true + download_translations: true + auto_approve_imported: true + import_eq_suggestions: true + export_only_approved: true + skip_untranslated_strings: true + localization_branch_name: l10n_crowdin_translations + create_pull_request: true + pull_request_title: 'New Crowdin Translations' + pull_request_body: 'New Crowdin translations by [Crowdin GH Action](https://github.com/crowdin/github-action)' + pull_request_base_branch_name: 'canary' + config: packages/frontend/i18n/crowdin.yml + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CROWDIN_PROJECT_ID: ${{ secrets.CROWDIN_PROJECT_ID }} + CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }} diff --git a/packages/frontend/i18n/crowdin.yml b/packages/frontend/i18n/crowdin.yml new file mode 100644 index 0000000000000..1d10ad4fcf672 --- /dev/null +++ b/packages/frontend/i18n/crowdin.yml @@ -0,0 +1,12 @@ +'base_path': '.' +'base_url': 'https://api.crowdin.com' + +'preserve_hierarchy': true + +'files': + [ + { + 'source': '/src/resources/en.json', + 'translation': '/src/resources/%locale%.json', + }, + ] diff --git a/packages/frontend/i18n/package.json b/packages/frontend/i18n/package.json index 575ba285ad053..d28bf040e38ce 100644 --- a/packages/frontend/i18n/package.json +++ b/packages/frontend/i18n/package.json @@ -8,10 +8,7 @@ }, "scripts": { "build": "node build.mjs", - "dev": "node dev.mjs", - "sync-languages": "node --loader ts-node/esm/transpile-only src/scripts/sync.ts", - "sync-languages:check": "yarn run sync-languages --check", - "download-resources": "node --loader ts-node/esm/transpile-only src/scripts/download.ts" + "dev": "node dev.mjs" }, "keywords": [], "repository": { diff --git a/packages/frontend/i18n/src/scripts/api.ts b/packages/frontend/i18n/src/scripts/api.ts deleted file mode 100644 index a16ebb1bb2b66..0000000000000 --- a/packages/frontend/i18n/src/scripts/api.ts +++ /dev/null @@ -1,212 +0,0 @@ -import type { Response } from 'undici-types'; - -// cSpell:ignore Tolgee -import { fetchTolgee } from './request.js'; - -/** - * Returns all project languages - * - * See https://tolgee.io/api#operation/getAll_6 - * @example - * ```ts - * const languages = [ - * { - * id: 1000016008, - * name: 'English', - * tag: 'en', - * originalName: 'English', - * flagEmoji: '🇬🇧', - * base: true - * }, - * { - * id: 1000016013, - * name: 'Spanish', - * tag: 'es', - * originalName: 'español', - * flagEmoji: '🇪🇸', - * base: false - * }, - * { - * id: 1000016009, - * name: 'Simplified Chinese', - * tag: 'zh-Hans', - * originalName: '简体中文', - * flagEmoji: '🇨🇳', - * base: false - * }, - * { - * id: 1000016012, - * name: 'Traditional Chinese', - * tag: 'zh-Hant', - * originalName: '繁體中文', - * flagEmoji: '🇭🇰', - * base: false - * } - * ] - * ``` - */ - -export const getAllProjectLanguages = async ( - size = 1000 -): Promise< - { - id: number; - name: string; - tag: string; - originalName: string; - flagEmoji: string; - base: boolean; - }[] -> => { - const url = `/languages?size=${size}`; - const resp = await fetchTolgee(url); - if (resp.status < 200 || resp.status >= 300) { - throw new Error(url + ' ' + resp.status + '\n' + (await resp.text())); - } - const json: { - _embedded: { - languages: { - id: number; - name: string; - tag: string; - originalName: string; - flagEmoji: string; - base: boolean; - }[]; - }; - page: unknown; - } = (await resp.json()) as any; - return json._embedded.languages; -}; - -/** - * Returns translations in project - * - * See https://tolgee.io/api#operation/getTranslations_ - */ -export const getTranslations = async (): Promise => { - const url = '/translations'; - const resp = await fetchTolgee(url); - if (resp.status < 200 || resp.status >= 300) { - throw new Error(url + ' ' + resp.status + '\n' + (await resp.text())); - } - const json = await resp.json(); - return json; -}; - -/** - * Returns all translations for specified languages - * - * See https://tolgee.io/api#operation/getAllTranslations_1 - */ -export const getLanguagesTranslations = async ( - languages: T -): Promise<{ [key in T]?: Record }> => { - const url = `/translations/${languages}`; - const resp = await fetchTolgee(url); - if (resp.status < 200 || resp.status >= 300) { - throw new Error(url + ' ' + resp.status + '\n' + (await resp.text())); - } - const json = await resp.json(); - return json as { [key in T]?: Record }; -}; - -export const getRemoteTranslations = async ( - languages: string -): Promise> => { - const translations = await getLanguagesTranslations(languages); - if (!(languages in translations)) { - return {}; - } - // The assert is safe because we checked above - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - return translations[languages]!; -}; - -/** - * Creates new key - * - * See https://tolgee.io/api#operation/create_2 - */ -export const createsNewKey = async ( - key: string, - translations: Record -): Promise => { - const url = '/translations/keys/create'; - const resp = await fetchTolgee(url, { - method: 'POST', - body: JSON.stringify({ name: key, translations }), - }); - if (resp.status < 200 || resp.status >= 300) { - /** - * There are some problems in the i18n backend, - * which is used to temporarily solve the ci error. - */ - console.warn(url + ' ' + resp.status + '\n' + (await resp.text())); - return; - } - const json = await resp.json(); - return json; -}; - -/** - * Tags a key with tag. If tag with provided name doesn't exist, it is created - * - * See https://tolgee.io/api#operation/tagKey_1 - */ -export const addTag = async ( - keyId: string, - tagName: string -): Promise => { - const url = `/keys/${keyId}/tags`; - const resp = await fetchTolgee(url, { - method: 'PUT', - body: JSON.stringify({ name: tagName }), - }); - if (resp.status < 200 || resp.status >= 300) { - throw new Error(url + ' ' + resp.status + '\n' + (await resp.text())); - } - const json = await resp.json(); - return json; -}; - -/** - * Tags a key with tag. If tag with provided name doesn't exist, it is created - * - * See https://tolgee.io/api#operation/tagKey_1 - */ -export const removeTag = async ( - keyId: string, - tagId: number -): Promise => { - const url = `/keys/${keyId}/tags/${tagId}`; - const resp = await fetchTolgee(url, { - method: 'DELETE', - }); - if (resp.status < 200 || resp.status >= 300) { - throw new Error(url + ' ' + resp.status + '\n' + (await resp.text())); - } - const json = await resp.json(); - return json; -}; - -// export const addTagByKey = async (key: string, tag: string) => { -// // TODO get key id by key name -// // const keyId = -// // addTag(keyId, tag); -// }; - -/** - * Exports data - * - * See https://tolgee.io/api#operation/export_1 - */ -export const exportResources = async (): Promise => { - const url = `/export`; - const resp = await fetchTolgee(url); - - if (resp.status < 200 || resp.status >= 300) { - throw new Error(url + ' ' + resp.status + '\n' + (await resp.text())); - } - return resp; -}; diff --git a/packages/frontend/i18n/src/scripts/download.ts b/packages/frontend/i18n/src/scripts/download.ts deleted file mode 100644 index 71110951e85aa..0000000000000 --- a/packages/frontend/i18n/src/scripts/download.ts +++ /dev/null @@ -1,155 +0,0 @@ -// cSpell:ignore Tolgee -import * as fs from 'node:fs/promises'; -import * as path from 'node:path'; - -import { format } from 'prettier'; - -import { getAllProjectLanguages, getRemoteTranslations } from './api.js'; -import type { TranslationRes } from './utils.js'; -import { flattenTranslation } from './utils.js'; - -const INDENT = 2; -const RES_DIR = path.resolve(process.cwd(), 'src', 'resources'); - -const countKeys = (obj: TranslationRes | null) => { - if (!obj) { - return 0; - } - let count = 0; - // eslint-disable-next-line @typescript-eslint/no-unused-vars - Object.entries(obj).forEach(([_, value]) => { - if (typeof value === 'string') { - count++; - } else { - count += countKeys(value); - } - }); - return count; -}; - -const getBaseTranslations = async (baseLanguage: { tag: string }) => { - try { - const baseTranslationsStr = await fs.readFile( - path.resolve(RES_DIR, `${baseLanguage.tag}.json`), - { encoding: 'utf8' } - ); - const baseTranslations = JSON.parse(baseTranslationsStr); - return baseTranslations; - } catch (e) { - console.error('base language:', JSON.stringify(baseLanguage)); - console.error('Failed to read base language', e); - const translations = await getRemoteTranslations(baseLanguage.tag); - await fs.writeFile( - path.resolve(RES_DIR, `${baseLanguage.tag}.json`), - JSON.stringify(translations, null, 4) - ); - } -}; - -const main = async () => { - try { - await fs.access(RES_DIR); - } catch { - fs.mkdir(RES_DIR).catch(console.error); - console.log('Create directory', RES_DIR); - } - console.log('Loading project languages...'); - const languages = await getAllProjectLanguages(); - const baseLanguage = languages.find(language => language.base); - if (!baseLanguage) { - console.error(JSON.stringify(languages)); - throw new Error('Could not find base language'); - } - console.log(`Loading ${baseLanguage.tag} languages translations as base...`); - - const baseTranslations = await getBaseTranslations(baseLanguage); - const baseKeyNum = countKeys(baseTranslations); - const languagesWithTranslations = await Promise.all( - languages.map(async language => { - console.log(`Loading ${language.tag} translations...`); - const translations = await getRemoteTranslations(language.tag); - const keyNum = countKeys(translations); - const completeRate = Number((keyNum / baseKeyNum).toFixed(3)); - console.log( - `Load ${language.name} ${ - completeRate * 100 - }, %(${keyNum}/${baseKeyNum}) complete` - ); - - return { - ...language, - translations, - completeRate, - }; - }) - ); - - const availableLanguages = languagesWithTranslations.filter( - language => language.completeRate > 0.2 - ); - - for (const language of availableLanguages - // skip base language - .filter(i => !i.base)) { - await fs.writeFile( - path.resolve(RES_DIR, `${language.tag}.json`), - JSON.stringify( - { - '// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.': - '', - ...flattenTranslation(language.translations), - }, - null, - INDENT - ) + '\n' - ); - } - - console.log('Generating meta data...'); - const code = `// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. - // Run \`yarn run download-resources\` to regenerate. - // If you need to update the code, please edit \`i18n/src/scripts/download.ts\` inside your project. - ${availableLanguages - .map( - language => - `import ${language.tag.replaceAll('-', '_')} from './${ - language.tag - }.json'` - ) - .sort() - .join('\n')} - - export const LOCALES = [ - ${availableLanguages - // eslint-disable-next-line @typescript-eslint/no-unused-vars -- omit key - .map(({ translations, ...language }) => - JSON.stringify({ - ...language, - // a trick to generate a string without quotation marks - res: '__RES_PLACEHOLDER', - }).replace( - '"__RES_PLACEHOLDER"', - language.tag.replaceAll('-', '_') - ) - ) - .join(',\n')} - ] as const; - `; - - await fs.writeFile( - path.resolve(RES_DIR, 'index.ts'), - await format(code, { - parser: 'typescript', - singleQuote: true, - trailingComma: 'es5', - tabWidth: INDENT, - arrowParens: 'avoid', - }) - ); - console.log('Done'); -}; - -main().catch(e => { - console.error(e); - process.exit(1); -}); diff --git a/packages/frontend/i18n/src/scripts/request.ts b/packages/frontend/i18n/src/scripts/request.ts deleted file mode 100644 index 9aaea89824102..0000000000000 --- a/packages/frontend/i18n/src/scripts/request.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { Headers } from 'undici'; -import type { fetch, Request, RequestInfo } from 'undici-types'; - -// cSpell:ignore Tolgee -const TOLGEE_API_KEY = process.env['TOLGEE_API_KEY']; -const TOLGEE_API_URL = 'https://i18n.affine.pro'; - -if (!TOLGEE_API_KEY) { - throw new Error(`Please set "TOLGEE_API_KEY" as environment variable!`); -} - -const withTolgee = (f: typeof fetch): typeof fetch => { - const baseUrl = `${TOLGEE_API_URL}/v2/projects`; - const headers = new Headers({ - 'X-API-Key': TOLGEE_API_KEY, - 'Content-Type': 'application/json', - }); - - const isRequest = (input: RequestInfo): input is Request => { - return typeof input === 'object' && !('href' in input); - }; - - return new Proxy(f, { - apply(target, thisArg: unknown, argArray: Parameters) { - if (isRequest(argArray[0])) { - // Request - if (!argArray[0].headers) { - argArray[0] = { - ...argArray[0], - url: `${baseUrl}${argArray[0].url}`, - headers, - }; - } - } else { - // URL or URLLike + ?RequestInit - if (typeof argArray[0] === 'string') { - argArray[0] = `${baseUrl}${argArray[0]}`; - } - if (!argArray[1]) { - argArray[1] = {}; - } - if (!argArray[1].headers) { - argArray[1].headers = headers; - } - } - // console.log('fetch', argArray); - return target.apply(thisArg, argArray); - }, - }); -}; - -export const fetchTolgee = withTolgee(globalThis.fetch as typeof fetch); diff --git a/packages/frontend/i18n/src/scripts/sync.ts b/packages/frontend/i18n/src/scripts/sync.ts deleted file mode 100644 index fa546eb77477d..0000000000000 --- a/packages/frontend/i18n/src/scripts/sync.ts +++ /dev/null @@ -1,153 +0,0 @@ -// cSpell:ignore Tolgee -import { readFile } from 'node:fs/promises'; -import { resolve } from 'node:path'; - -import { createsNewKey, getRemoteTranslations } from './api.js'; -import type { TranslationRes } from './utils.js'; - -const BASE_JSON_PATH = resolve(process.cwd(), 'src', 'resources', 'en.json'); -const BASE_LANGUAGES = 'en' as const; - -/** - * - * @example - * ```ts - * flatRes({ a: { b: 'c' } }); // { 'a.b': 'c' } - * ``` - */ -const flatRes = (obj: TranslationRes) => { - const getEntries = (o: TranslationRes, prefix = ''): [string, string][] => - Object.entries(o).flatMap<[string, string]>(([k, v]) => - typeof v !== 'string' - ? getEntries(v, `${prefix}${k}.`) - : [[`${prefix}${k}`, v]] - ); - return Object.fromEntries(getEntries(obj)); -}; - -const differenceObject = ( - newObj: Record, - oldObj: Record -) => { - const add: string[] = []; - const remove: string[] = []; - const modify: string[] = []; - const both: string[] = []; - - Object.keys(newObj).forEach(key => { - if (!(key in oldObj)) { - add.push(key); - } else { - both.push(key); - } - }); - - Object.keys(oldObj).forEach(key => { - if (!(key in newObj)) { - remove.push(key); - } - }); - - both.forEach(key => { - if (!(key in newObj) || !(key in oldObj)) { - throw new Error('Unreachable'); - } - const newVal = newObj[key]; - const oldVal = oldObj[key]; - if (newVal !== oldVal) { - modify.push(key); - } - }); - return { add, remove, modify }; -}; - -function warnDiff(diff: { add: string[]; remove: string[]; modify: string[] }) { - if (diff.add.length) { - console.log('New keys found:', diff.add.join(', ')); - //See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-a-notice-message - process.env['CI'] && - console.log( - `::notice file=${BASE_JSON_PATH},line=1,title=New keys::${diff.add.join( - ', ' - )}` - ); - } - if (diff.remove.length) { - console.warn('[WARN]', 'Unused keys found:', diff.remove.join(', ')); - process.env['CI'] && - console.warn( - `::notice file=${BASE_JSON_PATH},line=1,title=Unused keys::${diff.remove.join( - ', ' - )}` - ); - } - if (diff.modify.length) { - console.warn('[WARN]', 'Inconsistent keys found:', diff.modify.join(', ')); - process.env['CI'] && - console.warn( - `::warning file=${BASE_JSON_PATH},line=1,title=Inconsistent keys::${diff.modify.join( - ', ' - )}` - ); - } -} - -const main = async () => { - console.log('Loading local base translations...'); - const baseLocalTranslations = JSON.parse( - await readFile(BASE_JSON_PATH, { - encoding: 'utf8', - }) - ); - const flatLocalTranslations = flatRes(baseLocalTranslations); - console.log( - `Loading local base translations success! Total ${ - Object.keys(flatLocalTranslations).length - } keys` - ); - - console.log('Fetch remote base translations...'); - const baseRemoteTranslations = await getRemoteTranslations(BASE_LANGUAGES); - const flatRemoteTranslations = flatRes(baseRemoteTranslations); - console.log( - `Fetch remote base translations success! Total ${ - Object.keys(flatRemoteTranslations).length - } keys` - ); - - const diff = differenceObject(flatLocalTranslations, flatRemoteTranslations); - - console.log(''); // new line - warnDiff(diff); - console.log(''); // new line - - if (process.argv.slice(2).includes('--check')) { - // check mode - return; - } - - for (const key of diff.add) { - const val = flatLocalTranslations[key]; - console.log(`Creating new key: ${key} -> ${val}`); - await createsNewKey(key, { [BASE_LANGUAGES]: val }); - } - - // TODO remove unused tags from used keys - - // diff.remove.forEach(key => { - // // TODO set unused tag - // // console.log(`Add ${DEPRECATED_TAG_NAME} to ${key}`); - // addTagByKey(key, DEPRECATED_TAG_NAME); - // }); - - // diff.modify.forEach(key => { - // // TODO warn different between local and remote base translations - // }); - - // TODO send notification -}; - -main().catch(e => { - console.error(e); - process.exit(1); -}); diff --git a/packages/frontend/i18n/src/scripts/utils.ts b/packages/frontend/i18n/src/scripts/utils.ts deleted file mode 100644 index 00689866f7904..0000000000000 --- a/packages/frontend/i18n/src/scripts/utils.ts +++ /dev/null @@ -1,35 +0,0 @@ -export interface TranslationRes { - [x: string]: string | TranslationRes; -} - -/** - * Recursively flattens a JSON object using dot notation. - * - * NOTE: input must be an object as described by JSON spec. Arbitrary - * JS objects (e.g. {a: () => 42}) may result in unexpected output. - * MOREOVER, it removes keys with empty objects/arrays as value (see - * examples bellow). - * - * @example - * flattenTranslation({a: 1, b: [{c: 2, d: {e: 3}}, 4]}) - * // {a: 1, b.0.c: 2, b.0.d.e: 3, b.1: 4} - * flattenTranslation({a: 1, b: [{c: 2, d: {e: [true, false, {f: 1}]}}]}) - * // {a: 1, b.0.c: 2, b.0.d.e.0: true, b.0.d.e.1: false, b.0.d.e.2.f: 1} - * flattenTranslation({a: 1, b: [], c: {}}) - * // {a: 1} - * - * @param obj item to be flattened - */ -export const flattenTranslation = ( - obj: string | TranslationRes, - path?: string -): TranslationRes => { - if (!(obj instanceof Object)) return { [path ?? '']: obj }; - - return Object.keys(obj).reduce((output, key) => { - return Object.assign( - output, - flattenTranslation(obj[key], path ? path + '.' + key : key) - ); - }, {}); -}; diff --git a/packages/frontend/i18n/tsconfig.json b/packages/frontend/i18n/tsconfig.json index b41dcecd43add..66e008d309ff3 100644 --- a/packages/frontend/i18n/tsconfig.json +++ b/packages/frontend/i18n/tsconfig.json @@ -10,9 +10,6 @@ "references": [ { "path": "./tsconfig.resources.json" - }, - { - "path": "./tsconfig.node.json" } ] } diff --git a/packages/frontend/i18n/tsconfig.node.json b/packages/frontend/i18n/tsconfig.node.json deleted file mode 100644 index 6deaabf23c59e..0000000000000 --- a/packages/frontend/i18n/tsconfig.node.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../../tsconfig.json", - "compilerOptions": { - "types": ["node"], - "outDir": "./lib/scripts" - }, - "include": ["./src/scripts"] -}