From 7d2c58fb6b0fbc181809365e42a311f60c3625e2 Mon Sep 17 00:00:00 2001 From: Timofei Iatsenko Date: Thu, 25 May 2023 17:08:44 +0200 Subject: [PATCH] feature(po-format): add `explicitIdAsDefault` for po-format for easier migration --- packages/format-po/README.md | 15 +++ packages/format-po/src/po.test.ts | 142 +++++++++++++++++++++++++++ packages/format-po/src/po.ts | 42 +++++++- website/docs/releases/migration-4.md | 57 ++++++----- 4 files changed, 227 insertions(+), 29 deletions(-) diff --git a/packages/format-po/README.md b/packages/format-po/README.md index 739c7c41d..daa5bcca9 100644 --- a/packages/format-po/README.md +++ b/packages/format-po/README.md @@ -61,6 +61,21 @@ export type PoFormatterOptions = { * @default false */ printLinguiId?: boolean + + /** + * By default, the po-formatter treats the pair `msgid` + `msgctx` as the source + * for generating an ID by hashing its value. + * + * For messages with explicit IDs, the formatter adds a special comment `js-lingui-explicit-id` as a flag. + * When this flag is present, the formatter will use the `msgid` as-is without any additional processing. + * + * Set this option to true if you exclusively use explicit-ids in your project. + * + * https://lingui.dev/tutorials/react-patterns#using-custom-id + * + * @default false + */ + explicitIdAsDefault?: boolean } ``` diff --git a/packages/format-po/src/po.test.ts b/packages/format-po/src/po.test.ts index 2cfd8dc93..0a83fd111 100644 --- a/packages/format-po/src/po.test.ts +++ b/packages/format-po/src/po.test.ts @@ -122,6 +122,148 @@ describe("pofile format", () => { expect(actual).toMatchObject(catalog) }) + describe("explicitIdAsDefault", () => { + const catalog: CatalogType = { + // with generated id + Dgzql1: { + message: "with generated id", + translation: "", + context: "my context", + }, + + "custom.id": { + message: "with explicit id", + translation: "", + }, + } + + it("should set `js-lingui-generated-id` for messages with generated id when [explicitIdAsDefault: true]", () => { + const format = createFormatter({ + origins: true, + explicitIdAsDefault: true, + }) + + const serialized = format.serialize( + catalog, + defaultSerializeCtx + ) as string + + expect(serialized).toMatchInlineSnapshot(` + msgid "" + msgstr "" + "POT-Creation-Date: 2018-08-27 10:00+0000\\n" + "MIME-Version: 1.0\\n" + "Content-Type: text/plain; charset=utf-8\\n" + "Content-Transfer-Encoding: 8bit\\n" + "X-Generator: @lingui/cli\\n" + "Language: en\\n" + + #. js-lingui-generated-id + msgctxt "my context" + msgid "with generated id" + msgstr "" + + msgid "custom.id" + msgstr "" + + `) + + const actual = format.parse(serialized, defaultParseCtx) + expect(actual).toMatchInlineSnapshot(` + { + Dgzql1: { + comments: [ + js-lingui-generated-id, + ], + context: my context, + extra: { + flags: [], + translatorComments: [], + }, + message: with generated id, + obsolete: false, + origin: [], + translation: , + }, + custom.id: { + comments: [], + context: null, + extra: { + flags: [], + translatorComments: [], + }, + obsolete: false, + origin: [], + translation: , + }, + } + `) + }) + + it("should set `js-explicit-id` for messages with explicit id when [explicitIdAsDefault: false]", () => { + const format = createFormatter({ + origins: true, + explicitIdAsDefault: false, + }) + + const serialized = format.serialize( + catalog, + defaultSerializeCtx + ) as string + + expect(serialized).toMatchInlineSnapshot(` + msgid "" + msgstr "" + "POT-Creation-Date: 2018-08-27 10:00+0000\\n" + "MIME-Version: 1.0\\n" + "Content-Type: text/plain; charset=utf-8\\n" + "Content-Transfer-Encoding: 8bit\\n" + "X-Generator: @lingui/cli\\n" + "Language: en\\n" + + msgctxt "my context" + msgid "with generated id" + msgstr "" + + #. js-lingui-explicit-id + msgid "custom.id" + msgstr "" + + `) + + const actual = format.parse(serialized, defaultParseCtx) + expect(actual).toMatchInlineSnapshot(` + { + Dgzql1: { + comments: [], + context: my context, + extra: { + flags: [], + translatorComments: [], + }, + message: with generated id, + obsolete: false, + origin: [], + translation: , + }, + custom.id: { + comments: [ + js-lingui-explicit-id, + ], + context: null, + extra: { + flags: [], + translatorComments: [], + }, + obsolete: false, + origin: [], + translation: , + }, + } + `) + }) + }) + it("should print lingui id if printLinguiId = true", () => { const format = createFormatter({ origins: true, printLinguiId: true }) diff --git a/packages/format-po/src/po.ts b/packages/format-po/src/po.ts index 6cf018f15..01c99652d 100644 --- a/packages/format-po/src/po.ts +++ b/packages/format-po/src/po.ts @@ -43,6 +43,21 @@ export type PoFormatterOptions = { * @default false */ printLinguiId?: boolean + + /** + * By default, the po-formatter treats the pair `msgid` + `msgctx` as the source + * for generating an ID by hashing its value. + * + * For messages with explicit IDs, the formatter adds a special comment `js-lingui-explicit-id` as a flag. + * When this flag is present, the formatter will use the `msgid` as-is without any additional processing. + * + * Set this option to true if you exclusively use explicit-ids in your project. + * + * https://lingui.dev/tutorials/react-patterns#using-custom-id + * + * @default false + */ + explicitIdAsDefault?: boolean } function isGeneratedId(id: string, message: MessageType): boolean { @@ -61,6 +76,7 @@ function getCreateHeaders(language: string): PO["headers"] { } const EXPLICIT_ID_FLAG = "js-lingui-explicit-id" +const GENERATED_ID_FLAG = "js-lingui-generated-id" const serialize = (catalog: CatalogType, options: PoFormatterOptions) => { return Object.keys(catalog).map((id) => { @@ -84,15 +100,24 @@ const serialize = (catalog: CatalogType, options: PoFormatterOptions) => { if (_isGeneratedId) { item.msgid = message.message + if (options.explicitIdAsDefault) { + if (!item.extractedComments.includes(GENERATED_ID_FLAG)) { + item.extractedComments.push(GENERATED_ID_FLAG) + } + } + if (options.printLinguiId) { if (!item.extractedComments.find((c) => c.includes("js-lingui-id"))) { item.extractedComments.push(`js-lingui-id: ${id}`) } } } else { - if (!item.extractedComments.includes(EXPLICIT_ID_FLAG)) { - item.extractedComments.push(EXPLICIT_ID_FLAG) + if (!options.explicitIdAsDefault) { + if (!item.extractedComments.includes(EXPLICIT_ID_FLAG)) { + item.extractedComments.push(EXPLICIT_ID_FLAG) + } } + item.msgid = id } @@ -116,7 +141,10 @@ const serialize = (catalog: CatalogType, options: PoFormatterOptions) => { }) } -function deserialize(items: POItem[]): CatalogType { +function deserialize( + items: POItem[], + options: PoFormatterOptions +): CatalogType { return items.reduce>((catalog, item) => { const message: MessageType = { translation: item.msgstr[0], @@ -133,7 +161,11 @@ function deserialize(items: POItem[]): CatalogType { let id = item.msgid // if generated id, recreate it - if (!item.extractedComments.includes(EXPLICIT_ID_FLAG)) { + if ( + options.explicitIdAsDefault + ? item.extractedComments.includes(GENERATED_ID_FLAG) + : !item.extractedComments.includes(EXPLICIT_ID_FLAG) + ) { id = generateMessageId(item.msgid, item.msgctxt) message.message = item.msgid } @@ -156,7 +188,7 @@ export function formatter(options: PoFormatterOptions = {}): CatalogFormatter { parse(content): CatalogType { const po = PO.parse(content) - return deserialize(po.items) + return deserialize(po.items, options) }, serialize(catalog, ctx): string { diff --git a/website/docs/releases/migration-4.md b/website/docs/releases/migration-4.md index 4c86fdc96..75cf3bb78 100644 --- a/website/docs/releases/migration-4.md +++ b/website/docs/releases/migration-4.md @@ -41,39 +41,48 @@ No migration steps are necessary for components provided by Lingui, such as `Tra ### Hash-based message ID generation and Context feature -The previous implementation had a flaw: there is an original message in the bundle at least 2 times + 1 translation. +Starting from Lingui v4, hash-based IDs are used internally for message lookups. -For the line "Hello world" it'll exist in the source code as ID in i18n call, then as a key in the message catalog, and then as a translation itself. Strings could be very long, not just a couple of words, so this may bring more kB to the bundle. +If you use natural language as an ID in your project, for example: +```ts +const message = t`My Message` +``` +you will benefit significantly from this change. Your bundles will become smaller because the source message will be removed from the bundle in favor of a short generated ID. -A much better option is generating a "stable" ID based on the msg + context as a hash with a fixed length. +If you use natural language as an ID, you don't need to do anything special to migrate. -Hash would be calculated at build time by macros. So macros instead of: +However, if you use explicit IDs, like this: -```js -const message = t({ - context: 'My context', - message: `Hello` -}) +```ts +const message = t({id: "my.message", message: `My Message`}) +``` -// ↓ ↓ ↓ ↓ ↓ ↓ +there are some changes you need to make to your catalogs to migrate properly. In order to distinguish between generated IDs and explicit IDs in the PO format, Lingui adds a special comment for messages with explicit IDs called `js-lingui-explicit-id`. -import { i18n } from "@lingui/core" -const message = i18n._(/*i18n*/{ - context: 'My context', - id: `Hello` -}) +Here's an example of the comment in a PO file: +```gettext +#. js-lingui-explicit-id +msgid "custom.id" +msgstr "" ``` -now generates: +You need to add this comment manually to all your messages with explicit IDs. + +If you exclusively use explicit IDs in your project, you may consider enabling a different processing mode for the PO formatter. This can be done in your Lingui config file: +```ts title="lingui.config.ts" +import { formatter } from '@lingui/po-format' +import { LinguiConfig } from '@lingui/config' -```js -import { i18n } from "@lingui/core" -const message = i18n._(/*i18n*/{ - id: "", - message: `Hello`, -}) +const config: LinguiConfig = { + // ... + format: formatter({ explicitIdAsDefault: true }), +} ``` +Enabling this mode will swap the logic, and the formatter will treat all messages as having explicit IDs without the need for the explicit flag comment. + +You can read more about the motivation behind this change in the [original RFC](https://github.com/lingui/js-lingui/issues/1360) + Also, we've added a possibility to provide a context for the message. For more details, see the [Providing a context for a message](/docs/tutorials/react-patterns.md#providing-a-context-for-a-message). The context feature affects the message ID generation and adds the `msgctxt` parameter in case of the PO catalog format extraction. @@ -137,13 +146,13 @@ Extractor supports TypeScript out of the box. Please delete it from your configu If your extract command looks like: ```bash -NODE_ENV=development lingui-extract +NODE_ENV=development lingui extract ``` Now you can safely change it to just: ```bash -lingui-extract +lingui extract ``` ### Public interface of `ExtractorType` was changed