feat(po-format): add explicitIdAsDefault for po-format for easier m…

…igration (#1672) * feat(po-format): add `explicitIdAsDefault` for po-format for easier migration * docs: add lazy translations migrations notes
lingui · May 26, 2023 · 3303228 · 3303228 · vercel · May 26, 2023
1 parent f06cdf5
commit 3303228
Show file tree

Hide file tree

Showing 6 changed files with 323 additions and 36 deletions.
diff --git a/packages/format-po/README.md b/packages/format-po/README.md
@@ -61,6 +61,21 @@ export type PoFormatterOptions = {
    * @default false
    */
   printLinguiId?: boolean
+
+  /**
+   * By default, the po-formatter treats the pair `msgid` + `msgctx` as the source
+   * for generating an ID by hashing its value.
+   *
+   * For messages with explicit IDs, the formatter adds a special comment `js-lingui-explicit-id` as a flag.
+   * When this flag is present, the formatter will use the `msgid` as-is without any additional processing.
+   *
+   * Set this option to true if you exclusively use explicit-ids in your project.
+   *
+   * https://lingui.dev/tutorials/react-patterns#using-custom-id
+   *
+   * @default false
+   */
+  explicitIdAsDefault?: boolean
 }
 ```
 

diff --git a/packages/format-po/src/po.test.ts b/packages/format-po/src/po.test.ts
@@ -122,6 +122,148 @@ describe("pofile format", () => {
     expect(actual).toMatchObject(catalog)
   })
 
+  describe("explicitIdAsDefault", () => {
+    const catalog: CatalogType = {
+      // with generated id
+      Dgzql1: {
+        message: "with generated id",
+        translation: "",
+        context: "my context",
+      },
+
+      "custom.id": {
+        message: "with explicit id",
+        translation: "",
+      },
+    }
+
+    it("should set `js-lingui-generated-id` for messages with generated id when [explicitIdAsDefault: true]", () => {
+      const format = createFormatter({
+        origins: true,
+        explicitIdAsDefault: true,
+      })
+
+      const serialized = format.serialize(
+        catalog,
+        defaultSerializeCtx
+      ) as string
+
+      expect(serialized).toMatchInlineSnapshot(`
+        msgid ""
+        msgstr ""
+        "POT-Creation-Date: 2018-08-27 10:00+0000\\n"
+        "MIME-Version: 1.0\\n"
+        "Content-Type: text/plain; charset=utf-8\\n"
+        "Content-Transfer-Encoding: 8bit\\n"
+        "X-Generator: @lingui/cli\\n"
+        "Language: en\\n"
+
+        #. js-lingui-generated-id
+        msgctxt "my context"
+        msgid "with generated id"
+        msgstr ""
+
+        msgid "custom.id"
+        msgstr ""
+
+      `)
+
+      const actual = format.parse(serialized, defaultParseCtx)
+      expect(actual).toMatchInlineSnapshot(`
+        {
+          Dgzql1: {
+            comments: [
+              js-lingui-generated-id,
+            ],
+            context: my context,
+            extra: {
+              flags: [],
+              translatorComments: [],
+            },
+            message: with generated id,
+            obsolete: false,
+            origin: [],
+            translation: ,
+          },
+          custom.id: {
+            comments: [],
+            context: null,
+            extra: {
+              flags: [],
+              translatorComments: [],
+            },
+            obsolete: false,
+            origin: [],
+            translation: ,
+          },
+        }
+      `)
+    })
+
+    it("should set `js-explicit-id` for messages with explicit id when [explicitIdAsDefault: false]", () => {
+      const format = createFormatter({
+        origins: true,
+        explicitIdAsDefault: false,
+      })
+
+      const serialized = format.serialize(
+        catalog,
+        defaultSerializeCtx
+      ) as string
+
+      expect(serialized).toMatchInlineSnapshot(`
+        msgid ""
+        msgstr ""
+        "POT-Creation-Date: 2018-08-27 10:00+0000\\n"
+        "MIME-Version: 1.0\\n"
+        "Content-Type: text/plain; charset=utf-8\\n"
+        "Content-Transfer-Encoding: 8bit\\n"
+        "X-Generator: @lingui/cli\\n"
+        "Language: en\\n"
+
+        msgctxt "my context"
+        msgid "with generated id"
+        msgstr ""
+
+        #. js-lingui-explicit-id
+        msgid "custom.id"
+        msgstr ""
+
+      `)
+
+      const actual = format.parse(serialized, defaultParseCtx)
+      expect(actual).toMatchInlineSnapshot(`
+        {
+          Dgzql1: {
+            comments: [],
+            context: my context,
+            extra: {
+              flags: [],
+              translatorComments: [],
+            },
+            message: with generated id,
+            obsolete: false,
+            origin: [],
+            translation: ,
+          },
+          custom.id: {
+            comments: [
+              js-lingui-explicit-id,
+            ],
+            context: null,
+            extra: {
+              flags: [],
+              translatorComments: [],
+            },
+            obsolete: false,
+            origin: [],
+            translation: ,
+          },
+        }
+      `)
+    })
+  })
+
   it("should print lingui id if printLinguiId = true", () => {
     const format = createFormatter({ origins: true, printLinguiId: true })
 

diff --git a/packages/format-po/src/po.ts b/packages/format-po/src/po.ts
@@ -43,6 +43,21 @@ export type PoFormatterOptions = {
    * @default false
    */
   printLinguiId?: boolean
+
+  /**
+   * By default, the po-formatter treats the pair `msgid` + `msgctx` as the source
+   * for generating an ID by hashing its value.
+   *
+   * For messages with explicit IDs, the formatter adds a special comment `js-lingui-explicit-id` as a flag.
+   * When this flag is present, the formatter will use the `msgid` as-is without any additional processing.
+   *
+   * Set this option to true if you exclusively use explicit-ids in your project.
+   *
+   * https://lingui.dev/tutorials/react-patterns#using-custom-id
+   *
+   * @default false
+   */
+  explicitIdAsDefault?: boolean
 }
 
 function isGeneratedId(id: string, message: MessageType): boolean {
@@ -61,6 +76,7 @@ function getCreateHeaders(language: string): PO["headers"] {
 }
 
 const EXPLICIT_ID_FLAG = "js-lingui-explicit-id"
+const GENERATED_ID_FLAG = "js-lingui-generated-id"
 
 const serialize = (catalog: CatalogType, options: PoFormatterOptions) => {
   return Object.keys(catalog).map((id) => {
@@ -84,15 +100,24 @@ const serialize = (catalog: CatalogType, options: PoFormatterOptions) => {
     if (_isGeneratedId) {
       item.msgid = message.message
 
+      if (options.explicitIdAsDefault) {
+        if (!item.extractedComments.includes(GENERATED_ID_FLAG)) {
+          item.extractedComments.push(GENERATED_ID_FLAG)
+        }
+      }
+
       if (options.printLinguiId) {
         if (!item.extractedComments.find((c) => c.includes("js-lingui-id"))) {
           item.extractedComments.push(`js-lingui-id: ${id}`)
         }
       }
     } else {
-      if (!item.extractedComments.includes(EXPLICIT_ID_FLAG)) {
-        item.extractedComments.push(EXPLICIT_ID_FLAG)
+      if (!options.explicitIdAsDefault) {
+        if (!item.extractedComments.includes(EXPLICIT_ID_FLAG)) {
+          item.extractedComments.push(EXPLICIT_ID_FLAG)
+        }
       }
+
       item.msgid = id
     }
 
@@ -116,7 +141,10 @@ const serialize = (catalog: CatalogType, options: PoFormatterOptions) => {
   })
 }
 
-function deserialize(items: POItem[]): CatalogType {
+function deserialize(
+  items: POItem[],
+  options: PoFormatterOptions
+): CatalogType {
   return items.reduce<CatalogType<POCatalogExtra>>((catalog, item) => {
     const message: MessageType<POCatalogExtra> = {
       translation: item.msgstr[0],
@@ -133,7 +161,11 @@ function deserialize(items: POItem[]): CatalogType {
     let id = item.msgid
 
     // if generated id, recreate it
-    if (!item.extractedComments.includes(EXPLICIT_ID_FLAG)) {
+    if (
+      options.explicitIdAsDefault
+        ? item.extractedComments.includes(GENERATED_ID_FLAG)
+        : !item.extractedComments.includes(EXPLICIT_ID_FLAG)
+    ) {
       id = generateMessageId(item.msgid, item.msgctxt)
       message.message = item.msgid
     }
@@ -156,7 +188,7 @@ export function formatter(options: PoFormatterOptions = {}): CatalogFormatter {
 
     parse(content): CatalogType {
       const po = PO.parse(content)
-      return deserialize(po.items)
+      return deserialize(po.items, options)
     },
 
     serialize(catalog, ctx): string {

diff --git a/website/docs/releases/migration-4.md b/website/docs/releases/migration-4.md
@@ -41,45 +41,85 @@ No migration steps are necessary for components provided by Lingui, such as `Tra
 
 ### Hash-based message ID generation and Context feature
 
-The previous implementation had a flaw: there is an original message in the bundle at least 2 times + 1 translation.
+Starting from Lingui v4, hash-based IDs are used internally for message lookups.
 
-For the line "Hello world" it'll exist in the source code as ID in i18n call, then as a key in the message catalog, and then as a translation itself. Strings could be very long, not just a couple of words, so this may bring more kB to the bundle.
+If you use natural language as an ID in your project, for example:
+```ts
+const message = t`My Message`
+```
+you will benefit significantly from this change. Your bundles will become smaller because the source message will be removed from the bundle in favor of a short generated ID.
 
-A much better option is generating a "stable" ID based on the msg + context as a hash with a fixed length.
+If you use natural language as an ID, you don't need to do anything special to migrate.
 
-Hash would be calculated at build time by macros. So macros instead of:
+However, if you use explicit IDs, like this:
 
-```js
-const message = t({
-   context: 'My context',
-   message: `Hello`
-})
+```ts
+const message = t({id: "my.message", message: `My Message`})
+```
 
-// ↓ ↓ ↓ ↓ ↓ ↓
+there are some changes you need to make to your catalogs to migrate properly. In order to distinguish between generated IDs and explicit IDs in the PO format, Lingui adds a special comment for messages with explicit IDs called `js-lingui-explicit-id`.
 
-import { i18n } from "@lingui/core"
-const message = i18n._(/*i18n*/{
-   context: 'My context',
-   id: `Hello`
-})
+Here's an example of the comment in a PO file:
+```gettext
+#. js-lingui-explicit-id
+msgid "custom.id"
+msgstr ""
 ```
 
-now generates:
+You need to add this comment manually to all your messages with explicit IDs.
+
+If you exclusively use explicit IDs in your project, you may consider enabling a different processing mode for the PO formatter. This can be done in your Lingui config file:
+```ts title="lingui.config.ts"
+import { formatter } from '@lingui/po-format'
+import { LinguiConfig } from '@lingui/config'
 
-```js
-import { i18n } from "@lingui/core"
-const message = i18n._(/*i18n*/{
-   id: "<hash(message + context)>",
-   message: `Hello`,
-})
+const config: LinguiConfig = {
+  // ...
+  format: formatter({ explicitIdAsDefault: true }),
+}
 ```
 
+Enabling this mode will swap the logic, and the formatter will treat all messages as having explicit IDs without the need for the explicit flag comment.
+
+You can read more about the motivation behind this change in the [original RFC](https://github.com/lingui/js-lingui/issues/1360)
+
 Also, we've added a possibility to provide a context for the message. For more details, see the [Providing a context for a message](/docs/tutorials/react-patterns.md#providing-a-context-for-a-message).
 
 The context feature affects the message ID generation and adds the `msgctxt` parameter in case of the PO catalog format extraction.
 
 This also affects the `orderBy` with `messageId` as now the generated id is used when custom id is absent. To avoid confusion, we switched the default `orderBy` to use the source message (`message`) instead.
 
+### Translation outside React components migration
+
+If you have been using the following pattern in your code:
+
+```tsx
+import { t } from "@lingui/macro"
+
+const myMsg = t`Hello world!`
+
+export function Greeting(props: {}) {
+  return <h1>{t(myMsg)}</h1>
+}
+```
+You will need to make some changes as this is a misuse of the library that actually worked in v3.
+
+Due to the changes caused by hash-based message ID feature described earlier, this approach will no longer work.
+
+Instead, please use [recommended](/docs/tutorials/react-patterns.md#lazy-translations) pattern for such translations:
+```tsx
+import { t } from "@lingui/macro"
+import { useLingui } from "@lingui/react"
+
+const myMsg = msg`Hello world!`
+
+export function Greeting(props: {}) {
+  const { i18n } = useLingui()
+
+  return <h1>{i18n._(myMsg)}</h1>
+}
+```
+
 ### Change in generated ICU messages for nested JSX Macros
 
 We have made a small change in how Lingui generates ICU messages for nested JSX Macros. We have removed leading spaces from the texts in all cases.
@@ -137,13 +177,13 @@ Extractor supports TypeScript out of the box. Please delete it from your configu
 If your extract command looks like:
 
 ```bash
-NODE_ENV=development lingui-extract
+NODE_ENV=development lingui extract
 ```
 
 Now you can safely change it to just:
 
 ```bash
-lingui-extract
+lingui extract
 ```
 
 ### Public interface of `ExtractorType` was changed