feat(cli): Extract - Flatten ICU messages

jest.config.integration.js

@@ -8,8 +8,6 @@ module.exports = {
 
   roots: ["<rootDir>/packages/"],
   testPathIgnorePatterns: ["/node_modules/"],
-  // Redirect imports to the compiled bundles
-  moduleNameMapper: {},
   setupFiles: ["set-tz/utc"],
 
   // Exclude the build output from transforms

packages/cli/src/api/__snapshots__/catalog.test.ts.snap

@@ -244,6 +244,78 @@ Object {
 }
 `;
 
+exports[`Catalog make should collect and write catalogs with flatten ICU messages 1`] = `
+Object {
+  cs: null,
+  en: null,
+}
+`;
+
+exports[`Catalog make should collect and write catalogs with flatten ICU messages 2`] = `
+Object {
+  cs: Object {
+    {count, plural, one {This is my #st cat.} two {This is my #nd cat.} other {This is my #rd cat.}}: Object {
+      comments: Array [],
+      context: null,
+      extractedComments: Array [],
+      flags: Array [],
+      obsolete: false,
+      origin: Array [
+        Array [
+          collect-flatten/component.js,
+          1,
+        ],
+      ],
+      translation: ,
+    },
+    {gender, select, female {She added a new image to the system.} male {He added a new image to the system.} other {They added a new image to the system.}}: Object {
+      comments: Array [],
+      context: null,
+      extractedComments: Array [],
+      flags: Array [],
+      obsolete: false,
+      origin: Array [
+        Array [
+          collect-flatten/component.js,
+          2,
+        ],
+      ],
+      translation: ,
+    },
+  },
+  en: Object {
+    {count, plural, one {This is my #st cat.} two {This is my #nd cat.} other {This is my #rd cat.}}: Object {
+      comments: Array [],
+      context: null,
+      extractedComments: Array [],
+      flags: Array [],
+      obsolete: false,
+      origin: Array [
+        Array [
+          collect-flatten/component.js,
+          1,
+        ],
+      ],
+      translation: ,
+    },
+    {gender, select, female {She added a new image to the system.} male {He added a new image to the system.} other {They added a new image to the system.}}: Object {
+      comments: Array [],
+      context: null,
+      extractedComments: Array [],
+      flags: Array [],
+      obsolete: false,
+      origin: Array [
+        Array [
+          collect-flatten/component.js,
+          2,
+        ],
+      ],
+      translation: ,
+    },
+  },
+}
+`;
+
 exports[`Catalog make should merge with existing catalogs 1`] = `
 Object {
   cs: Object {
@@ -546,6 +618,41 @@ Object {
 }
 `;
 
+exports[`Catalog makeTemplate should collect and write a template with flatten ICU messages 1`] = `null`;
+
+exports[`Catalog makeTemplate should collect and write a template with flatten ICU messages 2`] = `
+Object {
+  {count, plural, one {This is my #st cat.} two {This is my #nd cat.} other {This is my #rd cat.}}: Object {
+    comments: Array [],
+    context: null,
+    extractedComments: Array [],
+    flags: Array [],
+    obsolete: false,
+    origin: Array [
+      Array [
+        collect-flatten/component.js,
+        1,
+      ],
+    ],
+    translation: ,
+  },
+  {gender, select, female {She added a new image to the system.} male {He added a new image to the system.} other {They added a new image to the system.}}: Object {
+    comments: Array [],
+    context: null,
+    extractedComments: Array [],
+    flags: Array [],
+    obsolete: false,
+    origin: Array [
+      Array [
+        collect-flatten/component.js,
+        2,
+      ],
+    ],
+    translation: ,
+  },
+}
+`;
+
 exports[`Catalog read should read file in given format 1`] = `
 Object {
   obsolete: Object {

packages/cli/src/api/catalog.test.ts

@@ -111,6 +111,27 @@ describe("Catalog", () => {
       await catalog.make(defaultMakeOptions)
       expect(catalog.readAll()).toMatchSnapshot()
     })
+
+    it("should collect and write catalogs with flatten ICU messages", async () => {
+      const localeDir = copyFixture(fixture("locales", "initial"))
+      const catalog = new Catalog(
+        {
+          name: "messages",
+          path: path.join(localeDir, "{locale}", "messages"),
+          include: [fixture("collect-flatten/")],
+          exclude: [],
+        },
+        mockConfig({
+          locales: ["en", "cs"],
+        })
+      )
+
+      // Everything should be empty
+      expect(catalog.readAll()).toMatchSnapshot()
+
+      await catalog.make({ ...defaultMakeOptions, flatten: true })
+      expect(catalog.readAll()).toMatchSnapshot()
+    })
   })
 
   describe("makeTemplate", () => {
@@ -137,6 +158,27 @@ describe("Catalog", () => {
       await catalog.makeTemplate(defaultMakeTemplateOptions)
       expect(catalog.readTemplate()).toMatchSnapshot()
     })
+
+    it("should collect and write a template with flatten ICU messages", async () => {
+      const localeDir = copyFixture(fixture("locales", "initial"))
+      const catalog = new Catalog(
+        {
+          name: "messages",
+          path: path.join(localeDir, "{locale}", "messages"),
+          include: [fixture("collect-flatten/")],
+          exclude: [],
+        },
+        mockConfig({
+          locales: ["en", "cs"],
+        })
+      )
+
+      // Everything should be empty
+      expect(catalog.readTemplate()).toMatchSnapshot()
+
+      await catalog.makeTemplate({ ...defaultMakeOptions, flatten: true })
+      expect(catalog.readTemplate()).toMatchSnapshot()
+    })
   })
 
   describe("POT Flow", () => {

packages/cli/src/api/catalog.ts

@@ -20,6 +20,7 @@ import extract, { ExtractedMessage } from "./extractors"
 import { CliExtractOptions } from "../lingui-extract"
 import { CliExtractTemplateOptions } from "../lingui-extract-template"
 import { CompiledCatalogNamespace } from "./compile"
+import { flattenMessage } from "@lingui/core/flatten"
 import { prettyOrigin } from "./utils"
 import chalk from "chalk"
 
@@ -177,33 +178,38 @@ export class Catalog {
       const fileSuccess = await extract(
         filename,
         (next: ExtractedMessage) => {
-          if (!messages[next.id]) {
-            messages[next.id] = {
-              message: next.message,
+          const nextId = options.flatten ? flattenMessage(next.id) : next.id
+          const nextMessage = options.flatten
+            ? flattenMessage(next.message)
+            : next.message
+
+          if (!messages[nextId]) {
+            messages[nextId] = {
+              message: nextMessage,
               extractedComments: [],
               origin: [],
             }
           }
 
-          const prev = messages[next.id]
+          const prev = messages[nextId]
 
           const filename = path
             .relative(this.config.rootDir, next.origin[0])
             .replace(/\\/g, "/")
 
           const origin: MessageOrigin = [filename, next.origin[1]]
 
-          if (prev.message && next.message && prev.message !== next.message) {
+          if (prev.message && nextMessage && prev.message !== nextMessage) {
             throw new Error(
               `Encountered different default translations for message ${chalk.yellow(
-                next.id
+                nextId
               )}` +
                 `\n${chalk.yellow(prettyOrigin(prev.origin))} ${prev.message}` +
-                `\n${chalk.yellow(prettyOrigin([origin]))} ${next.message}`
+                `\n${chalk.yellow(prettyOrigin([origin]))} ${nextMessage}`
             )
           }
 
-          messages[next.id] = {
+          messages[nextId] = {
             ...prev,
             extractedComments: next.comment
               ? [...prev.extractedComments, next.comment]

packages/cli/src/api/fixtures/collect-flatten/component.js

@@ -0,0 +1,2 @@
+/*i18n*/ i18n._("This is my {count, plural, one {#st} two {#nd} other {#rd}} cat.", { count })
+/*i18n*/ i18n._("{gender, select, female {She} male {He} other {They}} added a new image to the system.", { gender })

packages/cli/src/lingui-extract-template.ts

@@ -12,6 +12,7 @@ export type CliExtractTemplateOptions = {
   configPath: string
   extractors?: ExtractorType[]
   files?: string[]
+  flatten?: boolean
 }
 
 export default async function command(
@@ -61,6 +62,7 @@ export default async function command(
 if (require.main === module) {
   program
     .option("--config <path>", "Path to the config file")
+    .option("--flatten", "Flattens ICU messages")
     .option("--verbose", "Verbose output")
     .parse(process.argv)
 
@@ -69,6 +71,7 @@ if (require.main === module) {
   })
 
   const result = command(config, {
+    flatten: program.flatten || false,
     verbose: program.verbose || false,
     configPath: program.config || process.env.LINGUI_CONFIG,
   }).then(() => {

packages/cli/src/lingui-extract.ts

@@ -20,6 +20,7 @@ export type CliExtractOptions = {
   locale: string
   prevFormat: string | null
   watch?: boolean
+  flatten?: boolean
 }
 
 export default async function command(
@@ -108,6 +109,7 @@ if (require.main === module) {
       "Convert from previous format of message catalogs"
     )
     .option("--watch", "Enables Watch Mode")
+    .option("--flatten", "Flattens ICU messages")
     // Obsolete options
     .option(
       "--babelOptions",
@@ -169,6 +171,7 @@ if (require.main === module) {
       locale: program.locale,
       configPath: program.config || process.env.LINGUI_CONFIG,
       watch: program.watch || false,
+      flatten: program.flatten || false,
       files: filePath?.length ? filePath : undefined,
       prevFormat,
     })

packages/cli/src/tests.ts

@@ -30,6 +30,7 @@ export const defaultMakeOptions: MakeOptions = {
   prevFormat: null,
   configPath: null,
   orderBy: "messageId",
+  flatten: false,
 }
 
 export const defaultMakeTemplateOptions: MakeTemplateOptions = {

packages/core/flatten.entry.ts

@@ -0,0 +1,4 @@
+/**
+ * this is Rollup Entry
+ */
+export * from "./src/compile"

packages/core/flatten.js

@@ -0,0 +1,5 @@
+/**
+ * This entry is for old runtimes which do not support `exports` field in package.json
+ * https://github.com/facebook/metro/issues/670
+ */
+module.exports = require("./build/cjs/flatten.js")

packages/core/package.json

@@ -40,6 +40,16 @@
         "default": "./build/esm/index.js"
       }
     },
+    "./flatten": {
+      "require": {
+        "types": "./build/flatten.d.ts",
+        "default": "./build/cjs/flatten.js"
+      },
+      "import": {
+        "types": "./build/flatten.d.ts",
+        "default": "./build/esm/flatten.js"
+      }
+    },
     "./compile": {
       "require": {
         "types": "./build/compile.d.ts",

packages/core/src/flatten/flattenMessage.test.ts

@@ -0,0 +1,50 @@
+import { mockConsole } from "@lingui/jest-mocks"
+import { flattenMessage as flatten } from "./flattenMessage"
+
+describe("flatten", () => {
+  it("should flatten selectordinal", () => {
+    const message = flatten(
+      "It's my dog's {year, selectordinal, one {#st} two {#nd} few {#rd} other {#th}} birthday!"
+    )
+    expect(message).toEqual(
+      "{year, selectordinal, one {It's my dog's #st birthday!} two {It's my dog's #nd birthday!} few {It's my dog's #rd birthday!} other {It's my dog's #th birthday!}}"
+    )
+  })
+
+  it("should flatten plural", () => {
+    const message = flatten(
+      "I have {value, plural, one {{value} book} other {# books}}"
+    )
+    expect(message).toEqual(
+      "{value, plural, one {I have {value} book} other {I have # books}}"
+    )
+  })
+
+  it("should flatten select with a placeholder in a previous sentence", () => {
+    const message = flatten(
+      "Hello, Your friend {friend} is now online. {gender, select, female {She} male {He} other {They}} added a new image!"
+    )
+    expect(message).toEqual(
+      "{gender, select, female {Hello, Your friend {friend} is now online. She added a new image!} male {Hello, Your friend {friend} is now online. He added a new image!} other {Hello, Your friend {friend} is now online. They added a new image!}}"
+    )
+  })
+
+  it("should flatten plural with number", () => {
+    const message = flatten(
+      "You have {count, plural, one {{count, number} dog} other {{count, number} dogs}}"
+    )
+    expect(message).toEqual(
+      "{count, plural, one {You have {count, number} dog} other {You have {count, number} dogs}}"
+    )
+  })
+
+  it("should throw error for invalid ICU message", () => {
+    mockConsole((console) => {
+      flatten("{foo, plural, =a{e1} other{baz}}")
+
+      expect(console.error).toBeCalledWith(
+        expect.stringContaining(`invalid syntax`)
+      )
+    })
+  })
+})