feat(): Experimental Deps extractor

.github/workflows/main-suite.yml

@@ -29,7 +29,7 @@ jobs:
           scope: '@lingui'
 
       - name: Install dependencies if needed
-        run: yarn install
+        run: yarn install --network-timeout 1000000
 
       - name: Unit Testing
         run: yarn test:ci

.prettierignore

@@ -2,9 +2,11 @@
 **/dist/*
 **/test/fixtures/**
 **/test/**/actual.js
+**/test/**/actual/**
 **/test/**/expected.js
 **/locale/*
 **/fixtures/*
+**/expected/*
 coverage/
 website/
 .github

packages/cli/package.json

@@ -62,6 +62,7 @@
     "commander": "^10.0.0",
     "convert-source-map": "^2.0.0",
     "date-fns": "^2.16.1",
+    "esbuild": "^0.17.10",
     "glob": "^7.1.4",
     "inquirer": "^7.3.3",
     "micromatch": "4.0.2",
@@ -70,6 +71,7 @@
     "normalize-path": "^3.0.0",
     "ora": "^5.1.0",
     "papaparse": "^5.3.0",
+    "pkg-up": "^3.1.0",
     "plurals-cldr": "^1.0.4",
     "pofile": "^1.1.4",
     "pseudolocale": "^1.1.0",

packages/cli/src/api/catalog.ts

@@ -99,11 +99,8 @@ export class Catalog {
 
     const sortedCatalogs = cleanAndSort(catalogs)
 
-    if (options.locale) {
-      this.write(options.locale, sortedCatalogs[options.locale])
-    } else {
-      this.writeAll(sortedCatalogs)
-    }
+    const locales = options.locale ? [options.locale] : this.locales
+    locales.forEach((locale) => this.write(locale, sortedCatalogs[locale]))
 
     return sortedCatalogs
   }
@@ -188,10 +185,6 @@ export class Catalog {
     return [created, filename]
   }
 
-  writeAll(catalogs: AllCatalogsType): void {
-    this.locales.forEach((locale) => this.write(locale, catalogs[locale]))
-  }
-
   writeTemplate(messages: CatalogType) {
     const filename = this.templateFile
     const options: CatalogFormatOptionsInternal = {
@@ -298,9 +291,7 @@ export function order<T extends ExtractedCatalogType>(
  * Object keys are in the same order as they were created
  * https://stackoverflow.com/a/31102605/1535540
  */
-export function orderByMessageId<T extends ExtractedCatalogType>(
-  messages: T
-): T {
+function orderByMessageId<T extends ExtractedCatalogType>(messages: T): T {
   return Object.keys(messages)
     .sort()
     .reduce((acc, key) => {
@@ -309,7 +300,7 @@ export function orderByMessageId<T extends ExtractedCatalogType>(
     }, {} as T)
 }
 
-export function orderByOrigin<T extends ExtractedCatalogType>(messages: T): T {
+function orderByOrigin<T extends ExtractedCatalogType>(messages: T): T {
   function getFirstOrigin(messageKey: string) {
     const sortedOrigins = messages[messageKey].origin.sort((a, b) => {
       if (a[0] < b[0]) return -1

packages/cli/src/api/catalog/getCatalogs.ts

@@ -5,6 +5,7 @@ import { Catalog } from "../catalog"
 import { normalizeRelativePath, PATHSEP, replacePlaceholders } from "../utils"
 import micromatch from "micromatch"
 import { getFormat } from "../formats"
+import { getExperimentalCatalogs } from "../../extract-experimental/getExperimentalCatalogs"
 
 const NAME_PH = "{name}"
 const LOCALE_PH = "{locale}"
@@ -78,6 +79,10 @@ export function getCatalogs(config: LinguiConfigNormalized): Catalog[] {
     })
   })
 
+  if (config.experimental?.extractor?.entries.length) {
+    catalogs.push(...getExperimentalCatalogs(config))
+  }
+
   return catalogs
 }
 

packages/cli/src/api/catalog/mergeCatalog.ts

@@ -7,7 +7,7 @@ export function mergeCatalog(
   nextCatalog: ExtractedCatalogType,
   forSourceLocale: boolean,
   options: MergeOptions
-) {
+): CatalogType {
   const nextKeys = Object.keys(nextCatalog)
 
   const prevKeys = R.keys(prevCatalog).map(String)

packages/cli/src/api/utils.test.ts

@@ -22,6 +22,14 @@ describe("replacePlaceholders", () => {
 
     expect(actual).toBe("/foo/bar/{a}/b-value/{a}")
   })
+
+  it("should replace to empty string", () => {
+    const actual = replacePlaceholders("/foo/bar/{a}.file", {
+      a: "",
+    })
+
+    expect(actual).toBe("/foo/bar/.file")
+  })
 })
 
 describe("normalizeRelativePath", () => {

packages/cli/src/api/utils.ts

@@ -17,7 +17,7 @@ export function replacePlaceholders(
   values: Record<string, string>
 ): string {
   return input.replace(/\{([^}]+)}/g, (m, placeholder) => {
-    return values[placeholder] || m
+    return values[placeholder] ?? m
   })
 }
 

packages/cli/src/extract-experimental/buildExternalizeFilter.test.ts

@@ -0,0 +1,84 @@
+import { buildExternalizeFilter } from "./buildExternalizeFilter"
+
+describe("buildExternalizeFilter", () => {
+  it("should exclude packages from package.json", () => {
+    const isExternal = buildExternalizeFilter({
+      includeDeps: [],
+      excludeDeps: [],
+      packageJson: {
+        dependencies: {
+          next: "*",
+        },
+        devDependencies: {},
+        optionalDependencies: {},
+        peerDependencies: {},
+      },
+    })
+
+    expect(isExternal("foo")).toBeFalsy()
+    expect(isExternal("next")).toBeTruthy()
+    expect(isExternal("next/head")).toBeTruthy()
+    expect(isExternal("@next/head")).toBeFalsy()
+  })
+
+  it("should respect all packageJson dependencies", () => {
+    const isExternal = buildExternalizeFilter({
+      includeDeps: [],
+      excludeDeps: [],
+      packageJson: {
+        dependencies: {
+          package1: "*",
+        },
+        devDependencies: {
+          package2: "*",
+        },
+        optionalDependencies: {
+          package3: "*",
+        },
+        peerDependencies: {
+          package4: "*",
+        },
+      },
+    })
+
+    expect(isExternal("non-deps")).toBeFalsy()
+    expect(isExternal("package1")).toBeTruthy()
+    expect(isExternal("package2")).toBeTruthy()
+    expect(isExternal("package3")).toBeTruthy()
+    expect(isExternal("package4")).toBeTruthy()
+  })
+
+  it("should not externalize packages from includeDeps", () => {
+    const isExternal = buildExternalizeFilter({
+      includeDeps: ["package1"],
+      excludeDeps: [],
+      packageJson: {
+        dependencies: {
+          package1: "*",
+          package2: "*",
+        },
+      },
+    })
+
+    expect(isExternal("package1")).toBeFalsy()
+    expect(isExternal("package2")).toBeTruthy()
+  })
+
+  it("should externalize deps from excludeDeps", () => {
+    const isExternal = buildExternalizeFilter({
+      includeDeps: ["package1"],
+      excludeDeps: ["package2"],
+      packageJson: {
+        dependencies: {
+          package1: "*",
+          package3: "*",
+        },
+      },
+    })
+
+    expect(isExternal("package1")).toBeFalsy()
+    expect(isExternal("package2")).toBeTruthy()
+    expect(isExternal("package3")).toBeTruthy()
+    expect(isExternal("non-deps")).toBeFalsy()
+  })
+})

packages/cli/src/extract-experimental/buildExternalizeFilter.ts

@@ -0,0 +1,60 @@
+type PackageJson = {
+  dependencies?: Record<string, string>
+  devDependencies?: Record<string, string>
+  peerDependencies?: Record<string, string>
+  optionalDependencies?: Record<string, string>
+}
+
+function createPackageRegExp(packageName: string) {
+  return new RegExp("^" + packageName + "(?:\\/.+)?")
+}
+
+function packages(
+  packages: Record<string, string>,
+  includeDeps: string[]
+): RegExp[] {
+  return Object.keys(packages || {})
+    .filter((packageName) => {
+      return !includeDeps.some((incl) => packageName.startsWith(incl))
+    })
+    .map(createPackageRegExp)
+}
+
+export function buildExternalizeFilter({
+  includeDeps,
+  excludeDeps,
+  packageJson,
+}: {
+  includeDeps: string[]
+  excludeDeps: string[]
+  packageJson: PackageJson
+}) {
+  const external = [
+    ...packages(packageJson.dependencies, includeDeps),
+    ...packages(packageJson.devDependencies, includeDeps),
+    ...packages(packageJson.peerDependencies, includeDeps),
+    ...packages(packageJson.optionalDependencies, includeDeps),
+    ...excludeDeps.map(createPackageRegExp),
+  ]
+
+  return (id: string) =>
+    external.some((regExp) => {
+      return regExp.test(id)
+    })
+}
+
+export async function getPackageJson(rootDir: string): Promise<PackageJson> {
+  const { default: pkgUp } = await import("pkg-up")
+  const packageJsonPath = await pkgUp({
+    cwd: rootDir,
+  })
+
+  if (!packageJsonPath) {
+    throw new Error(
+      "We could not able to find your package.json file. " +
+        "Check that `rootDir` is pointing to the folder with package.json"
+    )
+  }
+
+  return await import(packageJsonPath)
+}

packages/cli/src/extract-experimental/bundleSource.ts

@@ -0,0 +1,96 @@
+import { ExperimentalExtractorOptions } from "@lingui/conf"
+import { BuildOptions } from "esbuild"
+import {
+  buildExternalizeFilter,
+  getPackageJson,
+} from "./buildExternalizeFilter"
+
+function createExtRegExp(extensions: string[]) {
+  return new RegExp("\\.(?:" + extensions.join("|") + ")(?:\\?.*)?$")
+}
+
+export async function bundleSource(
+  config: ExperimentalExtractorOptions,
+  entryPoints: string[],
+  outDir: string,
+  rootDir: string
+) {
+  const esbuild = await import("esbuild")
+
+  const excludeExtensions = config.excludeExtensions || [
+    "ico",
+    "pot",
+    "xliff",
+    "woff2",
+    "woff",
+    "eot",
+    "gif",
+    "otf",
+    "ttf",
+    "mp4",
+    "svg",
+    "png",
+    "css",
+    "sass",
+    "less",
+    "jpg",
+  ]
+
+  const packageJson = await getPackageJson(rootDir)
+
+  const esbuildOptions: BuildOptions = {
+    entryPoints: entryPoints,
+    outExtension: { ".js": ".jsx" },
+    jsx: "preserve",
+    bundle: true,
+    platform: "node",
+    target: ["esnext"],
+    format: "esm",
+    splitting: false,
+    treeShaking: true,
+    outdir: outDir,
+    sourcemap: "inline",
+    sourceRoot: outDir,
+    sourcesContent: false,
+    outbase: rootDir,
+    metafile: true,
+    plugins: [
+      {
+        name: "externalize-deps",
+        setup(build) {
+          const isExternal = buildExternalizeFilter({
+            includeDeps: config.includeDeps || [],
+            excludeDeps: config.excludeDeps || [],
+            packageJson,
+          })
+
+          // externalize bare imports
+          build.onResolve({ filter: /^[^.].*/ }, async ({ path: id }) => {
+            if (isExternal(id)) {
+              return {
+                external: true,
+              }
+            }
+          })
+        },
+      },
+      {
+        name: "externalize-files",
+        setup(build) {
+          build.onResolve(
+            { filter: createExtRegExp(excludeExtensions) },
+            () => ({
+              external: true,
+            })
+          )
+        },
+      },
+    ],
+  }
+
+  return await esbuild.build(
+    config.resolveEsbuildOptions
+      ? config.resolveEsbuildOptions(esbuildOptions)
+      : esbuildOptions
+  )
+}

packages/cli/src/extract-experimental/constants.ts

@@ -0,0 +1,2 @@
+export const ENTRY_NAME_PH = "{entryName}"
+export const DEFAULT_TEMPLATE_NAME = "messages"

packages/cli/src/extract-experimental/getEntryPoints.ts

@@ -0,0 +1,7 @@
+import glob from "glob"
+
+export function getEntryPoints(entries: string[]) {
+  const patterns = entries.length > 1 ? `{${entries.join(",")}}` : entries[0]
+
+  return glob.sync(patterns, { mark: true })
+}