Add auto-generation of TypeScript definitions on build (#38990)

Summary:
Pull Request resolved: #38990

This PR adds auto-generation of Typescript definitions from Flow source code for packages using the shared monorepo build setup (#38718).

Today, these are the following Node.js packages:

- `packages/community-cli-plugin`
- `packages/dev-middleware` (⬅️ `emitTypeScriptDefs` enabled)

This also improves emitted Flow definitions (`.js.flow`), by using [`flow-api-translator`](https://www.npmjs.com/package/flow-api-translator) to strip implementations.

**All changes**

- Include `flow-api-translator` and configure this to emit type definitions as part of `yarn build`.
    - Add translation from Flow source to TypeScript definitions (`.d.ts`) adjacent to each built file.
    - Improve emitted Flow definitions (`.js.flow`), by using `flow-api-translator` to strip implementations (previously, source files were copied). The Flow and TS defs now mirror each other.
-  Add `emitFlowDefs` and `emitTypeScriptDefs` options to build config to configure the above.
- Integrate TypeScript compiler to perform program validation on emitted `.d.ts` files.
     - This is based on this guide: https://github.com/microsoft/TypeScript-wiki/blob/main/Using-the-Compiler-API.md#a-minimal-compiler.
- Throw an exception on the `rewritePackageExports` step if a package does not define an `"exports"` field.
- Add minimal `flow-typed` definitions for `typescript` 😄.

**Notes on [`flow-api-translator`](https://www.npmjs.com/package/flow-api-translator)**

This project is experimental but is in a more mature state than when we evaluated it earlier in 2023.
- It's now possible to run this tool on our new Node.js packages, since they are exclusively authored using `import`/`export` syntax (a requirement of the tool).
- As a safety net, we run the TypeScript compiler against the generated program, which will fail the build.

Changelog: [Internal]

Reviewed By: robhogan

Differential Revision: D48312463

fbshipit-source-id: 817edb35f911f52fa987946f2d8fc1a319078c9d

flow-typed/npm/@tsconfig/node18_v1.x.x.js

@@ -0,0 +1,13 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @flow
+ * @format
+ */
+
+declare module '@tsconfig/node18/tsconfig.json' {
+  declare module.exports: any;
+}

flow-typed/npm/typescript_v5.x.x.js

@@ -0,0 +1,56 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @flow
+ * @format
+ */
+
+declare module 'typescript' {
+  declare enum ModuleResolutionKind {
+    Classic = 'Classic',
+    NodeJs = 'NodeJs',
+    Node10 = 'Node10',
+    Node16 = 'Node16',
+    NodeNext = 'NodeNext',
+    Bundler = 'Bundler',
+  }
+
+  declare type SourceFile = $ReadOnly<{
+    fileName: string,
+    text: string,
+    ...
+  }>;
+
+  declare type Diagnostic = $ReadOnly<{
+    file?: SourceFile,
+    start?: number,
+    messageText: string,
+    ...
+  }>;
+
+  declare type EmitResult = $ReadOnly<{
+    diagnostics: Array<Diagnostic>,
+    ...
+  }>;
+
+  declare type Program = $ReadOnly<{
+    emit: () => EmitResult,
+    ...
+  }>;
+
+  declare type TypeScriptAPI = {
+    createProgram(files: Array<string>, compilerOptions: Object): Program,
+    flattenDiagnosticMessageText: (...messageText: Array<string>) => string,
+    getLineAndCharacterOfPosition(
+      file: SourceFile,
+      start?: number,
+    ): $ReadOnly<{line: number, character: number}>,
+    ModuleResolutionKind: typeof ModuleResolutionKind,
+    ...
+  };
+
+  declare module.exports: TypeScriptAPI;
+}

package.json

@@ -57,6 +57,7 @@
     "@pkgjs/parseargs": "^0.11.0",
     "@react-native/metro-babel-transformer": "^0.73.11",
     "@react-native/metro-config": "^0.73.0",
+    "@tsconfig/node18": "1.0.1",
     "@types/react": "^18.0.18",
     "@typescript-eslint/parser": "^5.57.1",
     "async": "^3.2.2",
@@ -81,6 +82,7 @@
     "eslint-plugin-react-native": "^4.0.0",
     "eslint-plugin-redundant-undefined": "^0.4.0",
     "eslint-plugin-relay": "^1.8.3",
+    "flow-api-translator": "0.15.0",
     "flow-bin": "^0.214.0",
     "glob": "^7.1.1",
     "hermes-eslint": "0.15.0",

packages/dev-middleware/src/index.js

@@ -17,4 +17,4 @@ if (!process.env.BUILD_EXCLUDE_BABEL_REGISTER) {
   require('../../../scripts/build/babel-register').registerForMonorepo();
 }
 
-module.exports = require('./index.flow');
+export * from './index.flow';

scripts/build/babel/node.config.js

@@ -0,0 +1,44 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @flow
+ * @format
+ * @oncall react_native
+ */
+
+/*::
+import type {BabelCoreOptions} from '@babel/core';
+*/
+
+const TARGET_NODE_VERSION = '18';
+
+const config /*: BabelCoreOptions */ = {
+  presets: [
+    '@babel/preset-flow',
+    [
+      '@babel/preset-env',
+      {
+        targets: {
+          node: TARGET_NODE_VERSION,
+        },
+      },
+    ],
+  ],
+  plugins: [
+    [
+      'transform-define',
+      {
+        'process.env.BUILD_EXCLUDE_BABEL_REGISTER': true,
+      },
+    ],
+    [
+      'minify-dead-code-elimination',
+      {keepFnName: true, keepFnArgs: true, keepClassName: true},
+    ],
+  ],
+};
+
+module.exports = config;

scripts/build/build.js

@@ -12,14 +12,22 @@
 const babel = require('@babel/core');
 const {parseArgs} = require('@pkgjs/parseargs');
 const chalk = require('chalk');
+const translate = require('flow-api-translator');
 const glob = require('glob');
 const micromatch = require('micromatch');
-const fs = require('fs');
+const {promises: fs} = require('fs');
 const path = require('path');
 const prettier = require('prettier');
-const {buildConfig, getBabelConfig} = require('./config');
-
-const PACKAGES_DIR /*: string */ = path.resolve(__dirname, '../../packages');
+const ts = require('typescript');
+const {
+  buildConfig,
+  getBabelConfig,
+  getBuildOptions,
+  getTypeScriptCompilerOptions,
+} = require('./config');
+
+const REPO_ROOT = path.resolve(__dirname, '../..');
+const PACKAGES_DIR /*: string */ = path.join(REPO_ROOT, 'packages');
 const SRC_DIR = 'src';
 const BUILD_DIR = 'dist';
 const JS_FILES_PATTERN = '**/*.js';
@@ -32,7 +40,7 @@ const config = {
   },
 };
 
-function build() {
+async function build() {
   const {
     positionals: packageNames,
     values: {help},
@@ -53,35 +61,48 @@ function build() {
 
   console.log('\n' + chalk.bold.inverse('Building packages') + '\n');
 
-  if (packageNames.length) {
-    packageNames
-      .filter(packageName => packageName in buildConfig.packages)
-      .forEach(buildPackage);
-  } else {
-    Object.keys(buildConfig.packages).forEach(buildPackage);
+  const packagesToBuild = packageNames.length
+    ? packageNames.filter(packageName => packageName in buildConfig.packages)
+    : Object.keys(buildConfig.packages);
+
+  for (const packageName of packagesToBuild) {
+    await buildPackage(packageName);
   }
 
   process.exitCode = 0;
 }
 
-function buildPackage(packageName /*: string */) {
+async function buildPackage(packageName /*: string */) {
+  const {emitTypeScriptDefs} = getBuildOptions(packageName);
   const files = glob.sync(
     path.resolve(PACKAGES_DIR, packageName, SRC_DIR, '**/*'),
     {nodir: true},
   );
-  const packageJsonPath = path.join(PACKAGES_DIR, packageName, 'package.json');
 
   process.stdout.write(
     `${packageName} ${chalk.dim('.').repeat(72 - packageName.length)} `,
   );
-  files.forEach(file => buildFile(path.normalize(file), true));
-  rewritePackageExports(packageJsonPath);
+
+  // Build all files matched for package
+  for (const file of files) {
+    await buildFile(path.normalize(file), true);
+  }
+
+  // Validate program for emitted .d.ts files
+  if (emitTypeScriptDefs) {
+    validateTypeScriptDefs(packageName);
+  }
+
+  // Rewrite package.json "exports" field (src -> dist)
+  await rewritePackageExports(packageName);
+
   process.stdout.write(chalk.reset.inverse.bold.green(' DONE ') + '\n');
 }
 
-function buildFile(file /*: string */, silent /*: boolean */ = false) {
+async function buildFile(file /*: string */, silent /*: boolean */ = false) {
   const packageName = getPackageName(file);
   const buildPath = getBuildPath(file);
+  const {emitFlowDefs, emitTypeScriptDefs} = getBuildOptions(packageName);
 
   const logResult = ({copied, desc} /*: {copied: boolean, desc?: string} */) =>
     silent ||
@@ -97,24 +118,43 @@ function buildFile(file /*: string */, silent /*: boolean */ = false) {
     return;
   }
 
-  fs.mkdirSync(path.dirname(buildPath), {recursive: true});
+  await fs.mkdir(path.dirname(buildPath), {recursive: true});
 
   if (!micromatch.isMatch(file, JS_FILES_PATTERN)) {
-    fs.copyFileSync(file, buildPath);
+    await fs.copyFile(file, buildPath);
     logResult({copied: true, desc: 'copy'});
-  } else {
-    const transformed = prettier.format(
-      babel.transformFileSync(file, getBabelConfig(packageName)).code,
-      {parser: 'babel'},
-    );
-    fs.writeFileSync(buildPath, transformed);
+    return;
+  }
 
-    if (/@flow/.test(fs.readFileSync(file, 'utf-8'))) {
-      fs.copyFileSync(file, buildPath + '.flow');
-    }
+  const source = await fs.readFile(file, 'utf-8');
+  const prettierConfig = {parser: 'babel'};
 
-    logResult({copied: true});
+  // Transform source file using Babel
+  const transformed = prettier.format(
+    (await babel.transformFileAsync(file, getBabelConfig(packageName))).code,
+    prettierConfig,
+  );
+  await fs.writeFile(buildPath, transformed);
+
+  // Translate source Flow types for each type definition target
+  if (/@flow/.test(source)) {
+    await Promise.all([
+      emitFlowDefs
+        ? fs.writeFile(
+            buildPath + '.flow',
+            await translate.translateFlowToFlowDef(source, prettierConfig),
+          )
+        : null,
+      emitTypeScriptDefs
+        ? fs.writeFile(
+            buildPath.replace(/\.js$/, '') + '.d.ts',
+            await translate.translateFlowToTSDef(source, prettierConfig),
+          )
+        : null,
+    ]);
   }
+
+  logResult({copied: true});
 }
 
 function getPackageName(file /*: string */) /*: string */ {
@@ -130,16 +170,22 @@ function getBuildPath(file /*: string */) /*: string */ {
   );
 }
 
-function rewritePackageExports(packageJsonPath /*: string */) {
-  const pkg = JSON.parse(fs.readFileSync(packageJsonPath, {encoding: 'utf8'}));
+async function rewritePackageExports(packageName /*: string */) {
+  const packageJsonPath = path.join(PACKAGES_DIR, packageName, 'package.json');
+  const pkg = JSON.parse(await fs.readFile(packageJsonPath, 'utf8'));
 
   if (pkg.exports == null) {
-    return;
+    throw new Error(
+      packageName +
+        ' does not define an "exports" field in its package.json. As part ' +
+        'of the build setup, this field must be used in order to rewrite ' +
+        'paths to built files in production.',
+    );
   }
 
   pkg.exports = rewriteExportsField(pkg.exports);
 
-  fs.writeFileSync(
+  await fs.writeFile(
     packageJsonPath,
     prettier.format(JSON.stringify(pkg), {parser: 'json'}),
   );
@@ -173,6 +219,49 @@ function rewriteExportsTarget(target /*: string */) /*: string */ {
   return target.replace('./' + SRC_DIR + '/', './' + BUILD_DIR + '/');
 }
 
+function validateTypeScriptDefs(packageName /*: string */) {
+  const files = glob.sync(
+    path.resolve(PACKAGES_DIR, packageName, BUILD_DIR, '**/*.d.ts'),
+  );
+  const compilerOptions = {
+    ...getTypeScriptCompilerOptions(packageName),
+    noEmit: true,
+    skipLibCheck: false,
+  };
+  const program = ts.createProgram(files, compilerOptions);
+  const emitResult = program.emit();
+
+  if (emitResult.diagnostics.length) {
+    for (const diagnostic of emitResult.diagnostics) {
+      if (diagnostic.file != null) {
+        let {line, character} = ts.getLineAndCharacterOfPosition(
+          diagnostic.file,
+          diagnostic.start,
+        );
+        let message = ts.flattenDiagnosticMessageText(
+          diagnostic.messageText,
+          '\n',
+        );
+        console.log(
+          // $FlowIssue[incompatible-use] Type refined above
+          `${diagnostic.file.fileName} (${line + 1},${
+            character + 1
+          }): ${message}`,
+        );
+      } else {
+        console.log(
+          ts.flattenDiagnosticMessageText(diagnostic.messageText, '\n'),
+        );
+      }
+    }
+
+    throw new Error(
+      'Failing build because TypeScript errors were encountered for ' +
+        'generated type definitions.',
+    );
+  }
+}
+
 module.exports = {
   buildFile,
   getBuildPath,
@@ -182,5 +271,6 @@ module.exports = {
 };
 
 if (require.main === module) {
-  build();
+  // eslint-disable-next-line no-void
+  void build();
 }