fix(cli): improve JSON stream performance (#4663)

packages/cli/generation/local-generation/local-workspace-runner/src/runGenerator.ts

@@ -157,7 +157,7 @@ async function writeIrToFile({
         tmpdir: workspaceTempDir.path
     });
     const absolutePathToIr = AbsoluteFilePath.of(irFile.path);
-    await streamObjectToFile(absolutePathToIr, ir, { pretty: true });
+    await streamObjectToFile(absolutePathToIr, ir, { pretty: false });
     context.logger.debug(`Wrote IR to ${absolutePathToIr}`);
     return absolutePathToIr;
 }

packages/commons/fs-utils/package.json

@@ -30,7 +30,7 @@
     "@fern-api/core-utils": "workspace:*",
     "glob": "^8.0.3",
     "glob-promise": "^5.0.0",
-    "json-stream-stringify": "^3.0.1",
+    "json-stream-stringify": "^3.1.4",
     "stream-json": "^1.8.0",
     "tmp-promise": "^3.0.3"
   },

packages/commons/fs-utils/src/streamObjectFromFile.ts

@@ -1,21 +1,17 @@
 import { createReadStream } from "fs";
 import { AbsoluteFilePath } from "./AbsoluteFilePath";
-import StreamObject from "stream-json/streamers/StreamObject.js";
+import StreamJSON from "stream-json";
+import Assembler from "stream-json/Assembler.js";
 
 export async function streamObjectFromFile(filePath: AbsoluteFilePath): Promise<unknown> {
-    const jsonStream = StreamObject.withParser();
-    const accumulator = {};
+    return new Promise((resolve, reject) => {
+        const jsonStream = StreamJSON.parser({
+            streamValues: false
+        });
 
-    jsonStream.on("data", ({ key, value }: { key: string; value: unknown }) =>
-        Object.assign(accumulator, { [key]: value })
-    );
+        const asm = Assembler.connectTo(jsonStream);
+        asm.on("done", ({ current }) => resolve(current));
 
-    return new Promise((resolve, reject) => {
-        createReadStream(filePath)
-            .pipe(jsonStream)
-            .on("error", reject)
-            .on("finish", () => {
-                resolve(accumulator);
-            });
+        createReadStream(filePath).pipe(jsonStream).on("error", reject);
     });
 }