feat(parser/napi): add flexbuffer to AST transfer (2x speedup) (#1680)

Hi! I have created a proof of concept of improving using oxc in
JavaScript. The method is not polished but it provides valuable insights
for future direction!

Feel free to close~ It is for reference only :)

# Context 

This is a proof of concept implementation of passing binary AST to
JavaScript. JavaScript can selectively read flexbuffers-based AST nodes
on demand to avoid the deserialization toll. More context
[here](https://dev.to/herrington_darkholme/benchmark-typescript-parsers-demystify-rust-tooling-performance-2go8).

# Changes

* Add a `parseSyncBuffer` napi method to return a binary AST from Rust
to JavaScript. The AST is in flexbuffer format.
* Add a `test_buffer.js` to test usage of flexbuffers in JavaScript. It
is in cjs format because flexbuffers does not support ESM :/

# Result
Some preliminary results, for reference only.

```
~ node test_buffer.js
testJSON: 4.043s
testBuffer: 2.395s
```

Buffer based API is 100% faster than JSON.

# Future Ideas
* Flexbuffers itself is slow. A better binary protocol is desired!
* Using binary reader to traverse AST is undesirable. A proxy-based API
to emulate object behavior will be nice.

napi/parser/Cargo.toml

@@ -26,7 +26,9 @@ oxc_parser    = { workspace = true }
 oxc_ast       = { workspace = true, features = ["serde"] }
 oxc_span      = { workspace = true }
 
+serde = { workspace = true }
 serde_json = { workspace = true }
+flexbuffers = { version = "2.0.0" }
 miette     = { workspace = true, features = ["fancy-no-backtrace"] }
 
 tokio       = { workspace = true }

napi/parser/index.d.ts

@@ -33,6 +33,11 @@ export function parseWithoutReturn(sourceText: string, options?: ParserOptions |
  * * Serde JSON serialization
  */
 export function parseSync(sourceText: string, options?: ParserOptions | undefined | null): ParseResult
+/**
+ * Returns a binary AST in flexbuffers format.
+ * This is a POC API. Error handling is not done yet.
+ */
+export function parseSyncBuffer(sourceText: string, options?: ParserOptions | undefined | null): Buffer
 /**
  * # Panics
  *

napi/parser/index.js

@@ -252,8 +252,9 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { parseWithoutReturn, parseSync, parseAsync } = nativeBinding
+const { parseWithoutReturn, parseSync, parseSyncBuffer, parseAsync } = nativeBinding
 
 module.exports.parseWithoutReturn = parseWithoutReturn
 module.exports.parseSync = parseSync
+module.exports.parseSyncBuffer = parseSyncBuffer
 module.exports.parseAsync = parseAsync

napi/parser/package.json

@@ -6,7 +6,8 @@
     "test": "node test.mjs"
   },
   "devDependencies": {
-    "@napi-rs/cli": "^2.15.2"
+    "@napi-rs/cli": "^2.15.2",
+    "flatbuffers": "^23.5.26"
   },
   "engines": {
     "node": ">=14.*"

napi/parser/src/lib.rs

@@ -2,12 +2,15 @@
 
 use std::sync::Arc;
 
+use flexbuffers::FlexbufferSerializer;
 use miette::NamedSource;
+use napi::bindgen_prelude::Buffer;
 use napi_derive::napi;
 use oxc_allocator::Allocator;
 pub use oxc_ast::ast::Program;
 use oxc_parser::{Parser, ParserReturn};
 use oxc_span::SourceType;
+use serde::Serialize;
 
 /// Babel Parser Options
 ///
@@ -87,6 +90,23 @@ pub fn parse_sync(source_text: String, options: Option<ParserOptions>) -> ParseR
     ParseResult { program, errors }
 }
 
+/// Returns a binary AST in flexbuffers format.
+/// This is a POC API. Error handling is not done yet.
+/// # Panics
+///
+/// * File extension is invalid
+/// * FlexbufferSerializer serialization error
+#[allow(clippy::needless_pass_by_value)]
+#[napi]
+pub fn parse_sync_buffer(source_text: String, options: Option<ParserOptions>) -> Buffer {
+    let options = options.unwrap_or_default();
+    let allocator = Allocator::default();
+    let ret = parse(&allocator, &source_text, &options);
+    let mut serializer = FlexbufferSerializer::new();
+    ret.program.serialize(&mut serializer).unwrap();
+    serializer.take_buffer().into()
+}
+
 /// # Panics
 ///
 /// * Tokio crashes

napi/parser/test_buffer.js

@@ -0,0 +1,32 @@
+const oxc = require('./index');
+const assert = require('assert');
+const flexbuffers = require('flatbuffers/js/flexbuffers');
+const file = require('fs').readFileSync(__dirname + '/index.js', 'utf8');
+
+function testBuffer() {
+  const buffer = oxc.parseSyncBuffer(file);
+  const ref = flexbuffers.toReference(buffer.buffer);
+  assert(ref.isMap());
+  assert.equal(ref.get('type').stringValue(), 'Program');
+  const body = ref.get('body');
+  assert(body.isVector());
+}
+
+function testJSON() {
+  const ret = oxc.parseSync(file);
+  const program = JSON.parse(ret.program);
+  assert(typeof program === 'object');
+  assert.equal(program.type, 'Program');
+  assert(Array.isArray(program.body));
+}
+
+function benchmark(func, time) {
+  console.time(func.name);
+  for (let i = 0; i < time; i++) {
+    func();
+  }
+  console.timeEnd(func.name)
+}
+
+benchmark(testJSON, 10000);
+benchmark(testBuffer, 10000);