GH-37657: [JS] Run bin scripts with ts-node (#38500)

(rebased version of #37668, yarn.lock conflicts resolved in favor of main) * update bin scripts to use ts-node in transpile-only mode * remove ts-node config from top level tsconfig.json * fix tsc errors in bin scripts * Closes: #37657 Authored-by: ptaylor <paul.e.taylor@me.com> Signed-off-by: Jacob Wujciak-Jens <jacob@wujciak.de>
apache · Nov 2, 2023 · d8aeb42 · d8aeb42
1 parent 7281851
commit d8aeb42
Show file tree

Hide file tree

Showing 23 changed files with 1,830 additions and 1,678 deletions.
diff --git a/dev/archery/archery/integration/tester.py b/dev/archery/archery/integration/tester.py
@@ -225,11 +225,12 @@ def __init__(self, debug=False, **args):
         self.args = args
         self.debug = debug
 
-    def run_shell_command(self, cmd):
+    def run_shell_command(self, cmd, **kwargs):
         cmd = ' '.join(cmd)
         if self.debug:
             log(cmd)
-        subprocess.check_call(cmd, shell=True)
+        kwargs.update(shell=True)
+        subprocess.check_call(cmd, **kwargs)
 
     def json_to_file(self, json_path, arrow_path):
         """

diff --git a/dev/archery/archery/integration/tester_js.py b/dev/archery/archery/integration/tester_js.py
@@ -22,11 +22,12 @@
 from ..utils.source import ARROW_ROOT_DEFAULT
 
 
-_EXE_PATH = os.path.join(ARROW_ROOT_DEFAULT, 'js/bin')
-_VALIDATE = os.path.join(_EXE_PATH, 'integration.js')
-_JSON_TO_ARROW = os.path.join(_EXE_PATH, 'json-to-arrow.js')
-_STREAM_TO_FILE = os.path.join(_EXE_PATH, 'stream-to-file.js')
-_FILE_TO_STREAM = os.path.join(_EXE_PATH, 'file-to-stream.js')
+ARROW_JS_ROOT = os.path.join(ARROW_ROOT_DEFAULT, 'js')
+_EXE_PATH = os.path.join(ARROW_JS_ROOT, 'bin')
+_VALIDATE = os.path.join(_EXE_PATH, 'integration.ts')
+_JSON_TO_ARROW = os.path.join(_EXE_PATH, 'json-to-arrow.ts')
+_STREAM_TO_FILE = os.path.join(_EXE_PATH, 'stream-to-file.ts')
+_FILE_TO_STREAM = os.path.join(_EXE_PATH, 'file-to-stream.ts')
 
 
 class JSTester(Tester):
@@ -50,26 +51,25 @@ def _run(self, exe_cmd, arrow_path=None, json_path=None,
         if self.debug:
             log(' '.join(cmd))
 
-        run_cmd(cmd)
+        run_cmd(cmd, cwd=ARROW_JS_ROOT)
 
     def validate(self, json_path, arrow_path, quirks=None):
         return self._run(_VALIDATE, arrow_path, json_path, 'VALIDATE')
 
     def json_to_file(self, json_path, arrow_path):
-        cmd = ['node',
-               '--no-warnings', _JSON_TO_ARROW,
+        cmd = [_JSON_TO_ARROW,
                '-a', arrow_path,
                '-j', json_path]
-        self.run_shell_command(cmd)
+        self.run_shell_command(cmd, cwd=ARROW_JS_ROOT)
 
     def stream_to_file(self, stream_path, file_path):
-        cmd = ['node', '--no-warnings', _STREAM_TO_FILE,
+        cmd = [_STREAM_TO_FILE,
                '<', stream_path,
                '>', file_path]
-        self.run_shell_command(cmd)
+        self.run_shell_command(cmd, cwd=ARROW_JS_ROOT)
 
     def file_to_stream(self, file_path, stream_path):
-        cmd = ['node', '--no-warnings', _FILE_TO_STREAM,
+        cmd = [_FILE_TO_STREAM,
                '<', file_path,
                '>', stream_path]
-        self.run_shell_command(cmd)
+        self.run_shell_command(cmd, cwd=ARROW_JS_ROOT)
diff --git a/dev/archery/archery/integration/util.py b/dev/archery/archery/integration/util.py
@@ -127,12 +127,13 @@ def frombytes(o):
     return o
 
 
-def run_cmd(cmd):
+def run_cmd(cmd, **kwargs):
     if isinstance(cmd, str):
         cmd = cmd.split(' ')
 
     try:
-        output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+        kwargs.update(stderr=subprocess.STDOUT)
+        output = subprocess.check_output(cmd, **kwargs)
     except subprocess.CalledProcessError as e:
         # this avoids hiding the stdout / stderr of failed processes
         sio = io.StringIO()

diff --git a/js/.vscode/launch.json b/js/.vscode/launch.json
@@ -31,7 +31,7 @@
       "args": {
         "cwd": "${workspaceFolder}",
         "description": "Select a file to debug",
-        "command": "./node_modules/.bin/jest --listTests | sed -r \"s@$PWD/test/@@g\"",
+        "command": "node_modules/.bin/jest --listTests | sed -r \"s@$PWD/test/@@g\"",
       }
     },
     {
@@ -98,25 +98,23 @@
       "request": "launch",
       "name": "Debug Integration Tests",
       "cwd": "${workspaceFolder}",
-      "program": "${workspaceFolder}/bin/integration.js",
+      "program": "${workspaceFolder}/bin/integration.ts",
       "skipFiles": [
         "<node_internals>/**/*.js",
         "${workspaceFolder}/node_modules/**/*.js"
       ],
       "env": {
         "NODE_NO_WARNINGS": "1",
-        "ARROW_JS_DEBUG": "src",
         "TS_NODE_CACHE": "false"
       },
       "runtimeArgs": [
-        "-r",
-        "ts-node/register"
+        "--loader", "ts-node/esm/transpile-only"
       ],
       "args": [
         "--mode",
         "VALIDATE",
         "-j", "test/data/json/unions.json",
-        "-a", "./test/data/cpp/stream/struct_example.arrow"
+        "-a", "test/data/cpp/stream/unions.arrow"
       ]
     },
     {
@@ -140,8 +138,7 @@
         "${workspaceFolder}/node_modules/**/*.js"
       ],
       "runtimeArgs": [
-        "--loader",
-        "ts-node/esm/transpile-only"
+        "--loader", "ts-node/esm/transpile-only"
       ]
     },
     {
@@ -150,12 +147,10 @@
       "name": "Debug bin/arrow2csv",
       "cwd": "${workspaceFolder}",
       "env": {
-        "ARROW_JS_DEBUG": "src",
         "TS_NODE_CACHE": "false"
       },
       "runtimeArgs": [
-        "-r",
-        "ts-node/register"
+        "--loader", "ts-node/esm/transpile-only"
       ],
       "console": "integratedTerminal",
       "skipFiles": [
@@ -165,7 +160,7 @@
       "args": [
         "${workspaceFolder}/src/bin/arrow2csv.ts",
         "-f",
-        "./test/data/cpp/stream/simple.arrow"
+        "test/data/cpp/stream/simple.arrow"
       ]
     },
     {
@@ -174,21 +169,19 @@
       "name": "Debug bin/file-to-stream",
       "cwd": "${workspaceFolder}",
       "env": {
-        "ARROW_JS_DEBUG": "src",
         "TS_NODE_CACHE": "false"
       },
       "runtimeArgs": [
-        "-r",
-        "ts-node/register"
+        "--loader", "ts-node/esm/transpile-only"
       ],
       "skipFiles": [
         "<node_internals>/**/*.js",
         "${workspaceFolder}/node_modules/**/*.js"
       ],
       "args": [
-        "${workspaceFolder}/bin/file-to-stream.js",
-        "./test/data/cpp/file/struct_example.arrow",
-        "./struct_example-stream-out.arrow",
+        "${workspaceFolder}/bin/file-to-stream.ts",
+        "test/data/cpp/file/struct_example.arrow",
+        "struct_example-stream-out.arrow",
       ]
     },
     {
@@ -197,21 +190,19 @@
       "name": "Debug bin/stream-to-file",
       "cwd": "${workspaceFolder}",
       "env": {
-        "ARROW_JS_DEBUG": "src",
         "TS_NODE_CACHE": "false"
       },
       "runtimeArgs": [
-        "-r",
-        "ts-node/register"
+        "--loader", "ts-node/esm/transpile-only"
       ],
       "skipFiles": [
         "<node_internals>/**/*.js",
         "${workspaceFolder}/node_modules/**/*.js"
       ],
       "args": [
-        "${workspaceFolder}/bin/stream-to-file.js",
-        "./test/data/cpp/stream/struct_example.arrow",
-        "./struct_example-file-out.arrow",
+        "${workspaceFolder}/bin/stream-to-file.ts",
+        "test/data/cpp/stream/struct_example.arrow",
+        "struct_example-file-out.arrow",
       ]
     },
     {
@@ -220,23 +211,21 @@
       "name": "Debug bin/json-to-arrow",
       "cwd": "${workspaceFolder}",
       "env": {
-        "ARROW_JS_DEBUG": "src",
         "TS_NODE_CACHE": "false"
       },
       "runtimeArgs": [
-        "-r",
-        "ts-node/register"
+        "--loader", "ts-node/esm/transpile-only"
       ],
       "skipFiles": [
         "<node_internals>/**/*.js",
         "${workspaceFolder}/node_modules/**/*.js"
       ],
       "args": [
-        "${workspaceFolder}/bin/json-to-arrow.js",
+        "${workspaceFolder}/bin/json-to-arrow.ts",
         "-j",
-        "./test/data/json/struct_example.json",
+        "test/data/json/struct_example.json",
         "-a",
-        "./struct_example-stream-out.arrow",
+        "struct_example-stream-out.arrow",
         "-f",
         "stream"
       ]
@@ -247,20 +236,18 @@
       "name": "Debug bin/print-buffer-alignment",
       "cwd": "${workspaceFolder}",
       "env": {
-        "ARROW_JS_DEBUG": "src",
         "TS_NODE_CACHE": "false"
       },
       "runtimeArgs": [
-        "-r",
-        "ts-node/register"
+        "--loader", "ts-node/esm/transpile-only"
       ],
       "skipFiles": [
         "<node_internals>/**/*.js",
         "${workspaceFolder}/node_modules/**/*.js"
       ],
       "args": [
-        "${workspaceFolder}/bin/print-buffer-alignment.js",
-        "./test/data/cpp/stream/struct_example.arrow"
+        "${workspaceFolder}/bin/print-buffer-alignment.ts",
+        "test/data/cpp/stream/struct_example.arrow"
       ]
     },
     {

diff --git a/js/bin/arrow2csv.js → js/bin/arrow2csv.cjs b/js/bin/arrow2csv.js → js/bin/arrow2csv.cjs
diff --git a/js/bin/file-to-stream.js → js/bin/file-to-stream.ts b/js/bin/file-to-stream.js → js/bin/file-to-stream.ts
@@ -1,4 +1,4 @@
-#! /usr/bin/env node
+#! /usr/bin/env -S node --no-warnings --loader ts-node/esm/transpile-only
 
 // Licensed to the Apache Software Foundation (ASF) under one
 // or more contributor license agreements.  See the NOTICE file
@@ -17,18 +17,15 @@
 // specific language governing permissions and limitations
 // under the License.
 
-// @ts-check
-
-const fs = require('fs');
-const path = require('path');
-const eos = require('util').promisify(require('stream').finished);
-const extension = process.env.ARROW_JS_DEBUG === 'src' ? '.ts' : '.cjs';
-const { RecordBatchReader, RecordBatchStreamWriter } = require(`../index${extension}`);
+import * as fs from 'fs';
+import * as Path from 'path';
+import { finished as eos } from 'stream/promises';
+import { RecordBatchReader, RecordBatchStreamWriter } from '../index.ts';
 
 (async () => {
 
-    const readable = process.argv.length < 3 ? process.stdin : fs.createReadStream(path.resolve(process.argv[2]));
-    const writable = process.argv.length < 4 ? process.stdout : fs.createWriteStream(path.resolve(process.argv[3]));
+    const readable = process.argv.length < 3 ? process.stdin : fs.createReadStream(Path.resolve(process.argv[2]));
+    const writable = process.argv.length < 4 ? process.stdout : fs.createWriteStream(Path.resolve(process.argv[3]));
 
     const fileToStream = readable
         .pipe(RecordBatchReader.throughNode())