build: pass directory instead of list of files to js2c.py

node.gyp

@@ -27,251 +27,15 @@
     'node_lib_target_name%': 'libnode',
     'node_intermediate_lib_type%': 'static_library',
     'node_builtin_modules_path%': '',
+    # We list the deps/ files out instead of globbing them in js2c.py since we
+    # only include a subset of all the files under these directories.
+    # The lengths of their file names combined should not exceed the
+    # Windows command length limit or there would be an error.
+    # See https://docs.microsoft.com/en-us/troubleshoot/windows-client/shell-experience/command-line-string-limitation
     'library_files': [
-      'lib/internal/bootstrap/environment.js',
-      'lib/internal/bootstrap/loaders.js',
-      'lib/internal/bootstrap/node.js',
-      'lib/internal/bootstrap/pre_execution.js',
-      'lib/internal/bootstrap/switches/does_own_process_state.js',
-      'lib/internal/bootstrap/switches/does_not_own_process_state.js',
-      'lib/internal/bootstrap/switches/is_main_thread.js',
-      'lib/internal/bootstrap/switches/is_not_main_thread.js',
-      'lib/internal/per_context/primordials.js',
-      'lib/internal/per_context/domexception.js',
-      'lib/internal/per_context/messageport.js',
-      'lib/async_hooks.js',
-      'lib/assert.js',
-      'lib/assert/strict.js',
-      'lib/buffer.js',
-      'lib/child_process.js',
-      'lib/console.js',
-      'lib/constants.js',
-      'lib/crypto.js',
-      'lib/cluster.js',
-      'lib/diagnostics_channel.js',
-      'lib/dgram.js',
-      'lib/dns.js',
-      'lib/dns/promises.js',
-      'lib/domain.js',
-      'lib/events.js',
-      'lib/fs.js',
-      'lib/fs/promises.js',
-      'lib/http.js',
-      'lib/http2.js',
-      'lib/_http_agent.js',
-      'lib/_http_client.js',
-      'lib/_http_common.js',
-      'lib/_http_incoming.js',
-      'lib/_http_outgoing.js',
-      'lib/_http_server.js',
-      'lib/https.js',
-      'lib/inspector.js',
-      'lib/module.js',
-      'lib/net.js',
-      'lib/os.js',
-      'lib/path.js',
-      'lib/path/posix.js',
-      'lib/path/win32.js',
-      'lib/perf_hooks.js',
-      'lib/process.js',
-      'lib/punycode.js',
-      'lib/querystring.js',
-      'lib/readline.js',
-      'lib/repl.js',
-      'lib/stream.js',
-      'lib/stream/promises.js',
-      'lib/_stream_readable.js',
-      'lib/_stream_writable.js',
-      'lib/_stream_duplex.js',
-      'lib/_stream_transform.js',
-      'lib/_stream_passthrough.js',
-      'lib/_stream_wrap.js',
-      'lib/string_decoder.js',
-      'lib/sys.js',
-      'lib/timers/promises.js',
-      'lib/timers.js',
-      'lib/tls.js',
-      'lib/_tls_common.js',
-      'lib/_tls_wrap.js',
-      'lib/trace_events.js',
-      'lib/tty.js',
-      'lib/url.js',
-      'lib/util.js',
-      'lib/util/types.js',
-      'lib/v8.js',
-      'lib/vm.js',
-      'lib/wasi.js',
-      'lib/worker_threads.js',
-      'lib/zlib.js',
-      'lib/internal/abort_controller.js',
-      'lib/internal/assert.js',
-      'lib/internal/assert/assertion_error.js',
-      'lib/internal/assert/calltracker.js',
-      'lib/internal/async_hooks.js',
-      'lib/internal/blob.js',
-      'lib/internal/blocklist.js',
-      'lib/internal/buffer.js',
-      'lib/internal/cli_table.js',
-      'lib/internal/child_process.js',
-      'lib/internal/child_process/serialization.js',
-      'lib/internal/cluster/child.js',
-      'lib/internal/cluster/primary.js',
-      'lib/internal/cluster/round_robin_handle.js',
-      'lib/internal/cluster/shared_handle.js',
-      'lib/internal/cluster/utils.js',
-      'lib/internal/cluster/worker.js',
-      'lib/internal/console/constructor.js',
-      'lib/internal/console/global.js',
-      'lib/internal/crypto/aes.js',
-      'lib/internal/crypto/certificate.js',
-      'lib/internal/crypto/cipher.js',
-      'lib/internal/crypto/diffiehellman.js',
-      'lib/internal/crypto/dsa.js',
-      'lib/internal/crypto/ec.js',
-      'lib/internal/crypto/hash.js',
-      'lib/internal/crypto/hashnames.js',
-      'lib/internal/crypto/hkdf.js',
-      'lib/internal/crypto/keygen.js',
-      'lib/internal/crypto/keys.js',
-      'lib/internal/crypto/mac.js',
-      'lib/internal/crypto/pbkdf2.js',
-      'lib/internal/crypto/random.js',
-      'lib/internal/crypto/rsa.js',
-      'lib/internal/crypto/scrypt.js',
-      'lib/internal/crypto/sig.js',
-      'lib/internal/crypto/util.js',
-      'lib/internal/crypto/webcrypto.js',
-      'lib/internal/crypto/x509.js',
-      'lib/internal/constants.js',
-      'lib/internal/debugger/_inspect.js',
-      'lib/internal/debugger/inspect_client.js',
-      'lib/internal/debugger/inspect_repl.js',
-      'lib/internal/dgram.js',
-      'lib/internal/dns/promises.js',
-      'lib/internal/dns/utils.js',
-      'lib/internal/dtrace.js',
-      'lib/internal/encoding.js',
-      'lib/internal/errors.js',
-      'lib/internal/error_serdes.js',
-      'lib/internal/event_target.js',
-      'lib/internal/fixed_queue.js',
-      'lib/internal/freelist.js',
-      'lib/internal/freeze_intrinsics.js',
-      'lib/internal/fs/dir.js',
-      'lib/internal/fs/promises.js',
-      'lib/internal/fs/read_file_context.js',
-      'lib/internal/fs/rimraf.js',
-      'lib/internal/fs/streams.js',
-      'lib/internal/fs/sync_write_stream.js',
-      'lib/internal/fs/utils.js',
-      'lib/internal/fs/watchers.js',
-      'lib/internal/http.js',
-      'lib/internal/heap_utils.js',
-      'lib/internal/histogram.js',
-      'lib/internal/idna.js',
-      'lib/internal/inspector_async_hook.js',
-      'lib/internal/js_stream_socket.js',
-      'lib/internal/legacy/processbinding.js',
-      'lib/internal/linkedlist.js',
-      'lib/internal/main/check_syntax.js',
-      'lib/internal/main/eval_string.js',
-      'lib/internal/main/eval_stdin.js',
-      'lib/internal/main/inspect.js',
-      'lib/internal/main/print_help.js',
-      'lib/internal/main/prof_process.js',
-      'lib/internal/main/repl.js',
-      'lib/internal/main/run_main_module.js',
-      'lib/internal/main/worker_thread.js',
-      'lib/internal/modules/run_main.js',
-      'lib/internal/modules/package_json_reader.js',
-      'lib/internal/modules/cjs/helpers.js',
-      'lib/internal/modules/cjs/loader.js',
-      'lib/internal/modules/esm/loader.js',
-      'lib/internal/modules/esm/create_dynamic_module.js',
-      'lib/internal/modules/esm/get_format.js',
-      'lib/internal/modules/esm/get_source.js',
-      'lib/internal/modules/esm/module_job.js',
-      'lib/internal/modules/esm/module_map.js',
-      'lib/internal/modules/esm/resolve.js',
-      'lib/internal/modules/esm/transform_source.js',
-      'lib/internal/modules/esm/translators.js',
-      'lib/internal/net.js',
-      'lib/internal/options.js',
-      'lib/internal/perf/perf.js',
-      'lib/internal/perf/nodetiming.js',
-      'lib/internal/perf/usertiming.js',
-      'lib/internal/perf/observe.js',
-      'lib/internal/perf/event_loop_delay.js',
-      'lib/internal/perf/event_loop_utilization.js',
-      'lib/internal/perf/timerify.js',
-      'lib/internal/policy/manifest.js',
-      'lib/internal/policy/sri.js',
-      'lib/internal/priority_queue.js',
-      'lib/internal/process/esm_loader.js',
-      'lib/internal/process/execution.js',
-      'lib/internal/process/per_thread.js',
-      'lib/internal/process/policy.js',
-      'lib/internal/process/promises.js',
-      'lib/internal/process/warning.js',
-      'lib/internal/process/worker_thread_only.js',
-      'lib/internal/process/report.js',
-      'lib/internal/process/signal.js',
-      'lib/internal/process/task_queues.js',
-      'lib/internal/querystring.js',
-      'lib/internal/readline/callbacks.js',
-      'lib/internal/readline/emitKeypressEvents.js',
-      'lib/internal/readline/utils.js',
-      'lib/internal/repl.js',
-      'lib/internal/repl/await.js',
-      'lib/internal/repl/history.js',
-      'lib/internal/repl/utils.js',
-      'lib/internal/socketaddress.js',
-      'lib/internal/socket_list.js',
-      'lib/internal/source_map/prepare_stack_trace.js',
-      'lib/internal/source_map/source_map.js',
-      'lib/internal/source_map/source_map_cache.js',
-      'lib/internal/test/binding.js',
-      'lib/internal/test/transfer.js',
-      'lib/internal/timers.js',
-      'lib/internal/tls.js',
-      'lib/internal/trace_events_async_hooks.js',
-      'lib/internal/tty.js',
-      'lib/internal/url.js',
-      'lib/internal/util.js',
-      'lib/internal/util/comparisons.js',
-      'lib/internal/util/debuglog.js',
-      'lib/internal/util/inspect.js',
-      'lib/internal/util/inspector.js',
-      'lib/internal/util/iterable_weak_map.js',
-      'lib/internal/util/types.js',
-      'lib/internal/http2/core.js',
-      'lib/internal/http2/compat.js',
-      'lib/internal/http2/util.js',
-      'lib/internal/v8_prof_polyfill.js',
-      'lib/internal/v8_prof_processor.js',
-      'lib/internal/validators.js',
-      'lib/internal/stream_base_commons.js',
-      'lib/internal/vm/module.js',
-      'lib/internal/worker.js',
-      'lib/internal/worker/io.js',
-      'lib/internal/worker/js_transferable.js',
-      'lib/internal/watchdog.js',
-      'lib/internal/streams/lazy_transform.js',
-      'lib/internal/streams/add-abort-signal.js',
-      'lib/internal/streams/buffer_list.js',
-      'lib/internal/streams/duplexpair.js',
-      'lib/internal/streams/from.js',
-      'lib/internal/streams/legacy.js',
-      'lib/internal/streams/readable.js',
-      'lib/internal/streams/writable.js',
-      'lib/internal/streams/duplex.js',
-      'lib/internal/streams/passthrough.js',
-      'lib/internal/streams/transform.js',
-      'lib/internal/streams/destroy.js',
-      'lib/internal/streams/state.js',
-      'lib/internal/streams/pipeline.js',
-      'lib/internal/streams/end-of-stream.js',
-      'lib/internal/streams/utils.js',
+      '<!@(python tools/search_files.py --ext js lib)',
+    ],
+    'deps_files': [
       'deps/v8/tools/splaytree.mjs',
       'deps/v8/tools/codemap.mjs',
       'deps/v8/tools/consarray.mjs',
@@ -794,6 +558,7 @@
         'deps/v8/include/v8.h',
         # javascript files to make for an even more pleasant IDE experience
         '<@(library_files)',
+        '<@(deps_files)',
         # node.gyp is added by default, common.gypi is added for change detection
         'common.gypi',
       ],
@@ -1047,14 +812,21 @@
             # Put the code first so it's a dependency and can be used for invocation.
             'tools/js2c.py',
             '<@(library_files)',
+            '<@(deps_files)',
             'config.gypi'
           ],
           'outputs': [
             '<(SHARED_INTERMEDIATE_DIR)/node_javascript.cc',
           ],
           'action': [
-            'python', '<@(_inputs)',
-            '--target', '<@(_outputs)',
+            'python',
+            'tools/js2c.py',
+            '--directory',
+            'lib',
+            '--target',
+            '<@(_outputs)',
+            'config.gypi',
+            '<@(deps_files)',
           ],
         },
       ],
@@ -1598,6 +1370,7 @@
           ],
           'sources': [
             '<@(library_files)',
+            '<@(deps_files)',
             'common.gypi',
           ],
           'direct_dependent_settings': {

tools/js2c.py

@@ -36,6 +36,7 @@
 import re
 import functools
 import codecs
+import utils
 
 def ReadFile(filename):
   if is_verbose:
@@ -204,12 +205,23 @@ def main():
     fromfile_prefix_chars='@'
   )
   parser.add_argument('--target', help='output file')
+  parser.add_argument(
+      '--directory',
+      default=None,
+      help='input file directory')
   parser.add_argument('--verbose', action='store_true', help='output file')
   parser.add_argument('sources', nargs='*', help='input files')
   options = parser.parse_args()
   global is_verbose
   is_verbose = options.verbose
-  source_files = functools.reduce(SourceFileByExt, options.sources, {})
+  sources = options.sources
+  if options.directory is not None:
+    js_files = utils.SearchFiles(options.directory, 'js')
+    mjs_files = utils.SearchFiles(options.directory, 'mjs')
+    sources = js_files + mjs_files + options.sources
+
+  source_files = functools.reduce(SourceFileByExt, sources, {})
+
   # Should have exactly 3 types: `.js`, `.mjs` and `.gypi`
   assert len(source_files) == 3
   # Currently config.gypi is the only `.gypi` file allowed

tools/search_files.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+
+"""
+This is a utility for recursively searching files under
+a specified directory
+"""
+
+import argparse
+import utils
+
+def main():
+  parser = argparse.ArgumentParser(
+    description='Search files with a specific extension under a directory',
+    fromfile_prefix_chars='@'
+  )
+  parser.add_argument('--ext', required=True, help='extension to search for')
+  parser.add_argument('directory', help='input directory')
+  options = parser.parse_args()
+  print('\n'.join(utils.SearchFiles(options.directory, options.ext)))
+
+if __name__ == "__main__":
+  main()

tools/utils.py

@@ -26,8 +26,10 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
+import glob
 import platform
 import re
+import sys
 
 
 # Reads a .list file into an array of strings
@@ -106,3 +108,10 @@ def GuessArchitecture():
 
 def IsWindows():
   return GuessOS() == 'win32'
+
+
+def SearchFiles(dir, ext):
+  list = glob.glob(dir+ '/**/*.' + ext, recursive=True)
+  if sys.platform == 'win32':
+    list = [ x.replace('\\', '/')for x in list]
+  return list