From 59117317f3a08a10484a933918171cd725707cd7 Mon Sep 17 00:00:00 2001 From: Cheng Zhao Date: Thu, 16 Nov 2023 16:30:16 +0900 Subject: [PATCH] build,tools: make addons tests work with GN PR-URL: https://github.com/nodejs/node/pull/50737 Reviewed-By: Joyee Cheung Reviewed-By: Benjamin Gruenbaum Reviewed-By: Yagiz Nizipli --- Makefile | 48 ++- deps/openssl/unofficial.gni | 5 + deps/uv/unofficial.gni | 5 + doc/contributing/collaborator-guide.md | 1 - tools/build-addons.mjs | 65 ---- tools/build_addons.py | 145 +++++++ tools/install.py | 498 +++++++++++++------------ unofficial.gni | 20 +- vcbuild.bat | 13 +- 9 files changed, 469 insertions(+), 331 deletions(-) delete mode 100755 tools/build-addons.mjs create mode 100755 tools/build_addons.py diff --git a/Makefile b/Makefile index b7871bf218572c..1113873a21a298 100644 --- a/Makefile +++ b/Makefile @@ -187,11 +187,11 @@ config.gypi: configure configure.py src/node_version.h .PHONY: install install: all ## Installs node into $PREFIX (default=/usr/local). - $(PYTHON) tools/install.py $@ '$(DESTDIR)' '$(PREFIX)' + $(PYTHON) tools/install.py $@ --dest-dir '$(DESTDIR)' --prefix '$(PREFIX)' .PHONY: uninstall uninstall: ## Uninstalls node from $PREFIX (default=/usr/local). - $(PYTHON) tools/install.py $@ '$(DESTDIR)' '$(PREFIX)' + $(PYTHON) tools/install.py $@ --dest-dir '$(DESTDIR)' --prefix '$(PREFIX)' .PHONY: clean .NOTPARALLEL: clean @@ -379,6 +379,28 @@ test/addons/.docbuildstamp: $(DOCBUILDSTAMP_PREREQS) tools/doc/node_modules [ $$? -eq 0 ] && touch $@; \ fi +# All files that will be included in headers tarball should be listed as deps +# for generating headers. The list is manually synchronized with install.py. +ADDONS_HEADERS_PREREQS := tools/install.py \ + config.gypi common.gypi \ + $(wildcard deps/openssl/config/*.h) \ + $(wildcard deps/openssl/openssl/include/openssl/*.h) \ + $(wildcard deps/uv/include/*.h) \ + $(wildcard deps/uv/include/*/*.h) \ + $(wildcard deps/v8/include/*.h) \ + $(wildcard deps/v8/include/*/*.h) \ + deps/zlib/zconf.h deps/zlib/zlib.h \ + src/node.h src/node_api.h src/js_native_api.h src/js_native_api_types.h \ + src/node_api_types.h src/node_buffer.h src/node_object_wrap.h \ + src/node_version.h + +ADDONS_HEADERS_DIR = out/$(BUILDTYPE)/addons_headers + +# Generate node headers which will be used for building addons. +test/addons/.headersbuildstamp: $(ADDONS_HEADERS_PREREQS) + $(PYTHON) tools/install.py install --headers-only --dest-dir '$(ADDONS_HEADERS_DIR)' --prefix '/' + @touch $@ + ADDONS_BINDING_GYPS := \ $(filter-out test/addons/??_*/binding.gyp, \ $(wildcard test/addons/*/binding.gyp)) @@ -387,16 +409,11 @@ ADDONS_BINDING_SOURCES := \ $(filter-out test/addons/??_*/*.cc, $(wildcard test/addons/*/*.cc)) \ $(filter-out test/addons/??_*/*.h, $(wildcard test/addons/*/*.h)) -ADDONS_PREREQS := config.gypi \ - deps/npm/node_modules/node-gyp/package.json tools/build-addons.mjs \ - deps/uv/include/*.h deps/v8/include/*.h \ - src/node.h src/node_buffer.h src/node_object_wrap.h src/node_version.h +ADDONS_PREREQS := test/addons/.headersbuildstamp \ + deps/npm/node_modules/node-gyp/package.json tools/build_addons.py define run_build_addons -env npm_config_loglevel=$(LOGLEVEL) npm_config_nodedir="$$PWD" \ - npm_config_python="$(PYTHON)" $(NODE) "$$PWD/tools/build-addons.mjs" \ - "$$PWD/deps/npm/node_modules/node-gyp/bin/node-gyp.js" \ - $1 +env $(PYTHON) "$$PWD/tools/build_addons.py" --loglevel=$(LOGLEVEL) --headers-dir "$(ADDONS_HEADERS_DIR)" $1 touch $2 endef @@ -429,8 +446,7 @@ JS_NATIVE_API_BINDING_SOURCES := \ # Implicitly depends on $(NODE_EXE), see the build-js-native-api-tests rule for rationale. test/js-native-api/.buildstamp: $(ADDONS_PREREQS) \ $(JS_NATIVE_API_BINDING_GYPS) $(JS_NATIVE_API_BINDING_SOURCES) \ - src/node_api.h src/node_api_types.h src/js_native_api.h \ - src/js_native_api_types.h src/js_native_api_v8.h src/js_native_api_v8_internals.h + src/js_native_api_v8.h src/js_native_api_v8_internals.h @$(call run_build_addons,"$$PWD/test/js-native-api",$@) .PHONY: build-js-native-api-tests @@ -454,8 +470,7 @@ NODE_API_BINDING_SOURCES := \ # Implicitly depends on $(NODE_EXE), see the build-node-api-tests rule for rationale. test/node-api/.buildstamp: $(ADDONS_PREREQS) \ $(NODE_API_BINDING_GYPS) $(NODE_API_BINDING_SOURCES) \ - src/node_api.h src/node_api_types.h src/js_native_api.h \ - src/js_native_api_types.h src/js_native_api_v8.h src/js_native_api_v8_internals.h + src/js_native_api_v8.h src/js_native_api_v8_internals.h @$(call run_build_addons,"$$PWD/test/node-api",$@) .PHONY: build-node-api-tests @@ -660,9 +675,10 @@ test-addons: test-build test-js-native-api test-node-api .PHONY: test-addons-clean .NOTPARALLEL: test-addons-clean test-addons-clean: + $(RM) -r "$(ADDONS_HEADERS_DIR)" $(RM) -r test/addons/??_*/ $(RM) -r test/addons/*/build - $(RM) test/addons/.buildstamp test/addons/.docbuildstamp + $(RM) test/addons/.buildstamp test/addons/.docbuildstamp test/addons/.headersbuildstamp $(MAKE) test-js-native-api-clean $(MAKE) test-node-api-clean @@ -1216,7 +1232,7 @@ $(TARBALL)-headers: release-only --tag=$(TAG) \ --release-urlbase=$(RELEASE_URLBASE) \ $(CONFIG_FLAGS) $(BUILD_RELEASE_FLAGS) - HEADERS_ONLY=1 $(PYTHON) tools/install.py install '$(TARNAME)' '/' + $(PYTHON) tools/install.py install --headers-only --dest-dir '$(TARNAME)' --prefix '/' find $(TARNAME)/ -type l | xargs $(RM) tar -cf $(TARNAME)-headers.tar $(TARNAME) $(RM) -r $(TARNAME) diff --git a/deps/openssl/unofficial.gni b/deps/openssl/unofficial.gni index 31f31043812427..5d58d1d55152f7 100644 --- a/deps/openssl/unofficial.gni +++ b/deps/openssl/unofficial.gni @@ -88,6 +88,11 @@ template("openssl_gn_build") { configs += [ ":openssl_internal_config" ] public_configs = [ ":openssl_external_config" ] + if (is_posix) { + configs -= [ "//build/config/gcc:symbol_visibility_hidden" ] + configs += [ "//build/config/gcc:symbol_visibility_default" ] + } + config_path_name = "" if (is_win) { if (target_cpu == "x86") { diff --git a/deps/uv/unofficial.gni b/deps/uv/unofficial.gni index bc29bb04f64958..ce30341044e907 100644 --- a/deps/uv/unofficial.gni +++ b/deps/uv/unofficial.gni @@ -64,6 +64,11 @@ template("uv_gn_build") { configs += [ ":uv_internal_config" ] public_configs = [ ":uv_external_config" ] + if (is_posix) { + configs -= [ "//build/config/gcc:symbol_visibility_hidden" ] + configs += [ "//build/config/gcc:symbol_visibility_default" ] + } + if (is_win) { libs = [ "advapi32.lib", diff --git a/doc/contributing/collaborator-guide.md b/doc/contributing/collaborator-guide.md index 2bf751b7076139..290b7b8500da3f 100644 --- a/doc/contributing/collaborator-guide.md +++ b/doc/contributing/collaborator-guide.md @@ -232,7 +232,6 @@ There are some other files that touch the build chain. Changes in the following files also qualify as affecting the `node` binary: * `tools/*.py` -* `tools/build-addons.mjs` * `*.gyp` * `*.gypi` * `configure` diff --git a/tools/build-addons.mjs b/tools/build-addons.mjs deleted file mode 100755 index dc30c549fa5fbf..00000000000000 --- a/tools/build-addons.mjs +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env node - -// Usage: e.g. node build-addons.mjs - -import child_process from 'node:child_process'; -import path from 'node:path'; -import fs from 'node:fs/promises'; -import util from 'node:util'; -import process from 'node:process'; -import os from 'node:os'; - -const execFile = util.promisify(child_process.execFile); - -const parallelization = +process.env.JOBS || os.availableParallelism(); -const nodeGyp = process.argv[2]; -const directory = process.argv[3]; - -async function buildAddon(dir) { - try { - // Only run for directories that have a `binding.gyp`. - // (https://github.com/nodejs/node/issues/14843) - await fs.stat(path.join(dir, 'binding.gyp')); - } catch (err) { - if (err.code === 'ENOENT' || err.code === 'ENOTDIR') - return; - throw err; - } - - console.log(`Building addon in ${dir}`); - const { stdout, stderr } = - await execFile(process.execPath, [nodeGyp, 'rebuild', `--directory=${dir}`], - { - stdio: 'inherit', - env: { ...process.env, MAKEFLAGS: '-j1' }, - }); - - // We buffer the output and print it out once the process is done in order - // to avoid interleaved output from multiple builds running at once. - process.stdout.write(stdout); - process.stderr.write(stderr); -} - -async function parallel(jobQueue, limit) { - const next = async () => { - if (jobQueue.length === 0) { - return; - } - const job = jobQueue.shift(); - await job(); - await next(); - }; - - const workerCnt = Math.min(limit, jobQueue.length); - await Promise.all(Array.from({ length: workerCnt }, next)); -} - -const jobs = []; -for await (const dirent of await fs.opendir(directory)) { - if (dirent.isDirectory()) { - jobs.push(() => buildAddon(path.join(directory, dirent.name))); - } else if (dirent.isFile() && dirent.name === 'binding.gyp') { - jobs.push(() => buildAddon(directory)); - } -} -await parallel(jobs, parallelization); diff --git a/tools/build_addons.py b/tools/build_addons.py new file mode 100755 index 00000000000000..8e445be5116c62 --- /dev/null +++ b/tools/build_addons.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 + +import argparse +import os +import shutil +import subprocess +import sys +import tempfile +from concurrent.futures import ThreadPoolExecutor + +ROOT_DIR = os.path.abspath(os.path.join(__file__, '..', '..')) + +# Run install.py to install headers. +def generate_headers(headers_dir, install_args): + print('Generating headers') + subprocess.check_call([ + sys.executable, + os.path.join(ROOT_DIR, 'tools/install.py'), + 'install', + '--silent', + '--headers-only', + '--prefix', '/', + '--dest-dir', headers_dir, + ] + install_args) + +# Rebuild addons in parallel. +def rebuild_addons(args): + headers_dir = os.path.abspath(args.headers_dir) + out_dir = os.path.abspath(args.out_dir) + node_bin = os.path.join(out_dir, 'node') + if args.is_win: + node_bin += '.exe' + + if os.path.isabs(args.node_gyp): + node_gyp = args.node_gyp + else: + node_gyp = os.path.join(ROOT_DIR, args.node_gyp) + + # Copy node.lib. + if args.is_win: + node_lib_dir = os.path.join(headers_dir, 'Release') + os.makedirs(node_lib_dir) + shutil.copy2(os.path.join(args.out_dir, 'node.lib'), + os.path.join(node_lib_dir, 'node.lib')) + + def node_gyp_rebuild(test_dir): + print('Building addon in', test_dir) + try: + process = subprocess.Popen([ + node_bin, + node_gyp, + 'rebuild', + '--directory', test_dir, + '--nodedir', headers_dir, + '--python', sys.executable, + '--loglevel', args.loglevel, + ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + # We buffer the output and print it out once the process is done in order + # to avoid interleaved output from multiple builds running at once. + return_code = process.wait() + stdout, stderr = process.communicate() + if return_code != 0: + print(f'Failed to build addon in {test_dir}:') + if stdout: + print(stdout.decode()) + if stderr: + print(stderr.decode()) + + except Exception as e: + print(f'Unexpected error when building addon in {test_dir}. Error: {e}') + + test_dirs = [] + skip_tests = args.skip_tests.split(',') + only_tests = args.only_tests.split(',') if args.only_tests else None + for child_dir in os.listdir(args.target): + full_path = os.path.join(args.target, child_dir) + if not os.path.isdir(full_path): + continue + if 'binding.gyp' not in os.listdir(full_path): + continue + if child_dir in skip_tests: + continue + if only_tests and child_dir not in only_tests: + continue + test_dirs.append(full_path) + + with ThreadPoolExecutor() as executor: + executor.map(node_gyp_rebuild, test_dirs) + +def get_default_out_dir(args): + default_out_dir = os.path.join('out', 'Release') + if not args.is_win: + # POSIX platforms only have one out dir. + return default_out_dir + # On Windows depending on the args of GYP and configure script, the out dir + # could be 'out/Release' or just 'Release'. + if os.path.exists(default_out_dir): + return default_out_dir + if os.path.exists('Release'): + return 'Release' + raise RuntimeError('Can not find out dir, did you run configure?') + +def main(): + if sys.platform == 'cygwin': + raise RuntimeError('This script does not support running with cygwin python.') + + parser = argparse.ArgumentParser( + description='Install headers and rebuild child directories') + parser.add_argument('target', help='target directory to build addons') + parser.add_argument('--headers-dir', + help='path to node headers directory, if not specified ' + 'new headers will be generated for building', + default=None) + parser.add_argument('--out-dir', help='path to the output directory', + default=None) + parser.add_argument('--loglevel', help='loglevel of node-gyp', + default='silent') + parser.add_argument('--skip-tests', help='skip building tests', + default='') + parser.add_argument('--only-tests', help='only build tests in the list', + default='') + parser.add_argument('--node-gyp', help='path to node-gyp script', + default='deps/npm/node_modules/node-gyp/bin/node-gyp.js') + parser.add_argument('--is-win', help='build for Windows target', + action='store_true', default=(sys.platform == 'win32')) + args, unknown_args = parser.parse_known_args() + + if not args.out_dir: + args.out_dir = get_default_out_dir(args) + + if args.headers_dir: + rebuild_addons(args) + else: + # When --headers-dir is not specified, generate headers into a temp dir and + # build with the new headers. + try: + args.headers_dir = tempfile.mkdtemp() + generate_headers(args.headers_dir, unknown_args) + rebuild_addons(args) + finally: + shutil.rmtree(args.headers_dir) + +if __name__ == '__main__': + main() diff --git a/tools/install.py b/tools/install.py index c2cd85e7dde4e7..196bfdf86045b4 100755 --- a/tools/install.py +++ b/tools/install.py @@ -1,7 +1,6 @@ -#!/usr/bin/env python - -from __future__ import print_function +#!/usr/bin/env python3 +import argparse import ast import errno import os @@ -9,19 +8,14 @@ import sys import re -# set at init time -node_prefix = '/usr/local' # PREFIX variable from Makefile -install_path = '' # base target directory (DESTDIR + PREFIX from Makefile) -target_defaults = None -variables = None - def abspath(*args): path = os.path.join(*args) return os.path.abspath(path) -def load_config(): - with open('config.gypi') as f: - return ast.literal_eval(f.read()) +def is_child_dir(child, parent): + p = os.path.abspath(parent) + c = os.path.abspath(child) + return c.startswith(p) and c != p def try_unlink(path): try: @@ -29,8 +23,9 @@ def try_unlink(path): except OSError as e: if e.errno != errno.ENOENT: raise -def try_symlink(source_path, link_path): - print('symlinking %s -> %s' % (source_path, link_path)) +def try_symlink(options, source_path, link_path): + if not options.silent: + print('symlinking %s -> %s' % (source_path, link_path)) try_unlink(link_path) try_mkdir_r(os.path.dirname(link_path)) os.symlink(source_path, link_path) @@ -41,9 +36,9 @@ def try_mkdir_r(path): except OSError as e: if e.errno != errno.EEXIST: raise -def try_rmdir_r(path): +def try_rmdir_r(options, path): path = abspath(path) - while path.startswith(install_path): + while is_child_dir(path, options.install_path): try: os.rmdir(path) except OSError as e: @@ -52,67 +47,74 @@ def try_rmdir_r(path): raise path = abspath(path, '..') -def mkpaths(path, dst): - if dst.endswith('/'): - target_path = abspath(install_path, dst, os.path.basename(path)) +def mkpaths(options, path, dest): + if dest.endswith('/') or dest.endswith('\\'): + target_path = abspath(options.install_path, dest, os.path.basename(path)) else: - target_path = abspath(install_path, dst) - return path, target_path + target_path = abspath(options.install_path, dest) + if os.path.isabs(path): + source_path = path + else: + source_path = abspath(options.root_dir, path) + return source_path, target_path -def try_copy(path, dst): - source_path, target_path = mkpaths(path, dst) - print('installing %s' % target_path) +def try_copy(options, path, dest): + source_path, target_path = mkpaths(options, path, dest) + if not options.silent: + print('installing %s' % target_path) try_mkdir_r(os.path.dirname(target_path)) try_unlink(target_path) # prevent ETXTBSY errors return shutil.copy2(source_path, target_path) -def try_remove(path, dst): - source_path, target_path = mkpaths(path, dst) - print('removing %s' % target_path) +def try_remove(options, path, dest): + source_path, target_path = mkpaths(options, path, dest) + if not options.silent: + print('removing %s' % target_path) try_unlink(target_path) - try_rmdir_r(os.path.dirname(target_path)) + try_rmdir_r(options, os.path.dirname(target_path)) -def install(paths, dst): +def install(options, paths, dest): for path in paths: - try_copy(path, dst) + try_copy(options, path, dest) -def uninstall(paths, dst): +def uninstall(options, paths, dest): for path in paths: - try_remove(path, dst) + try_remove(options, path, dest) -def package_files(action, name, bins): - target_path = 'lib/node_modules/' + name + '/' +def package_files(options, action, name, bins): + target_path = os.path.join('lib/node_modules', name) # don't install npm if the target path is a symlink, it probably means # that a dev version of npm is installed there - if os.path.islink(abspath(install_path, target_path)): return + if os.path.islink(abspath(options.install_path, target_path)): return # npm has a *lot* of files and it'd be a pain to maintain a fixed list here # so we walk its source directory instead... - root = 'deps/' + name + root = os.path.join('deps', name) for dirname, subdirs, basenames in os.walk(root, topdown=True): subdirs[:] = [subdir for subdir in subdirs if subdir != 'test'] paths = [os.path.join(dirname, basename) for basename in basenames] - action(paths, target_path + dirname[len(root) + 1:] + '/') + action(options, paths, + os.path.join(target_path, dirname[len(root) + 1:]) + os.path.sep) # create/remove symlinks for bin_name, bin_target in bins.items(): - link_path = abspath(install_path, 'bin/' + bin_name) + link_path = abspath(options.install_path, os.path.join('bin', bin_name)) if action == uninstall: - action([link_path], 'bin/' + bin_name) + action(options, [link_path], os.path.join('bin', bin_name)) elif action == install: - try_symlink('../lib/node_modules/' + name + '/' + bin_target, link_path) + try_symlink(options, os.path.join('../lib/node_modules', name, bin_target), link_path) else: assert 0 # unhandled action type -def npm_files(action): - package_files(action, 'npm', { +def npm_files(options, action): + package_files(options, action, 'npm', { 'npm': 'bin/npm-cli.js', 'npx': 'bin/npx-cli.js', }) -def corepack_files(action): - package_files(action, 'corepack', { +def corepack_files(options, action): + package_files(options, action, 'corepack', { 'corepack': 'dist/corepack.js', # Not the default just yet: # 'yarn': 'dist/yarn.js', @@ -124,194 +126,200 @@ def corepack_files(action): # On z/OS, we install node-gyp for convenience, as some vendors don't have # external access and may want to build native addons. if sys.platform == 'zos': - link_path = abspath(install_path, 'bin/node-gyp') + link_path = abspath(options.install_path, 'bin/node-gyp') if action == uninstall: - action([link_path], 'bin/node-gyp') + action(options, [link_path], 'bin/node-gyp') elif action == install: - try_symlink('../lib/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js', link_path) + try_symlink(options, '../lib/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js', link_path) else: assert 0 # unhandled action type -def subdir_files(path, dest, action): +def subdir_files(options, path, dest, action): + source_path, _ = mkpaths(options, path, dest) ret = {} - for dirpath, dirnames, filenames in os.walk(path): - files_in_path = [dirpath + '/' + f for f in filenames if f.endswith('.h')] - ret[dest + dirpath.replace(path, '')] = files_in_path + for dirpath, dirnames, filenames in os.walk(source_path): + files_in_path = [os.path.join(os.path.relpath(dirpath, options.root_dir), f) for f in filenames if f.endswith('.h')] + ret[os.path.join(dest, os.path.relpath(dirpath, source_path))] = files_in_path for subdir, files_in_path in ret.items(): - action(files_in_path, subdir + '/') - -def files(action): - is_windows = sys.platform == 'win32' - output_file = 'node' - output_prefix = 'out/Release/' - - if is_windows: - output_file += '.exe' - action([output_prefix + output_file], 'bin/' + output_file) - - if 'true' == variables.get('node_shared'): - if is_windows: - action([output_prefix + 'libnode.dll'], 'bin/libnode.dll') - action([output_prefix + 'libnode.lib'], 'lib/libnode.lib') + action(options, files_in_path, subdir + os.path.sep) + +def files(options, action): + node_bin = 'node' + if options.is_win: + node_bin += '.exe' + action(options, [os.path.join(options.build_dir, node_bin)], os.path.join('bin', node_bin)) + + if 'true' == options.variables.get('node_shared'): + if options.is_win: + action(options, [os.path.join(options.build_dir, 'libnode.dll')], 'bin/libnode.dll') + action(options, [os.path.join(options.build_dir, 'libnode.lib')], 'lib/libnode.lib') elif sys.platform == 'zos': # GYP will output to lib.target; see _InstallableTargetInstallPath # function in tools/gyp/pylib/gyp/generator/make.py - output_prefix += 'lib.target/' + output_prefix = os.path.join(options.build_dir, 'lib.target') - output_lib = 'libnode.' + variables.get('shlib_suffix') - action([output_prefix + output_lib], 'lib/' + output_lib) + output_lib = 'libnode.' + options.variables.get('shlib_suffix') + action(options, [os.path.join(output_prefix, output_lib)], os.path.join('lib', output_lib)) # create libnode.x that references libnode.so (C++ addons compat) os.system(os.path.dirname(os.path.realpath(__file__)) + '/zos/modifysidedeck.sh ' + - abspath(install_path, 'lib/' + output_lib) + ' ' + - abspath(install_path, 'lib/libnode.x') + ' libnode.so') + abspath(options.install_path, 'lib', output_lib) + ' ' + + abspath(options.install_path, 'lib/libnode.x') + ' libnode.so') # install libnode.version.so - so_name = 'libnode.' + re.sub(r'\.x$', '.so', variables.get('shlib_suffix')) - action([output_prefix + so_name], variables.get('libdir') + '/' + so_name) + so_name = 'libnode.' + re.sub(r'\.x$', '.so', options.variables.get('shlib_suffix')) + action(options, [os.path.join(output_prefix, so_name)], options.variables.get('libdir') + '/' + so_name) # create symlink of libnode.so -> libnode.version.so (C++ addons compat) - link_path = abspath(install_path, 'lib/libnode.so') - try_symlink(so_name, link_path) + link_path = abspath(options.install_path, 'lib/libnode.so') + try_symlink(options, so_name, link_path) else: - output_lib = 'libnode.' + variables.get('shlib_suffix') - action([output_prefix + output_lib], variables.get('libdir') + '/' + output_lib) + output_lib = 'libnode.' + options.variables.get('shlib_suffix') + action(options, [os.path.join(output_prefix, output_lib)], + os.path.join(options.variables.get('libdir'), output_lib)) - action(['deps/v8/tools/gdbinit'], 'share/doc/node/') - action(['deps/v8/tools/lldb_commands.py'], 'share/doc/node/') + action(options, [os.path.join(options.v8_dir, 'tools/gdbinit')], 'share/doc/node/') + action(options, [os.path.join(options.v8_dir, 'tools/lldb_commands.py')], 'share/doc/node/') if 'freebsd' in sys.platform or 'openbsd' in sys.platform: - action(['doc/node.1'], 'man/man1/') + action(options, ['doc/node.1'], 'man/man1/') else: - action(['doc/node.1'], 'share/man/man1/') + action(options, ['doc/node.1'], 'share/man/man1/') - if 'true' == variables.get('node_install_npm'): - npm_files(action) + if 'true' == options.variables.get('node_install_npm'): + npm_files(options, action) - if 'true' == variables.get('node_install_corepack'): - corepack_files(action) + if 'true' == options.variables.get('node_install_corepack'): + corepack_files(options, action) - headers(action) + headers(options, action) -def headers(action): - def wanted_v8_headers(files_arg, dest): +def headers(options, action): + def wanted_v8_headers(options, files_arg, dest): v8_headers = [ # The internal cppgc headers are depended on by the public # ones, so they need to be included as well. - 'deps/v8/include/cppgc/internal/api-constants.h', - 'deps/v8/include/cppgc/internal/atomic-entry-flag.h', - 'deps/v8/include/cppgc/internal/base-page-handle.h', - 'deps/v8/include/cppgc/internal/caged-heap-local-data.h', - 'deps/v8/include/cppgc/internal/caged-heap.h', - 'deps/v8/include/cppgc/internal/compiler-specific.h', - 'deps/v8/include/cppgc/internal/finalizer-trait.h', - 'deps/v8/include/cppgc/internal/gc-info.h', - 'deps/v8/include/cppgc/internal/logging.h', - 'deps/v8/include/cppgc/internal/member-storage.h', - 'deps/v8/include/cppgc/internal/name-trait.h', - 'deps/v8/include/cppgc/internal/persistent-node.h', - 'deps/v8/include/cppgc/internal/pointer-policies.h', - 'deps/v8/include/cppgc/internal/write-barrier.h', + 'include/cppgc/internal/api-constants.h', + 'include/cppgc/internal/atomic-entry-flag.h', + 'include/cppgc/internal/base-page-handle.h', + 'include/cppgc/internal/caged-heap-local-data.h', + 'include/cppgc/internal/caged-heap.h', + 'include/cppgc/internal/compiler-specific.h', + 'include/cppgc/internal/finalizer-trait.h', + 'include/cppgc/internal/gc-info.h', + 'include/cppgc/internal/logging.h', + 'include/cppgc/internal/member-storage.h', + 'include/cppgc/internal/name-trait.h', + 'include/cppgc/internal/persistent-node.h', + 'include/cppgc/internal/pointer-policies.h', + 'include/cppgc/internal/write-barrier.h', # cppgc headers - 'deps/v8/include/cppgc/allocation.h', - 'deps/v8/include/cppgc/common.h', - 'deps/v8/include/cppgc/cross-thread-persistent.h', - 'deps/v8/include/cppgc/custom-space.h', - 'deps/v8/include/cppgc/default-platform.h', - 'deps/v8/include/cppgc/ephemeron-pair.h', - 'deps/v8/include/cppgc/explicit-management.h', - 'deps/v8/include/cppgc/garbage-collected.h', - 'deps/v8/include/cppgc/heap-consistency.h', - 'deps/v8/include/cppgc/heap-handle.h', - 'deps/v8/include/cppgc/heap-state.h', - 'deps/v8/include/cppgc/heap-statistics.h', - 'deps/v8/include/cppgc/heap.h', - 'deps/v8/include/cppgc/liveness-broker.h', - 'deps/v8/include/cppgc/macros.h', - 'deps/v8/include/cppgc/member.h', - 'deps/v8/include/cppgc/name-provider.h', - 'deps/v8/include/cppgc/object-size-trait.h', - 'deps/v8/include/cppgc/persistent.h', - 'deps/v8/include/cppgc/platform.h', - 'deps/v8/include/cppgc/prefinalizer.h', - 'deps/v8/include/cppgc/process-heap-statistics.h', - 'deps/v8/include/cppgc/sentinel-pointer.h', - 'deps/v8/include/cppgc/source-location.h', - 'deps/v8/include/cppgc/testing.h', - 'deps/v8/include/cppgc/trace-trait.h', - 'deps/v8/include/cppgc/type-traits.h', - 'deps/v8/include/cppgc/visitor.h', + 'include/cppgc/allocation.h', + 'include/cppgc/common.h', + 'include/cppgc/cross-thread-persistent.h', + 'include/cppgc/custom-space.h', + 'include/cppgc/default-platform.h', + 'include/cppgc/ephemeron-pair.h', + 'include/cppgc/explicit-management.h', + 'include/cppgc/garbage-collected.h', + 'include/cppgc/heap-consistency.h', + 'include/cppgc/heap-handle.h', + 'include/cppgc/heap-state.h', + 'include/cppgc/heap-statistics.h', + 'include/cppgc/heap.h', + 'include/cppgc/liveness-broker.h', + 'include/cppgc/macros.h', + 'include/cppgc/member.h', + 'include/cppgc/name-provider.h', + 'include/cppgc/object-size-trait.h', + 'include/cppgc/persistent.h', + 'include/cppgc/platform.h', + 'include/cppgc/prefinalizer.h', + 'include/cppgc/process-heap-statistics.h', + 'include/cppgc/sentinel-pointer.h', + 'include/cppgc/source-location.h', + 'include/cppgc/testing.h', + 'include/cppgc/trace-trait.h', + 'include/cppgc/type-traits.h', + 'include/cppgc/visitor.h', # libplatform headers - 'deps/v8/include/libplatform/libplatform-export.h', - 'deps/v8/include/libplatform/libplatform.h', - 'deps/v8/include/libplatform/v8-tracing.h', + 'include/libplatform/libplatform-export.h', + 'include/libplatform/libplatform.h', + 'include/libplatform/v8-tracing.h', # v8 headers - 'deps/v8/include/v8-array-buffer.h', - 'deps/v8/include/v8-callbacks.h', - 'deps/v8/include/v8-container.h', - 'deps/v8/include/v8-context.h', - 'deps/v8/include/v8-cppgc.h', - 'deps/v8/include/v8-data.h', - 'deps/v8/include/v8-date.h', - 'deps/v8/include/v8-debug.h', - 'deps/v8/include/v8-embedder-heap.h', - 'deps/v8/include/v8-embedder-state-scope.h', - 'deps/v8/include/v8-exception.h', - 'deps/v8/include/v8-extension.h', - 'deps/v8/include/v8-external.h', - 'deps/v8/include/v8-forward.h', - 'deps/v8/include/v8-function-callback.h', - 'deps/v8/include/v8-function.h', - 'deps/v8/include/v8-handle-base.h', - 'deps/v8/include/v8-initialization.h', - 'deps/v8/include/v8-internal.h', - 'deps/v8/include/v8-isolate.h', - 'deps/v8/include/v8-json.h', - 'deps/v8/include/v8-local-handle.h', - 'deps/v8/include/v8-locker.h', - 'deps/v8/include/v8-maybe.h', - 'deps/v8/include/v8-memory-span.h', - 'deps/v8/include/v8-message.h', - 'deps/v8/include/v8-microtask-queue.h', - 'deps/v8/include/v8-microtask.h', - 'deps/v8/include/v8-object.h', - 'deps/v8/include/v8-persistent-handle.h', - 'deps/v8/include/v8-platform.h', - 'deps/v8/include/v8-primitive-object.h', - 'deps/v8/include/v8-primitive.h', - 'deps/v8/include/v8-profiler.h', - 'deps/v8/include/v8-promise.h', - 'deps/v8/include/v8-proxy.h', - 'deps/v8/include/v8-regexp.h', - 'deps/v8/include/v8-script.h', - 'deps/v8/include/v8-snapshot.h', - 'deps/v8/include/v8-source-location.h', - 'deps/v8/include/v8-statistics.h', - 'deps/v8/include/v8-template.h', - 'deps/v8/include/v8-traced-handle.h', - 'deps/v8/include/v8-typed-array.h', - 'deps/v8/include/v8-unwinder.h', - 'deps/v8/include/v8-value-serializer.h', - 'deps/v8/include/v8-value.h', - 'deps/v8/include/v8-version.h', - 'deps/v8/include/v8-wasm.h', - 'deps/v8/include/v8-weak-callback-info.h', - 'deps/v8/include/v8.h', - 'deps/v8/include/v8config.h', + 'include/v8-array-buffer.h', + 'include/v8-callbacks.h', + 'include/v8-container.h', + 'include/v8-context.h', + 'include/v8-cppgc.h', + 'include/v8-data.h', + 'include/v8-date.h', + 'include/v8-debug.h', + 'include/v8-embedder-heap.h', + 'include/v8-embedder-state-scope.h', + 'include/v8-exception.h', + 'include/v8-extension.h', + 'include/v8-external.h', + 'include/v8-forward.h', + 'include/v8-function-callback.h', + 'include/v8-function.h', + 'include/v8-handle-base.h', + 'include/v8-initialization.h', + 'include/v8-internal.h', + 'include/v8-isolate.h', + 'include/v8-json.h', + 'include/v8-local-handle.h', + 'include/v8-locker.h', + 'include/v8-maybe.h', + 'include/v8-memory-span.h', + 'include/v8-message.h', + 'include/v8-microtask-queue.h', + 'include/v8-microtask.h', + 'include/v8-object.h', + 'include/v8-persistent-handle.h', + 'include/v8-platform.h', + 'include/v8-primitive-object.h', + 'include/v8-primitive.h', + 'include/v8-profiler.h', + 'include/v8-promise.h', + 'include/v8-proxy.h', + 'include/v8-regexp.h', + 'include/v8-script.h', + 'include/v8-snapshot.h', + 'include/v8-source-location.h', + 'include/v8-statistics.h', + 'include/v8-template.h', + 'include/v8-traced-handle.h', + 'include/v8-typed-array.h', + 'include/v8-unwinder.h', + 'include/v8-value-serializer.h', + 'include/v8-value.h', + 'include/v8-version.h', + 'include/v8-wasm.h', + 'include/v8-weak-callback-info.h', + 'include/v8.h', + 'include/v8config.h', ] - files_arg = [name for name in files_arg if name in v8_headers] - action(files_arg, dest) + if sys.platform == 'win32': + # Native win32 python uses \ for path separator. + v8_headers = [os.path.normpath(path) for path in v8_headers] + if os.path.isabs(options.v8_dir): + rel_v8_dir = os.path.relpath(options.v8_dir, options.root_dir) + else: + rel_v8_dir = options.v8_dir + files_arg = [name for name in files_arg if os.path.relpath(name, rel_v8_dir) in v8_headers] + action(options, files_arg, dest) - def wanted_zoslib_headers(files_arg, dest): + def wanted_zoslib_headers(options, files_arg, dest): import glob zoslib_headers = glob.glob(zoslibinc + '/*.h') files_arg = [name for name in files_arg if name in zoslib_headers] - action(files_arg, dest) + action(options, files_arg, dest) - action([ + action(options, [ + options.config_gypi_path, 'common.gypi', - 'config.gypi', 'src/node.h', 'src/node_api.h', 'src/js_native_api.h', @@ -324,21 +332,21 @@ def wanted_zoslib_headers(files_arg, dest): # Add the expfile that is created on AIX if sys.platform.startswith('aix') or sys.platform == "os400": - action(['out/Release/node.exp'], 'include/node/') + action(options, ['out/Release/node.exp'], 'include/node/') - subdir_files('deps/v8/include', 'include/node/', wanted_v8_headers) + subdir_files(options, os.path.join(options.v8_dir, 'include'), 'include/node/', wanted_v8_headers) - if 'false' == variables.get('node_shared_libuv'): - subdir_files('deps/uv/include', 'include/node/', action) + if 'false' == options.variables.get('node_shared_libuv'): + subdir_files(options, 'deps/uv/include', 'include/node/', action) - if 'true' == variables.get('node_use_openssl') and \ - 'false' == variables.get('node_shared_openssl'): - subdir_files('deps/openssl/openssl/include/openssl', 'include/node/openssl/', action) - subdir_files('deps/openssl/config/archs', 'include/node/openssl/archs', action) - subdir_files('deps/openssl/config', 'include/node/openssl', action) + if 'true' == options.variables.get('node_use_openssl') and \ + 'false' == options.variables.get('node_shared_openssl'): + subdir_files(options, 'deps/openssl/openssl/include/openssl', 'include/node/openssl/', action) + subdir_files(options, 'deps/openssl/config/archs', 'include/node/openssl/archs', action) + subdir_files(options, 'deps/openssl/config', 'include/node/openssl', action) - if 'false' == variables.get('node_shared_zlib'): - action([ + if 'false' == options.variables.get('node_shared_zlib'): + action(options, [ 'deps/zlib/zconf.h', 'deps/zlib/zlib.h', ], 'include/node/') @@ -349,47 +357,63 @@ def wanted_zoslib_headers(files_arg, dest): raise RuntimeError('Environment variable ZOSLIB_INCLUDES is not set\n') if not os.path.isfile(zoslibinc + '/zos-base.h'): raise RuntimeError('ZOSLIB_INCLUDES is not set to a valid location\n') - subdir_files(zoslibinc, 'include/node/zoslib/', wanted_zoslib_headers) - -def run(args): - global node_prefix, install_path, target_defaults, variables - - # chdir to the project's top-level directory - os.chdir(abspath(os.path.dirname(__file__), '..')) - - conf = load_config() - variables = conf['variables'] - target_defaults = conf['target_defaults'] - - # argv[2] is a custom install prefix for packagers (think DESTDIR) - # argv[3] is a custom install prefix (think PREFIX) - # Difference is that dst_dir won't be included in shebang lines etc. - dst_dir = args[2] if len(args) > 2 else '' - - if len(args) > 3: - node_prefix = args[3] - - # install_path thus becomes the base target directory. - install_path = dst_dir + node_prefix + '/' - - cmd = args[1] if len(args) > 1 else 'install' + subdir_files(options, zoslibinc, 'include/node/zoslib/', wanted_zoslib_headers) - if os.environ.get('HEADERS_ONLY'): - if cmd == 'install': - headers(install) +def run(options): + if options.headers_only: + if options.command == 'install': + headers(options, install) return - if cmd == 'uninstall': - headers(uninstall) + if options.command == 'uninstall': + headers(options, uninstall) return else: - if cmd == 'install': - files(install) + if options.command == 'install': + files(options, install) return - if cmd == 'uninstall': - files(uninstall) + if options.command == 'uninstall': + files(options, uninstall) return - raise RuntimeError('Bad command: %s\n' % cmd) + raise RuntimeError('Bad command: %s\n' % options.command) + +def parse_options(args): + parser = argparse.ArgumentParser( + description='Install headers and binaries into filesystem') + parser.add_argument('command', choices=['install', 'uninstall']) + parser.add_argument('--dest-dir', help='custom install prefix for packagers, i.e. DESTDIR', + default=os.getcwd()) + parser.add_argument('--prefix', help='custom install prefix, i.e. PREFIX', + default='/usr/local') + parser.add_argument('--headers-only', help='only install headers', + action='store_true', default=False) + parser.add_argument('--root-dir', help='the root directory of source code', + default=os.getcwd()) + parser.add_argument('--build-dir', help='the location of built binaries', + default='out/Release') + parser.add_argument('--v8-dir', help='the location of V8', + default='deps/v8') + parser.add_argument('--config-gypi-path', help='the location of config.gypi', + default='config.gypi') + parser.add_argument('--is-win', help='build for Windows target', + action='store_true', + default=(sys.platform in ['win32', 'cygwin'])) + parser.add_argument('--silent', help='do not output log', + action='store_true', default=False) + options = parser.parse_args(args) + + # |dest_dir| is a custom install prefix for packagers (think DESTDIR) + # |prefix| is a custom install prefix (think PREFIX) + # Difference is that dest_dir won't be included in shebang lines etc. + # |install_path| thus becomes the base target directory. + options.install_path = os.path.join(options.dest_dir + options.prefix) + + # Read variables from the config.gypi. + with open(options.config_gypi_path) as f: + config = ast.literal_eval(f.read()) + options.variables = config['variables'] + + return options if __name__ == '__main__': - run(sys.argv[:]) + run(parse_options(sys.argv[1:])) diff --git a/unofficial.gni b/unofficial.gni index 6047dfd9c0d8a1..aa2645a475b525 100644 --- a/unofficial.gni +++ b/unofficial.gni @@ -37,8 +37,6 @@ template("node_gn_build") { } if (v8_enable_i18n_support) { defines += [ "NODE_HAVE_I18N_SUPPORT=1" ] - } else { - defines += [ "NODE_HAVE_I18N_SUPPORT=0" ] } } @@ -170,6 +168,10 @@ template("node_gn_build") { if (is_mac) { frameworks = [ "CoreFoundation.framework" ] } + if (is_posix) { + configs -= [ "//build/config/gcc:symbol_visibility_hidden" ] + configs += [ "//build/config/gcc:symbol_visibility_default" ] + } if (v8_enable_i18n_support) { deps += [ "//third_party/icu" ] @@ -202,7 +204,10 @@ template("node_gn_build") { forward_variables_from(invoker, "*") sources = [ "src/node_main.cc" ] - deps = [ ":libnode" ] + deps = [ + ":libnode", + "//build/win:default_exe_manifest", + ] if (node_use_node_snapshot) { sources += [ "$target_gen_dir/node_snapshot.cc" ] deps += [ ":run_node_mksnapshot" ] @@ -216,6 +221,15 @@ template("node_gn_build") { sources += [ "src/node_snapshot_stub.cc" ] } output_name = "node" + + if (is_apple) { + # Default optimization on apple adds "-dead_strip" to ldflags, which would + # strip exported V8 and NAPI symbols, has to remove it to make native + # modules work. + # There is almost no performance penalty since we are only removing + # optimization on the node_main.cc. + configs -= [ "//build/config/compiler:default_optimization" ] + } } if (node_use_node_snapshot) { diff --git a/vcbuild.bat b/vcbuild.bat index c3852c89cc5e18..e1607504dcb7ee 100644 --- a/vcbuild.bat +++ b/vcbuild.bat @@ -429,10 +429,8 @@ if defined dll ( copy /Y node.def %TARGET_NAME%\Release\ > nul if errorlevel 1 echo Cannot copy node.def && goto package_error - set HEADERS_ONLY=1 - python ..\tools\install.py install %CD%\%TARGET_NAME% \ > nul + python ..\tools\install.py install --dest-dir %CD%\%TARGET_NAME% --prefix \ --headers-only --silent if errorlevel 1 echo Cannot install headers && goto package_error - set HEADERS_ONLY= ) cd .. @@ -560,8 +558,7 @@ for /d %%F in (test\addons\??_*) do ( if %errorlevel% neq 0 exit /b %errorlevel% :: building addons setlocal -set npm_config_nodedir=%~dp0 -"%node_exe%" "%~dp0tools\build-addons.mjs" "%~dp0deps\npm\node_modules\node-gyp\bin\node-gyp.js" "%~dp0test\addons" +python "%~dp0tools\build_addons.py" "%~dp0test\addons" if errorlevel 1 exit /b 1 endlocal @@ -578,8 +575,7 @@ for /d %%F in (test\js-native-api\??_*) do ( ) :: building js-native-api setlocal -set npm_config_nodedir=%~dp0 -"%node_exe%" "%~dp0tools\build-addons.mjs" "%~dp0deps\npm\node_modules\node-gyp\bin\node-gyp.js" "%~dp0test\js-native-api" +python "%~dp0tools\build_addons.py" "%~dp0test\js-native-api" if errorlevel 1 exit /b 1 endlocal goto build-node-api-tests @@ -597,8 +593,7 @@ for /d %%F in (test\node-api\??_*) do ( ) :: building node-api setlocal -set npm_config_nodedir=%~dp0 -"%node_exe%" "%~dp0tools\build-addons.mjs" "%~dp0deps\npm\node_modules\node-gyp\bin\node-gyp.js" "%~dp0test\node-api" +python "%~dp0tools\build_addons.py" "%~dp0test\node-api" if errorlevel 1 exit /b 1 endlocal goto run-tests