From 1c1c911e7c3327b2f50ab129c37dacf0ef37c075 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 1 Nov 2021 15:09:28 -0700 Subject: [PATCH 01/52] re-enble --- tests/common.py | 4 ---- tools/building.py | 10 +++------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/tests/common.py b/tests/common.py index ef6ed9ba49b5a..115b6ad921cdc 100644 --- a/tests/common.py +++ b/tests/common.py @@ -399,10 +399,6 @@ def setUp(self): # Increate stack trace limit to maximise usefulness of test failure reports '--stack-trace-limit=50', # Opt in to node v15 default behaviour: - # https://nodejs.org/api/cli.html#cli_unhandled_rejections_mode - '--unhandled-rejections=throw', - # Include backtrace for all uncuaght exceptions (not just Error). - '--trace-uncaught', ] self.v8_args = [] self.env = {} diff --git a/tools/building.py b/tools/building.py index 8ede8b4dd7cd1..3f79aeb1980ad 100644 --- a/tools/building.py +++ b/tools/building.py @@ -647,13 +647,9 @@ def acorn_optimizer(filename, passes, extra_info=None, return_output=False): # evals ctors. if binaryen_bin is provided, it is the dir of the binaryen tool # for this, and we are in wasm mode def eval_ctors(js_file, binary_file, debug_info=False): # noqa - logger.debug('Ctor evalling in the wasm backend is disabled due to https://github.com/emscripten-core/emscripten/issues/9527') - return - # TODO re-enable - # cmd = [PYTHON, path_from_root('tools/ctor_evaller.py'), js_file, binary_file, str(settings.INITIAL_MEMORY), str(settings.TOTAL_STACK), str(settings.GLOBAL_BASE), binaryen_bin, str(int(debug_info))] - # if binaryen_bin: - # cmd += get_binaryen_feature_flags() - # check_call(cmd) + cmd = [PYTHON, path_from_root('tools/ctor_evaller.py'), js_file, binary_file, str(settings.INITIAL_MEMORY), str(settings.TOTAL_STACK), str(settings.GLOBAL_BASE), get_binaryen_bin(), str(int(debug_info))] + cmd += get_binaryen_feature_flags() + check_call(cmd) def get_closure_compiler(): From 95b67e7231d048c88c94dce2c208ccdd17b92597 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 1 Nov 2021 15:21:04 -0700 Subject: [PATCH 02/52] test --- .circleci/config.yml | 9 +++++++++ tests/test_core.py | 1 - tests/test_other.py | 1 - 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2aa19545da2ad..6345ac4f876e8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -374,6 +374,12 @@ jobs: - run-tests: # also add a little select testing for wasm2js in -O3 test_targets: "wasm3 wasm2js3.test_memorygrowth_2" + test-wasmz: + executor: bionic + steps: + - run-tests: + # also add a little select testing for wasm2js in -O3 + test_targets: "wasmz" test-wasm2js1: executor: bionic steps: @@ -470,6 +476,9 @@ workflows: - test-wasm3: requires: - build-linux + - test-wasmz: + requires: + - build-linux - test-wasm2js1: requires: - build-linux diff --git a/tests/test_core.py b/tests/test_core.py index a590a43b3d804..d7644831f75ea 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -6719,7 +6719,6 @@ def test_tracing(self): self.emcc_args += ['--tracing'] self.do_core_test('test_tracing.c') - @disabled('https://github.com/emscripten-core/emscripten/issues/9527') def test_eval_ctors(self): if '-O2' not in str(self.emcc_args) or '-O1' in str(self.emcc_args): self.skipTest('need js optimizations') diff --git a/tests/test_other.py b/tests/test_other.py index 851a95da17be7..3c5966d1847ee 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -6593,7 +6593,6 @@ def test_eval_ctors_non_terminating(self): create_file('src.cpp', src) self.run_process([EMXX, 'src.cpp', '-O2', '-s', 'EVAL_CTORS', '-profiling-funcs', '-s', 'WASM=%d' % wasm]) - @disabled('EVAL_CTORS is currently disabled') def test_eval_ctors(self): for wasm in (1, 0): print('wasm', wasm) From 495db3e4c8f11b1dfc3e5dc12d8664e730002db0 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 15 Nov 2021 16:58:49 -0800 Subject: [PATCH 03/52] Get basic logic working again --- tests/test_core.py | 8 ++--- tools/ctor_evaller.py | 75 +++++++++++++------------------------------ 2 files changed, 26 insertions(+), 57 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index f6d4e279909c2..883c39f9e020d 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -6757,16 +6757,16 @@ def test_eval_ctors(self): def get_code_size(): if self.is_wasm(): # Use number of functions as a for code size - return self.count_wasm_contents('hello_libcxx.wasm', 'funcs') + return self.count_wasm_contents('src.wasm', 'funcs') else: - return os.path.getsize('hello_libcxx.js') + return os.path.getsize('src.js') def get_mem_size(): if self.is_wasm(): # Use number of functions as a for code size - return self.count_wasm_contents('hello_libcxx.wasm', 'memory-data') + return self.count_wasm_contents('src.wasm', 'memory-data') if self.uses_memory_init_file(): - return os.path.getsize('hello_libcxx.js.mem') + return os.path.getsize('src.js.mem') # otherwise we ignore memory size return 0 diff --git a/tools/ctor_evaller.py b/tools/ctor_evaller.py index dfef685ae34f2..923b2972bed63 100755 --- a/tools/ctor_evaller.py +++ b/tools/ctor_evaller.py @@ -37,78 +37,47 @@ # helpers +CTOR_NAME = '__wasm_call_ctors' -def find_ctors(js): - ctors_start = js.find('__ATINIT__.push(') - if ctors_start < 0: - return (-1, -1) - ctors_end = js.find(');', ctors_start) - assert ctors_end > 0 - ctors_end += 3 - return (ctors_start, ctors_end) - - -def find_ctors_data(js, num): - ctors_start, ctors_end = find_ctors(js) - assert ctors_start > 0 - ctors_text = js[ctors_start:ctors_end] - all_ctors = [ctor for ctor in ctors_text.split(' ') if ctor.endswith('()') and not ctor == 'function()' and '.' not in ctor] - all_ctors = [ctor.replace('()', '') for ctor in all_ctors] - assert all(ctor.startswith('_') for ctor in all_ctors) - all_ctors = [ctor[1:] for ctor in all_ctors] - assert len(all_ctors) - ctors = all_ctors[:num] - return ctors_start, ctors_end, all_ctors, ctors - - -def eval_ctors(js, wasm_file, num): - ctors_start, ctors_end, all_ctors, ctors = find_ctors_data(js, num) - cmd = [os.path.join(binaryen_bin, 'wasm-ctor-eval'), wasm_file, '-o', wasm_file, '--ctors=' + ','.join(ctors)] +CTOR_ADD_PATTERN = '''addOnInit(Module['asm']['%s']);''' % CTOR_NAME + +def has_ctor(js): + return CTOR_ADD_PATTERN in js + + +def eval_ctors(js, wasm_file): + cmd = [os.path.join(binaryen_bin, 'wasm-ctor-eval'), wasm_file, '-o', wasm_file, '--ctors=' + CTOR_NAME] cmd += extra_args if debug_info: cmd += ['-g'] - logger.debug('wasm ctor cmd: ' + str(cmd)) + logger.warning('wasm ctor cmd: ' + str(cmd)) try: - err = subprocess.run(cmd, stderr=subprocess.PIPE, timeout=10).stdout + err = subprocess.run(cmd, stderr=subprocess.PIPE, timeout=10, text=True).stderr except subprocess.TimeoutExpired: - logger.debug('ctors timed out\n') + logger.warning('ctors timed out\n') return 0, js + logger.warning(err) num_successful = err.count('success on') - logger.debug(err) - if len(ctors) == num_successful: - new_ctors = '' - else: - elements = [] - for ctor in all_ctors[num_successful:]: - elements.append('{ func: function() { %s() } }' % ctor) - new_ctors = '__ATINIT__.push(' + ', '.join(elements) + ');' - js = js[:ctors_start] + new_ctors + js[ctors_end:] + if num_successful: + js = js.replace(CTOR_ADD_PATTERN, '') return num_successful, js # main def main(): + logger.warning('ctor_evaller: waka') js = utils.read_file(js_file) - ctors_start, ctors_end = find_ctors(js) - if ctors_start < 0: - logger.debug('ctor_evaller: no ctors') + if not has_ctor(js): + logger.warning('ctor_evaller: no ctors') sys.exit(0) - ctors_text = js[ctors_start:ctors_end] - if ctors_text.count('(') == 1: - logger.debug('ctor_evaller: push, but no ctors') - sys.exit(0) - - num_ctors = ctors_text.count('function()') - logger.debug('ctor_evaller: %d ctors, from |%s|' % (num_ctors, ctors_text)) - wasm_file = binary_file - logger.debug('ctor_evaller (wasm): trying to eval %d global constructors' % num_ctors) - num_successful, new_js = eval_ctors(js, wasm_file, num_ctors) + logger.warning('ctor_evaller (wasm): trying to eval global ctor') + num_successful, new_js = eval_ctors(js, wasm_file) if num_successful == 0: - logger.debug('ctor_evaller: not successful') + logger.warning('ctor_evaller: not successful') sys.exit(0) - logger.debug('ctor_evaller: we managed to remove %d ctors' % num_successful) + logger.warning('ctor_evaller: we managed to remove the ctors') utils.write_file(js_file, new_js) From efdf2aad3a2a6b21e28e6884505d71ae840caac1 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 7 Jan 2022 10:06:54 -0800 Subject: [PATCH 04/52] don't eval ctors by default any more --- emcc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/emcc.py b/emcc.py index bfbf47c551724..ecd0c04c63140 100755 --- a/emcc.py +++ b/emcc.py @@ -1442,8 +1442,6 @@ def default_setting(name, new_default): if settings.OPT_LEVEL >= 1: default_setting('ASSERTIONS', 0) - if settings.SHRINK_LEVEL >= 2: - default_setting('EVAL_CTORS', 1) if options.emrun: options.pre_js.append(utils.path_from_root('src/emrun_prejs.js')) From c40287314df7864b95e9f240c8fa7e6e82763632 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 7 Jan 2022 10:13:46 -0800 Subject: [PATCH 05/52] temp [ci skip] --- tools/building.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/building.py b/tools/building.py index a64033ed7655f..8d8fce6558f88 100644 --- a/tools/building.py +++ b/tools/building.py @@ -653,6 +653,7 @@ def acorn_optimizer(filename, passes, extra_info=None, return_output=False): # for this, and we are in wasm mode def eval_ctors(js_file, binary_file, debug_info=False): # noqa cmd = [PYTHON, path_from_root('tools/ctor_evaller.py'), js_file, binary_file, str(settings.INITIAL_MEMORY), str(settings.TOTAL_STACK), str(settings.GLOBAL_BASE), get_binaryen_bin(), str(int(debug_info))] + cmd += ['--ignore-external-input'] # TODO: option cmd += get_binaryen_feature_flags() check_call(cmd) From 4b5447ab2c2938906d40604be93361a4c1b2dd0b Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 7 Jan 2022 10:26:26 -0800 Subject: [PATCH 06/52] force test --- src/settings.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/settings.js b/src/settings.js index 6ee898933677d..eff420b0dd1aa 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1567,7 +1567,7 @@ var PTHREADS_DEBUG = 0; // to optimize ctors with lowest priority. We do know that, and can optimize all // the ctors. // [link] -var EVAL_CTORS = 0; +var EVAL_CTORS = 1; // Is enabled, use the JavaScript TextDecoder API for string marshalling. // Enabled by default, set this to 0 to disable. From 269a69adbc5b87367349d90045306960027a6f01 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 7 Jan 2022 12:10:01 -0800 Subject: [PATCH 07/52] fornow --- tools/building.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/building.py b/tools/building.py index 8d8fce6558f88..8d5ef5d097246 100644 --- a/tools/building.py +++ b/tools/building.py @@ -652,6 +652,8 @@ def acorn_optimizer(filename, passes, extra_info=None, return_output=False): # evals ctors. if binaryen_bin is provided, it is the dir of the binaryen tool # for this, and we are in wasm mode def eval_ctors(js_file, binary_file, debug_info=False): # noqa + if settings.WASM2JS: + exit_with_error('EVAL_CTORS is not supported with wasm2js yet. see #XXXXX') # code size/memory and correctness issues cmd = [PYTHON, path_from_root('tools/ctor_evaller.py'), js_file, binary_file, str(settings.INITIAL_MEMORY), str(settings.TOTAL_STACK), str(settings.GLOBAL_BASE), get_binaryen_bin(), str(int(debug_info))] cmd += ['--ignore-external-input'] # TODO: option cmd += get_binaryen_feature_flags() From df56bb8cd4fb9a5ab7f6c3760f7c79432cf9ac05 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 7 Jan 2022 12:56:04 -0800 Subject: [PATCH 08/52] work [ci skip] --- emcc.py | 14 ++++++++++++++ tools/building.py | 2 -- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/emcc.py b/emcc.py index ecd0c04c63140..57b9554fe1555 100755 --- a/emcc.py +++ b/emcc.py @@ -2165,6 +2165,20 @@ def check_memory_setting(setting): if settings.SINGLE_FILE: settings.GENERATE_SOURCE_MAP = 0 + if settings.EVAL_CTORS: + if settings.WASM2JS: + #diagnostics.warning('emcc', 'disabling EVAL_CTORS due to wasm2js. see #XXXXX') # code size/memory and correctness issues + print('emcc', 'disabling EVAL_CTORS due to wasm2js. see #XXXXX') # code size/memory and correctness issues + settings.EVAL_CTORS = 0 + elif settings.USE_PTHREADS: + #diagnostics.warning('emcc', 'disabling EVAL_CTORS due to pthreads (passive segments)') + print('emcc', 'disabling EVAL_CTORS due to pthreads (passive segments)') + settings.EVAL_CTORS = 0 + elif settings.RELOCATABLE: + #diagnostics.warning('emcc', 'disabling EVAL_CTORS due to relocatable (movable segments)') + print('emcc', 'disabling EVAL_CTORS due to relocatable (movable segments)') + settings.EVAL_CTORS = 0 + if options.use_closure_compiler == 2 and not settings.WASM2JS: exit_with_error('closure compiler mode 2 assumes the code is asm.js, so not meaningful for wasm') diff --git a/tools/building.py b/tools/building.py index 8d5ef5d097246..8d8fce6558f88 100644 --- a/tools/building.py +++ b/tools/building.py @@ -652,8 +652,6 @@ def acorn_optimizer(filename, passes, extra_info=None, return_output=False): # evals ctors. if binaryen_bin is provided, it is the dir of the binaryen tool # for this, and we are in wasm mode def eval_ctors(js_file, binary_file, debug_info=False): # noqa - if settings.WASM2JS: - exit_with_error('EVAL_CTORS is not supported with wasm2js yet. see #XXXXX') # code size/memory and correctness issues cmd = [PYTHON, path_from_root('tools/ctor_evaller.py'), js_file, binary_file, str(settings.INITIAL_MEMORY), str(settings.TOTAL_STACK), str(settings.GLOBAL_BASE), get_binaryen_bin(), str(int(debug_info))] cmd += ['--ignore-external-input'] # TODO: option cmd += get_binaryen_feature_flags() From d6512b40e20aa327c96ec5ebe7c0640edc27daa8 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 7 Jan 2022 13:24:06 -0800 Subject: [PATCH 09/52] work --- emcc.py | 6 +-- tools/building.py | 35 ++++++++++++++--- tools/ctor_evaller.py | 87 ------------------------------------------- 3 files changed, 33 insertions(+), 95 deletions(-) delete mode 100755 tools/ctor_evaller.py diff --git a/emcc.py b/emcc.py index 57b9554fe1555..f05a62e57c6db 100755 --- a/emcc.py +++ b/emcc.py @@ -3246,7 +3246,6 @@ def phase_binaryen(target, options, wasm_target): passes += ['--strip-debug'] if strip_producers: passes += ['--strip-producers'] - building.save_intermediate(wasm_target, 'pre-byn.wasm') # if asyncify is used, we will use it in the next stage, and so if it is # the only reason we need intermediate debug info, we can stop keeping it if settings.ASYNCIFY: @@ -3255,16 +3254,17 @@ def phase_binaryen(target, options, wasm_target): wasm_target, args=passes, debug=intermediate_debug_info) + building.save_intermediate(wasm_target, 'byn.wasm') elif strip_debug or strip_producers: # we are not running wasm-opt. if we need to strip certain sections # then do so using llvm-objcopy which is fast and does not rewrite the # code (which is better for debug info) - building.save_intermediate(wasm_target, 'pre-strip.wasm') building.strip(wasm_target, wasm_target, debug=strip_debug, producers=strip_producers) + building.save_intermediate(wasm_target, 'strip.wasm') if settings.EVAL_CTORS: - building.save_intermediate(wasm_target, 'pre-ctors.wasm') building.eval_ctors(final_js, wasm_target, debug_info=intermediate_debug_info) + building.save_intermediate(wasm_target, 'ctors.wasm') # after generating the wasm, do some final operations diff --git a/tools/building.py b/tools/building.py index 8d8fce6558f88..6f970fded3632 100644 --- a/tools/building.py +++ b/tools/building.py @@ -652,10 +652,35 @@ def acorn_optimizer(filename, passes, extra_info=None, return_output=False): # evals ctors. if binaryen_bin is provided, it is the dir of the binaryen tool # for this, and we are in wasm mode def eval_ctors(js_file, binary_file, debug_info=False): # noqa - cmd = [PYTHON, path_from_root('tools/ctor_evaller.py'), js_file, binary_file, str(settings.INITIAL_MEMORY), str(settings.TOTAL_STACK), str(settings.GLOBAL_BASE), get_binaryen_bin(), str(int(debug_info))] - cmd += ['--ignore-external-input'] # TODO: option - cmd += get_binaryen_feature_flags() - check_call(cmd) + CTOR_NAME = '__wasm_call_ctors' + CTOR_ADD_PATTERN = '''addOnInit(Module['asm']['%s']);''' % CTOR_NAME + + def has_ctor(js): + return CTOR_ADD_PATTERN in js + + def do_eval_ctors(js, wasm_file): + args = ['--ctors=' + CTOR_NAME] + args += ['--ignore-external-input'] # TODO: option + out = run_binaryen_command('wasm-ctor-eval', wasm_file, wasm_file, args=args, stdout=PIPE) + logger.warning(out) + num_successful = out.count('success on') + if num_successful: + js = js.replace(CTOR_ADD_PATTERN, '') + return num_successful, js + + js = utils.read_file(js_file) + if not has_ctor(js): + logger.warning('ctor_evaller: no ctors') + sys.exit(0) + + wasm_file = binary_file + logger.warning('ctor_evaller (wasm): trying to eval global ctor') + num_successful, new_js = do_eval_ctors(js, wasm_file) + if num_successful == 0: + logger.warning('ctor_evaller: not successful') + sys.exit(0) + logger.warning('ctor_evaller: we managed to remove the ctors') + utils.write_file(js_file, new_js) def get_closure_compiler(): @@ -1469,7 +1494,7 @@ def run_binaryen_command(tool, infile, outfile=None, args=[], debug=False, stdou cmd += get_binaryen_feature_flags() # if we are emitting a source map, every time we load and save the wasm # we must tell binaryen to update it - if settings.GENERATE_SOURCE_MAP and outfile: + if settings.GENERATE_SOURCE_MAP and outfile and tool in ['wasm-opt', 'wasm-emscripten-finalize']: cmd += [f'--input-source-map={infile}.map'] cmd += [f'--output-source-map={outfile}.map'] ret = check_call(cmd, stdout=stdout).stdout diff --git a/tools/ctor_evaller.py b/tools/ctor_evaller.py deleted file mode 100755 index 2a67a457602ea..0000000000000 --- a/tools/ctor_evaller.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2016 The Emscripten Authors. All rights reserved. -# Emscripten is available under two separate licenses, the MIT license and the -# University of Illinois/NCSA Open Source License. Both these licenses can be -# found in the LICENSE file. - -"""Tries to evaluate global constructors, applying their effects ahead of time. - -This is an LTO-like operation, and to avoid parsing the entire tree (we might -fail to parse a massive project, we operate on the text in python. -""" - -import logging -import os -import subprocess -import sys - -__scriptdir__ = os.path.dirname(os.path.abspath(__file__)) -__rootdir__ = os.path.dirname(__scriptdir__) -sys.path.append(__rootdir__) - -from tools import utils - - -js_file = sys.argv[1] -binary_file = sys.argv[2] # mem init for js, wasm binary for wasm -total_memory = int(sys.argv[3]) -total_stack = int(sys.argv[4]) -global_base = int(sys.argv[5]) -binaryen_bin = sys.argv[6] -debug_info = int(sys.argv[7]) -extra_args = sys.argv[8:] - -wasm = bool(binaryen_bin) - -assert global_base > 0 - -logger = logging.getLogger('ctor_evaller') - -# helpers - -CTOR_NAME = '__wasm_call_ctors' - -CTOR_ADD_PATTERN = '''addOnInit(Module['asm']['%s']);''' % CTOR_NAME - -def has_ctor(js): - return CTOR_ADD_PATTERN in js - - -def eval_ctors(js, wasm_file): - cmd = [os.path.join(binaryen_bin, 'wasm-ctor-eval'), wasm_file, '-o', wasm_file, '--ctors=' + CTOR_NAME] - cmd += extra_args - if debug_info: - cmd += ['-g'] - logger.warning('wasm ctor cmd: ' + str(cmd)) - try: - err = subprocess.run(cmd, stderr=subprocess.PIPE, timeout=10, text=True).stderr - except subprocess.TimeoutExpired: - logger.warning('ctors timed out\n') - return 0, js - logger.warning(err) - num_successful = err.count('success on') - if num_successful: - js = js.replace(CTOR_ADD_PATTERN, '') - return num_successful, js - - -# main -def main(): - logger.warning('ctor_evaller: waka') - js = utils.read_file(js_file) - if not has_ctor(js): - logger.warning('ctor_evaller: no ctors') - sys.exit(0) - - wasm_file = binary_file - logger.warning('ctor_evaller (wasm): trying to eval global ctor') - num_successful, new_js = eval_ctors(js, wasm_file) - if num_successful == 0: - logger.warning('ctor_evaller: not successful') - sys.exit(0) - logger.warning('ctor_evaller: we managed to remove the ctors') - utils.write_file(js_file, new_js) - - -if __name__ == '__main__': - sys.exit(main()) From ec504baa23e78826d07903d3a721cfc4f0b7768f Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 7 Jan 2022 13:25:18 -0800 Subject: [PATCH 10/52] nicer [ci skip] --- tools/building.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/building.py b/tools/building.py index 6f970fded3632..a3c26741b67e9 100644 --- a/tools/building.py +++ b/tools/building.py @@ -662,7 +662,7 @@ def do_eval_ctors(js, wasm_file): args = ['--ctors=' + CTOR_NAME] args += ['--ignore-external-input'] # TODO: option out = run_binaryen_command('wasm-ctor-eval', wasm_file, wasm_file, args=args, stdout=PIPE) - logger.warning(out) + logger.warning('\n\n' + out) num_successful = out.count('success on') if num_successful: js = js.replace(CTOR_ADD_PATTERN, '') From a136023e287da1325006efe52bd239eab761a746 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 7 Jan 2022 13:38:55 -0800 Subject: [PATCH 11/52] fix --- tests/test_core.py | 4 ++-- tools/building.py | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index e0efb339da284..01f232d1f2ce6 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -8233,8 +8233,8 @@ def test_ubsan_full_stack_trace(self, g_flag, expected_output): if g_flag == '-gsource-map': if not self.is_wasm(): self.skipTest('wasm2js has no source map support') - elif '-Oz' in self.emcc_args: - self.skipTest('-Oz breaks stack traces') + elif self.get_setting('EVAL_CTORS'): + self.skipTest('EVAL_CTORS does not support source maps') create_file('pre.js', 'Module = {UBSAN_OPTIONS: "print_stacktrace=1"};') self.emcc_args += ['-fsanitize=null', g_flag, '--pre-js=pre.js'] diff --git a/tools/building.py b/tools/building.py index a3c26741b67e9..b252eb6e4c637 100644 --- a/tools/building.py +++ b/tools/building.py @@ -676,10 +676,6 @@ def do_eval_ctors(js, wasm_file): wasm_file = binary_file logger.warning('ctor_evaller (wasm): trying to eval global ctor') num_successful, new_js = do_eval_ctors(js, wasm_file) - if num_successful == 0: - logger.warning('ctor_evaller: not successful') - sys.exit(0) - logger.warning('ctor_evaller: we managed to remove the ctors') utils.write_file(js_file, new_js) From 279a23a3841a92b1ba2fa9797bfa78e5267e717d Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 7 Jan 2022 13:39:48 -0800 Subject: [PATCH 12/52] fix --- tools/building.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/building.py b/tools/building.py index b252eb6e4c637..408167862df48 100644 --- a/tools/building.py +++ b/tools/building.py @@ -671,7 +671,7 @@ def do_eval_ctors(js, wasm_file): js = utils.read_file(js_file) if not has_ctor(js): logger.warning('ctor_evaller: no ctors') - sys.exit(0) + return wasm_file = binary_file logger.warning('ctor_evaller (wasm): trying to eval global ctor') From 86312e976e9302020e042028359bf9bc91cf0aa2 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 7 Jan 2022 13:44:35 -0800 Subject: [PATCH 13/52] work [ci skip] --- tools/building.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/building.py b/tools/building.py index 408167862df48..d4c92e7c569b4 100644 --- a/tools/building.py +++ b/tools/building.py @@ -653,7 +653,7 @@ def acorn_optimizer(filename, passes, extra_info=None, return_output=False): # for this, and we are in wasm mode def eval_ctors(js_file, binary_file, debug_info=False): # noqa CTOR_NAME = '__wasm_call_ctors' - CTOR_ADD_PATTERN = '''addOnInit(Module['asm']['%s']);''' % CTOR_NAME + CTOR_ADD_PATTERN = "addOnInit(Module['asm']['%s']);" % CTOR_NAME def has_ctor(js): return CTOR_ADD_PATTERN in js From 111bfc6195e4f27eb93359f8b968f6b2d8d13e7d Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 7 Jan 2022 13:50:29 -0800 Subject: [PATCH 14/52] test [ci skip]. --- tests/other/metadce/minimal_Oz_EVAL_CTORS.exports | 3 +++ tests/other/metadce/minimal_Oz_EVAL_CTORS.funcs | 1 + tests/other/metadce/minimal_Oz_EVAL_CTORS.imports | 1 + tests/other/metadce/minimal_Oz_EVAL_CTORS.jssize | 1 + tests/other/metadce/minimal_Oz_EVAL_CTORS.sent | 1 + tests/other/metadce/minimal_Oz_EVAL_CTORS.size | 1 + tests/test_other.py | 2 ++ 7 files changed, 10 insertions(+) create mode 100644 tests/other/metadce/minimal_Oz_EVAL_CTORS.exports create mode 100644 tests/other/metadce/minimal_Oz_EVAL_CTORS.funcs create mode 100644 tests/other/metadce/minimal_Oz_EVAL_CTORS.imports create mode 100644 tests/other/metadce/minimal_Oz_EVAL_CTORS.jssize create mode 100644 tests/other/metadce/minimal_Oz_EVAL_CTORS.sent create mode 100644 tests/other/metadce/minimal_Oz_EVAL_CTORS.size diff --git a/tests/other/metadce/minimal_Oz_EVAL_CTORS.exports b/tests/other/metadce/minimal_Oz_EVAL_CTORS.exports new file mode 100644 index 0000000000000..de980441c3ab0 --- /dev/null +++ b/tests/other/metadce/minimal_Oz_EVAL_CTORS.exports @@ -0,0 +1,3 @@ +a +b +c diff --git a/tests/other/metadce/minimal_Oz_EVAL_CTORS.funcs b/tests/other/metadce/minimal_Oz_EVAL_CTORS.funcs new file mode 100644 index 0000000000000..ebfabbe4ba30c --- /dev/null +++ b/tests/other/metadce/minimal_Oz_EVAL_CTORS.funcs @@ -0,0 +1 @@ +$0 diff --git a/tests/other/metadce/minimal_Oz_EVAL_CTORS.imports b/tests/other/metadce/minimal_Oz_EVAL_CTORS.imports new file mode 100644 index 0000000000000..8b137891791fe --- /dev/null +++ b/tests/other/metadce/minimal_Oz_EVAL_CTORS.imports @@ -0,0 +1 @@ + diff --git a/tests/other/metadce/minimal_Oz_EVAL_CTORS.jssize b/tests/other/metadce/minimal_Oz_EVAL_CTORS.jssize new file mode 100644 index 0000000000000..63b0ca1b611d8 --- /dev/null +++ b/tests/other/metadce/minimal_Oz_EVAL_CTORS.jssize @@ -0,0 +1 @@ +11893 diff --git a/tests/other/metadce/minimal_Oz_EVAL_CTORS.sent b/tests/other/metadce/minimal_Oz_EVAL_CTORS.sent new file mode 100644 index 0000000000000..8b137891791fe --- /dev/null +++ b/tests/other/metadce/minimal_Oz_EVAL_CTORS.sent @@ -0,0 +1 @@ + diff --git a/tests/other/metadce/minimal_Oz_EVAL_CTORS.size b/tests/other/metadce/minimal_Oz_EVAL_CTORS.size new file mode 100644 index 0000000000000..a8fa06e1be7da --- /dev/null +++ b/tests/other/metadce/minimal_Oz_EVAL_CTORS.size @@ -0,0 +1 @@ +62 diff --git a/tests/test_other.py b/tests/test_other.py index cc8ae54aa990d..e10b303d2eb9f 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -7246,6 +7246,8 @@ def strip_numeric_suffixes(funcname): 'Os': (['-Os'], [], []), # noqa 'Oz': (['-Oz'], [], []), # noqa 'Os_mr': (['-Os', '-s', 'MINIMAL_RUNTIME'], [], [], 74), # noqa + # EVAL_CTORS also removes the __wasm_call_ctors function + 'Oz-ctors': (['-Oz', '-s', 'EVAL_CTORS'], [], []), # noqa }) def test_metadce_minimal(self, *args): self.run_metadce_test('minimal.c', *args) From dced5fb42a44754fca013acdf4e051db14cdb6d9 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 7 Jan 2022 13:51:16 -0800 Subject: [PATCH 15/52] restore --- src/settings.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/settings.js b/src/settings.js index eff420b0dd1aa..6ee898933677d 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1567,7 +1567,7 @@ var PTHREADS_DEBUG = 0; // to optimize ctors with lowest priority. We do know that, and can optimize all // the ctors. // [link] -var EVAL_CTORS = 1; +var EVAL_CTORS = 0; // Is enabled, use the JavaScript TextDecoder API for string marshalling. // Enabled by default, set this to 0 to disable. From af0358fc80bbe134b46ee55bbccb6c46b61f71e6 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 7 Jan 2022 14:03:48 -0800 Subject: [PATCH 16/52] better --- .circleci/config.yml | 6 ------ tests/test_core.py | 5 +++-- tools/building.py | 2 +- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8ef540b981b8a..8641df2b3b97b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -380,12 +380,6 @@ jobs: - run-tests: # also add a little select testing for wasm2js in -O3 test_targets: "wasm3 wasm2js3.test_memorygrowth_2" - test-wasmz: - executor: bionic - steps: - - run-tests: - # also add a little select testing for wasm2js in -O3 - test_targets: "wasmz" test-wasm2js1: executor: bionic steps: diff --git a/tests/test_core.py b/tests/test_core.py index 01f232d1f2ce6..8d485745b0d00 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -7783,10 +7783,11 @@ def verify_broken(args=['0']): # verify that by changing the first wasm to throw in that function found_foo_end = break_wasm('emscripten_lazy_load_code.wasm') if not conditional and self.is_optimizing(): - self.assertFalse(found_foo_end, 'should have optimizd out $foo_end') + self.assertFalse(found_foo_end, 'should have optimized out $foo_end') verify_working() # but breaking the second wasm actually breaks us - break_wasm('emscripten_lazy_load_code.wasm.lazy.wasm') + if not break_wasm('emscripten_lazy_load_code.wasm.lazy.wasm'): + raise Exception('could not break lazy wasm - missing expected code') verify_broken() # restore diff --git a/tools/building.py b/tools/building.py index d4c92e7c569b4..c0a62467a82bf 100644 --- a/tools/building.py +++ b/tools/building.py @@ -660,7 +660,7 @@ def has_ctor(js): def do_eval_ctors(js, wasm_file): args = ['--ctors=' + CTOR_NAME] - args += ['--ignore-external-input'] # TODO: option + #args += ['--ignore-external-input'] # TODO: option out = run_binaryen_command('wasm-ctor-eval', wasm_file, wasm_file, args=args, stdout=PIPE) logger.warning('\n\n' + out) num_successful = out.count('success on') From bfd2561c68ae9761399cccf3c745054a6581c78e Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 7 Jan 2022 14:06:11 -0800 Subject: [PATCH 17/52] work --- src/settings.js | 7 +++++++ tools/building.py | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/settings.js b/src/settings.js index 6ee898933677d..d027a1cf3ca9d 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1566,6 +1566,13 @@ var PTHREADS_DEBUG = 0; // that it doesn't know that we will not link in further code, so it only tries // to optimize ctors with lowest priority. We do know that, and can optimize all // the ctors. +// +// If set to a value of 2, this also makes some "unsafe" assumptions, +// specifically that there is no input received while evalling ctors. That means +// we ignore args to main() as well as assume no environment vars are readable. +// This allows more programs to be optimized, but you need to make sure your +// program does not depend on those features. +// // [link] var EVAL_CTORS = 0; diff --git a/tools/building.py b/tools/building.py index c0a62467a82bf..7120f39eaedc8 100644 --- a/tools/building.py +++ b/tools/building.py @@ -660,7 +660,8 @@ def has_ctor(js): def do_eval_ctors(js, wasm_file): args = ['--ctors=' + CTOR_NAME] - #args += ['--ignore-external-input'] # TODO: option + if settings.EVAL_CTORS == 2: + args += ['--ignore-external-input'] out = run_binaryen_command('wasm-ctor-eval', wasm_file, wasm_file, args=args, stdout=PIPE) logger.warning('\n\n' + out) num_successful = out.count('success on') From bb68e121b23174f4083fa49409db1bd2327156ab Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 7 Jan 2022 14:10:08 -0800 Subject: [PATCH 18/52] test [ci skip] --- .../metadce/hello_libcxx_O2_EVAL_CTORS.exports | 13 +++++++++++++ .../metadce/hello_libcxx_O2_EVAL_CTORS.imports | 11 +++++++++++ .../other/metadce/hello_libcxx_O2_EVAL_CTORS.jssize | 1 + tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.sent | 12 ++++++++++++ tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.size | 1 + .../metadce/hello_libcxx_O2_EVAL_CTORS_2.exports | 12 ++++++++++++ .../metadce/hello_libcxx_O2_EVAL_CTORS_2.imports | 9 +++++++++ .../metadce/hello_libcxx_O2_EVAL_CTORS_2.jssize | 1 + .../other/metadce/hello_libcxx_O2_EVAL_CTORS_2.sent | 12 ++++++++++++ .../other/metadce/hello_libcxx_O2_EVAL_CTORS_2.size | 1 + tests/test_other.py | 4 ++++ 11 files changed, 77 insertions(+) create mode 100644 tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.exports create mode 100644 tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.imports create mode 100644 tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.jssize create mode 100644 tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.sent create mode 100644 tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.size create mode 100644 tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.exports create mode 100644 tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.imports create mode 100644 tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.jssize create mode 100644 tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.sent create mode 100644 tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.size diff --git a/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.exports b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.exports new file mode 100644 index 0000000000000..2702de847e70f --- /dev/null +++ b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.exports @@ -0,0 +1,13 @@ +__errno_location +__indirect_function_table +__wasm_call_ctors +dynCall_iiiiiijj +dynCall_iiiiij +dynCall_iiiiijj +dynCall_jiji +dynCall_viijii +main +memory +stackAlloc +stackRestore +stackSave diff --git a/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.imports b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.imports new file mode 100644 index 0000000000000..3cfc4e50e30cb --- /dev/null +++ b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.imports @@ -0,0 +1,11 @@ +env.abort +env.emscripten_memcpy_big +env.emscripten_resize_heap +env.setTempRet0 +env.strftime_l +wasi_snapshot_preview1.environ_get +wasi_snapshot_preview1.environ_sizes_get +wasi_snapshot_preview1.fd_close +wasi_snapshot_preview1.fd_read +wasi_snapshot_preview1.fd_seek +wasi_snapshot_preview1.fd_write diff --git a/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.jssize b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.jssize new file mode 100644 index 0000000000000..a71fabff39235 --- /dev/null +++ b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.jssize @@ -0,0 +1 @@ +98361 diff --git a/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.sent b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.sent new file mode 100644 index 0000000000000..943c0ca713dc4 --- /dev/null +++ b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.sent @@ -0,0 +1,12 @@ +__cxa_atexit +abort +emscripten_memcpy_big +emscripten_resize_heap +environ_get +environ_sizes_get +fd_close +fd_read +fd_seek +fd_write +setTempRet0 +strftime_l diff --git a/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.size b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.size new file mode 100644 index 0000000000000..8412b58d02298 --- /dev/null +++ b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.size @@ -0,0 +1 @@ +124687 diff --git a/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.exports b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.exports new file mode 100644 index 0000000000000..023e1ead1581b --- /dev/null +++ b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.exports @@ -0,0 +1,12 @@ +__errno_location +__indirect_function_table +dynCall_iiiiiijj +dynCall_iiiiij +dynCall_iiiiijj +dynCall_jiji +dynCall_viijii +main +memory +stackAlloc +stackRestore +stackSave diff --git a/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.imports b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.imports new file mode 100644 index 0000000000000..0292c83ba04a8 --- /dev/null +++ b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.imports @@ -0,0 +1,9 @@ +env.abort +env.emscripten_memcpy_big +env.emscripten_resize_heap +env.setTempRet0 +env.strftime_l +wasi_snapshot_preview1.fd_close +wasi_snapshot_preview1.fd_read +wasi_snapshot_preview1.fd_seek +wasi_snapshot_preview1.fd_write diff --git a/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.jssize b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.jssize new file mode 100644 index 0000000000000..a70c904e54784 --- /dev/null +++ b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.jssize @@ -0,0 +1 @@ +98259 diff --git a/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.sent b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.sent new file mode 100644 index 0000000000000..943c0ca713dc4 --- /dev/null +++ b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.sent @@ -0,0 +1,12 @@ +__cxa_atexit +abort +emscripten_memcpy_big +emscripten_resize_heap +environ_get +environ_sizes_get +fd_close +fd_read +fd_seek +fd_write +setTempRet0 +strftime_l diff --git a/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.size b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.size new file mode 100644 index 0000000000000..3bc4dfb478e0a --- /dev/null +++ b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.size @@ -0,0 +1 @@ +122346 diff --git a/tests/test_other.py b/tests/test_other.py index e10b303d2eb9f..7d45e4740a6bd 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -7263,6 +7263,10 @@ def test_metadce_minimal_pthreads(self): # exceptions does not pull in demangling by default, which increases code size 'mangle': (['-O2', '-fexceptions', '-s', 'DEMANGLE_SUPPORT'], [], ['waka']), # noqa + # eval_ctors 1 can partially optimize, but runs into getenv() for locale + # code. mode 2 ignores those and fully optimizes out the ctors + 'ctors1': (['-O2', '-sEVAL_CTORS'], [], ['waka']), # noqa + 'ctors2': (['-O2', '-sEVAL_CTORS=2'], [], ['waka']), # noqa }) def test_metadce_cxx(self, *args): # do not check functions in this test as there are a lot of libc++ functions From 0cd741c1b3b22f917da0b1d20d5c2201f2a5c91c Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 7 Jan 2022 14:46:16 -0800 Subject: [PATCH 19/52] test [ci skip] --- tests/test_core.py | 80 +++++++--------------------------------------- 1 file changed, 12 insertions(+), 68 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index 8d485745b0d00..f25bf21b98bea 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -6788,35 +6788,21 @@ def test_eval_ctors(self): int main() {} ''', "constructing!\n") - def get_code_size(): - if self.is_wasm(): - # Use number of functions as a for code size - return self.count_wasm_contents('src.wasm', 'funcs') - else: - return os.path.getsize('src.js') - - def get_mem_size(): - if self.is_wasm(): - # Use number of functions as a for code size - return self.count_wasm_contents('src.wasm', 'memory-data') - if self.uses_memory_init_file(): - return os.path.getsize('src.js.mem') - - # otherwise we ignore memory size - return 0 - - def do_test(test): - self.set_setting('EVAL_CTORS') + def do_test(test, level=1, prefix='src'): + def get_code_size(): + if self.is_wasm(): + # this also includes the memory, but it is close enough for our + # purposes + return os.path.getsize(prefix + '.wasm') + else: + return os.path.getsize(prefix + '.js') + + self.set_setting('EVAL_CTORS', level) test() ec_code_size = get_code_size() - ec_mem_size = get_mem_size() self.clear_setting('EVAL_CTORS') test() code_size = get_code_size() - mem_size = get_mem_size() - if mem_size: - print('mem: ', mem_size, '=>', ec_mem_size) - self.assertGreater(ec_mem_size, mem_size) print('code:', code_size, '=>', ec_code_size) self.assertLess(ec_code_size, code_size) @@ -6841,56 +6827,14 @@ def test1(): do_test(test1) - # The wasm backend currently exports a single initalizer so the ctor - # evaluation is all or nothing. As well as that it doesn't currently - # do DCE of libcxx symbols (because the are marked as visibility(defaault) - # and because of that we end up not being able to eval ctors unless all - # libcxx constrcutors can be eval'd - print('libcxx - remove 2 ctors from iostream code') output = 'hello, world!' def test2(): self.do_runf(test_file('hello_libcxx.cpp'), output) - do_test(test2) - - print('assertions too') - self.set_setting('ASSERTIONS') - self.do_runf(test_file('hello_libcxx.cpp'), output) - self.set_setting('ASSERTIONS', 0) - print('remove just some, leave others') - - def test3(): - self.do_run(r''' -#include -#include - -class std_string { -public: - std_string(): ptr(nullptr) { std::cout << "std_string()\n"; } - std_string(const char* s): ptr(s) { std::cout << "std_string(const char* s)" << std::endl; } - std_string(const std_string& s): ptr(s.ptr) { std::cout << "std_string(const std_string& s) " << std::endl; } - const char* data() const { return ptr; } -private: - const char* ptr; -}; - -const std_string txtTestString("212121\0"); -const std::string s2text("someweirdtext"); - -int main() { - std::cout << s2text << std::endl; - std::cout << txtTestString.data() << std::endl; - std::cout << txtTestString.data() << std::endl; - return 0; -} - ''', '''std_string(const char* s) -someweirdtext -212121 -212121 -''') # noqa - do_test(test3) + do_test(test2, level=1, prefix='hello_libcxx') + do_test(test2, level=2, prefix='hello_libcxx') def test_embind(self): self.emcc_args += ['--bind'] From 9a5a3cfea6c764a31cecb6bc890b2d4551cf862f Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 7 Jan 2022 14:52:41 -0800 Subject: [PATCH 20/52] fix --- tests/test_other.py | 139 +++++++++++++------------------------------- 1 file changed, 41 insertions(+), 98 deletions(-) diff --git a/tests/test_other.py b/tests/test_other.py index 7d45e4740a6bd..8e8a1873717d7 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -6620,110 +6620,53 @@ def test_EM_ASM_i64(self): ''') self.expect_fail([EMXX, 'src.cpp', '-Oz']) - def test_eval_ctors_non_terminating(self): - for wasm in (1, 0): - print('wasm', wasm) + def test_eval_ctor_ordering(self): + # ensure order of execution remains correct, even with a bad ctor + def test(p1, p2, p3, last, expected): src = r''' + #include + #include + volatile int total = 0; struct C { - C() { - volatile int y = 0; - while (y == 0) {} + C(int x) { + volatile int y = x; + y++; + y--; + if (y == 0xf) { + // A printf can't be evalled ahead of time, so this will stop + // us. + printf("you can't eval me ahead of time\n"); + } + total <<= 4; + total += int(y); } }; - C always; - int main() {} - ''' + C __attribute__((init_priority(%d))) c1(0x5); + C __attribute__((init_priority(%d))) c2(0x8); + C __attribute__((init_priority(%d))) c3(%d); + int main() { + printf("total is 0x%%x.\n", total); + } + ''' % (p1, p2, p3, last) create_file('src.cpp', src) - self.run_process([EMXX, 'src.cpp', '-O2', '-s', 'EVAL_CTORS', '-profiling-funcs', '-s', 'WASM=%d' % wasm]) - - def test_eval_ctors(self): - for wasm in (1, 0): - print('wasm', wasm) - print('check no ctors is ok') - - # on by default in -Oz, but user-overridable + self.run_process([EMXX, 'src.cpp', '-O2', '-s', 'EVAL_CTORS', '-profiling-funcs', '-s']) + self.assertContained('total is %s.' % hex(expected), self.run_js('a.out.js')) + shutil.copyfile('a.out.js', 'x' + hex(expected) + '.js') + shutil.copyfile('a.out.wasm', 'x' + hex(expected) + '.wasm') + return os.path.getsize('a.out.wasm') - def get_size(args): - print('get_size', args) - self.run_process([EMXX, test_file('hello_libcxx.cpp'), '-s', 'WASM=%d' % wasm] + args) - self.assertContained('hello, world!', self.run_js('a.out.js')) - if wasm: - codesize = self.count_wasm_contents('a.out.wasm', 'funcs') - memsize = self.count_wasm_contents('a.out.wasm', 'memory-data') - else: - codesize = os.path.getsize('a.out.js') - memsize = os.path.getsize('a.out.js.mem') - return (codesize, memsize) - - def check_size(left, right): - # can't measure just the mem out of the wasm, so ignore [1] for wasm - if left[0] == right[0] and left[1] == right[1]: - return 0 - if left[0] < right[0] and left[1] > right[1]: - return -1 # smaller code, bigger mem - if left[0] > right[0] and left[1] < right[1]: - return 1 - assert False, [left, right] - - o2_size = get_size(['-O2']) - assert check_size(get_size(['-O2']), o2_size) == 0, 'deterministic' - assert check_size(get_size(['-O2', '-s', 'EVAL_CTORS']), o2_size) < 0, 'eval_ctors works if user asks for it' - oz_size = get_size(['-Oz']) - assert check_size(get_size(['-Oz']), oz_size) == 0, 'deterministic' - assert check_size(get_size(['-Oz', '-s', 'EVAL_CTORS']), oz_size) == 0, 'eval_ctors is on by default in oz' - assert check_size(get_size(['-Oz', '-s', 'EVAL_CTORS=0']), oz_size) == 1, 'eval_ctors can be turned off' - - linkable_size = get_size(['-Oz', '-s', 'EVAL_CTORS', '-s', 'LINKABLE']) - assert check_size(get_size(['-Oz', '-s', 'EVAL_CTORS=0', '-s', 'LINKABLE']), linkable_size) == 1, 'noticeable difference in linkable too' - - def test_eval_ctor_ordering(self): - # ensure order of execution remains correct, even with a bad ctor - def test(p1, p2, p3, last, expected): - src = r''' - #include - #include - volatile int total = 0; - struct C { - C(int x) { - volatile int y = x; - y++; - y--; - if (y == 0xf) { - printf("you can't eval me ahead of time\n"); // bad ctor - } - total <<= 4; - total += int(y); - } - }; - C __attribute__((init_priority(%d))) c1(0x5); - C __attribute__((init_priority(%d))) c2(0x8); - C __attribute__((init_priority(%d))) c3(%d); - int main() { - printf("total is 0x%%x.\n", total); - } - ''' % (p1, p2, p3, last) - create_file('src.cpp', src) - self.run_process([EMXX, 'src.cpp', '-O2', '-s', 'EVAL_CTORS', '-profiling-funcs', '-s', 'WASM=%d' % wasm]) - self.assertContained('total is %s.' % hex(expected), self.run_js('a.out.js')) - shutil.copyfile('a.out.js', 'x' + hex(expected) + '.js') - if wasm: - shutil.copyfile('a.out.wasm', 'x' + hex(expected) + '.wasm') - return self.count_wasm_contents('a.out.wasm', 'funcs') - else: - return read_file('a.out.js').count('function _') - - print('no bad ctor') - first = test(1000, 2000, 3000, 0xe, 0x58e) # noqa - second = test(3000, 1000, 2000, 0xe, 0x8e5) # noqa - third = test(2000, 3000, 1000, 0xe, 0xe58) # noqa - print(first, second, third) - assert first == second and second == third - print('with bad ctor') - first = test(1000, 2000, 3000, 0xf, 0x58f) # noqa; 2 will succeed - second = test(3000, 1000, 2000, 0xf, 0x8f5) # noqa; 1 will succedd - third = test(2000, 3000, 1000, 0xf, 0xf58) # noqa; 0 will succeed - print(first, second, third) - assert first < second and second < third, [first, second, third] + print('no bad ctor') + first = test(1000, 2000, 3000, 0xe, 0x58e) # noqa + second = test(3000, 1000, 2000, 0xe, 0x8e5) # noqa + third = test(2000, 3000, 1000, 0xe, 0xe58) # noqa + print(first, second, third) + assert first == second and second == third + print('with bad ctor') + first = test(1000, 2000, 3000, 0xf, 0x58f) # noqa; 2 will succeed + second = test(3000, 1000, 2000, 0xf, 0x8f5) # noqa; 1 will succedd + third = test(2000, 3000, 1000, 0xf, 0xf58) # noqa; 0 will succeed + print(first, second, third) + assert first < second and second < third, [first, second, third] @uses_canonical_tmp @with_env_modify({'EMCC_DEBUG': '1'}) From 40cc46d6d86bc76af0598735157f03b190153bd5 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 7 Jan 2022 14:55:56 -0800 Subject: [PATCH 21/52] fix --- tests/test_other.py | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/tests/test_other.py b/tests/test_other.py index 8e8a1873717d7..24f70458d3cf9 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -6671,27 +6671,23 @@ def test(p1, p2, p3, last, expected): @uses_canonical_tmp @with_env_modify({'EMCC_DEBUG': '1'}) def test_eval_ctors_debug_output(self): - for wasm in (1, 0): - print('wasm', wasm) - create_file('lib.js', r''' + create_file('lib.js', r''' mergeInto(LibraryManager.library, { - external_thing: function() {} +external_thing: function() {} }); ''') - create_file('src.cpp', r''' - extern "C" void external_thing(); - struct C { - C() { external_thing(); } // don't remove this! - }; - C c; - int main() {} - ''') - err = self.run_process([EMXX, 'src.cpp', '--js-library', 'lib.js', '-Oz', '-s', 'WASM=%d' % wasm], stderr=PIPE).stderr - # disabled in the wasm backend - self.assertContained('Ctor evalling in the wasm backend is disabled', err) - self.assertNotContained('ctor_evaller: not successful', err) # with logging - # TODO(sbc): Re-enable onece ctor evaluation is working with llvm backend. - # self.assertContained('external_thing', err) # the failing call should be mentioned + create_file('src.cpp', r''' +extern "C" void external_thing(); +struct C { + C() { external_thing(); } // don't remove this! +}; +C c; +int main() {} + ''') + err = self.run_process([EMXX, 'src.cpp', '--js-library', 'lib.js', '-O2', '-s', 'EVAL_CTORS'], stderr=PIPE).stderr + # logging should show we failed, and why + self.assertNotContained('ctor_evaller: not successful', err) + self.assertContained('stopping since could not eval: call import: env.external_thing', err) def test_override_js_execution_environment(self): create_file('main.cpp', r''' From 025acaa9d56941efcb5a5aff21fd93cb68e3993c Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 7 Jan 2022 14:56:34 -0800 Subject: [PATCH 22/52] todo [ci skip] --- tools/building.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/building.py b/tools/building.py index 7120f39eaedc8..7003c19b26efb 100644 --- a/tools/building.py +++ b/tools/building.py @@ -658,7 +658,7 @@ def eval_ctors(js_file, binary_file, debug_info=False): # noqa def has_ctor(js): return CTOR_ADD_PATTERN in js - def do_eval_ctors(js, wasm_file): + def do_eval_ctors(js, wasm_file): # TODO: remove func args = ['--ctors=' + CTOR_NAME] if settings.EVAL_CTORS == 2: args += ['--ignore-external-input'] From 3fa026e089b41c92da7cbb81203e93133976962f Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 11 Jan 2022 16:22:47 -0800 Subject: [PATCH 23/52] wip [ci skip] --- tests/test_benchmark.py | 11 ++++++----- tools/building.py | 5 ++++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index c0ac6758a9c84..bc5e4f067a25a 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -33,7 +33,7 @@ # 5: 10 seconds DEFAULT_ARG = '4' -TEST_REPS = 5 +TEST_REPS = 0 # by default, run just core benchmarks CORE_BENCHMARKS = True @@ -211,7 +211,7 @@ def build(self, parent, filename, args, shared_args, emcc_args, native_args, nat if common.EMTEST_FORCE64: cmd += ['--profiling'] else: - cmd += ['--closure=1', '-sMINIMAL_RUNTIME'] + cmd += ['--closure=0', '-sMINIMAL_RUNTIME'] # add additional emcc args at the end, which may override other things # above, such as minimal runtime cmd += emcc_args + self.extra_args @@ -361,7 +361,7 @@ def cleanup(self): if not common.EMTEST_FORCE64: benchmarkers += [ - NativeBenchmarker('clang', [CLANG_CC], [CLANG_CXX]), + # NativeBenchmarker('clang', [CLANG_CC], [CLANG_CXX]), # NativeBenchmarker('gcc', ['gcc', '-no-pie'], ['g++', '-no-pie']) ] @@ -377,8 +377,9 @@ def cleanup(self): ] else: benchmarkers += [ - EmscriptenBenchmarker(default_v8_name, aot_v8), - EmscriptenBenchmarker(default_v8_name + '-lto', aot_v8, ['-flto']), + EmscriptenBenchmarker('norm', aot_v8), + EmscriptenBenchmarker('eval', aot_v8, ['-sEVAL_CTORS=2']), + #EmscriptenBenchmarker(default_v8_name + '-lto', aot_v8, ['-flto']), # EmscriptenWasm2CBenchmarker('wasm2c') ] if os.path.exists(CHEERP_BIN): diff --git a/tools/building.py b/tools/building.py index 7003c19b26efb..78585c9a272b1 100644 --- a/tools/building.py +++ b/tools/building.py @@ -653,7 +653,10 @@ def acorn_optimizer(filename, passes, extra_info=None, return_output=False): # for this, and we are in wasm mode def eval_ctors(js_file, binary_file, debug_info=False): # noqa CTOR_NAME = '__wasm_call_ctors' - CTOR_ADD_PATTERN = "addOnInit(Module['asm']['%s']);" % CTOR_NAME + if settings.MINIMAL_RUNTIME: + CTOR_ADD_PATTERN = f"asm['{CTOR_NAME}']();" # TODO test + else: + CTOR_ADD_PATTERN = f"addOnInit(Module['asm']['{CTOR_NAME}']);" def has_ctor(js): return CTOR_ADD_PATTERN in js From d17a1841119700efd9063d455b58a4a8c4f3e329 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 11 Jan 2022 16:30:32 -0800 Subject: [PATCH 24/52] rebaseline [ci skip] --- tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.jssize | 2 +- tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.sent | 1 - tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.size | 2 +- tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.jssize | 2 +- tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.sent | 1 - tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.size | 2 +- tests/other/metadce/minimal_Oz_EVAL_CTORS.jssize | 2 +- 7 files changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.jssize b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.jssize index a71fabff39235..52882ff120b26 100644 --- a/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.jssize +++ b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.jssize @@ -1 +1 @@ -98361 +98089 diff --git a/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.sent b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.sent index 943c0ca713dc4..25e3f929f2fb9 100644 --- a/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.sent +++ b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.sent @@ -1,4 +1,3 @@ -__cxa_atexit abort emscripten_memcpy_big emscripten_resize_heap diff --git a/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.size b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.size index 8412b58d02298..b388504765b4d 100644 --- a/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.size +++ b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS.size @@ -1 +1 @@ -124687 +124645 diff --git a/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.jssize b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.jssize index a70c904e54784..c43ab37452f05 100644 --- a/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.jssize +++ b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.jssize @@ -1 +1 @@ -98259 +97987 diff --git a/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.sent b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.sent index 943c0ca713dc4..25e3f929f2fb9 100644 --- a/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.sent +++ b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.sent @@ -1,4 +1,3 @@ -__cxa_atexit abort emscripten_memcpy_big emscripten_resize_heap diff --git a/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.size b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.size index 3bc4dfb478e0a..91e83ccb98875 100644 --- a/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.size +++ b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.size @@ -1 +1 @@ -122346 +122294 diff --git a/tests/other/metadce/minimal_Oz_EVAL_CTORS.jssize b/tests/other/metadce/minimal_Oz_EVAL_CTORS.jssize index 63b0ca1b611d8..7e11609cbce95 100644 --- a/tests/other/metadce/minimal_Oz_EVAL_CTORS.jssize +++ b/tests/other/metadce/minimal_Oz_EVAL_CTORS.jssize @@ -1 +1 @@ -11893 +11845 From aab7908c2762fd4856233c7d7e4909ebeccc3196 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 11 Jan 2022 16:49:15 -0800 Subject: [PATCH 25/52] main too! --- tools/building.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tools/building.py b/tools/building.py index 78585c9a272b1..4d16e62d1c1bd 100644 --- a/tools/building.py +++ b/tools/building.py @@ -652,17 +652,21 @@ def acorn_optimizer(filename, passes, extra_info=None, return_output=False): # evals ctors. if binaryen_bin is provided, it is the dir of the binaryen tool # for this, and we are in wasm mode def eval_ctors(js_file, binary_file, debug_info=False): # noqa - CTOR_NAME = '__wasm_call_ctors' + WASM_CALL_CTORS = '__wasm_call_ctors' if settings.MINIMAL_RUNTIME: - CTOR_ADD_PATTERN = f"asm['{CTOR_NAME}']();" # TODO test + CTOR_ADD_PATTERN = f"asm['{WASM_CALL_CTORS}']();" # TODO test else: - CTOR_ADD_PATTERN = f"addOnInit(Module['asm']['{CTOR_NAME}']);" + CTOR_ADD_PATTERN = f"addOnInit(Module['asm']['{WASM_CALL_CTORS}']);" def has_ctor(js): return CTOR_ADD_PATTERN in js def do_eval_ctors(js, wasm_file): # TODO: remove func - args = ['--ctors=' + CTOR_NAME] + # eval the ctor caller as well as main. note that we must keep main around + # even if we eval it (we could in theory remove the call from the JS) + # TODO: handle standalone mode and reactors + args = ['--ctors=' + WASM_CALL_CTORS + ',main', + '--kept-exports=main'] if settings.EVAL_CTORS == 2: args += ['--ignore-external-input'] out = run_binaryen_command('wasm-ctor-eval', wasm_file, wasm_file, args=args, stdout=PIPE) From de94af727d3feb1718df81799881578f14f1f717 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 12 Jan 2022 08:27:44 -0800 Subject: [PATCH 26/52] fix [ci skip] --- tools/building.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tools/building.py b/tools/building.py index 4d16e62d1c1bd..98bebdbd612a0 100644 --- a/tools/building.py +++ b/tools/building.py @@ -664,9 +664,17 @@ def has_ctor(js): def do_eval_ctors(js, wasm_file): # TODO: remove func # eval the ctor caller as well as main. note that we must keep main around # even if we eval it (we could in theory remove the call from the JS) - # TODO: handle standalone mode and reactors - args = ['--ctors=' + WASM_CALL_CTORS + ',main', - '--kept-exports=main'] + if not settings.STANDALONE_WASM: + if settings.HAS_MAIN: + args = ['--ctors=' + WASM_CALL_CTORS + ',main', + '--kept-exports=main'] + else: + args = ['--ctors=' + WASM_CALL_CTORS] + else: + if settings.EXPECT_MAIN: + args = ['--ctors=_start'] + else: + args = ['--ctors=_instantiate'] if settings.EVAL_CTORS == 2: args += ['--ignore-external-input'] out = run_binaryen_command('wasm-ctor-eval', wasm_file, wasm_file, args=args, stdout=PIPE) From 0f7b08d0b2a4ca0ce458dd6041f665d4d97489f5 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 12 Jan 2022 08:45:16 -0800 Subject: [PATCH 27/52] progress [ci skip] --- tests/test_core.py | 3 +- tools/building.py | 70 ++++++++++++++++++++++++---------------------- 2 files changed, 38 insertions(+), 35 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index 01be18aa129bb..5a8e2e8932012 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -6771,9 +6771,10 @@ def test_tracing(self): self.emcc_args += ['--tracing'] self.do_core_test('test_tracing.c') + @also_with_standalone_wasm() def test_eval_ctors(self): if '-O2' not in str(self.emcc_args) or '-O1' in str(self.emcc_args): - self.skipTest('need js optimizations') + self.skipTest('need opts') if not self.is_wasm(): self.skipTest('this test uses wasm binaries') diff --git a/tools/building.py b/tools/building.py index 98bebdbd612a0..5048dfd19a8d2 100644 --- a/tools/building.py +++ b/tools/building.py @@ -651,48 +651,50 @@ def acorn_optimizer(filename, passes, extra_info=None, return_output=False): # evals ctors. if binaryen_bin is provided, it is the dir of the binaryen tool # for this, and we are in wasm mode -def eval_ctors(js_file, binary_file, debug_info=False): # noqa +def eval_ctors(js_file, wasm_file, debug_info=False): # noqa WASM_CALL_CTORS = '__wasm_call_ctors' if settings.MINIMAL_RUNTIME: CTOR_ADD_PATTERN = f"asm['{WASM_CALL_CTORS}']();" # TODO test else: CTOR_ADD_PATTERN = f"addOnInit(Module['asm']['{WASM_CALL_CTORS}']);" - def has_ctor(js): - return CTOR_ADD_PATTERN in js + js = utils.read_file(js_file) - def do_eval_ctors(js, wasm_file): # TODO: remove func - # eval the ctor caller as well as main. note that we must keep main around - # even if we eval it (we could in theory remove the call from the JS) - if not settings.STANDALONE_WASM: - if settings.HAS_MAIN: - args = ['--ctors=' + WASM_CALL_CTORS + ',main', - '--kept-exports=main'] - else: - args = ['--ctors=' + WASM_CALL_CTORS] + has_wasm_call_ctors = False + + # eval the ctor caller as well as main, or, in standalone mode, the proper + # entry/init function + if not settings.STANDALONE_WASM: + ctors = [] + kept_ctors = [] + has_wasm_call_ctors = CTOR_ADD_PATTERN in js + if has_wasm_call_ctors: + ctors += [WASM_CALL_CTORS] + if settings.HAS_MAIN: + ctors += ['main'] + # TODO perhaps remove the call to main from the JS? or is this an abi + # we want to preserve? + kept_ctors += ['main'] + if not ctors: + logger.warning('ctor_evaller: no ctors') + return + args = ['--ctors=' + ','.join(ctors)] + if kept_ctors: + args += ['--kept-exports=' + ','.join(kept_ctors)] + else: + if settings.EXPECT_MAIN: + args = ['--ctors=_start'] else: - if settings.EXPECT_MAIN: - args = ['--ctors=_start'] - else: - args = ['--ctors=_instantiate'] - if settings.EVAL_CTORS == 2: - args += ['--ignore-external-input'] - out = run_binaryen_command('wasm-ctor-eval', wasm_file, wasm_file, args=args, stdout=PIPE) - logger.warning('\n\n' + out) - num_successful = out.count('success on') - if num_successful: - js = js.replace(CTOR_ADD_PATTERN, '') - return num_successful, js - - js = utils.read_file(js_file) - if not has_ctor(js): - logger.warning('ctor_evaller: no ctors') - return - - wasm_file = binary_file - logger.warning('ctor_evaller (wasm): trying to eval global ctor') - num_successful, new_js = do_eval_ctors(js, wasm_file) - utils.write_file(js_file, new_js) + args = ['--ctors=_initialize'] + if settings.EVAL_CTORS == 2: + args += ['--ignore-external-input'] + logger.warning('ctor_evaller (wasm): trying to eval global ctors (' + ' '.join(args) + ')') + out = run_binaryen_command('wasm-ctor-eval', wasm_file, wasm_file, args=args, stdout=PIPE) + logger.warning('\n\n' + out) + num_successful = out.count('success on') + if num_successful and has_wasm_call_ctors: + js = js.replace(CTOR_ADD_PATTERN, '') + utils.write_file(js_file, js) def get_closure_compiler(): From d94fc70131bfb4b1df33695c7e276e7e09f8cac9 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 12 Jan 2022 08:52:21 -0800 Subject: [PATCH 28/52] test passes [ci skip] --- tests/common.py | 5 +++++ tests/test_core.py | 9 +++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/common.py b/tests/common.py index 2b4a97a66b119..0617cde783ac9 100644 --- a/tests/common.py +++ b/tests/common.py @@ -642,6 +642,11 @@ def is_exported_in_wasm(self, name, wasm): wat = self.get_wasm_text(wasm) return ('(export "%s"' % name) in wat + def measure_wasm_code_lines(self, wasm): + wat_lines = self.get_wasm_text(wasm).splitlines() + non_data_lines = [line for line in wat_lines if '(data ' not in line] + return len(non_data_lines) + def run_js(self, filename, engine=None, args=[], output_nicerizer=None, assert_returncode=0, diff --git a/tests/test_core.py b/tests/test_core.py index 5a8e2e8932012..873fbbafddf82 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -6794,7 +6794,7 @@ def get_code_size(): if self.is_wasm(): # this also includes the memory, but it is close enough for our # purposes - return os.path.getsize(prefix + '.wasm') + return self.measure_wasm_code_lines(prefix + '.wasm') else: return os.path.getsize(prefix + '.js') @@ -6834,7 +6834,12 @@ def test1(): def test2(): self.do_runf(test_file('hello_libcxx.cpp'), output) - do_test(test2, level=1, prefix='hello_libcxx') + # in standalone more there is more usage of WASI APIs, which mode 2 is + # needed to avoid in order to fully optimize, so do not test mode 1 in + # that mode. + if not self.get_setting('STANDALONE_WASM'): + do_test(test2, level=1, prefix='hello_libcxx') + do_test(test2, level=2, prefix='hello_libcxx') def test_embind(self): From bae78c223fd7df069c6e19e56e39cbc8596efa57 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 12 Jan 2022 08:57:03 -0800 Subject: [PATCH 29/52] test [ci skip] --- tests/test_core.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_core.py b/tests/test_core.py index 873fbbafddf82..89c851626d7e9 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1529,6 +1529,12 @@ def test_ctors_no_main(self): self.emcc_args.append('--no-entry') self.do_core_test('test_ctors_no_main.cpp') + @also_with_standalone_wasm(impure=True) + def test_eval_ctors_no_main(self): + self.set_setting('EVAL_CTORS') + self.emcc_args.append('--no-entry') + self.do_core_test('test_ctors_no_main.cpp') + def test_class(self): self.do_core_test('test_class.cpp') From bf24890bdcd8550649ef9383f27fc1541d49e856 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 12 Jan 2022 13:02:40 -0800 Subject: [PATCH 30/52] nicer --- emcc.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/emcc.py b/emcc.py index 8d3d326560313..26a282eff0aca 100755 --- a/emcc.py +++ b/emcc.py @@ -2165,16 +2165,19 @@ def check_memory_setting(setting): if settings.EVAL_CTORS: if settings.WASM2JS: - #diagnostics.warning('emcc', 'disabling EVAL_CTORS due to wasm2js. see #XXXXX') # code size/memory and correctness issues - print('emcc', 'disabling EVAL_CTORS due to wasm2js. see #XXXXX') # code size/memory and correctness issues + diagnostics.warning('emcc', 'disabling EVAL_CTORS due to wasm2js. see #XXXXX') # code size/memory and correctness issues settings.EVAL_CTORS = 0 elif settings.USE_PTHREADS: - #diagnostics.warning('emcc', 'disabling EVAL_CTORS due to pthreads (passive segments)') - print('emcc', 'disabling EVAL_CTORS due to pthreads (passive segments)') + diagnostics.warning('emcc', 'disabling EVAL_CTORS due to pthreads (passive segments)') settings.EVAL_CTORS = 0 elif settings.RELOCATABLE: + diagnostics.warning('emcc', 'disabling EVAL_CTORS due to relocatable (movable segments)') + settings.EVAL_CTORS = 0 + elif settings.ASYNCIFY: #diagnostics.warning('emcc', 'disabling EVAL_CTORS due to relocatable (movable segments)') - print('emcc', 'disabling EVAL_CTORS due to relocatable (movable segments)') + # In Asyncify exports can be called more than once, and this seems to not + # work properly yet (see test_emscripten_scan_registers). + diagnostics.warning('emcc', 'disabling EVAL_CTORS due to asyncify') settings.EVAL_CTORS = 0 if options.use_closure_compiler == 2 and not settings.WASM2JS: From 26fc1321de49d479579bf7f76b13ff1b4a1edeba Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 12 Jan 2022 13:16:32 -0800 Subject: [PATCH 31/52] fix [ci skip] --- tools/building.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/building.py b/tools/building.py index 5048dfd19a8d2..056326f43793e 100644 --- a/tools/building.py +++ b/tools/building.py @@ -689,7 +689,7 @@ def eval_ctors(js_file, wasm_file, debug_info=False): # noqa if settings.EVAL_CTORS == 2: args += ['--ignore-external-input'] logger.warning('ctor_evaller (wasm): trying to eval global ctors (' + ' '.join(args) + ')') - out = run_binaryen_command('wasm-ctor-eval', wasm_file, wasm_file, args=args, stdout=PIPE) + out = run_binaryen_command('wasm-ctor-eval', wasm_file, wasm_file, args=args, stdout=PIPE, debug=debug_info) logger.warning('\n\n' + out) num_successful = out.count('success on') if num_successful and has_wasm_call_ctors: From 6314ac1e69fe1c501e677f588951cf93b22fea36 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 12 Jan 2022 13:18:12 -0800 Subject: [PATCH 32/52] comment --- emcc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/emcc.py b/emcc.py index 26a282eff0aca..25ba9fbaf59b9 100755 --- a/emcc.py +++ b/emcc.py @@ -2165,7 +2165,8 @@ def check_memory_setting(setting): if settings.EVAL_CTORS: if settings.WASM2JS: - diagnostics.warning('emcc', 'disabling EVAL_CTORS due to wasm2js. see #XXXXX') # code size/memory and correctness issues + # code size/memory and correctness issues, file issue + diagnostics.warning('emcc', 'disabling EVAL_CTORS due to wasm2js. see #XXXXX') settings.EVAL_CTORS = 0 elif settings.USE_PTHREADS: diagnostics.warning('emcc', 'disabling EVAL_CTORS due to pthreads (passive segments)') From 71c8503833e9e40b045a1fbf3f0259f414ee1803 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 12 Jan 2022 13:26:17 -0800 Subject: [PATCH 33/52] update --- tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.size | 2 +- tests/other/metadce/minimal_Oz_EVAL_CTORS.funcs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.size b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.size index 91e83ccb98875..881de2bcd54db 100644 --- a/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.size +++ b/tests/other/metadce/hello_libcxx_O2_EVAL_CTORS_2.size @@ -1 +1 @@ -122294 +122060 diff --git a/tests/other/metadce/minimal_Oz_EVAL_CTORS.funcs b/tests/other/metadce/minimal_Oz_EVAL_CTORS.funcs index ebfabbe4ba30c..66ddb2638a16b 100644 --- a/tests/other/metadce/minimal_Oz_EVAL_CTORS.funcs +++ b/tests/other/metadce/minimal_Oz_EVAL_CTORS.funcs @@ -1 +1 @@ -$0 +$add From 2a525ed657d0da6dc994091305ba62ea94b460d4 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 12 Jan 2022 13:31:46 -0800 Subject: [PATCH 34/52] docs [ci skip] --- src/settings.js | 47 ++++++++++++++++++++--------------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/src/settings.js b/src/settings.js index 5da25880b9ce9..a125d185ef76a 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1535,28 +1535,20 @@ var ALLOW_BLOCKING_ON_MAIN_THREAD = 1; // [link] var PTHREADS_DEBUG = 0; -// This tries to evaluate global ctors at compile-time, applying their effects -// into the mem init file. This saves running code during startup, and also -// allows removing the global ctor functions and other code that only they used, -// so this is also good for reducing code size. However, this does make the -// compile step much slower. -// -// This basically runs the ctors during compile time, seeing if they execute -// safely in a sandbox. Any ffi access out of wasm causes failure, as it could -// do something nondeterministic and/or alter some other state we don't see. If -// all the global ctor does is pure computation inside wasm, it should be ok. -// Run with EMCC_DEBUG=1 in the env to see logging, and errors when it fails to -// eval (you'll see a message, or a stack trace; in the latter case, the -// functions on the stack should give you an idea of what ffi was called and -// why, and perhaps you can refactor your code to avoid it, e.g., remove -// mallocs, printfs in global ctors). -// -// This optimization can increase the size of the mem init file, because ctors -// can write to memory that would otherwise be in a zeroinit area. This may not -// be a significant increase after gzip, if there are mostly zeros in there, and -// in any case the mem init increase would be offset by a code size decrease. -// (Unless you have a small ctor that writes 'random' data to memory, which -// would reduce little code but add potentially lots of uncompressible data.) +// This tries to evaluate code at compile time. The main use case is to eval +// global ctor functions, which are those that run before main(), but main() +// itself or parts of it can also be evalled. Evaluating code this way can avoid +// work at runtime, as it applies the results of the execution to memory and +// globals and so forth, "snapshotting" the wasm and then just running it from +// there when it is loaded. +// +// This will stop when it sees something it cannot eval at compile time, like a +// call to an import. When running with this option you will see logging that +// indicates what is evalled and where it stops. +// +// This optimization can either reduce or increase code size. If a small amount +// of code generates many changes in memory, for example, then overall size may +// increase. // // LLVM's GlobalOpt *almost* does this operation. It does in simple cases, where // LLVM IR is not too complex for its logic to evaluate, but it isn't powerful @@ -1565,16 +1557,17 @@ var PTHREADS_DEBUG = 0; // GlobalOpt to have a full interpreter, plus a way to write back into LLVM IR // global objects. At the wasm level, however, everything has been lowered // into a simple low level, and we also just need to write bytes into an array, -// so this is easy for us to do, but not for LLVM. A further issue for LLVM is -// that it doesn't know that we will not link in further code, so it only tries -// to optimize ctors with lowest priority. We do know that, and can optimize all -// the ctors. +// so this is easy for us to do. A further issue for LLVM is that it doesn't +// know that we will not link in further code, so it only tries to optimize +// ctors with lowest priority (we do know that, if dynamic linking is not +// enabled). // // If set to a value of 2, this also makes some "unsafe" assumptions, // specifically that there is no input received while evalling ctors. That means // we ignore args to main() as well as assume no environment vars are readable. // This allows more programs to be optimized, but you need to make sure your -// program does not depend on those features. +// program does not depend on those features - even just checking the value of +// argc can lead to problems. // // [link] var EVAL_CTORS = 0; From 36ad34af4cb736d193dba14b3f648ecdf90b0469 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 12 Jan 2022 13:46:37 -0800 Subject: [PATCH 35/52] docs [ci skip] --- .../docs/optimizing/Optimizing-Code.rst | 52 +++++++++++++++++++ tools/building.py | 2 +- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/site/source/docs/optimizing/Optimizing-Code.rst b/site/source/docs/optimizing/Optimizing-Code.rst index ec9fe020c998c..eec8d4558cab1 100644 --- a/site/source/docs/optimizing/Optimizing-Code.rst +++ b/site/source/docs/optimizing/Optimizing-Code.rst @@ -119,6 +119,58 @@ linker can handle a mix wasm object files and LTO object files. Passing Thus, to allow maximal LTO opportunities with the LLVM wasm backend, build all source files with ``-flto`` and also link with ``flto``. +EVAL_CTORS +========== + +Building with ``-s EVAL_CTORS`` will evaluate as much code as possible at +compile time. That includes both the "global ctor" functions (functions LLVM +emits that run before ``main()``) as well as ``main()`` itself. As much as can +be evaluated will be, and the resulting state is then "snapshotted" into the +wasm. Then when the program is run it will begin from that state, and not need +to execute that code, which can save time. + +This optimization can either reduce or increase code size. If a small amount +of code generates many changes in memory, for example, then overall size may +increase. It is best to build with this flag and then measure and see if the +tradeoff is worthwhile in your program. + +You can make an effort to write EVAL_CTORS-friendly code, by deferring things +that cannot be evalled as much as possible. For example, calls to imports stop +this optimization, and so if you have a game engine that creates a GL context +and then does some pure computation to set up unrelated data structures in +memory, then you could reverse that order. Then the pure computation could run +first, and be evalled away, and the GL context creation call to an import would +not prevent that. Other things you can do are to avoid using ``argc/argv``, to +avoid using ``getenv()``, and so forth. + +Logging is shown when using this option so that you can see whether things can +be improved. Here is an example of output from ``emcc -s EVAL_CTORS``: + +:: + + trying to eval __wasm_call_ctors + ...partial evalling successful, but stopping since could not eval: call import: wasi_snapshot_preview1.environ_sizes_get + recommendation: consider --ignore-external-input + ...stopping + +The first line indicates an attempt to eval LLVM's function that runs global +ctors. It fails on the WASI import ``environ_sizes_get``, which means it is +trying to read from the environment. As the output says, you can tell +``EVAL_CTORS`` to ignore external input, which will ignore such things. You +can enable that with mode ``2``, that is, build with ``emcc -s EVAL_CTORS-2``: + +:: + + trying to eval __wasm_call_ctors + ...success on __wasm_call_ctors. + trying to eval main + ...stopping (in block) since could not eval: call import: wasi_snapshot_preview1.fd_write + ...stopping + +Now it has succeeded to eval ``__wasm_call_ctors`` completely. It then moved on +to ``main``, where it stopped because of a call to WASI's ``fd_write``, that is, +a call to print something. + Very large codebases ==================== diff --git a/tools/building.py b/tools/building.py index 984c12fede132..95dde46d759be 100644 --- a/tools/building.py +++ b/tools/building.py @@ -697,7 +697,7 @@ def eval_ctors(js_file, wasm_file, debug_info=False): # noqa args = ['--ctors=_initialize'] if settings.EVAL_CTORS == 2: args += ['--ignore-external-input'] - logger.warning('ctor_evaller (wasm): trying to eval global ctors (' + ' '.join(args) + ')') + logger.warning('ctor_evaller: trying to eval global ctors (' + ' '.join(args) + ')') out = run_binaryen_command('wasm-ctor-eval', wasm_file, wasm_file, args=args, stdout=PIPE, debug=debug_info) logger.warning('\n\n' + out) num_successful = out.count('success on') From d40b95d5297abbc1d12d127bdc1e2c0b86190100 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 12 Jan 2022 14:09:46 -0800 Subject: [PATCH 36/52] text [ci skip] --- site/source/docs/optimizing/Optimizing-Code.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/site/source/docs/optimizing/Optimizing-Code.rst b/site/source/docs/optimizing/Optimizing-Code.rst index eec8d4558cab1..5ced5c2835dc0 100644 --- a/site/source/docs/optimizing/Optimizing-Code.rst +++ b/site/source/docs/optimizing/Optimizing-Code.rst @@ -131,8 +131,8 @@ to execute that code, which can save time. This optimization can either reduce or increase code size. If a small amount of code generates many changes in memory, for example, then overall size may -increase. It is best to build with this flag and then measure and see if the -tradeoff is worthwhile in your program. +increase. It is best to build with this flag and then measure code and startup +speed and see if the tradeoff is worthwhile in your program. You can make an effort to write EVAL_CTORS-friendly code, by deferring things that cannot be evalled as much as possible. For example, calls to imports stop From 22c33baa23b60585884074c561c9634676d834fa Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 13 Jan 2022 06:52:29 -0800 Subject: [PATCH 37/52] fixes --- site/source/docs/optimizing/Optimizing-Code.rst | 8 ++++++++ tests/test_core.py | 4 ++-- tools/building.py | 4 ++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/site/source/docs/optimizing/Optimizing-Code.rst b/site/source/docs/optimizing/Optimizing-Code.rst index 5ced5c2835dc0..0b5a08ca4baf6 100644 --- a/site/source/docs/optimizing/Optimizing-Code.rst +++ b/site/source/docs/optimizing/Optimizing-Code.rst @@ -171,6 +171,14 @@ Now it has succeeded to eval ``__wasm_call_ctors`` completely. It then moved on to ``main``, where it stopped because of a call to WASI's ``fd_write``, that is, a call to print something. +Another form of output that you may see is + +:: + + ...partial evalling successful, but stopping since could not eval: call import: wasi_snapshot_preview1.fd_write + +That indicates that part of the function was evalled but not all of it. + Very large codebases ==================== diff --git a/tests/test_core.py b/tests/test_core.py index a822b614b2587..8e50e9e1fb61d 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1533,6 +1533,7 @@ def test_ctors_no_main(self): self.emcc_args.append('--no-entry') self.do_core_test('test_ctors_no_main.cpp') + @no_wasm2js('eval_ctors not supported yet') @also_with_standalone_wasm(impure=True) def test_eval_ctors_no_main(self): self.set_setting('EVAL_CTORS') @@ -6789,12 +6790,11 @@ def test_tracing(self): self.emcc_args += ['--tracing'] self.do_core_test('test_tracing.c') + @no_wasm2js('eval_ctors not supported yet') @also_with_standalone_wasm() def test_eval_ctors(self): if '-O2' not in str(self.emcc_args) or '-O1' in str(self.emcc_args): self.skipTest('need opts') - if not self.is_wasm(): - self.skipTest('this test uses wasm binaries') print('leave printf in ctor') self.set_setting('EVAL_CTORS') diff --git a/tools/building.py b/tools/building.py index 95dde46d759be..4a562f3ffc637 100644 --- a/tools/building.py +++ b/tools/building.py @@ -692,9 +692,9 @@ def eval_ctors(js_file, wasm_file, debug_info=False): # noqa args += ['--kept-exports=' + ','.join(kept_ctors)] else: if settings.EXPECT_MAIN: - args = ['--ctors=_start'] + args = ['--ctors=_start', '--kept-exports=_start'] else: - args = ['--ctors=_initialize'] + args = ['--ctors=_initialize', '--kept-exports=_initialize'] if settings.EVAL_CTORS == 2: args += ['--ignore-external-input'] logger.warning('ctor_evaller: trying to eval global ctors (' + ' '.join(args) + ')') From 67dd2daf8ede4eaf49fe83efd7af4c98f3686dbd Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 13 Jan 2022 06:53:48 -0800 Subject: [PATCH 38/52] fix --- tools/building.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/building.py b/tools/building.py index 4a562f3ffc637..4b6a374103e14 100644 --- a/tools/building.py +++ b/tools/building.py @@ -692,9 +692,10 @@ def eval_ctors(js_file, wasm_file, debug_info=False): # noqa args += ['--kept-exports=' + ','.join(kept_ctors)] else: if settings.EXPECT_MAIN: - args = ['--ctors=_start', '--kept-exports=_start'] + ctor = '_start' else: - args = ['--ctors=_initialize', '--kept-exports=_initialize'] + ctor = '_initialize' + args = ['--ctors=' + ctor, '--kept-exports=' + ctor] if settings.EVAL_CTORS == 2: args += ['--ignore-external-input'] logger.warning('ctor_evaller: trying to eval global ctors (' + ' '.join(args) + ')') From ff718431da373b8da043a9730da66ba459947bba Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 13 Jan 2022 06:55:48 -0800 Subject: [PATCH 39/52] comments --- src/settings.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/settings.js b/src/settings.js index a125d185ef76a..acea60a2cb405 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1553,14 +1553,14 @@ var PTHREADS_DEBUG = 0; // LLVM's GlobalOpt *almost* does this operation. It does in simple cases, where // LLVM IR is not too complex for its logic to evaluate, but it isn't powerful // enough for e.g. libc++ iostream ctors. It is just hard to do at the LLVM IR -// level - LLVM IR is complex and getting more complex, this would require +// level - LLVM IR is complex and getting more complex, so this would require // GlobalOpt to have a full interpreter, plus a way to write back into LLVM IR // global objects. At the wasm level, however, everything has been lowered // into a simple low level, and we also just need to write bytes into an array, // so this is easy for us to do. A further issue for LLVM is that it doesn't // know that we will not link in further code, so it only tries to optimize -// ctors with lowest priority (we do know that, if dynamic linking is not -// enabled). +// ctors with lowest priority (while we do know explicitly if dynamic linking is +// enabled or not). // // If set to a value of 2, this also makes some "unsafe" assumptions, // specifically that there is no input received while evalling ctors. That means From 8f5c13ddc853fc1c1af4944f149db4b34ff124a4 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 13 Jan 2022 07:00:12 -0800 Subject: [PATCH 40/52] text --- site/source/docs/optimizing/Optimizing-Code.rst | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/site/source/docs/optimizing/Optimizing-Code.rst b/site/source/docs/optimizing/Optimizing-Code.rst index 0b5a08ca4baf6..aed446835490e 100644 --- a/site/source/docs/optimizing/Optimizing-Code.rst +++ b/site/source/docs/optimizing/Optimizing-Code.rst @@ -154,10 +154,11 @@ be improved. Here is an example of output from ``emcc -s EVAL_CTORS``: ...stopping The first line indicates an attempt to eval LLVM's function that runs global -ctors. It fails on the WASI import ``environ_sizes_get``, which means it is -trying to read from the environment. As the output says, you can tell -``EVAL_CTORS`` to ignore external input, which will ignore such things. You -can enable that with mode ``2``, that is, build with ``emcc -s EVAL_CTORS-2``: +ctors. It evalled some of the function but then it stopped on the WASI import +``environ_sizes_get``, which means it is trying to read from the environment. +As the output says, you can tell ``EVAL_CTORS`` to ignore external input, which +will ignore such things. You can enable that with mode ``2``, that is, build +with ``emcc -s EVAL_CTORS=2``: :: @@ -171,14 +172,6 @@ Now it has succeeded to eval ``__wasm_call_ctors`` completely. It then moved on to ``main``, where it stopped because of a call to WASI's ``fd_write``, that is, a call to print something. -Another form of output that you may see is - -:: - - ...partial evalling successful, but stopping since could not eval: call import: wasi_snapshot_preview1.fd_write - -That indicates that part of the function was evalled but not all of it. - Very large codebases ==================== From 3ab442b8c03461a57ca6928736702832264d8aae Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 13 Jan 2022 07:59:15 -0800 Subject: [PATCH 41/52] unbench --- tests/test_benchmark.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index bc5e4f067a25a..c0ac6758a9c84 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -33,7 +33,7 @@ # 5: 10 seconds DEFAULT_ARG = '4' -TEST_REPS = 0 +TEST_REPS = 5 # by default, run just core benchmarks CORE_BENCHMARKS = True @@ -211,7 +211,7 @@ def build(self, parent, filename, args, shared_args, emcc_args, native_args, nat if common.EMTEST_FORCE64: cmd += ['--profiling'] else: - cmd += ['--closure=0', '-sMINIMAL_RUNTIME'] + cmd += ['--closure=1', '-sMINIMAL_RUNTIME'] # add additional emcc args at the end, which may override other things # above, such as minimal runtime cmd += emcc_args + self.extra_args @@ -361,7 +361,7 @@ def cleanup(self): if not common.EMTEST_FORCE64: benchmarkers += [ - # NativeBenchmarker('clang', [CLANG_CC], [CLANG_CXX]), + NativeBenchmarker('clang', [CLANG_CC], [CLANG_CXX]), # NativeBenchmarker('gcc', ['gcc', '-no-pie'], ['g++', '-no-pie']) ] @@ -377,9 +377,8 @@ def cleanup(self): ] else: benchmarkers += [ - EmscriptenBenchmarker('norm', aot_v8), - EmscriptenBenchmarker('eval', aot_v8, ['-sEVAL_CTORS=2']), - #EmscriptenBenchmarker(default_v8_name + '-lto', aot_v8, ['-flto']), + EmscriptenBenchmarker(default_v8_name, aot_v8), + EmscriptenBenchmarker(default_v8_name + '-lto', aot_v8, ['-flto']), # EmscriptenWasm2CBenchmarker('wasm2c') ] if os.path.exists(CHEERP_BIN): From 4fd28692fa68a1101e8806863dd27dd8a8f1ba7f Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 13 Jan 2022 07:59:48 -0800 Subject: [PATCH 42/52] bench --- tests/test_benchmark.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index c0ac6758a9c84..091b0f4f8bc30 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -379,6 +379,7 @@ def cleanup(self): benchmarkers += [ EmscriptenBenchmarker(default_v8_name, aot_v8), EmscriptenBenchmarker(default_v8_name + '-lto', aot_v8, ['-flto']), + EmscriptenBenchmarker(default_v8_name + '-ctors', aot_v8, ['-sEVAL_CTORS']), # EmscriptenWasm2CBenchmarker('wasm2c') ] if os.path.exists(CHEERP_BIN): From 0589371fc3695df131a3d7eea2524e39e39ef9d7 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 13 Jan 2022 09:48:13 -0800 Subject: [PATCH 43/52] cleanup --- emcc.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/emcc.py b/emcc.py index 25ba9fbaf59b9..5db7a8002a6d8 100755 --- a/emcc.py +++ b/emcc.py @@ -2165,8 +2165,8 @@ def check_memory_setting(setting): if settings.EVAL_CTORS: if settings.WASM2JS: - # code size/memory and correctness issues, file issue - diagnostics.warning('emcc', 'disabling EVAL_CTORS due to wasm2js. see #XXXXX') + # code size/memory and correctness issues TODO + diagnostics.warning('emcc', 'disabling EVAL_CTORS due to wasm2js') settings.EVAL_CTORS = 0 elif settings.USE_PTHREADS: diagnostics.warning('emcc', 'disabling EVAL_CTORS due to pthreads (passive segments)') @@ -2175,7 +2175,6 @@ def check_memory_setting(setting): diagnostics.warning('emcc', 'disabling EVAL_CTORS due to relocatable (movable segments)') settings.EVAL_CTORS = 0 elif settings.ASYNCIFY: - #diagnostics.warning('emcc', 'disabling EVAL_CTORS due to relocatable (movable segments)') # In Asyncify exports can be called more than once, and this seems to not # work properly yet (see test_emscripten_scan_registers). diagnostics.warning('emcc', 'disabling EVAL_CTORS due to asyncify') From 28b394521fbcc3bdab8a4e5a8b39aa29ed6f2b78 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 13 Jan 2022 09:55:03 -0800 Subject: [PATCH 44/52] changelog [ci skip] --- ChangeLog.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ChangeLog.md b/ChangeLog.md index b6a00459d5947..f07e1c692b029 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -20,6 +20,10 @@ See docs/process.md for more on how version tagging works. 3.1.2 ----- +- `EVAL_CTORS` has been rewritten and improved. The main differences from before + are that it is much more capable (it can now eval parts of functions and not + just all or nothing, etc.), and now it is not run by default - you must run it + manually with `-s EVAL_CTORS`. See `settings.js` for more details. (#) - `wasmX` test suites that are defined in `test_core.py` have been renamed to `coreX` to better reflect where they are defined. The old suite names such as `wasm2` will continue to work for now as aliases. From d80f2a436441837a4c080fd29eb70bcf054397d9 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 13 Jan 2022 12:35:52 -0800 Subject: [PATCH 45/52] comment [ci skip] --- ChangeLog.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index f07e1c692b029..3fc47096f5ae6 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -22,8 +22,9 @@ See docs/process.md for more on how version tagging works. ----- - `EVAL_CTORS` has been rewritten and improved. The main differences from before are that it is much more capable (it can now eval parts of functions and not - just all or nothing, etc.), and now it is not run by default - you must run it - manually with `-s EVAL_CTORS`. See `settings.js` for more details. (#) + just all or nothing, and it can eval more wasm constructs like globals). It is + no longer run by default, so to use it you should build with `-s EVAL_CTORS`. + See `settings.js` for more details. (#) - `wasmX` test suites that are defined in `test_core.py` have been renamed to `coreX` to better reflect where they are defined. The old suite names such as `wasm2` will continue to work for now as aliases. From cf66fc8869ef2f16ef0b45388c1dceee0c34d6a2 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 13 Jan 2022 12:52:04 -0800 Subject: [PATCH 46/52] add PR # --- ChangeLog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog.md b/ChangeLog.md index 3fc47096f5ae6..14693ec060a85 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -24,7 +24,7 @@ See docs/process.md for more on how version tagging works. are that it is much more capable (it can now eval parts of functions and not just all or nothing, and it can eval more wasm constructs like globals). It is no longer run by default, so to use it you should build with `-s EVAL_CTORS`. - See `settings.js` for more details. (#) + See `settings.js` for more details. (#16011) - `wasmX` test suites that are defined in `test_core.py` have been renamed to `coreX` to better reflect where they are defined. The old suite names such as `wasm2` will continue to work for now as aliases. From 19a7caa51871f0dbfc62f588ace182990ebe2caa Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 14 Jan 2022 09:58:00 -0800 Subject: [PATCH 47/52] warnings => errors --- emcc.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/emcc.py b/emcc.py index 5db7a8002a6d8..4547ebe997def 100755 --- a/emcc.py +++ b/emcc.py @@ -2166,19 +2166,15 @@ def check_memory_setting(setting): if settings.EVAL_CTORS: if settings.WASM2JS: # code size/memory and correctness issues TODO - diagnostics.warning('emcc', 'disabling EVAL_CTORS due to wasm2js') - settings.EVAL_CTORS = 0 + exit_with_error('EVAL_CTORS is not compatible with wasm2js yet') elif settings.USE_PTHREADS: - diagnostics.warning('emcc', 'disabling EVAL_CTORS due to pthreads (passive segments)') - settings.EVAL_CTORS = 0 + exit_with_error('EVAL_CTORS is not compatible with pthreads yet (passive segments)') elif settings.RELOCATABLE: - diagnostics.warning('emcc', 'disabling EVAL_CTORS due to relocatable (movable segments)') - settings.EVAL_CTORS = 0 + exit_with_error('EVAL_CTORS is not compatible with relocatable yet (movable segments)') elif settings.ASYNCIFY: # In Asyncify exports can be called more than once, and this seems to not # work properly yet (see test_emscripten_scan_registers). - diagnostics.warning('emcc', 'disabling EVAL_CTORS due to asyncify') - settings.EVAL_CTORS = 0 + exit_with_error('EVAL_CTORS is not compatible with asyncify yet') if options.use_closure_compiler == 2 and not settings.WASM2JS: exit_with_error('closure compiler mode 2 assumes the code is asm.js, so not meaningful for wasm') From 22511837e4b07af1b0481480807c61f58397e62e Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 14 Jan 2022 10:00:36 -0800 Subject: [PATCH 48/52] testfix --- tests/test_other.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_other.py b/tests/test_other.py index e3461dde40a7c..3130ea9ef2d91 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -6659,7 +6659,8 @@ def test(p1, p2, p3, last, expected): second = test(3000, 1000, 2000, 0xf, 0x8f5) # noqa; 1 will succedd third = test(2000, 3000, 1000, 0xf, 0xf58) # noqa; 0 will succeed print(first, second, third) - assert first < second and second < third, [first, second, third] + self.assertLess(first, second) + self.assertLess(second, third) @uses_canonical_tmp @with_env_modify({'EMCC_DEBUG': '1'}) From ef875b4a4df82896eea2009f30bdb57afdf41cf8 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 14 Jan 2022 10:00:47 -0800 Subject: [PATCH 49/52] unify WASM_CALL_CTORS --- emscripten.py | 6 ++---- tools/building.py | 4 +++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/emscripten.py b/emscripten.py index b2859dbc6ac1a..2a8c15a3937b3 100644 --- a/emscripten.py +++ b/emscripten.py @@ -34,14 +34,12 @@ logger = logging.getLogger('emscripten') -WASM_INIT_FUNC = '__wasm_call_ctors' - def compute_minimal_runtime_initializer_and_exports(post, exports, receiving): # Declare all exports out to global JS scope so that JS library functions can access them in a # way that minifies well with Closure # e.g. var a,b,c,d,e,f; - exports_that_are_not_initializers = [x for x in exports if x not in WASM_INIT_FUNC] + exports_that_are_not_initializers = [x for x in exports if x not in building.WASM_CALL_CTORS] # In Wasm backend the exports are still unmangled at this point, so mangle the names here exports_that_are_not_initializers = [asmjs_mangle(x) for x in exports_that_are_not_initializers] @@ -701,7 +699,7 @@ def create_receiving(exports): if not settings.DECLARE_ASM_MODULE_EXPORTS: return '' - exports_that_are_not_initializers = [x for x in exports if x != WASM_INIT_FUNC] + exports_that_are_not_initializers = [x for x in exports if x != building.WASM_CALL_CTORS] receiving = [] diff --git a/tools/building.py b/tools/building.py index 4b6a374103e14..6de28361b5915 100644 --- a/tools/building.py +++ b/tools/building.py @@ -658,10 +658,12 @@ def acorn_optimizer(filename, passes, extra_info=None, return_output=False): return output +WASM_CALL_CTORS = '__wasm_call_ctors' + + # evals ctors. if binaryen_bin is provided, it is the dir of the binaryen tool # for this, and we are in wasm mode def eval_ctors(js_file, wasm_file, debug_info=False): # noqa - WASM_CALL_CTORS = '__wasm_call_ctors' if settings.MINIMAL_RUNTIME: CTOR_ADD_PATTERN = f"asm['{WASM_CALL_CTORS}']();" # TODO test else: From ae7b9e2d6b66742e34dec61584ec89bcb412d378 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 14 Jan 2022 10:01:41 -0800 Subject: [PATCH 50/52] loggingfix --- tools/building.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/building.py b/tools/building.py index 6de28361b5915..04b2ca66d55ba 100644 --- a/tools/building.py +++ b/tools/building.py @@ -687,7 +687,7 @@ def eval_ctors(js_file, wasm_file, debug_info=False): # noqa # we want to preserve? kept_ctors += ['main'] if not ctors: - logger.warning('ctor_evaller: no ctors') + logger.info('ctor_evaller: no ctors') return args = ['--ctors=' + ','.join(ctors)] if kept_ctors: @@ -700,9 +700,9 @@ def eval_ctors(js_file, wasm_file, debug_info=False): # noqa args = ['--ctors=' + ctor, '--kept-exports=' + ctor] if settings.EVAL_CTORS == 2: args += ['--ignore-external-input'] - logger.warning('ctor_evaller: trying to eval global ctors (' + ' '.join(args) + ')') + logger.info('ctor_evaller: trying to eval global ctors (' + ' '.join(args) + ')') out = run_binaryen_command('wasm-ctor-eval', wasm_file, wasm_file, args=args, stdout=PIPE, debug=debug_info) - logger.warning('\n\n' + out) + logger.info('\n\n' + out) num_successful = out.count('success on') if num_successful and has_wasm_call_ctors: js = js.replace(CTOR_ADD_PATTERN, '') From b1707c60d4fa9038e42d357dc0a4cda7f2f6a076 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 14 Jan 2022 10:02:44 -0800 Subject: [PATCH 51/52] comment --- tools/building.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/building.py b/tools/building.py index 04b2ca66d55ba..5276a7f16dac0 100644 --- a/tools/building.py +++ b/tools/building.py @@ -1520,6 +1520,8 @@ def run_binaryen_command(tool, infile, outfile=None, args=[], debug=False, stdou cmd += get_binaryen_feature_flags() # if we are emitting a source map, every time we load and save the wasm # we must tell binaryen to update it + # TODO: all tools should support source maps; wasm-ctor-eval does not atm, + # for example if settings.GENERATE_SOURCE_MAP and outfile and tool in ['wasm-opt', 'wasm-emscripten-finalize']: cmd += [f'--input-source-map={infile}.map'] cmd += [f'--output-source-map={outfile}.map'] From facd964361a578cc0fc4549b4fd53979bfa11163 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 14 Jan 2022 10:03:39 -0800 Subject: [PATCH 52/52] -sEVAL_CTORS, no space --- site/source/docs/optimizing/Optimizing-Code.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/site/source/docs/optimizing/Optimizing-Code.rst b/site/source/docs/optimizing/Optimizing-Code.rst index aed446835490e..96135808254fc 100644 --- a/site/source/docs/optimizing/Optimizing-Code.rst +++ b/site/source/docs/optimizing/Optimizing-Code.rst @@ -122,7 +122,7 @@ source files with ``-flto`` and also link with ``flto``. EVAL_CTORS ========== -Building with ``-s EVAL_CTORS`` will evaluate as much code as possible at +Building with ``-sEVAL_CTORS`` will evaluate as much code as possible at compile time. That includes both the "global ctor" functions (functions LLVM emits that run before ``main()``) as well as ``main()`` itself. As much as can be evaluated will be, and the resulting state is then "snapshotted" into the @@ -144,7 +144,7 @@ not prevent that. Other things you can do are to avoid using ``argc/argv``, to avoid using ``getenv()``, and so forth. Logging is shown when using this option so that you can see whether things can -be improved. Here is an example of output from ``emcc -s EVAL_CTORS``: +be improved. Here is an example of output from ``emcc -sEVAL_CTORS``: :: @@ -158,7 +158,7 @@ ctors. It evalled some of the function but then it stopped on the WASI import ``environ_sizes_get``, which means it is trying to read from the environment. As the output says, you can tell ``EVAL_CTORS`` to ignore external input, which will ignore such things. You can enable that with mode ``2``, that is, build -with ``emcc -s EVAL_CTORS=2``: +with ``emcc -sEVAL_CTORS=2``: ::