From c14ab14ce1e9745972b38370637f6bdb1f80b69d Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Tue, 19 Dec 2017 21:00:55 +0100 Subject: [PATCH 01/30] Script that creates web-platform-test wrappers for test262's tests The generated tests run the target test262 within different agent. Currently supported agents are: IFrame, Window, DedicatedWorker and SharedWorker. For each agent two tests are generated, one in strict mode and another in non-strict mode, unless the test indicates otherwise. The generated tests are placed at js/test262/. --- resources/test262-agent-harness.js | 335 ++++++++++++++++++++++++ resources/test262-harness.js | 39 +++ test262-to-wpt | 404 +++++++++++++++++++++++++++++ 3 files changed, 778 insertions(+) create mode 100644 resources/test262-agent-harness.js create mode 100644 resources/test262-harness.js create mode 100755 test262-to-wpt diff --git a/resources/test262-agent-harness.js b/resources/test262-agent-harness.js new file mode 100644 index 00000000000000..e94e8acacf4f1d --- /dev/null +++ b/resources/test262-agent-harness.js @@ -0,0 +1,335 @@ +// Helper file to run a test262 within an agent. +// +// The target test262 is in text format. The harnessing code composes an HTML +// page containing the test262, runs it and reports for errors if there was +// any. Similarly if the target agent is a Worker, the harnessing code embedes +// the test code within the target worker, runs the test and checks out if +// there were any errors. +// +// Currently supported agents are IFrame, Window, Worker and SharedWorker. + +// IFrame. +function run_in_iframe_strict(test262, attrs, t) { + let opts = {} + opts.strict = true; + run_in_iframe(test262, attrs, t, opts); +} + +function run_in_iframe(test262, attrs, t, opts) { + opts = opts || {}; + // Rethrow error from iframe. + window.addEventListener('message', t.step_func(function(e) { + if (e.data[0] == 'error') { + throw new Error(e.data[1]); + } + })); + let iframe = document.createElement('iframe'); + iframe.style = 'display: none'; + content = test262_as_html(test262, attrs, opts.strict); + let blob = new Blob([content], {type: 'text/html'}); + iframe.src = URL.createObjectURL(blob); + document.body.appendChild(iframe); + + let w = iframe.contentWindow; + // Finish test on completed event. + w.addEventListener('completed', t.step_func(function(e) { + t.done(); + })); + // If test failed, rethrow error. + let FAILED = 'iframe-failed' + opts.strict ? 'strict' : ''; + window.addEventListener(FAILED, t.step_func(function(e) { + t.set_status(t.FAIL); + throw new Error(e.detail); + })); + // In case of error send it to parent window. + w.addEventListener('error', function(e) { + e.preventDefault(); + // If the test failed due to a SyntaxError but phase was 'early', then the + // test should actually pass. + if (e.message.startsWith("SyntaxError")) { + if (attrs.type == 'SyntaxError' && attrs.phase == 'early') { + t.done(); + return; + } + } + top.dispatchEvent(new CustomEvent(FAILED, {detail: e.message})); + }); +} + +let HEADER = ` + + + + + + + + + + + + + + + + +''') + +ASYNC_TEST = trim(''' +async_test(function(t) { + ###TEST_CALL###(test262, attrs, t); +}, '###TITLE###'); +''') + +def run_in_iframe(title): + output = ASYNC_TEST + output = re.sub('###TITLE###', 'IFrame: ' + title, output) + output = re.sub('###TEST_CALL###', 'run_in_iframe', output) + return output + +def run_in_iframe_strict(title): + output = ASYNC_TEST + output = re.sub('###TITLE###', 'IFrame (strict): ' + title, output) + output = re.sub('###TEST_CALL###', 'run_in_iframe_strict', output) + return output + +def run_in_window(title): + output = ASYNC_TEST + output = re.sub('###TITLE###', 'Window: ' + title, output) + output = re.sub('###TEST_CALL###', 'run_in_window', output) + return output + +def run_in_window_strict(title): + output = ASYNC_TEST + output = re.sub('###TITLE###', 'Window (strict): ' + title, output) + output = re.sub('###TEST_CALL###', 'run_in_window_strict', output) + return output + +def run_in_worker(title): + output = ASYNC_TEST + output = re.sub('###TITLE###', 'Worker: ' + title, output) + output = re.sub('###TEST_CALL###', 'run_in_worker', output) + return output + +def run_in_worker_strict(title): + output = ASYNC_TEST + output = re.sub('###TITLE###', 'Worker (strict): ' + title, output) + output = re.sub('###TEST_CALL###', 'run_in_worker_strict', output) + return output + +def run_in_shared_worker(title): + output = ASYNC_TEST + output = re.sub('###TITLE###', 'SharedWorker: ' + title, output) + output = re.sub('###TEST_CALL###', 'run_in_shared_worker', output) + return output + +def run_in_shared_worker_strict(title): + output = ASYNC_TEST + output = re.sub('###TITLE###', 'SharedWorker (strict): ' + title, output) + output = re.sub('###TEST_CALL###', 'run_in_shared_worker_strict', output) + return output + +''' +Converts a dictionary to a JSON data structure. It is possible to pass a list +of attributes in opts.exclude so they don't get serialized to JSON. +TODO: Use Python's JSON module? +''' +def tojson(d, opts): + def stringify(list): + if list == None or len(list) == 0: + return '[]' + l = [] + for i in range(0, len(list)): + elem = "'" + str(list[i]) + "'" + l.append(elem) + return '[%s]' % ",".join(l) + ret = [] + exclude = opts['exclude'] or [] + def append(string): + ret.append(string) + def flush(): + return "\n".join(ret) + def quote(v): + if isinstance(v, str): + if len(v) == 0: + return "''" + first, last = [v[0], v[len(v)-1]] + if first == "'" and last == "'": + return v + if first == '"' and last == '"': + return v + return "'%s'" % str(v) + def escape(v): + return re.sub(r"'", r"\'", v) + append('{') + for k, v in d.iteritems(): + if k in exclude: + continue + if isinstance(v, list): + append("\t{key}: {value},".format(key=k, value=stringify(v))) + else: + append("\t{key}: {value},".format(key=k, value=quote(escape(v)))) + append('}') + return flush() + +# Indents content n tabs. A tab is 2 spaces (TAB_WIDTH). +def indent(num_tabs, content): + # Duplicates char n times. + def dup(char, times): + ret = [] + for i in range(0, times): + ret.append(char) + return "".join(ret) + ret = [] + space = dup(" ", num_tabs * TAB_WIDTH) + for line in content.split("\n"): + ret.append(space + line) + return "\n".join(ret) + +''' +Generates web-platform-test wrapper for test262 tests. + +If a directory is passed as origin for the tests, the class recursively reads +out al the JavaScript files contained and converts them to web-platform-tests. + +If a file is passed as origin, only that tests is converted. + +The resulting tests are placed inside an output directory named 'js'. +''' +class WPTestBuilder(object): + + def __init__(self, path): + self.path = path + + def generate(self): + path = self.path + mode = os.stat(self.path)[ST_MODE] + if S_ISDIR(mode): + # Check root folder contains 'test'. + test_dir = os.path.join(path, "test") + mode = os.stat(test_dir)[ST_MODE] + if not S_ISDIR(mode): + print("Could not locate 'test' folder at %s" % path) + exit(1) + print "Generating web-platform-test wrappers for test262" + for each in self.listall(test_dir, {'ext': 'js', 'skip_hidden': True}): + self.generate_single_wpt(each) + print "Output: " + DEST_PATH + elif S_ISREG(mode): + pos = path.find("test262/test") + if pos < 0: + print("Not a test262 file: '%'" % path) + exit(1) + print "Generating web-platform-tests wrapper for {0}".format(path) + print "Output: %s" % self.generate_single_wpt(path) + else: + print "Skipping %s" % path + + def destination_path(self, path): + filename = re.sub(".*?test262/test", "", path) + filename = re.sub('.js$', '.html', filename) + return DEST_PATH + filename + + def generate_single_wpt(self, src): + dst = self.destination_path(src) + self.ensure_destination_path(dst) + return self.savefile(dst, self.build(dst, self.readfile(src))) + + def ensure_destination_path(self, path): + dirname = subprocess.check_output(("dirname {0}".format(path)).split()) + dirname = re.sub('\n', '', dirname) + process = subprocess.Popen(("mkdir -p {0}".format(dirname)).split()) + process.wait() + + def savefile(self, output, content): + with open(output, 'wt') as fd: + fd.write(content) + return fd.name + + def build(self, title, content): + test262 = Test262Parser(content) + def header(): + opts = { 'exclude': ['description', 'info', 'esid', 'es6id'] } + ret = HEADER + ret = re.sub('###TITLE###', title, ret) + ret = ret.replace('###ATTRS###', indent(2, "let attrs = %s;" % tojson(test262.attrs, opts))) + ret = ret.replace('###HEADER###', indent(2, test262.header)) + return ret + def body(): + def escape(text): + text = re.sub(r'\\', r'\\\\', text) + text = re.sub(r'"', r'\"', text) + return text + def quote(line): + return "\"" + line + "\\n\"" + def format(text): + output = [] + lines = text.split("\n"); + for line in text.split("\n"): + if len(line) > 0: + output.append(quote(line)) + return " + \n".join(output) + return indent(4, "return \"\" +\n%s;" % format(escape(test262.body))) + def footer(): + # By default tests are run in strict and non-strict modes, + # unless flags says otherwise. + ret = [] + flags = 'flags' in test262.attrs and test262.attrs['flags'] or [] + if 'onlyStrict' in flags: + ret.append(run_in_iframe_strict(title)) + ret.append(run_in_window_strict(title)) + ret.append(run_in_worker_strict(title)) + ret.append(run_in_shared_worker_strict(title)) + elif 'noStrict' in flags: + ret.append(run_in_iframe(title)) + ret.append(run_in_window(title)) + ret.append(run_in_worker(title)) + ret.append(run_in_shared_worker(title)) + else: + ret.append(run_in_iframe_strict(title)) + ret.append(run_in_iframe(title)) + ret.append(run_in_window_strict(title)) + ret.append(run_in_window(title)) + ret.append(run_in_worker_strict(title)) + ret.append(run_in_worker(title)) + ret.append(run_in_shared_worker_strict(title)) + ret.append(run_in_shared_worker(title)) + return FOOTER.replace("###TESTS###", indent(3, "\n".join(ret))) + ret = [] + ret.append(header()) + ret.append(body()) + ret.append(footer()) + return "\n".join(ret) + + def readfile(self, path): + with open(path) as fd: + content = fd.read() + # Remove carriage return if any. + return re.sub("\x0D", "", content) + + def listall(self, root, opts): + opts = opts or {} + ret = [] + ext = opts['ext'] + skip_hidden = opts['skip_hidden'] + for root, dirs, files in os.walk(root): + for filename in files: + if skip_hidden and filename.startswith('.'): + continue + if ext and filename.endswith(ext): + ret.append(os.path.join(root, filename)) + return ret + +def main(): + if len(sys.argv) < 2: + usage() + path = sys.argv[1] + WPTestBuilder(path).generate() + +if __name__ == "__main__": + main() From 323d4cadaccb832dd0b8210afebb1ebc2515ff67 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Thu, 11 Jan 2018 10:37:24 +0100 Subject: [PATCH 02/30] Remove tabs --- resources/test262-harness.js | 46 ++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/resources/test262-harness.js b/resources/test262-harness.js index ebe76827fead73..47edc54e8f60cf 100644 --- a/resources/test262-harness.js +++ b/resources/test262-harness.js @@ -1,32 +1,32 @@ // Source: https://github.com/bakkot/test262-web-runner function installAPI(global) { - global.$262 = { - createRealm: function() { - var iframe = global.document.createElement('iframe'); + global.$262 = { + createRealm: function() { + var iframe = global.document.createElement('iframe'); iframe.style.cssText = "display: none"; - iframe.src = ""; // iframeSrc; - global.document.body.appendChild(iframe); - return installAPI(iframe.contentWindow); - }, - evalScript: function(src) { - var script = global.document.createElement('script'); - script.text = src; - global.document.body.appendChild(script); - }, - detachArrayBuffer: function(buffer) { - if (typeof postMessage !== 'function') { - throw new Error('No method available to detach an ArrayBuffer'); - } else { + iframe.src = ""; // iframeSrc; + global.document.body.appendChild(iframe); + return installAPI(iframe.contentWindow); + }, + evalScript: function(src) { + var script = global.document.createElement('script'); + script.text = src; + global.document.body.appendChild(script); + }, + detachArrayBuffer: function(buffer) { + if (typeof postMessage !== 'function') { + throw new Error('No method available to detach an ArrayBuffer'); + } else { postMessage(null, '*', [buffer]); - /* - See https://html.spec.whatwg.org/multipage/comms.html#dom-window-postmessage - which calls https://html.spec.whatwg.org/multipage/infrastructure.html#structuredclonewithtransfer - which calls https://html.spec.whatwg.org/multipage/infrastructure.html#transfer-abstract-op - which calls the DetachArrayBuffer abstract operation https://tc39.github.io/ecma262/#sec-detacharraybuffer + /* + See https://html.spec.whatwg.org/multipage/comms.html#dom-window-postmessage + which calls https://html.spec.whatwg.org/multipage/infrastructure.html#structuredclonewithtransfer + which calls https://html.spec.whatwg.org/multipage/infrastructure.html#transfer-abstract-op + which calls the DetachArrayBuffer abstract operation https://tc39.github.io/ecma262/#sec-detacharraybuffer */ - } + } }, - global: global + global: global }; return global.$262; } From 2b0fee3d3e96dd973953988e995f1744766a9c57 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Thu, 11 Jan 2018 10:40:18 +0100 Subject: [PATCH 03/30] Remove unneeded meta tags --- test262-to-wpt | 2 -- 1 file changed, 2 deletions(-) diff --git a/test262-to-wpt b/test262-to-wpt index b93d1abcab0c29..dce6129d0dc8cf 100755 --- a/test262-to-wpt +++ b/test262-to-wpt @@ -125,8 +125,6 @@ HEADER = trim(''' ###TITLE### - - From 459584bd66a854111544cd6b29c838a5c39df623 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Thu, 11 Jan 2018 10:41:30 +0100 Subject: [PATCH 04/30] Include test262-harness at global level --- resources/test262-agent-harness.js | 3 --- test262-to-wpt | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/resources/test262-agent-harness.js b/resources/test262-agent-harness.js index e94e8acacf4f1d..3893f3e93c119d 100644 --- a/resources/test262-agent-harness.js +++ b/resources/test262-agent-harness.js @@ -63,9 +63,6 @@ let HEADER = ` - - + From d1c2076ef0a9dc2c1c0732a907b0962746dfb300 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Thu, 11 Jan 2018 10:57:04 +0100 Subject: [PATCH 05/30] Parametrized host and port --- resources/test262-agent-harness.js | 8 ++++---- test262-to-wpt | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/resources/test262-agent-harness.js b/resources/test262-agent-harness.js index 3893f3e93c119d..7694a070b7c4ee 100644 --- a/resources/test262-agent-harness.js +++ b/resources/test262-agent-harness.js @@ -64,8 +64,8 @@ let HEADER = ` - - + From a9400b4fadc9281a9a2425b6f86e2256f1d261cb Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Thu, 11 Jan 2018 13:03:02 +0100 Subject: [PATCH 06/30] Blacklist harness and copy _FIXTURE.js files --- test262-to-wpt | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/test262-to-wpt b/test262-to-wpt index 9b779481cfbbc3..54efac21c84bf2 100755 --- a/test262-to-wpt +++ b/test262-to-wpt @@ -8,7 +8,7 @@ import sys import tempfile import threading -from random import randint +from shutil import copyfile from stat import * ''' @@ -285,8 +285,20 @@ class WPTestBuilder(object): print("Could not locate 'test' folder at %s" % path) exit(1) print "Generating web-platform-test wrappers for test262" - for each in self.listall(test_dir, {'ext': 'js', 'skip_hidden': True}): - self.generate_single_wpt(each) + # Blacklist 'harness' from root folders. + folders = [] + for each in os.listdir(test_dir): + if each == 'harness': + continue + folders.append(os.path.join(test_dir, each)) + # Generate wrappers. + for filename in self.listall(folders, {'ext': 'js', 'skip_hidden': True}): + if re.search('_FIXTURE.js$', filename): + dest = DEST_PATH + re.sub(".*test262/test", "", filename) + self.ensure_destination_path(dest) + copyfile(filename, dest) + else: + self.generate_single_wpt(filename) print "Output: " + DEST_PATH elif S_ISREG(mode): pos = path.find("test262/test") @@ -380,17 +392,18 @@ class WPTestBuilder(object): # Remove carriage return if any. return re.sub("\x0D", "", content) - def listall(self, root, opts): + def listall(self, folders, opts): opts = opts or {} ret = [] ext = opts['ext'] skip_hidden = opts['skip_hidden'] - for root, dirs, files in os.walk(root): - for filename in files: - if skip_hidden and filename.startswith('.'): - continue - if ext and filename.endswith(ext): - ret.append(os.path.join(root, filename)) + for folder in folders: + for root, dirs, files in os.walk(folder): + for filename in files: + if skip_hidden and filename.startswith('.'): + continue + if ext and filename.endswith(ext): + ret.append(os.path.join(root, filename)) return ret def main(): From 6a749ca01720cab11f5e2b8ae22c8e6f794d5109 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 12 Jan 2018 13:20:25 +0100 Subject: [PATCH 07/30] Use Python's YAML and JSON libraries --- test262-to-wpt | 73 +++++++++++++++----------------------------------- 1 file changed, 22 insertions(+), 51 deletions(-) diff --git a/test262-to-wpt b/test262-to-wpt index 54efac21c84bf2..3e98608cfcf00f 100755 --- a/test262-to-wpt +++ b/test262-to-wpt @@ -7,6 +7,8 @@ import subprocess import sys import tempfile import threading +import yaml +import json from shutil import copyfile from stat import * @@ -61,60 +63,24 @@ class Test262Parser(object): else: self.header = "" self.body = text - self.attrs = {} - self.parse_header() + self.attrs = self.parse_header() def parse_header(self): if len(self.header) == 0: - return - value = [] - last_attr = None + return {} + return yaml.load(self.yaml_section()) or {} + + def yaml_section(self): + is_yaml = False + ret = [] for line in self.header.split('\n'): - # End of the attribute section? - if re.search(r'---\*/', line): - self.store_attribute(last_attr, "\n".join(value)) + if re.search(r'/*---', line): + is_yaml = True + elif is_yaml and re.search(r'---*/', line): break - - # Attribute? (attr: ) - match = re.search(r'^\s*(\w+):', line) - if match: - attr = match.group(1) - # Name of attribute changed? - if attr != last_attr: - # Should store attribute? - if last_attr is not None: - self.store_attribute(last_attr, "".join(value)) - match = re.search(': (.*)\n?', line) - if match: - value = [match.group(1)] - else: - value = [] - last_attr = attr - else: - value.append(line) - self.store_attribute(last_attr, "".join(value)) - - def store_attribute(self, name, value): - if name in ['includes', 'flags', 'features']: - self.attrs[name] = self.parse_list(value) - else: - self.attrs[name] = value - - ''' - There are two ways of describing a list of properties: - attr: [prop1, prop2, ..., propn] - attr: - - prop1 - - prop2 - - ... - - propn - ''' - def parse_list(self, text): - match = re.search("\[", text) - if match: - return re.findall("[^\[, \]\n]+", text) - else: - return re.findall("[^- \n]+", text) + elif is_yaml: + ret.append(line) + return "\n".join(ret) ### @@ -334,12 +300,17 @@ class WPTestBuilder(object): def build(self, title, content): test262 = Test262Parser(content) def header(): - opts = { 'exclude': ['description', 'info', 'esid', 'es6id'] } + attrs = tojson(test262.attrs) ret = HEADER ret = re.sub('###TITLE###', title, ret) - ret = ret.replace('###ATTRS###', indent(2, "let attrs = %s;" % tojson(test262.attrs, opts))) + ret = ret.replace('###ATTRS###', indent(2, "let attrs = %s;" % tojson(test262.attrs))) ret = ret.replace('###HEADER###', indent(2, test262.header)) return ret + def tojson(attrs): + exclude = ['description', 'info', 'esid', 'es6id'] + for key in exclude: + attrs.pop(key, None) + return json.dumps(attrs) or '{}' def body(): def escape(text): text = re.sub(r'\\', r'\\\\', text) From 8ba46add0eedcb083ae9607340411e6648165105 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 12 Jan 2018 17:57:05 +0100 Subject: [PATCH 08/30] Simplify SyntaxError handling If a test expects a SyntaxError, regardless of the value of phase, the error is always managed in window.onerror. --- resources/test262-agent-harness.js | 67 +++++++++++------------------- 1 file changed, 25 insertions(+), 42 deletions(-) diff --git a/resources/test262-agent-harness.js b/resources/test262-agent-harness.js index 7694a070b7c4ee..ab7887ddb33acc 100644 --- a/resources/test262-agent-harness.js +++ b/resources/test262-agent-harness.js @@ -15,6 +15,11 @@ function run_in_iframe_strict(test262, attrs, t) { run_in_iframe(test262, attrs, t, opts); } +function is_syntax_error(message, attrs) { + let negative = attrs.negative || {}; + return message.startsWith('SyntaxError') && negative.type == 'SyntaxError'; +} + function run_in_iframe(test262, attrs, t, opts) { opts = opts || {}; // Rethrow error from iframe. @@ -46,11 +51,9 @@ function run_in_iframe(test262, attrs, t, opts) { e.preventDefault(); // If the test failed due to a SyntaxError but phase was 'early', then the // test should actually pass. - if (e.message.startsWith("SyntaxError")) { - if (attrs.type == 'SyntaxError' && attrs.phase == 'early') { - t.done(); - return; - } + if (is_syntax_error(e.message, attrs)) { + t.done(); + return; } top.dispatchEvent(new CustomEvent(FAILED, {detail: e.message})); }); @@ -77,7 +80,7 @@ let HEADER = ` -