From 2210203d0d3b7e5179162e4eba19bed8148d13d3 Mon Sep 17 00:00:00 2001 From: Rahul Gopinath Date: Sun, 28 Jan 2018 16:35:57 +0100 Subject: [PATCH 01/13] Add python 3.6 word code support --- byterun/pyobj.py | 4 ++++ byterun/pyvm2.py | 40 ++++++++++++++++++++++++++++++++-------- tox.ini | 2 +- 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/byterun/pyobj.py b/byterun/pyobj.py index f2924305..9f57b279 100644 --- a/byterun/pyobj.py +++ b/byterun/pyobj.py @@ -3,8 +3,10 @@ import collections import inspect import types +import dis import six +import sys PY3, PY2 = six.PY3, not six.PY3 @@ -137,6 +139,8 @@ def set(self, value): class Frame(object): def __init__(self, f_code, f_globals, f_locals, f_back): self.f_code = f_code + self.py36_opcodes = list(dis.get_instructions(self.f_code)) \ + if six.PY3 and sys.version_info.minor >= 6 else None self.f_globals = f_globals self.f_locals = f_locals self.f_back = f_back diff --git a/byterun/pyvm2.py b/byterun/pyvm2.py index 394e2e0b..d5252f88 100644 --- a/byterun/pyvm2.py +++ b/byterun/pyvm2.py @@ -166,18 +166,36 @@ def unwind_block(self, block): def parse_byte_and_args(self): """ Parse 1 - 3 bytes of bytecode into - an instruction and optionally arguments.""" + an instruction and optionally arguments. + In Python3.6 the format is 2 bytes per instruction.""" f = self.frame opoffset = f.f_lasti - byteCode = byteint(f.f_code.co_code[opoffset]) + if f.py36_opcodes: + currentOp = f.py36_opcodes[opoffset] + byteCode = currentOp.opcode + byteName = currentOp.opname + else: + byteCode = byteint(f.f_code.co_code[opoffset]) + byteName = dis.opname[byteCode] f.f_lasti += 1 - byteName = dis.opname[byteCode] arg = None arguments = [] + if f.py36_opcodes and byteCode == dis.EXTENDED_ARG: + # Prefixes any opcode which has an argument too big to fit into the + # default two bytes. ext holds two additional bytes which, taken + # together with the subsequent opcode’s argument, comprise a + # four-byte argument, ext being the two most-significant bytes. + # We simply ignore the EXTENDED_ARG because that calculation + # is already done by dis, and stored in next currentOp. + # Lib/dis.py:_unpack_opargs + return self.parse_byte_and_args() if byteCode >= dis.HAVE_ARGUMENT: - arg = f.f_code.co_code[f.f_lasti:f.f_lasti+2] - f.f_lasti += 2 - intArg = byteint(arg[0]) + (byteint(arg[1]) << 8) + if f.py36_opcodes: + intArg = currentOp.arg + else: + arg = f.f_code.co_code[f.f_lasti:f.f_lasti+2] + f.f_lasti += 2 + intArg = byteint(arg[0]) + (byteint(arg[1]) << 8) if byteCode in dis.hasconst: arg = f.f_code.co_consts[intArg] elif byteCode in dis.hasfree: @@ -189,9 +207,15 @@ def parse_byte_and_args(self): elif byteCode in dis.hasname: arg = f.f_code.co_names[intArg] elif byteCode in dis.hasjrel: - arg = f.f_lasti + intArg + if f.py36_opcodes: + arg = f.f_lasti + intArg//2 + else: + arg = f.f_lasti + intArg elif byteCode in dis.hasjabs: - arg = intArg + if f.py36_opcodes: + arg = intArg//2 + else: + arg = intArg elif byteCode in dis.haslocal: arg = f.f_code.co_varnames[intArg] else: diff --git a/tox.ini b/tox.ini index 4544cbc0..778fe570 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py27, py33 +envlist = py27, py33, py36 [testenv] commands = From b7db929f90f3c630222c36ec5437f54ee4d292fd Mon Sep 17 00:00:00 2001 From: Rahul Gopinath Date: Tue, 30 Jan 2018 11:54:52 +0100 Subject: [PATCH 02/13] tests/test_basic.py: Fix printing for python3 --- tests/test_basic.py | 41 +++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/tests/test_basic.py b/tests/test_basic.py index 7a42c07f..afe4cda9 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -12,21 +12,38 @@ class TestIt(vmtest.VmTestCase): def test_constant(self): self.assert_ok("17") - def test_globals(self): - self.assert_ok("""\ - global xyz - xyz=2106 + if PY2: + def test_globals(self): + self.assert_ok("""\ + global xyz + xyz=2106 - def abc(): + def abc(): + global xyz + xyz+=1 + print("Midst:",xyz) + + + print "Pre:",xyz + abc() + print "Post:",xyz + """) + elif PY3: + def test_globals(self): + self.assert_ok("""\ global xyz - xyz+=1 - print("Midst:",xyz) + xyz=2106 - - print "Pre:",xyz - abc() - print "Post:",xyz - """) + def abc(): + global xyz + xyz+=1 + print("Midst:",xyz) + + + print("Pre:",xyz) + abc() + print("Post:",xyz) + """) def test_for_loop(self): self.assert_ok("""\ From a429e8ff9548ef3648acfcf3d8b0393b69474c36 Mon Sep 17 00:00:00 2001 From: Rahul Gopinath Date: Tue, 30 Jan 2018 11:55:32 +0100 Subject: [PATCH 03/13] byterun/pyvm2.py: Fix BUILD_MAP and add BUILD_CONST_KEY_MAP --- byterun/pyvm2.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/byterun/pyvm2.py b/byterun/pyvm2.py index d5252f88..fa395cc6 100644 --- a/byterun/pyvm2.py +++ b/byterun/pyvm2.py @@ -603,9 +603,28 @@ def byte_BUILD_SET(self, count): elts = self.popn(count) self.push(set(elts)) - def byte_BUILD_MAP(self, size): - # size is ignored. - self.push({}) + def byte_BUILD_CONST_KEY_MAP(self, count): + # count values are consumed from the stack. + # The top element contains tuple of keys + # added in version 3.6 + keys = self.pop() + values = self.popn(count) + kvs = dict(zip(keys, values)) + self.push(kvs) + + def byte_BUILD_MAP(self, count): + # Pushes a new dictionary on to stack. + if not(six.PY3 and sys.version_info.minor >= 5): + self.push({}) + return + # Pop 2*count items so that + # dictionary holds count entries: {..., TOS3: TOS2, TOS1:TOS} + # updated in version 3.5 + kvs = {} + for i in range(0, count): + key, val = self.popn(2) + kvs[key] = val + self.push(kvs) def byte_STORE_MAP(self): the_map, val, key = self.popn(3) From 4011ef7ab49b054fcae93da2da0ca8b86b59186d Mon Sep 17 00:00:00 2001 From: Rahul Gopinath Date: Tue, 30 Jan 2018 14:32:31 +0100 Subject: [PATCH 04/13] byterun/pyvm2.py: Add BUILD_TUPLE_UNPACK and ..WITH_CALL --- byterun/pyvm2.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/byterun/pyvm2.py b/byterun/pyvm2.py index fa395cc6..fae4c0cc 100644 --- a/byterun/pyvm2.py +++ b/byterun/pyvm2.py @@ -590,6 +590,20 @@ def byte_DELETE_SUBSCR(self): ## Building + def byte_BUILD_TUPLE_UNPACK_WITH_CALL(self, count): + # This is similar to BUILD_TUPLE_UNPACK, but is used for f(*x, *y, *z) + # call syntax. The stack item at position count + 1 should be the + # corresponding callable f. + elts = self.popn(count) + self.push(tuple(e for l in elts for e in l)) + + def byte_BUILD_TUPLE_UNPACK(self, count): + # Pops count iterables from the stack, joins them in a single tuple, + # and pushes the result. Implements iterable unpacking in + # tuple displays (*x, *y, *z). + elts = self.popn(count) + self.push(tuple(e for l in elts for e in l)) + def byte_BUILD_TUPLE(self, count): elts = self.popn(count) self.push(tuple(elts)) From ac20f4645210e87490dc6e01552e866cb7488759 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=0F=C3=B6rn=20Mathis?= Date: Tue, 30 Jan 2018 17:10:44 +0100 Subject: [PATCH 05/13] added GET_YIELD_FROM_ITER --- byterun/pyvm2.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/byterun/pyvm2.py b/byterun/pyvm2.py index fae4c0cc..1a4fde68 100644 --- a/byterun/pyvm2.py +++ b/byterun/pyvm2.py @@ -9,6 +9,7 @@ import logging import operator import sys +import types import six from six.moves import reprlib @@ -767,6 +768,13 @@ def byte_SETUP_LOOP(self, dest): def byte_GET_ITER(self): self.push(iter(self.pop())) + def byte_GET_YIELD_FROM_ITER(self): + tos = self.top() + if isinstance(tos, types.GeneratorType) or isinstance(tos, types.CoroutineType): + return + tos = self.pop() + self.push(iter(tos)) + def byte_FOR_ITER(self, jump): iterobj = self.top() try: From 0c4d8fcb85a78404e896909ba68c61cb88aaa29f Mon Sep 17 00:00:00 2001 From: Rahul Gopinath Date: Wed, 31 Jan 2018 10:44:32 +0100 Subject: [PATCH 06/13] byterun/pyvm2.py: add WITH_CLEANUP_START .. FINISH --- byterun/pyvm2.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/byterun/pyvm2.py b/byterun/pyvm2.py index 1a4fde68..faba328c 100644 --- a/byterun/pyvm2.py +++ b/byterun/pyvm2.py @@ -917,6 +917,38 @@ def byte_SETUP_WITH(self, dest): self.push_block('finally', dest) self.push(ctxmgr_obj) + def byte_WITH_CLEANUP_START(self): + u = self.top() + v = None + w = None + if u is None: + exit_method = self.pop(1) + elif isinstance(u, str): + if u in {'return', 'continue'}: + exit_method = self.pop(2) + else: + exit_method = self.pop(1) + elif issubclass(u, BaseException): + w, v, u = self.popn(3) + tp, exc, tb = self.popn(3) + exit_method = self.pop() + self.push(tp, exc, tb) + self.push(None) + self.push(w, v, u) + block = self.pop_block() + assert block.type == 'except-handler' + self.push_block(block.type, block.handler, block.level-1) + + res = exit_method(u, v, w) + self.push(u) + self.push(res) + + def byte_WITH_CLEANUP_FINISH(self): + res = self.pop() + u = self.pop() + if type(u) is type and issubclass(u, BaseException) and res: + self.push("silenced") + def byte_WITH_CLEANUP(self): # The code here does some weird stack manipulation: the exit function # is buried in the stack, and where depends on what's on top of it. From d3b90887b2d16c152c28efd6c45022d3922d060f Mon Sep 17 00:00:00 2001 From: Rahul Gopinath Date: Wed, 31 Jan 2018 10:49:27 +0100 Subject: [PATCH 07/13] byterun/pyvm2.py: Fix CALL_FUNCTION_KW and add _EX (need #20 PR) --- byterun/pyvm2.py | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/byterun/pyvm2.py b/byterun/pyvm2.py index faba328c..8daf41bc 100644 --- a/byterun/pyvm2.py +++ b/byterun/pyvm2.py @@ -1018,16 +1018,45 @@ def byte_MAKE_CLOSURE(self, argc): fn = Function(name, code, globs, defaults, closure, self) self.push(fn) + def byte_CALL_FUNCTION_EX(self, arg): + # Calls a function. The lowest bit of flags indicates whether the + # var-keyword argument is placed at the top of the stack. Below + # the var-keyword argument, the var-positional argument is on the + # stack. Below the arguments, the function object to call is placed. + # Pops all function arguments, and the function itself off the stack, + # and pushes the return value. + # Note that this opcode pops at most three items from the stack. + #Var-positional and var-keyword arguments are packed by + #BUILD_TUPLE_UNPACK_WITH_CALL and BUILD_MAP_UNPACK_WITH_CALL. + # new in 3.6 + varkw = self.pop() if (arg & 0x1) else {} + varpos = self.pop() + return self.call_function(0, varpos, varkw) + def byte_CALL_FUNCTION(self, arg): + # Calls a function. argc indicates the number of positional arguments. + # The positional arguments are on the stack, with the right-most + # argument on top. Below the arguments, the function object to call is + # on the stack. Pops all function arguments, and the function itself + # off the stack, and pushes the return value. + # 3.6: Only used for calls with positional args return self.call_function(arg, [], {}) def byte_CALL_FUNCTION_VAR(self, arg): args = self.pop() return self.call_function(arg, args, {}) - def byte_CALL_FUNCTION_KW(self, arg): - kwargs = self.pop() - return self.call_function(arg, [], kwargs) + def byte_CALL_FUNCTION_KW(self, argc): + if not(six.PY3 and sys.version_info.minor >= 6): + kwargs = self.pop() + return self.call_function(arg, [], kwargs) + # changed in 3.6: keyword arguments are packed in a tuple instead + # of a dict. argc indicates total number of args. + kwargnames = self.pop() + lkwargs = len(kwargnames) + kwargs = self.popn(lkwargs) + arg = argc - lkwargs + return self.call_function(arg, [], dict(zip(kwargnames, kwargs))) def byte_CALL_FUNCTION_VAR_KW(self, arg): args, kwargs = self.popn(2) From 9c386c7014d19eab7f0e420033dde0b3408be755 Mon Sep 17 00:00:00 2001 From: Rahul Gopinath Date: Wed, 31 Jan 2018 11:18:11 +0100 Subject: [PATCH 08/13] byterun/pyvm2.py: MAKE_FUNCTION --- byterun/pyobj.py | 5 +++-- byterun/pyvm2.py | 16 +++++++++++++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/byterun/pyobj.py b/byterun/pyobj.py index 9f57b279..ee0c5a26 100644 --- a/byterun/pyobj.py +++ b/byterun/pyobj.py @@ -30,11 +30,12 @@ class Function(object): '_vm', '_func', ] - def __init__(self, name, code, globs, defaults, closure, vm): + def __init__(self, name, code, globs, defaults, kwdefaults, closure, vm): self._vm = vm self.func_code = code self.func_name = self.__name__ = name or code.co_name - self.func_defaults = tuple(defaults) + self.func_defaults = defaults \ + if PY3 and sys.version_info.minor >= 6 else tuple(defaults) self.func_globals = globs self.func_locals = self._vm.frame.f_locals self.__dict__ = {} diff --git a/byterun/pyvm2.py b/byterun/pyvm2.py index 8daf41bc..3bd61081 100644 --- a/byterun/pyvm2.py +++ b/byterun/pyvm2.py @@ -996,11 +996,21 @@ def byte_MAKE_FUNCTION(self, argc): if PY3: name = self.pop() else: + # Pushes a new function object on the stack. TOS is the code + # associated with the function. The function object is defined to + # have argc default parameters, which are found below TOS. name = None code = self.pop() - defaults = self.popn(argc) globs = self.frame.f_globals - fn = Function(name, code, globs, defaults, None, self) + if PY3 and sys.version_info.minor >= 6: + closure = self.pop() if (argc & 0x8) else None + ann = self.pop() if (argc & 0x4) else None + kwdefaults = self.pop() if (argc & 0x2) else None + defaults = self.pop() if (argc & 0x1) else None + fn = Function(name, code, globs, defaults, kwdefaults, closure, self) + else: + defaults = self.popn(argc) + fn = Function(name, code, globs, defaults, None, None, self) self.push(fn) def byte_LOAD_CLOSURE(self, name): @@ -1015,7 +1025,7 @@ def byte_MAKE_CLOSURE(self, argc): closure, code = self.popn(2) defaults = self.popn(argc) globs = self.frame.f_globals - fn = Function(name, code, globs, defaults, closure, self) + fn = Function(name, code, globs, defaults, None, closure, self) self.push(fn) def byte_CALL_FUNCTION_EX(self, arg): From fe1ba4f697bd4013fea0997ed92fe474a159272b Mon Sep 17 00:00:00 2001 From: Rahul Gopinath Date: Wed, 31 Jan 2018 11:32:02 +0100 Subject: [PATCH 09/13] authors --- AUTHORS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index 4680226d..0f41574f 100644 --- a/AUTHORS +++ b/AUTHORS @@ -2,3 +2,5 @@ Paul Swartz Ned Batchelder Allison Kaptur Laura Lindzey +Rahul Gopinath +Björn Mathis From a07642f9c9b0aa776da393828a6062a1e518d1ed Mon Sep 17 00:00:00 2001 From: Rahul Gopinath Date: Wed, 31 Jan 2018 11:19:42 +0100 Subject: [PATCH 10/13] PR #20 from Darius --- AUTHORS | 1 + byterun/pyobj.py | 33 ++++++++-------------- byterun/pyvm2.py | 61 +++++++++++++++++++++++++++++++++++++---- tests/test_functions.py | 12 ++++++++ tests/vmtest.py | 39 ++++++++++++++++++-------- 5 files changed, 109 insertions(+), 37 deletions(-) diff --git a/AUTHORS b/AUTHORS index 0f41574f..814e943e 100644 --- a/AUTHORS +++ b/AUTHORS @@ -4,3 +4,4 @@ Allison Kaptur Laura Lindzey Rahul Gopinath Björn Mathis +Darius Bacon diff --git a/byterun/pyobj.py b/byterun/pyobj.py index ee0c5a26..51a7c743 100644 --- a/byterun/pyobj.py +++ b/byterun/pyobj.py @@ -2,6 +2,7 @@ import collections import inspect +import re import types import dis @@ -64,17 +65,18 @@ def __get__(self, instance, owner): return self def __call__(self, *args, **kwargs): - if PY2 and self.func_name in ["", "", ""]: + if re.search(r'<(?:listcomp|setcomp|dictcomp|genexpr)>$', self.func_name): # D'oh! http://bugs.python.org/issue19611 Py2 doesn't know how to # inspect set comprehensions, dict comprehensions, or generator # expressions properly. They are always functions of one argument, - # so just do the right thing. + # so just do the right thing. Py3.4 also would fail without this + # hack, for list comprehensions too. (Haven't checked for other 3.x.) assert len(args) == 1 and not kwargs, "Surprising comprehension!" callargs = {".0": args[0]} else: callargs = inspect.getcallargs(self._func, *args, **kwargs) frame = self._vm.make_frame( - self.func_code, callargs, self.func_globals, {} + self.func_code, callargs, self.func_globals, {}, self.func_closure ) CO_GENERATOR = 32 # flag for "this code uses yield" if self.func_code.co_flags & CO_GENERATOR: @@ -138,7 +140,7 @@ def set(self, value): class Frame(object): - def __init__(self, f_code, f_globals, f_locals, f_back): + def __init__(self, f_code, f_globals, f_locals, f_closure, f_back): self.f_code = f_code self.py36_opcodes = list(dis.get_instructions(self.f_code)) \ if six.PY3 and sys.version_info.minor >= 6 else None @@ -156,24 +158,13 @@ def __init__(self, f_code, f_globals, f_locals, f_back): self.f_lineno = f_code.co_firstlineno self.f_lasti = 0 - if f_code.co_cellvars: - self.cells = {} - if not f_back.cells: - f_back.cells = {} - for var in f_code.co_cellvars: - # Make a cell for the variable in our locals, or None. - cell = Cell(self.f_locals.get(var)) - f_back.cells[var] = self.cells[var] = cell - else: - self.cells = None - + self.cells = {} if f_code.co_cellvars or f_code.co_freevars else None + for var in f_code.co_cellvars: + # Make a cell for the variable in our locals, or None. + self.cells[var] = Cell(self.f_locals.get(var)) if f_code.co_freevars: - if not self.cells: - self.cells = {} - for var in f_code.co_freevars: - assert self.cells is not None - assert f_back.cells, "f_back.cells: %r" % (f_back.cells,) - self.cells[var] = f_back.cells[var] + assert len(f_code.co_freevars) == len(f_closure) + self.cells.update(zip(f_code.co_freevars, f_closure)) self.block_stack = [] self.generator = None diff --git a/byterun/pyvm2.py b/byterun/pyvm2.py index 3bd61081..c202f7eb 100644 --- a/byterun/pyvm2.py +++ b/byterun/pyvm2.py @@ -16,7 +16,7 @@ PY3, PY2 = six.PY3, not six.PY3 -from .pyobj import Frame, Block, Method, Function, Generator +from .pyobj import Frame, Block, Method, Function, Generator, Cell log = logging.getLogger(__name__) @@ -91,7 +91,7 @@ def push_block(self, type, handler=None, level=None): def pop_block(self): return self.frame.block_stack.pop() - def make_frame(self, code, callargs={}, f_globals=None, f_locals=None): + def make_frame(self, code, callargs={}, f_globals=None, f_locals=None, f_closure=None): log.info("make_frame: code=%r, callargs=%s" % (code, repper(callargs))) if f_globals is not None: f_globals = f_globals @@ -108,7 +108,7 @@ def make_frame(self, code, callargs={}, f_globals=None, f_locals=None): '__package__': None, } f_locals.update(callargs) - frame = Frame(code, f_globals, f_locals, self.frame) + frame = Frame(code, f_globals, f_locals, f_closure, self.frame) return frame def push_frame(self, frame): @@ -446,7 +446,10 @@ def byte_LOAD_GLOBAL(self, name): elif name in f.f_builtins: val = f.f_builtins[name] else: - raise NameError("global name '%s' is not defined" % name) + if PY2: + raise NameError("global name '%s' is not defined" % name) + elif PY3: + raise NameError("name '%s' is not defined" % name) self.push(val) def byte_STORE_GLOBAL(self, name): @@ -1169,7 +1172,7 @@ def byte_BUILD_CLASS(self): elif PY3: def byte_LOAD_BUILD_CLASS(self): # New in py3 - self.push(__build_class__) + self.push(build_class) def byte_STORE_LOCALS(self): self.frame.f_locals = self.pop() @@ -1177,3 +1180,51 @@ def byte_STORE_LOCALS(self): if 0: # Not in py2.7 def byte_SET_LINENO(self, lineno): self.frame.f_lineno = lineno + +if PY3: + def build_class(func, name, *bases, **kwds): + "Like __build_class__ in bltinmodule.c, but running in the byterun VM." + if not isinstance(func, Function): + raise TypeError("func must be a function") + if not isinstance(name, str): + raise TypeError("name is not a string") + metaclass = kwds.pop('metaclass', None) + # (We don't just write 'metaclass=None' in the signature above + # because that's a syntax error in Py2.) + if metaclass is None: + metaclass = type(bases[0]) if bases else type + if isinstance(metaclass, type): + metaclass = calculate_metaclass(metaclass, bases) + + try: + prepare = metaclass.__prepare__ + except AttributeError: + namespace = {} + else: + namespace = prepare(name, bases, **kwds) + + # Execute the body of func. This is the step that would go wrong if + # we tried to use the built-in __build_class__, because __build_class__ + # does not call func, it magically executes its body directly, as we + # do here (except we invoke our VirtualMachine instead of CPython's). + frame = func._vm.make_frame(func.func_code, + f_globals=func.func_globals, + f_locals=namespace, + f_closure=func.func_closure) + cell = func._vm.run_frame(frame) + + cls = metaclass(name, bases, namespace) + if isinstance(cell, Cell): + cell.set(cls) + return cls + + def calculate_metaclass(metaclass, bases): + "Determine the most derived metatype." + winner = metaclass + for base in bases: + t = type(base) + if issubclass(t, winner): + winner = t + elif not issubclass(winner, t): + raise TypeError("metaclass conflict", winner, t) + return winner diff --git a/tests/test_functions.py b/tests/test_functions.py index f86fd131..7df69df7 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -207,6 +207,18 @@ def f4(g): assert answer == 54 """) + def test_closure_vars_from_static_parent(self): + self.assert_ok("""\ + def f(xs): + return lambda: xs[0] + + def g(h): + xs = 5 + lambda: xs + return h() + + assert g(f([42])) == 42 + """) class TestGenerators(vmtest.VmTestCase): def test_first(self): diff --git a/tests/vmtest.py b/tests/vmtest.py index 9e071838..763aa174 100644 --- a/tests/vmtest.py +++ b/tests/vmtest.py @@ -40,6 +40,19 @@ def assert_ok(self, code, raises=None): # Print the disassembly so we'll see it if the test fails. dis_code(code) + # Run the code through our VM and the real Python interpreter, for comparison. + vm_value, vm_exc, vm_stdout = self.run_in_byterun(code) + py_value, py_exc, py_stdout = self.run_in_real_python(code) + + self.assert_same_exception(vm_exc, py_exc) + self.assertEqual(vm_stdout.getvalue(), py_stdout.getvalue()) + self.assertEqual(vm_value, py_value) + if raises: + self.assertIsInstance(vm_exc, raises) + else: + self.assertIsNone(vm_exc) + + def run_in_byterun(self, code): real_stdout = sys.stdout # Run the code through our VM. @@ -64,32 +77,36 @@ def assert_ok(self, code, raises=None): raise vm_exc = e finally: + sys.stdout = real_stdout real_stdout.write("-- stdout ----------\n") real_stdout.write(vm_stdout.getvalue()) - # Run the code through the real Python interpreter, for comparison. + return vm_value, vm_exc, vm_stdout + + def run_in_real_python(self, code): + real_stdout = sys.stdout py_stdout = six.StringIO() sys.stdout = py_stdout py_value = py_exc = None - globs = {} + globs = { + '__builtins__': __builtins__, + '__name__': '__main__', + '__doc__': None, + '__package__': None, + } + try: py_value = eval(code, globs, globs) except AssertionError: # pragma: no cover raise except Exception as e: py_exc = e + finally: + sys.stdout = real_stdout - sys.stdout = real_stdout - - self.assert_same_exception(vm_exc, py_exc) - self.assertEqual(vm_stdout.getvalue(), py_stdout.getvalue()) - self.assertEqual(vm_value, py_value) - if raises: - self.assertIsInstance(vm_exc, raises) - else: - self.assertIsNone(vm_exc) + return py_value, py_exc, py_stdout def assert_same_exception(self, e1, e2): """Exceptions don't implement __eq__, check it ourselves.""" From 1765ce061e39db10dc8c37241da18e48e2490705 Mon Sep 17 00:00:00 2001 From: Rahul Gopinath Date: Mon, 12 Feb 2018 14:47:19 +0100 Subject: [PATCH 11/13] byte_CALL_FUNCTION_KW for py34 --- byterun/pyvm2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/byterun/pyvm2.py b/byterun/pyvm2.py index c202f7eb..108b1eef 100644 --- a/byterun/pyvm2.py +++ b/byterun/pyvm2.py @@ -1062,7 +1062,7 @@ def byte_CALL_FUNCTION_VAR(self, arg): def byte_CALL_FUNCTION_KW(self, argc): if not(six.PY3 and sys.version_info.minor >= 6): kwargs = self.pop() - return self.call_function(arg, [], kwargs) + return self.call_function(argc, [], kwargs) # changed in 3.6: keyword arguments are packed in a tuple instead # of a dict. argc indicates total number of args. kwargnames = self.pop() From 5fce7b946143a3acb63f3cd5e6c24bf2e3f4c451 Mon Sep 17 00:00:00 2001 From: Rahul Gopinath Date: Mon, 12 Feb 2018 15:05:35 +0100 Subject: [PATCH 12/13] Updated with changes suggested by the reviewer --- byterun/pyvm2.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/byterun/pyvm2.py b/byterun/pyvm2.py index 108b1eef..797ab049 100644 --- a/byterun/pyvm2.py +++ b/byterun/pyvm2.py @@ -598,8 +598,8 @@ def byte_BUILD_TUPLE_UNPACK_WITH_CALL(self, count): # This is similar to BUILD_TUPLE_UNPACK, but is used for f(*x, *y, *z) # call syntax. The stack item at position count + 1 should be the # corresponding callable f. - elts = self.popn(count) - self.push(tuple(e for l in elts for e in l)) + self.byte_BUILD_TUPLE_UNPACK(count) + def byte_BUILD_TUPLE_UNPACK(self, count): # Pops count iterables from the stack, joins them in a single tuple, @@ -612,6 +612,10 @@ def byte_BUILD_TUPLE(self, count): elts = self.popn(count) self.push(tuple(elts)) + def byte_BUILD_LIST_UNPACK(self, count): + elts = self.popn(count) + self.push([e for l in elts for e in l]) + def byte_BUILD_LIST(self, count): elts = self.popn(count) self.push(elts) @@ -632,14 +636,14 @@ def byte_BUILD_CONST_KEY_MAP(self, count): def byte_BUILD_MAP(self, count): # Pushes a new dictionary on to stack. - if not(six.PY3 and sys.version_info.minor >= 5): + if sys.version_info[:2] < (3, 5): self.push({}) return # Pop 2*count items so that # dictionary holds count entries: {..., TOS3: TOS2, TOS1:TOS} # updated in version 3.5 kvs = {} - for i in range(0, count): + for i in range(count): key, val = self.popn(2) kvs[key] = val self.push(kvs) From 7bbec1905950040d6051f522ed551f960ac86250 Mon Sep 17 00:00:00 2001 From: Rahul Gopinath Date: Wed, 31 Jan 2018 17:46:31 +0100 Subject: [PATCH 13/13] byterun/modules.py: Ensure that modules go through byterun --- byterun/pyobj.py | 5 ++- byterun/pyvm2.py | 97 ++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 97 insertions(+), 5 deletions(-) diff --git a/byterun/pyobj.py b/byterun/pyobj.py index 51a7c743..699f76c4 100644 --- a/byterun/pyobj.py +++ b/byterun/pyobj.py @@ -151,7 +151,10 @@ def __init__(self, f_code, f_globals, f_locals, f_closure, f_back): if f_back: self.f_builtins = f_back.f_builtins else: - self.f_builtins = f_locals['__builtins__'] + if hasattr(f_locals, '__builtins__'): + self.f_builtins = f_locals['__builtins__'] + else: + self.f_builtins = f_globals['__builtins__'] if hasattr(self.f_builtins, '__dict__'): self.f_builtins = self.f_builtins.__dict__ diff --git a/byterun/pyvm2.py b/byterun/pyvm2.py index 797ab049..078f7aa1 100644 --- a/byterun/pyvm2.py +++ b/byterun/pyvm2.py @@ -11,6 +11,12 @@ import sys import types +import os.path +import imp +NoSource = Exception +Loaded = {} + + import six from six.moves import reprlib @@ -1109,6 +1115,44 @@ def call_function(self, arg, args, kwargs): retval = func(*posargs, **namedargs) self.push(retval) + def import_module(self, m, fromList, level): + f = self.frame + res = self.import_python_module(m, f.f_globals, f.f_locals, fromList, level) + self.push(res) + + def import_python_module(self, modulename, glo, loc, fromlist, level, search=None): + """Import a python module. + `modulename` is the name of the module, possibly a dot-separated name. + `fromlist` is the list of things to imported from the module. + """ + try: + if '.' not in modulename: + mymod = find_module(modulename, search, level, True, glo, loc) + # Open the source file. + try: + with open(mymod.__file__, "rU") as source_file: + source = source_file.read() + if not source or source[-1] != '\n': source += '\n' + code = compile(source, mymod.__file__, "exec") + # Execute the source file. + self.run_code(code, f_globals=mymod.__dict__, f_locals=mymod.__dict__) + # strip it with fromlist + # get the defined module + return mymod + except IOError as e: + raise NoSource("module does not live in a file: %r" % modulename) + else: + pkgn, name = modulename.rsplit('.', 1) + pkg = find_module(pkgn, search, level, False, glo, loc) + mod = self.import_python_module(name, glo, loc, fromlist, level, pkg.__file__) + # mod is an attribute of pkg + setattr(pkg, mod.__name__, mod) + return pkg + except NoSource as e: + m = __import__(modulename, glo, loc, fromlist, level) + Loaded[modulename] = m + return m + def byte_RETURN_VALUE(self): self.return_value = self.pop() if self.frame.generator: @@ -1145,10 +1189,7 @@ def byte_YIELD_FROM(self): def byte_IMPORT_NAME(self, name): level, fromlist = self.popn(2) - frame = self.frame - self.push( - __import__(name, frame.f_globals, frame.f_locals, fromlist, level) - ) + self.import_module(name, fromlist, level) def byte_IMPORT_STAR(self): # TODO: this doesn't use __all__ properly. @@ -1232,3 +1273,51 @@ def calculate_metaclass(metaclass, bases): elif not issubclass(winner, t): raise TypeError("metaclass conflict", winner, t) return winner + + + +def find_module_absolute(name, searchpath, isfile): + # search path should really be appeneded to a list of paths + # that the interpreter knows about. For now, we only look in '.' + myname = name if not searchpath else "%s/%s" % (searchpath, name) + if isfile: + fname = "%s.py" % myname + return os.path.abspath(fname) if os.path.isfile(fname) else None + else: + return os.path.abspath(myname) if os.path.isdir(myname) else None + +def find_module_relative(name, searchpath): return None + +def find_module(name, searchpath, level, isfile=True, glo=None, loc=None): + """ + `level` specifies whether to use absolute and/or relative. + The default is -1 which is both absolute and relative + 0 means only absolute and positive values indicate number + parent directories to search relative to the directory of module + calling `__import__` + """ + if name in Loaded: return Loaded[name] + + assert level <= 0 # we dont implement relative yet + path = None + if level == 0: + path = find_module_absolute(name, searchpath, isfile) + elif level > 0: + path = find_module_relative(name, searchpath, isfile) + else: + res = find_module_absolute(name, searchpath, isfile) + path = find_module_relative(name, searchpath, isfile) if not res \ + else res + + if not path: + v = imp.find_module(name, searchpath) + if v and v[1]: + path = v[1] + else: + raise NoSource("<%s> was not found" % name) + mymod = types.ModuleType(name) + mymod.__file__ = path + mymod.__builtins__ = glo['__builtins__'] + # mark the module as being loaded + Loaded[name] = mymod + return mymod