From 4bbe415a6c23819b1a35573419334fedd1fb8bd6 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 27 Jul 2021 14:59:51 -0700 Subject: [PATCH 001/163] allow any ABI-encodable type as args --- vyper/old_codegen/parser_utils.py | 113 ++++++++---------------------- vyper/semantics/types/function.py | 7 -- 2 files changed, 28 insertions(+), 92 deletions(-) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 70fe0b3f82..a5c48f9208 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -404,105 +404,48 @@ def unwrap_location(orig): # Pack function arguments for a call @type_check_wrapper def pack_arguments(signature, args, context, stmt_expr, is_external_call): + # FLAG cyclic dep + from vyper.old_codegen.abi import abi_type_of, abi_encode pos = getpos(stmt_expr) - setters = [] - staticarray_offset = 0 - maxlen = sum([get_size_of_type(arg.typ) for arg in signature.args]) * 32 + # abi encoding just treats all args as a big tuple + args_tuple_t = TupleType([x.typ for x in args]) + args_as_tuple = LLLnode.from_list(["multi"]+ [x for x in args], typ=args_tuple_t) + args_abi_t = abi_type_of(args_tuple_t) + + maxlen = args_abi_t.dynamic_size_bound() + args_abi_t.static_size() if is_external_call: - maxlen += 32 + maxlen += 32 # padding for the method id + + buf_t = ByteArrayType(maxlen=maxlen) + buf = context.new_internal_variable(buf_t) - placeholder_typ = ByteArrayType(maxlen=maxlen) - placeholder = context.new_internal_variable(placeholder_typ) + mstore_method_id = [] if is_external_call: - setters.append(["mstore", placeholder, signature.method_id]) - placeholder += 32 + # layout: + # 32 bytes | args + # 0x..00 | args + mstore_method_id.append(["mstore", buf, signature.method_id]) + buf += 32 if len(signature.args) != len(args): return - # check for dynamic-length types - dynamic_remaining = len([i for i in signature.args if isinstance(i.typ, ByteArrayLike)]) - needpos = bool(dynamic_remaining) - - for i, (arg, typ) in enumerate(zip(args, [arg.typ for arg in signature.args])): - if isinstance(typ, BaseType): - setters.append( - make_setter( - LLLnode.from_list(placeholder + staticarray_offset + i * 32, typ=typ,), - arg, - "memory", - pos=pos, - in_function_call=True, - ) - ) - - elif isinstance(typ, ByteArrayLike): - dynamic_remaining -= 1 - setters.append(["mstore", placeholder + staticarray_offset + i * 32, "_poz"]) - arg_copy = LLLnode.from_list("_s", typ=arg.typ, location=arg.location) - target = LLLnode.from_list(["add", placeholder, "_poz"], typ=typ, location="memory",) - pos_setter = "pass" - - # if `arg.value` is None, this is a call to `empty()` - # if `arg.typ.maxlen` is 0, this is a literal "" or b"" - if arg.value is None or arg.typ.maxlen == 0: - if dynamic_remaining: - # only adjust the dynamic pointer if this is not the last dynamic type - pos_setter = ["set", "_poz", ["add", "_poz", 64]] - setters.append(["seq", mzero(target, 64), pos_setter]) - else: - if dynamic_remaining: - pos_setter = [ - "set", - "_poz", - ["add", 32, ["ceil32", ["add", "_poz", get_bytearray_length(arg_copy)]]], - ] - setters.append( - [ - "with", - "_s", - arg, - ["seq", make_byte_array_copier(target, arg_copy, pos), pos_setter], - ] - ) - - elif isinstance(typ, (StructType, ListType)): - if has_dynamic_data(typ): - return - target = LLLnode.from_list( - [placeholder + staticarray_offset + i * 32], typ=typ, location="memory", - ) - setters.append(make_setter(target, arg, "memory", pos=pos)) - staticarray_offset += 32 * (get_size_of_type(typ) - 1) - - else: - return + encode_args = abi_encode(buf, args_as_tuple, pos) if is_external_call: - returner = [[placeholder - 4]] - inargsize = placeholder_typ.maxlen - 28 + returner = [[buf - 4]] + inargsize = buf_t.maxlen - 28 else: # internal call does not use a returner or adjust max length for signature returner = [] - inargsize = placeholder_typ.maxlen - - if needpos: - return ( - LLLnode.from_list( - ["with", "_poz", len(args) * 32 + staticarray_offset, ["seq"] + setters + returner], - typ=placeholder_typ, - location="memory", - ), - inargsize, - placeholder, - ) - else: - return ( - LLLnode.from_list(["seq"] + setters + returner, typ=placeholder_typ, location="memory"), - inargsize, - placeholder, - ) + inargsize = buf_t.maxlen + + return ( + LLLnode.from_list(["seq"] + mstore_method_id + [encode_args] + returner, typ=buf_t, location="memory"), + inargsize, + buf, + ) def _make_array_index_setter(target, target_token, pos, location, offset): diff --git a/vyper/semantics/types/function.py b/vyper/semantics/types/function.py index df3cfb4fee..262eb7c70e 100644 --- a/vyper/semantics/types/function.py +++ b/vyper/semantics/types/function.py @@ -321,13 +321,6 @@ def from_FunctionDef( type_definition = get_type_from_annotation( arg.annotation, location=DataLocation.CALLDATA, is_immutable=True ) - if isinstance(type_definition, StructDefinition) and type_definition.is_dynamic_size: - # this is a temporary restriction and should be removed once support for dynamically - # sized structs is implemented - https://github.com/vyperlang/vyper/issues/2190 - raise ArgumentException( - "Struct with dynamically sized data cannot be used as a function input", arg - ) - if value is not None: if not check_constant(value): raise StateAccessViolation( From 6d27c3d0204e3a399162b3b9178f6dcf33934d97 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 27 Jul 2021 22:49:45 +0000 Subject: [PATCH 002/163] remove tests that blocked dynamic structs as args --- .../exceptions/test_argument_exception.py | 25 ------------------- 1 file changed, 25 deletions(-) diff --git a/tests/parser/exceptions/test_argument_exception.py b/tests/parser/exceptions/test_argument_exception.py index b922cc08d6..17f4a1bab4 100644 --- a/tests/parser/exceptions/test_argument_exception.py +++ b/tests/parser/exceptions/test_argument_exception.py @@ -89,31 +89,6 @@ def foo(): for i in range(1, 2, 3, 4): pass """, - """ -struct Foo: - a: Bytes[32] - -@external -def foo(a: Foo): - pass - """, - """ -struct Foo: - a: String[32] - -@external -def foo(a: Foo): - pass - """, - """ -struct Foo: - b: uint256 - a: String[32] - -@external -def foo(a: Foo): - pass - """, ] From 75c2040d178822ce256a4dfc595dc8a6add48ad7 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 27 Jul 2021 22:50:58 +0000 Subject: [PATCH 003/163] fix linter errors --- vyper/old_codegen/parser_utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index a5c48f9208..89ce12af88 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -21,7 +21,6 @@ TupleType, ceil32, get_size_of_type, - has_dynamic_data, is_base_type, ) from vyper.utils import GAS_IDENTITY, GAS_IDENTITYWORD, MemoryPositions @@ -405,12 +404,13 @@ def unwrap_location(orig): @type_check_wrapper def pack_arguments(signature, args, context, stmt_expr, is_external_call): # FLAG cyclic dep - from vyper.old_codegen.abi import abi_type_of, abi_encode + from vyper.old_codegen.abi import abi_encode, abi_type_of + pos = getpos(stmt_expr) # abi encoding just treats all args as a big tuple args_tuple_t = TupleType([x.typ for x in args]) - args_as_tuple = LLLnode.from_list(["multi"]+ [x for x in args], typ=args_tuple_t) + args_as_tuple = LLLnode.from_list(["multi"] + [x for x in args], typ=args_tuple_t) args_abi_t = abi_type_of(args_tuple_t) maxlen = args_abi_t.dynamic_size_bound() + args_abi_t.static_size() @@ -442,7 +442,9 @@ def pack_arguments(signature, args, context, stmt_expr, is_external_call): inargsize = buf_t.maxlen return ( - LLLnode.from_list(["seq"] + mstore_method_id + [encode_args] + returner, typ=buf_t, location="memory"), + LLLnode.from_list( + ["seq"] + mstore_method_id + [encode_args] + returner, typ=buf_t, location="memory" + ), inargsize, buf, ) From 53019b63964e202afbbfe9df617754adacce7b62 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 29 Jul 2021 03:43:55 +0000 Subject: [PATCH 004/163] add a note about alignment --- vyper/old_codegen/parser_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 89ce12af88..8372ab0494 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -425,6 +425,10 @@ def pack_arguments(signature, args, context, stmt_expr, is_external_call): # layout: # 32 bytes | args # 0x..00 | args + # the reason for the left padding is just so the alignment is easier. + # if we were only targeting constantinople, we could align + # to buf (and also keep code size small) by using + # (mstore buf (shl signature.method_id 224)) mstore_method_id.append(["mstore", buf, signature.method_id]) buf += 32 From 2f4869f535255c6c194d8f77e18f3bfd1b0a8be5 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 29 Jul 2021 19:39:37 -0700 Subject: [PATCH 005/163] change FLAG comments to NOTE text editors highlight them --- vyper/builtin_functions/functions.py | 2 +- vyper/old_codegen/parser_utils.py | 2 +- vyper/semantics/namespace.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/vyper/builtin_functions/functions.py b/vyper/builtin_functions/functions.py index 776b3162fb..74a5791197 100644 --- a/vyper/builtin_functions/functions.py +++ b/vyper/builtin_functions/functions.py @@ -1491,7 +1491,7 @@ def build_LLL(self, expr, context): def get_create_forwarder_to_bytecode(): - # FLAG cyclic import? + # NOTE cyclic import? from vyper.lll.compile_lll import assembly_to_evm loader_asm = [ diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 8372ab0494..eaefa8cfba 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -403,7 +403,7 @@ def unwrap_location(orig): # Pack function arguments for a call @type_check_wrapper def pack_arguments(signature, args, context, stmt_expr, is_external_call): - # FLAG cyclic dep + # NOTE cyclic dep from vyper.old_codegen.abi import abi_encode, abi_type_of pos = getpos(stmt_expr) diff --git a/vyper/semantics/namespace.py b/vyper/semantics/namespace.py index ed2a6b25e7..79a5c8ad8e 100644 --- a/vyper/semantics/namespace.py +++ b/vyper/semantics/namespace.py @@ -22,7 +22,7 @@ class Namespace(dict): def __init__(self): super().__init__() self._scopes = [] - # FLAG cyclic imports! + # NOTE cyclic imports! from vyper.builtin_functions.functions import get_builtin_functions from vyper.semantics import environment from vyper.semantics.types import get_types @@ -62,7 +62,7 @@ def enter_scope(self): Called as a context manager, e.g. `with namespace.enter_scope():` All items that are added within the context are removed upon exit. """ - # FLAG cyclic imports! + # NOTE cyclic imports! from vyper.semantics import environment self._scopes.append(set()) From 3faea5327fd479a824a27269ba184c9fd29b5ddb Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 30 Jul 2021 11:04:29 -0700 Subject: [PATCH 006/163] add a nested tuple test --- .../functional/codegen/test_struct_return.py | 24 +++++++------------ 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/tests/functional/codegen/test_struct_return.py b/tests/functional/codegen/test_struct_return.py index afe6c6a242..300f6e60c0 100644 --- a/tests/functional/codegen/test_struct_return.py +++ b/tests/functional/codegen/test_struct_return.py @@ -5,34 +5,26 @@ def test_nested_tuple(get_contract): code = """ struct Animal: location: address - fur: uint256 + fur: String[32] struct Human: location: address - height: uint256 + animal: Animal @external -def return_nested_tuple() -> (Animal, Human): - animal: Animal = Animal({ - location: 0x1234567890123456789012345678901234567890, - fur: 123 - }) - human: Human = Human({ - location: 0x1234567890123456789012345678900000000000, - height: 456 - }) +def modify_nested_tuple(_human: Human) -> Human: + human: Human = _human # do stuff, edit the structs - animal.fur += 1 - human.height += 1 + human.animal.fur = slice(concat(human.animal.fur, " is great"), 0, 32) - return animal, human + return human """ c = get_contract(code) addr1 = "0x1234567890123456789012345678901234567890" addr2 = "0x1234567890123456789012345678900000000000" - assert c.return_nested_tuple() == [(addr1, 124), (addr2, 457)] - + #assert c.modify_nested_tuple([addr1, 123], [addr2, 456]) == [[addr1, 124], [addr2, 457]] + assert c.modify_nested_tuple({"location": addr1, "animal": {"location": addr2, "fur": "wool"}}) == [(addr1, (addr2, "wool is great"))] @pytest.mark.parametrize("string", ["a", "abc", "abcde", "potato"]) def test_string_inside_tuple(get_contract, string): From 71b4993e3b1857cf1fa941acb07b8f9cd9e2b708 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 2 Aug 2021 16:36:32 -0700 Subject: [PATCH 007/163] remove storage_prehashed cases it no longer exists --- vyper/old_codegen/parser_utils.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index eaefa8cfba..3ba75a0add 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -300,12 +300,6 @@ def add_variable_offset(parent, key, pos, array_bounds_check=True): for i in range(index): offset += get_size_of_type(typ.members[attrs[i]]) return LLLnode.from_list(["add", parent, offset], typ=subtype, location="storage",) - elif location == "storage_prehashed": - return LLLnode.from_list( - ["add", parent, LLLnode.from_list(index, annotation=annotation)], - typ=subtype, - location="storage", - ) elif location in ("calldata", "memory"): offset = 0 for i in range(index): @@ -376,8 +370,6 @@ def add_variable_offset(parent, key, pos, array_bounds_check=True): return LLLnode.from_list( ["add", parent, ["mul", sub, offset]], typ=subtype, location="storage", pos=pos ) - elif location == "storage_prehashed": - return LLLnode.from_list(["add", parent, sub], typ=subtype, location="storage", pos=pos) elif location in ("calldata", "memory"): offset = 32 * get_size_of_type(subtype) return LLLnode.from_list( From 0fb8f925c1b7e321d1e27d5cb84dfc757108df17 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 3 Aug 2021 09:44:56 -0700 Subject: [PATCH 008/163] clarify tuple wrapping behavior --- docs/built-in-functions.rst | 2 +- vyper/builtin_functions/functions.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/built-in-functions.rst b/docs/built-in-functions.rst index 215171f329..a00f6d07cf 100644 --- a/docs/built-in-functions.rst +++ b/docs/built-in-functions.rst @@ -630,7 +630,7 @@ Utilities Once this function has seen more use we provisionally plan to put it into the ``ethereum.abi`` namespace. * ``*args``: Arbitrary arguments - * ``ensure_tuple``: If set to True, ensures that even a single argument is encoded as a tuple. In other words, ``bytes`` gets encoded as ``(bytes,)``. This is the calling convention for Vyper and Solidity functions. Except for very specific use cases, this should be set to True. Must be a literal. + * ``ensure_tuple``: If set to True, ensures that even a single argument is encoded as a tuple. In other words, ``bytes`` gets encoded as ``(bytes,)``, and ``(bytes,)`` gets encoded as ``((bytes,),)`` This is the calling convention for Vyper and Solidity functions. Except for very specific use cases, this should be set to True. Must be a literal. * ``method_id``: A literal hex or Bytes[4] value to append to the beginning of the abi-encoded bytestring. Returns a bytestring whose max length is determined by the arguments. For example, encoding a ``Bytes[32]`` results in a ``Bytes[64]`` (first word is the length of the bytestring variable). diff --git a/vyper/builtin_functions/functions.py b/vyper/builtin_functions/functions.py index 74a5791197..6cc138e0d0 100644 --- a/vyper/builtin_functions/functions.py +++ b/vyper/builtin_functions/functions.py @@ -1746,7 +1746,8 @@ class ABIEncode(_SimpleBuiltinFunction): # to handle varargs.) # explanation of ensure_tuple: # default is to force even a single value into a tuple, - # e.g. _abi_encode(bytes) -> abi_encode((bytes,)) + # e.g. _abi_encode(bytes) -> _abi_encode((bytes,)) + # _abi_encode((bytes,)) -> _abi_encode(((bytes,),)) # this follows the encoding convention for functions: # ://docs.soliditylang.org/en/v0.8.6/abi-spec.html#function-selector-and-argument-encoding # if this is turned off, then bytes will be encoded as bytes. From b986d468f6f39bf1a593197fdfc8ff4936bd0cd5 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 3 Aug 2021 10:58:47 -0700 Subject: [PATCH 009/163] complex type --- vyper/old_codegen/abi.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index 94a6878521..e3d9716709 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -48,8 +48,8 @@ def selector_name(self): # Whether the type is a tuple at the ABI level. # (This is important because if it does, it needs an offset. # Compare the difference in encoding between `bytes` and `(bytes,)`.) - def is_tuple(self): - raise NotImplementedError("ABIType.is_tuple") + def is_complex_type(self): + raise NotImplementedError("ABIType.is_complex_type") # uint: unsigned integer type of M bits, 0 < M <= 256, M % 8 == 0. e.g. uint32, uint8, uint256. @@ -71,7 +71,7 @@ def static_size(self): def selector_name(self): return ("" if self.signed else "u") + f"int{self.m_bits}" - def is_tuple(self): + def is_complex_type(self): return False @@ -122,7 +122,7 @@ def static_size(self): def selector_name(self): return ("" if self.signed else "u") + "fixed{self.m_bits}x{self.n_places}" - def is_tuple(self): + def is_complex_type(self): return False @@ -143,7 +143,7 @@ def static_size(self): def selector_name(self): return f"bytes{self.m_bytes}" - def is_tuple(self): + def is_complex_type(self): return False @@ -178,7 +178,7 @@ def dynamic_size_bound(self): def selector_name(self): return f"{self.subtyp.selector_name()}[{self.m_elems}]" - def is_tuple(self): + def is_complex_type(self): return True @@ -204,7 +204,7 @@ def dynamic_size_bound(self): def selector_name(self): return "bytes" - def is_tuple(self): + def is_complex_type(self): return False @@ -233,7 +233,7 @@ def dynamic_size_bound(self): def selector_name(self): return f"{self.subtyp.selector_name()}[]" - def is_tuple(self): + def is_complex_type(self): return False @@ -250,7 +250,7 @@ def static_size(self): def dynamic_size_bound(self): return sum([t.dynamic_size_bound() for t in self.subtyps]) - def is_tuple(self): + def is_complex_type(self): return True @@ -320,7 +320,7 @@ def abi_type_of2(t: vy.BasePrimitive) -> ABIType: # there are a lot of places in the calling convention where a tuple # must be passed, so here's a convenience function for that. def ensure_tuple(abi_typ): - if not abi_typ.is_tuple(): + if not abi_typ.is_complex_type(): return ABI_Tuple([abi_typ]) return abi_typ @@ -391,7 +391,7 @@ def abi_encode(dst, lll_node, pos=None, bufsz=None, returns_len=False): for i, o in enumerate(os): abi_t = abi_type_of(o.typ) - if parent_abi_t.is_tuple(): + if parent_abi_t.is_complex_type(): if abi_t.is_dynamic(): lll_ret.append(["mstore", dst_loc, dyn_ofst]) # recurse @@ -425,7 +425,7 @@ def abi_encode(dst, lll_node, pos=None, bufsz=None, returns_len=False): if returns_len: if not parent_abi_t.is_dynamic(): lll_ret.append(parent_abi_t.embedded_static_size()) - elif parent_abi_t.is_tuple(): + elif parent_abi_t.is_complex_type(): lll_ret.append("dyn_ofst") elif isinstance(lll_node.typ, ByteArrayLike): # for abi purposes, return zero-padded length @@ -434,7 +434,7 @@ def abi_encode(dst, lll_node, pos=None, bufsz=None, returns_len=False): else: raise CompilerPanic("unknown type {lll_node.typ}") - if not (parent_abi_t.is_dynamic() and parent_abi_t.is_tuple()): + if not (parent_abi_t.is_dynamic() and parent_abi_t.is_complex_type()): pass # optimize out dyn_ofst allocation if we don't need it else: dyn_section_start = parent_abi_t.static_size() @@ -456,7 +456,7 @@ def abi_decode(lll_node, src, pos=None): for i, o in enumerate(os): abi_t = abi_type_of(o.typ) src_loc = LLLnode("src_loc", typ=o.typ, location=src.location) - if parent_abi_t.is_tuple(): + if parent_abi_t.is_complex_type(): if abi_t.is_dynamic(): child_loc = ["add", "src", unwrap_location(src_loc)] child_loc = LLLnode.from_list(child_loc, typ=o.typ, location=src.location) From 04feca81b42dd82d978275933ab124f1464b838a Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 3 Aug 2021 08:37:16 -0700 Subject: [PATCH 010/163] simplify return logic and protect return buffers from internal calls --- vyper/old_codegen/abi.py | 26 ++-- vyper/old_codegen/expr.py | 9 +- vyper/old_codegen/parser_utils.py | 2 +- vyper/old_codegen/return_.py | 212 ++++++++++-------------------- vyper/old_codegen/stmt.py | 145 +------------------- vyper/old_codegen/types/check.py | 4 +- 6 files changed, 99 insertions(+), 299 deletions(-) diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index e3d9716709..cc411767e1 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -51,7 +51,6 @@ def selector_name(self): def is_complex_type(self): raise NotImplementedError("ABIType.is_complex_type") - # uint: unsigned integer type of M bits, 0 < M <= 256, M % 8 == 0. e.g. uint32, uint8, uint256. # int: two’s complement signed integer type of M bits, 0 < M <= 256, M % 8 == 0. class ABI_GIntM(ABIType): @@ -317,14 +316,6 @@ def abi_type_of2(t: vy.BasePrimitive) -> ABIType: raise CompilerPanic(f"Unrecognized type {t}") -# there are a lot of places in the calling convention where a tuple -# must be passed, so here's a convenience function for that. -def ensure_tuple(abi_typ): - if not abi_typ.is_complex_type(): - return ABI_Tuple([abi_typ]) - return abi_typ - - # turn an lll node into a list, based on its type. def o_list(lll_node, pos=None): lll_t = lll_node.typ @@ -382,12 +373,25 @@ def abi_encode(dst, lll_node, pos=None, bufsz=None, returns_len=False): if bufsz is not None and bufsz < 32 * size_bound: raise CompilerPanic("buffer provided to abi_encode not large enough") + + # fastpath: if there is no dynamic data, we can optimize the + # encoding by using make_setter, since our memory encoding happens + # to be identical to the ABI encoding. + if not parent_abi_t.is_dynamic(): + # cast the output buffer to something that make_setter accepts + dst = LLLnode(dst, typ=lll_node.typ, location="memory") + lll_ret = ["seq_unchecked", make_setter(dst, lll_node, "memory", pos)] + lll_ret.append(parent_abi_t.embedded_static_size()) + return LLLnode.from_list(lll_ret, pos=pos, annotation=f"abi_encode {lll_node.typ}") + + lll_ret = ["seq"] dyn_ofst = "dyn_ofst" # current offset in the dynamic section dst_begin = "dst" # pointer to beginning of buffer dst_loc = "dst_loc" # pointer to write location in static section os = o_list(lll_node, pos=pos) + for i, o in enumerate(os): abi_t = abi_type_of(o.typ) @@ -408,9 +412,11 @@ def abi_encode(dst, lll_node, pos=None, bufsz=None, returns_len=False): elif isinstance(o.typ, BaseType): d = LLLnode(dst_loc, typ=o.typ, location="memory") + # call into make_setter routine lll_ret.append(make_setter(d, o, location=d.location, pos=pos)) elif isinstance(o.typ, ByteArrayLike): d = LLLnode.from_list(dst_loc, typ=o.typ, location="memory") + # call into make_setter routinme lll_ret.append(["seq", make_setter(d, o, location=d.location, pos=pos), zero_pad(d)]) else: raise CompilerPanic(f"unreachable type: {o.typ}") @@ -442,7 +448,7 @@ def abi_encode(dst, lll_node, pos=None, bufsz=None, returns_len=False): lll_ret = ["with", dst_begin, dst, ["with", dst_loc, dst_begin, lll_ret]] - return LLLnode.from_list(lll_ret, pos=pos) + return LLLnode.from_list(lll_ret, pos=pos, annotation=f"abi_encode {lll_node.typ}") # lll_node is the destination LLL item, src is the input buffer. diff --git a/vyper/old_codegen/expr.py b/vyper/old_codegen/expr.py index e79d9ea010..df5862d8e8 100644 --- a/vyper/old_codegen/expr.py +++ b/vyper/old_codegen/expr.py @@ -994,10 +994,11 @@ def parse_List(self): return LLLnode.from_list(lll_node, typ=typ, pos=getpos(self.expr)) def parse_Tuple(self): - call_lll, multi_lll = parse_sequence(self.expr, self.expr.elements, self.context) - typ = TupleType([x.typ for x in multi_lll], is_literal=True) - multi_lll = LLLnode.from_list(["multi"] + multi_lll, typ=typ, pos=getpos(self.expr)) - if not call_lll: + #call_lll, multi_lll = parse_sequence(self.expr, self.expr.elements, self.context) + tuple_elements = [Expr(x, self.context).lll_node for x in self.expr.elements] + typ = TupleType([x.typ for x in tuple_elements], is_literal=True) + multi_lll = LLLnode.from_list(["multi"] + tuple_elements, typ=typ, pos=getpos(self.expr)) + if True: # if not call_lll: return multi_lll lll_node = ["seq_unchecked"] + call_lll + [multi_lll] diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 3ba75a0add..849fa1d2d3 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -461,7 +461,7 @@ def _make_array_index_setter(target, target_token, pos, location, offset): # Create an x=y statement, where the types may be compound @type_check_wrapper -def make_setter(left, right, location, pos, in_function_call=False): +def make_setter(left, right, location, pos): # Basic types if isinstance(left.typ, BaseType): right = unwrap_location(right) diff --git a/vyper/old_codegen/return_.py b/vyper/old_codegen/return_.py index d69bc2bcfe..27da3ee6ba 100644 --- a/vyper/old_codegen/return_.py +++ b/vyper/old_codegen/return_.py @@ -2,157 +2,81 @@ from vyper.old_codegen.function_definitions.utils import get_nonreentrant_lock from vyper.old_codegen.lll_node import LLLnode from vyper.old_codegen.parser_utils import getpos, make_setter -from vyper.old_codegen.types import BaseType, TupleType, get_size_of_type +from vyper.old_codegen.types import ByteArrayType, TupleType, get_size_of_type from vyper.old_codegen.types.check import check_assign +from vyper.old_codegen.context import Context from vyper.utils import MemoryPositions -from .abi import abi_encode, abi_type_of, ensure_tuple +from vyper.old_codegen.abi import lll_tuple_from_args, abi_encode, abi_type_of + +# something that's compatible with new_internal_variable +class FakeType: + def __init__(self, maxlen): + self.size_in_bytes = maxlen + +def allocate_return_buffer(context: Context) -> int: + maxlen = abi_type_of(context.return_type).size_bound() + return context.new_internal_variable(FakeType(maxlen=maxlen)) # Generate return code for stmt -def make_return_stmt(stmt, context, begin_pos, _size, loop_memory_position=None): - # TODO check this out +def make_return_stmt(return_buffer_ofst: int, lll_val: LLLnode, stmt: "Stmt", context: Context) -> LLLnode: + _pos = getpos(stmt) + + # sanity typecheck + _tmp = LLLnode(0, location="memory", typ=context.return_type) + check_assign(_tmp, lll_val, _pos) + func_type = stmt.get_ancestor(vy_ast.FunctionDef)._metadata["type"] - _, nonreentrant_post = get_nonreentrant_lock(func_type) + + _pre, nonreentrant_post = get_nonreentrant_lock(func_type) + if context.is_internal: - if loop_memory_position is None: - loop_memory_position = context.new_internal_variable(BaseType("uint256")) - - # Make label for stack push loop. - label_id = "_".join([str(x) for x in (context.method_id, stmt.lineno, stmt.col_offset)]) - exit_label = f"make_return_loop_exit_{label_id}" - start_label = f"make_return_loop_start_{label_id}" - - # Push prepared data onto the stack, - # in reverse order so it can be popped of in order. - if isinstance(begin_pos, int) and isinstance(_size, int): - # static values, unroll the mloads instead. - mloads = [["mload", pos] for pos in range(begin_pos, _size, 32)] - else: - mloads = [ - "seq_unchecked", - ["mstore", loop_memory_position, _size], - ["label", start_label], - [ # maybe exit loop / break. - "if", - ["le", ["mload", loop_memory_position], 0], - ["goto", exit_label], - ], - [ # push onto stack - "mload", - ["add", begin_pos, ["sub", ["mload", loop_memory_position], 32]], - ], - [ # decrement i by 32. - "mstore", - loop_memory_position, - ["sub", ["mload", loop_memory_position], 32], - ], - ["goto", start_label], - ["label", exit_label], - ] - - # if we are in a for loop, we have to exit prior to returning - exit_repeater = ["exit_repeater"] if context.forvars else [] - - return ( - ["seq_unchecked"] - + exit_repeater - + mloads - + nonreentrant_post - + [["jump", ["mload", context.callback_ptr]]] - ) + + os = ["seq_unchecked"] + # write into the return buffer, + # unlock any non-reentrant locks + # and JUMP out of here + os += [abi_encode(return_buffer_ofst, lll_val, pos=_pos)] + os += nonreentrant_post + os += [["jump", ["mload", context.callback_ptr]]] + else: - return ["seq_unchecked"] + nonreentrant_post + [["return", begin_pos, _size]] - - -# Generate code for returning a tuple or struct. -def gen_tuple_return(stmt, context, sub): - abi_typ = abi_type_of(context.return_type) - # according to the ABI, return types are ALWAYS tuples even if - # only one element is being returned. - # https://solidity.readthedocs.io/en/latest/abi-spec.html#function-selector-and-argument-encoding - # "and the return values v_1, ..., v_k of f are encoded as - # - # enc((v_1, ..., v_k)) - # i.e. the values are combined into a tuple and encoded. - # " - # therefore, wrap it in a tuple if it's not already a tuple. - # (big difference between returning `(bytes,)` and `bytes`. - abi_typ = ensure_tuple(abi_typ) - abi_bytes_needed = abi_typ.static_size() + abi_typ.dynamic_size_bound() - dst = context.memory_allocator.expand_memory(abi_bytes_needed) - return_buffer = LLLnode( - dst, location="memory", annotation="return_buffer", typ=context.return_type - ) - - check_assign(return_buffer, sub, pos=getpos(stmt)) - - if sub.value == "multi": - - if isinstance(context.return_type, TupleType) and not abi_typ.dynamic_size_bound(): - # for tuples where every value is of the same type and a fixed length, - # we can simplify the encoding by using make_setter, since - # our memory encoding happens to be identical to the ABI - # encoding. - new_sub = LLLnode.from_list( - context.new_internal_variable(context.return_type), - typ=context.return_type, - location="memory", - ) - setter = make_setter(new_sub, sub, "memory", pos=getpos(stmt)) - return LLLnode.from_list( - [ - "seq", - setter, - make_return_stmt( - stmt, context, new_sub, get_size_of_type(context.return_type) * 32, - ), - ], - typ=None, - pos=getpos(stmt), - ) - - # in case of multi we can't create a variable to store location of the return expression - # as multi can have data from multiple location like store, calldata etc - encode_out = abi_encode(return_buffer, sub, pos=getpos(stmt), returns_len=True) - load_return_len = ["mload", MemoryPositions.FREE_VAR_SPACE] - os = [ - "seq", - ["mstore", MemoryPositions.FREE_VAR_SPACE, encode_out], - make_return_stmt(stmt, context, return_buffer, load_return_len), - ] - return LLLnode.from_list(os, typ=None, pos=getpos(stmt), valency=0) - - # for tuple return types where a function is called inside the tuple, we - # process the calls prior to encoding the return data - if sub.value == "seq_unchecked" and sub.args[-1].value == "multi": - encode_out = abi_encode(return_buffer, sub.args[-1], pos=getpos(stmt), returns_len=True) - load_return_len = ["mload", MemoryPositions.FREE_VAR_SPACE] - os = ( - ["seq"] - + sub.args[:-1] - + [ - ["mstore", MemoryPositions.FREE_VAR_SPACE, encode_out], - make_return_stmt(stmt, context, return_buffer, load_return_len), - ] - ) - return LLLnode.from_list(os, typ=None, pos=getpos(stmt), valency=0) - - # for all othe cases we are creating a stack variable named sub_loc to store the location - # of the return expression. This is done so that the return expression does not get evaluated - # abi-encode uses a function named o_list which evaluate the expression multiple times - sub_loc = LLLnode("sub_loc", typ=sub.typ, location=sub.location) - encode_out = abi_encode(return_buffer, sub_loc, pos=getpos(stmt), returns_len=True) - load_return_len = ["mload", MemoryPositions.FREE_VAR_SPACE] - os = [ - "with", - "sub_loc", - sub, - [ - "seq", - ["mstore", MemoryPositions.FREE_VAR_SPACE, encode_out], - make_return_stmt(stmt, context, return_buffer, load_return_len), - ], - ] - return LLLnode.from_list(os, typ=None, pos=getpos(stmt), valency=0) + # abi-encode the data into the return buffer, + # unlock any non-reentrant locks + # and RETURN out of here + + # according to the ABI spec, return types are ALWAYS tuples even + # if only one element is being returned. + # https://solidity.readthedocs.io/en/latest/abi-spec.html#function-selector-and-argument-encoding + # "and the return values v_1, ..., v_k of f are encoded as + # + # enc((v_1, ..., v_k)) + # i.e. the values are combined into a tuple and encoded. + # " + # therefore, wrap it in a tuple if it's not already a tuple. + # for example, `bytes` is returned as abi-encoded (bytes,) + # and `(bytes,)` is returned as abi-encoded ((bytes,),) + + if isinstance(lll_val.typ, TupleType) and len(lll_val.typ.members) > 1: + # returning something like (int, bytes, string) + pass + else: + # `-> (bytes,)` gets returned as ((bytes,),) + # In general `-> X` gets returned as (X,) + # (Sorry this is so confusing. I didn't make these rules.) + lll_val = lll_tuple_from_args([lll_val]) + + # encode_out is cleverly a sequence which does the abi-encoding and + # also returns the length of the output as a stack element + encode_out = abi_encode(return_buffer_ofst, lll_val, pos=_pos, returns_len=True) + + # run encode_out before nonreentrant post, in case there are side-effects in encode_out + os = ["with", "return_buffer_len", encode_out, + ["seq"] + + nonreentrant_post + + [["return", return_buffer_ofst, "return_buffer_len"]] + ] + + return LLLnode.from_list(os, typ=None, pos=_pos, valency=0) diff --git a/vyper/old_codegen/stmt.py b/vyper/old_codegen/stmt.py index 77056d6392..207790f89b 100644 --- a/vyper/old_codegen/stmt.py +++ b/vyper/old_codegen/stmt.py @@ -13,7 +13,7 @@ unwrap_location, zero_pad, ) -from vyper.old_codegen.return_ import gen_tuple_return, make_return_stmt +from vyper.old_codegen.return_ import make_return_stmt, allocate_return_buffer from vyper.old_codegen.types import ( BaseType, ByteArrayLike, @@ -388,143 +388,12 @@ def parse_Break(self): return LLLnode.from_list("break", typ=None, pos=getpos(self.stmt)) def parse_Return(self): - if self.context.return_type is None: - if self.stmt.value: - return - return LLLnode.from_list( - make_return_stmt(self.stmt, self.context, 0, 0), - typ=None, - pos=getpos(self.stmt), - valency=0, - ) - - sub = Expr(self.stmt.value, self.context).lll_node - - # Returning a value (most common case) - if isinstance(sub.typ, BaseType): - sub = unwrap_location(sub) - - if sub.typ.is_literal and ( - self.context.return_type.typ == sub.typ - or "int" in self.context.return_type.typ - and "int" in sub.typ.typ - ): # noqa: E501 - if SizeLimits.in_bounds(self.context.return_type.typ, sub.value): - return LLLnode.from_list( - [ - "seq", - ["mstore", 0, sub], - make_return_stmt(self.stmt, self.context, 0, 32), - ], - typ=None, - pos=getpos(self.stmt), - valency=0, - ) - elif isinstance(sub.typ, BaseType): - return LLLnode.from_list( - ["seq", ["mstore", 0, sub], make_return_stmt(self.stmt, self.context, 0, 32)], - typ=None, - pos=getpos(self.stmt), - valency=0, - ) - return - # Returning a byte array - elif isinstance(sub.typ, ByteArrayLike): - if not sub.typ.eq_base(self.context.return_type): - return - if sub.typ.maxlen > self.context.return_type.maxlen: - return - - # loop memory has to be allocated first. - loop_memory_position = self.context.new_internal_variable(typ=BaseType("uint256")) - # len & bytez placeholder have to be declared after each other at all times. - len_placeholder = self.context.new_internal_variable(BaseType("uint256")) - bytez_placeholder = self.context.new_internal_variable(sub.typ) - - if sub.location in ("storage", "memory"): - return LLLnode.from_list( - [ - "seq", - make_byte_array_copier( - LLLnode(bytez_placeholder, location="memory", typ=sub.typ), - sub, - pos=getpos(self.stmt), - ), - zero_pad(bytez_placeholder), - ["mstore", len_placeholder, 32], - make_return_stmt( - self.stmt, - self.context, - len_placeholder, - ["ceil32", ["add", ["mload", bytez_placeholder], 64]], - loop_memory_position=loop_memory_position, - ), - ], - typ=None, - pos=getpos(self.stmt), - valency=0, - ) - return - - elif isinstance(sub.typ, ListType): - loop_memory_position = self.context.new_internal_variable(typ=BaseType("uint256")) - if sub.location == "memory" and sub.value != "multi": - return LLLnode.from_list( - make_return_stmt( - self.stmt, - self.context, - sub, - get_size_of_type(self.context.return_type) * 32, - loop_memory_position=loop_memory_position, - ), - typ=None, - pos=getpos(self.stmt), - valency=0, - ) - else: - new_sub = LLLnode.from_list( - self.context.new_internal_variable(self.context.return_type), - typ=self.context.return_type, - location="memory", - ) - setter = make_setter(new_sub, sub, "memory", pos=getpos(self.stmt)) - return LLLnode.from_list( - [ - "seq", - setter, - make_return_stmt( - self.stmt, - self.context, - new_sub, - get_size_of_type(self.context.return_type) * 32, - loop_memory_position=loop_memory_position, - ), - ], - typ=None, - pos=getpos(self.stmt), - ) - - # Returning a struct - elif isinstance(sub.typ, StructType): - retty = self.context.return_type - if isinstance(retty, StructType) and retty.name == sub.typ.name: - return gen_tuple_return(self.stmt, self.context, sub) - - # Returning a tuple. - elif isinstance(sub.typ, TupleType): - if not isinstance(self.context.return_type, TupleType): - return - - if len(self.context.return_type.members) != len(sub.typ.members): - return - - # check return type matches, sub type. - for i, ret_x in enumerate(self.context.return_type.members): - s_member = sub.typ.members[i] - sub_type = s_member if isinstance(s_member, NodeType) else s_member.typ - if type(sub_type) is not type(ret_x): - return - return gen_tuple_return(self.stmt, self.context, sub) + # allocate the return buffer BEFORE parsing the return value, + # (registering it with the local context variable dictionary) + # otherwise any private calls may clobber the buffer. + return_buf = allocate_return_buffer(self.context) + lll_val = Expr(self.stmt.value, self.context).lll_node + return make_return_stmt(return_buf, lll_val, self.stmt, self.context) def _get_target(self, target): if isinstance(target, vy_ast.Name) and target.id in self.context.forvars: diff --git a/vyper/old_codegen/types/check.py b/vyper/old_codegen/types/check.py index fdce3092b3..438ae8a66b 100644 --- a/vyper/old_codegen/types/check.py +++ b/vyper/old_codegen/types/check.py @@ -6,6 +6,6 @@ # Check assignment from rhs to lhs. # For now use make_setter for its typechecking side effects -def check_assign(lhs, rhs, pos, in_function_call=False): - make_setter(lhs, rhs, location="memory", pos=pos, in_function_call=in_function_call) +def check_assign(lhs, rhs, pos): + make_setter(lhs, rhs, location="memory", pos=pos) # TODO Refactor into an actual type-checking function From 02c41428bc9acf7b7e14e15d853ccd6b443311a2 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 3 Aug 2021 15:33:27 -0700 Subject: [PATCH 011/163] wip: rewrite private function frame code --- vyper/old_codegen/context.py | 5 ++ .../function_definitions/__init__.py | 4 +- .../function_definitions/common.py | 83 +++++++++++++++++++ ...ernal_function.py => external_function.py} | 2 +- ...ernal_function.py => internal_function.py} | 2 +- .../function_definitions/parse_function.py | 67 --------------- vyper/old_codegen/parser.py | 13 +-- vyper/old_codegen/self_call.py | 4 +- 8 files changed, 102 insertions(+), 78 deletions(-) create mode 100644 vyper/old_codegen/function_definitions/common.py rename vyper/old_codegen/function_definitions/{parse_external_function.py => external_function.py} (99%) rename vyper/old_codegen/function_definitions/{parse_internal_function.py => internal_function.py} (99%) delete mode 100644 vyper/old_codegen/function_definitions/parse_function.py diff --git a/vyper/old_codegen/context.py b/vyper/old_codegen/context.py index 96868d2fe5..89c04cef74 100644 --- a/vyper/old_codegen/context.py +++ b/vyper/old_codegen/context.py @@ -64,6 +64,8 @@ def __init__( # Not intended to be accessed directly self.memory_allocator = memory_allocator + self._callee_frame_sizes = [] + # Intermented values, used for internal IDs self._internal_var_iter = 0 self._scope_id_iter = 0 @@ -71,6 +73,9 @@ def __init__( def is_constant(self): return self.constancy is Constancy.Constant or self.in_assertion or self.in_range_expr + def register_callee(frame_size): + self._callee_frame_sizes.append(frame_size) + # # Context Managers # - Context managers are used to ensure proper wrapping of scopes and context states. diff --git a/vyper/old_codegen/function_definitions/__init__.py b/vyper/old_codegen/function_definitions/__init__.py index 125ef0fa4b..b78d58a9f3 100644 --- a/vyper/old_codegen/function_definitions/__init__.py +++ b/vyper/old_codegen/function_definitions/__init__.py @@ -1,5 +1,5 @@ -from .parse_function import ( # noqa +from .common import ( # noqa is_default_func, is_initializer, - parse_function, + generate_lll_for_function, ) diff --git a/vyper/old_codegen/function_definitions/common.py b/vyper/old_codegen/function_definitions/common.py new file mode 100644 index 0000000000..aee43b8c41 --- /dev/null +++ b/vyper/old_codegen/function_definitions/common.py @@ -0,0 +1,83 @@ +# can't use from [module] import [object] because it breaks mocks in testing +from vyper.ast.signatures import FunctionSignature +from vyper.old_codegen import context as ctx +from vyper.old_codegen.context import Constancy +from vyper.old_codegen.function_definitions.external_function import ( + generate_lll_for_external_function, +) +from vyper.old_codegen.function_definitions.internal_function import ( + generate_lll_for_internal_function, +) +from vyper.old_codegen.memory_allocator import MemoryAllocator +from vyper.utils import calc_mem_gas + + +# Is a function the initializer? +def is_initializer(code): + return code.name == "__init__" + + +# Is a function the default function? +def is_default_func(code): + return code.name == "__default__" + + +def generate_lll_for_function(code, sigs, global_ctx, check_nonpayable, _vars=None): + """ + Parse a function and produce LLL code for the function, includes: + - Signature method if statement + - Argument handling + - Clamping and copying of arguments + - Function body + """ + if _vars is None: + _vars = {} + sig = FunctionSignature.from_definition(code, sigs=sigs, custom_structs=global_ctx._structs,) + + # Validate return statements. + sig.validate_return_statement_balance() + + # in order to statically allocate function frames, + # we codegen functions in two passes. + # one pass is just called for its side effects on the context/memory + # allocator. once that pass is finished, we inspect the context + # to see what the max frame size of any callee in the function was, + # then we run the codegen again with the max frame size as + # the start of the frame for this function. + def _run_pass(memory_allocator=None): + # Create a local (per function) context. + if memory_allocator is None: + memory_allocator = MemoryAllocator() + _vars = _vars.copy() # these will get clobbered in produce_* functions + sig = copy.deepcopy(sig) # just in case + context = ctx.Context( + vars=_vars, + global_ctx=global_ctx, + sigs=sigs, + memory_allocator=memory_allocator, + return_type=sig.output_type, + constancy=Constancy.Constant if sig.mutability in ("view", "pure") else Constancy.Mutable, + is_payable=sig.mutability == "payable", + is_internal=sig.internal, + method_id=sig.method_id, + sig=sig, + ) + + if sig.internal: + o = parse_internal_function(code=code, sig=sig, context=context,) + else: + o = parse_external_function( + code=code, sig=sig, context=context, check_nonpayable=check_nonpayable + ) + return o, context + + _, context = _run_pass(None) + allocate_start = max(ctx.callee_frame_sizes) + o, context = _run_pass(MemoryAllocator(allocate_start)) + + frame_size = context.memory_allocator.size_of_mem + + o.context = context + o.total_gas = o.gas + calc_mem_gas(frame_size) + o.func_name = sig.name + return o, frame_size diff --git a/vyper/old_codegen/function_definitions/parse_external_function.py b/vyper/old_codegen/function_definitions/external_function.py similarity index 99% rename from vyper/old_codegen/function_definitions/parse_external_function.py rename to vyper/old_codegen/function_definitions/external_function.py index 417e6033db..fd1d95b999 100644 --- a/vyper/old_codegen/function_definitions/parse_external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -32,7 +32,7 @@ def get_external_arg_copier( return copier -def parse_external_function( +def generate_lll_for_external_function( code: vy_ast.FunctionDef, sig: FunctionSignature, context: Context, check_nonpayable: bool, ) -> LLLnode: """ diff --git a/vyper/old_codegen/function_definitions/parse_internal_function.py b/vyper/old_codegen/function_definitions/internal_function.py similarity index 99% rename from vyper/old_codegen/function_definitions/parse_internal_function.py rename to vyper/old_codegen/function_definitions/internal_function.py index 289fc72674..15e0cc62b9 100644 --- a/vyper/old_codegen/function_definitions/parse_internal_function.py +++ b/vyper/old_codegen/function_definitions/internal_function.py @@ -38,7 +38,7 @@ def get_internal_arg_copier(total_size: int, memory_dest: int) -> List[Any]: return copier -def parse_internal_function( +def generate_lll_for_internal_function( code: vy_ast.FunctionDef, sig: FunctionSignature, context: Context ) -> LLLnode: """ diff --git a/vyper/old_codegen/function_definitions/parse_function.py b/vyper/old_codegen/function_definitions/parse_function.py deleted file mode 100644 index 71be255bbe..0000000000 --- a/vyper/old_codegen/function_definitions/parse_function.py +++ /dev/null @@ -1,67 +0,0 @@ -# can't use from [module] import [object] because it breaks mocks in testing -from vyper.ast.signatures import FunctionSignature -from vyper.old_codegen import context as ctx -from vyper.old_codegen.context import Constancy -# NOTE black/isort conflict >> -from vyper.old_codegen.function_definitions.parse_external_function import ( - parse_external_function, -) -# NOTE black/isort conflict >> -from vyper.old_codegen.function_definitions.parse_internal_function import ( - parse_internal_function, -) -from vyper.old_codegen.memory_allocator import MemoryAllocator -from vyper.utils import calc_mem_gas - - -# Is a function the initializer? -def is_initializer(code): - return code.name == "__init__" - - -# Is a function the default function? -def is_default_func(code): - return code.name == "__default__" - - -def parse_function(code, sigs, global_ctx, check_nonpayable, _vars=None): - """ - Parses a function and produces LLL code for the function, includes: - - Signature method if statement - - Argument handling - - Clamping and copying of arguments - - Function body - """ - if _vars is None: - _vars = {} - sig = FunctionSignature.from_definition(code, sigs=sigs, custom_structs=global_ctx._structs,) - - # Validate return statements. - sig.validate_return_statement_balance() - - # Create a local (per function) context. - memory_allocator = MemoryAllocator() - context = ctx.Context( - vars=_vars, - global_ctx=global_ctx, - sigs=sigs, - memory_allocator=memory_allocator, - return_type=sig.output_type, - constancy=Constancy.Constant if sig.mutability in ("view", "pure") else Constancy.Mutable, - is_payable=sig.mutability == "payable", - is_internal=sig.internal, - method_id=sig.method_id, - sig=sig, - ) - - if sig.internal: - o = parse_internal_function(code=code, sig=sig, context=context,) - else: - o = parse_external_function( - code=code, sig=sig, context=context, check_nonpayable=check_nonpayable - ) - - o.context = context - o.total_gas = o.gas + calc_mem_gas(o.context.memory_allocator.size_of_mem) - o.func_name = sig.name - return o diff --git a/vyper/old_codegen/parser.py b/vyper/old_codegen/parser.py index 6279f4c1a5..540332bde5 100644 --- a/vyper/old_codegen/parser.py +++ b/vyper/old_codegen/parser.py @@ -11,7 +11,7 @@ from vyper.old_codegen.function_definitions import ( is_default_func, is_initializer, - parse_function, + generate_lll_for_function, ) from vyper.old_codegen.global_context import GlobalContext from vyper.old_codegen.lll_node import LLLnode @@ -109,7 +109,7 @@ def parse_other_functions(o, otherfuncs, sigs, external_interfaces, global_ctx, for func_node in otherfuncs: func_type = func_node._metadata["type"] - func_lll = parse_function( + func_lll, frame_size = generate_lll_for_function( func_node, {**{"self": sigs}, **external_interfaces}, global_ctx, check_per_function ) if func_type.visibility == FunctionVisibility.INTERNAL: @@ -121,13 +121,15 @@ def parse_other_functions(o, otherfuncs, sigs, external_interfaces, global_ctx, external_func_sub.append(func_lll) add_gas += 30 func_lll.total_gas += add_gas + # update sigs with metadata gathered from compiling the function for sig in sig_utils.generate_default_arg_sigs(func_node, external_interfaces, global_ctx): sig.gas = func_lll.total_gas + sig.frame_size = frame_size sigs[sig.sig] = sig # generate LLL for fallback function if default_function: - fallback_lll = parse_function( + fallback_lll, _frame_size = generate_lll_for_function( default_function, {**{"self": sigs}, **external_interfaces}, global_ctx, @@ -196,11 +198,10 @@ def parse_tree_to_lll(global_ctx: GlobalContext) -> Tuple[LLLnode, LLLnode]: # If there is an init func... if initfunc: o.append(init_func_init_lll()) - o.append( - parse_function( + init_func_lll, _frame_size = generate_lll_for_function( initfunc[0], {**{"self": sigs}, **external_interfaces}, global_ctx, False, ) - ) + o.append(init_func_lll) # If there are regular functions... if otherfuncs or defaultfunc: diff --git a/vyper/old_codegen/self_call.py b/vyper/old_codegen/self_call.py index d8eaffd974..4674f33fe0 100644 --- a/vyper/old_codegen/self_call.py +++ b/vyper/old_codegen/self_call.py @@ -45,7 +45,6 @@ def _call_make_placeholder(stmt_expr, context, sig): raise TypeCheckFailure(f"Invalid output type: {sig.output_type}") return output_placeholder, returner, output_size - def make_call(stmt_expr, context): # ** Internal Call ** # Steps: @@ -68,6 +67,9 @@ def make_call(stmt_expr, context): pre_init, expr_args = parse_sequence(stmt_expr, stmt_expr.args, context) sig = FunctionSignature.lookup_sig(context.sigs, method_name, expr_args, stmt_expr, context,) + # register callee to help calculate our starting frame offset + context.register_callee(sig.frame_size) + if context.is_constant() and sig.mutability not in ("view", "pure"): raise StateAccessViolation( f"May not call state modifying function " From b6436e39ff6b075661fd229228746b75c64f794c Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sun, 29 Aug 2021 14:05:18 -0700 Subject: [PATCH 012/163] rewrite internal function calling convention --- vyper/ast/signatures/function_signature.py | 10 + vyper/old_codegen/context.py | 9 +- .../function_definitions/common.py | 2 +- .../function_definitions/internal_function.py | 232 +++------------- .../old_codegen/function_definitions/utils.py | 24 -- vyper/old_codegen/parser.py | 3 +- vyper/old_codegen/return_.py | 11 +- vyper/old_codegen/self_call.py | 261 ++---------------- vyper/old_codegen/stmt.py | 4 - vyper/semantics/types/function.py | 14 + vyper/semantics/types/value/array_value.py | 3 +- 11 files changed, 103 insertions(+), 470 deletions(-) diff --git a/vyper/ast/signatures/function_signature.py b/vyper/ast/signatures/function_signature.py index 0e8d3bca4b..44d1fbdf7e 100644 --- a/vyper/ast/signatures/function_signature.py +++ b/vyper/ast/signatures/function_signature.py @@ -87,6 +87,16 @@ def __str__(self): return input_name + " -> " + str(self.output_type) + ":" return input_name + ":" + @property + def internal_function_label(self): + assert self.internal, "why are you doing this" + + def mkalphanum(s): + return "".join([c if c.isalnumeric() else "_"] for c in s) + + # we could do a bit better than this but it just needs to be unique + return mkalphanum(str(self)) + def calculate_arg_totals(self): """ Calculate base arguments, and totals. """ diff --git a/vyper/old_codegen/context.py b/vyper/old_codegen/context.py index 89c04cef74..7449cc46e1 100644 --- a/vyper/old_codegen/context.py +++ b/vyper/old_codegen/context.py @@ -133,7 +133,7 @@ def block_scope(self): # Remove block scopes self._scopes.remove(scope_id) - def _new_variable(self, name: str, typ: NodeType, var_size: int, is_internal: bool) -> int: + def _new_variable(self, name: str, typ: NodeType, var_size: int, is_internal: bool, is_mutable: bool True) -> int: if is_internal: var_pos = self.memory_allocator.expand_memory(var_size) else: @@ -142,13 +142,13 @@ def _new_variable(self, name: str, typ: NodeType, var_size: int, is_internal: bo name=name, pos=var_pos, typ=typ, - mutable=True, + mutable=is_mutable, blockscopes=self._scopes.copy(), is_internal=is_internal, ) return var_pos - def new_variable(self, name: str, typ: NodeType, pos: VyperNode = None) -> int: + def new_variable(self, name: str, typ: NodeType, pos: VyperNode = None, is_mutable: bool = True) -> int: """ Allocate memory for a user-defined variable. @@ -173,8 +173,9 @@ def new_variable(self, name: str, typ: NodeType, pos: VyperNode = None) -> int: var_size = typ.size_in_bytes # type: ignore else: var_size = 32 * get_size_of_type(typ) - return self._new_variable(name, typ, var_size, False) + return self._new_variable(name, typ, var_size, False, is_mutable=is_mutable) + # do we ever allocate immutable internal variables? def new_internal_variable(self, typ: NodeType) -> int: """ Allocate memory for an internal variable. diff --git a/vyper/old_codegen/function_definitions/common.py b/vyper/old_codegen/function_definitions/common.py index aee43b8c41..e4f01390f2 100644 --- a/vyper/old_codegen/function_definitions/common.py +++ b/vyper/old_codegen/function_definitions/common.py @@ -80,4 +80,4 @@ def _run_pass(memory_allocator=None): o.context = context o.total_gas = o.gas + calc_mem_gas(frame_size) o.func_name = sig.name - return o, frame_size + return o, allocate_start, frame_size diff --git a/vyper/old_codegen/function_definitions/internal_function.py b/vyper/old_codegen/function_definitions/internal_function.py index 15e0cc62b9..1f7ff84547 100644 --- a/vyper/old_codegen/function_definitions/internal_function.py +++ b/vyper/old_codegen/function_definitions/internal_function.py @@ -22,22 +22,6 @@ from vyper.utils import MemoryPositions -def get_internal_arg_copier(total_size: int, memory_dest: int) -> List[Any]: - """ - Copy arguments. - For internal functions, MSTORE arguments and callback pointer from the stack. - - :param total_size: total size to copy - :param memory_dest: base memory position to copy to - :return: LLL list that copies total_size of memory - """ - - copier: List[Any] = ["seq"] - for pos in range(0, total_size, 32): - copier.append(["mstore", memory_dest + pos, "pass"]) - return copier - - def generate_lll_for_internal_function( code: vy_ast.FunctionDef, sig: FunctionSignature, context: Context ) -> LLLnode: @@ -46,197 +30,61 @@ def generate_lll_for_internal_function( :param sig: the FuntionSignature :param code: ast of function - :return: full sig compare & function body + :param context: current calling context + :return: function body in LLL """ + # The calling convention is: + # Caller fills in argument buffer + # Caller provides return address, return buffer on the stack + # Callee runs its code, fills in return buffer provided by caller + # Callee jumps back to caller + + # The reason caller fills argument buffer is so there is less + # complication with passing args on the stack; the caller is better + # suited to optimize the copy operation. Also it avoids the callee + # having to handle default args; that is easier left to the caller + # as well. Meanwhile, the reason the callee fills the return buffer + # is first, similarly, the callee is more suited to optimize the copy + # operation. Second, it allows the caller to allocate the return + # buffer in a way which reduces the number of copies. Third, it + # reduces the potential for bugs since it forces the caller to have + # the return data copied into a preallocated location. Otherwise, a + # situation like the following is easy to bork: + # x: T[2] = [self.generate_T(), self.generate_T()] + func_type = code._metadata["type"] # Get nonreentrant lock - nonreentrant_pre, nonreentrant_post = get_nonreentrant_lock(func_type) - # Create callback_ptr, this stores a destination in the bytecode for a internal - # function to jump to after a function has executed. - clampers: List[LLLnode] = [] + for arg in sig.args: + # allocate a variable for every arg, setting mutability + # to False to comply with vyper semantics, function arguments are immutable + context.new_variable(arg.name, arg.typ, is_mutable=False) - # Allocate variable space. - context.memory_allocator.expand_memory(sig.max_copy_size) + nonreentrant_pre, nonreentrant_post = get_nonreentrant_lock(func_type) - _post_callback_ptr = f"{sig.name}_{sig.method_id}_post_callback_ptr" - context.callback_ptr = context.new_internal_variable(typ=BaseType("uint256")) - clampers.append( - LLLnode.from_list( - ["mstore", context.callback_ptr, "pass"], annotation="pop callback pointer", - ) - ) - if sig.total_default_args > 0: - clampers.append(LLLnode.from_list(["label", _post_callback_ptr])) + # pseudocode + def mkidentifier(s): + "".join(c if c.isalnumeric() else "_" for c in s) + function_label = func_type.internal_function_label - # internal functions without return types need to jump back to - # the calling function, as there is no return statement to handle the - # jump. + # internal functions without return types need to jump back to the calling + # function, as there is no guarantee there is a user-provided return + # statement (which would generate the jump) if sig.output_type is None: - stop_func = [["jump", ["mload", context.callback_ptr]]] + stop_func = [["jump", "pass"]] else: - stop_func = [["stop"]] - - # Generate copiers - if len(sig.base_args) == 0: - copier = ["pass"] - clampers.append(LLLnode.from_list(copier)) - elif sig.total_default_args == 0: - copier = get_internal_arg_copier( - total_size=sig.base_copy_size, memory_dest=MemoryPositions.RESERVED_MEMORY - ) - clampers.append(LLLnode.from_list(copier)) - - # Fill variable positions - for arg in sig.args: - if isinstance(arg.typ, ByteArrayLike): - mem_pos = context.memory_allocator.expand_memory(32 * get_size_of_type(arg.typ)) - context.vars[arg.name] = VariableRecord(arg.name, mem_pos, arg.typ, False) - else: - context.vars[arg.name] = VariableRecord( - arg.name, MemoryPositions.RESERVED_MEMORY + arg.pos, arg.typ, False, - ) - - # internal function copiers. No clamping for internal functions. - dyn_variable_names = [a.name for a in sig.base_args if isinstance(a.typ, ByteArrayLike)] - if dyn_variable_names: - i_placeholder = context.new_internal_variable(typ=BaseType("uint256")) - unpackers: List[Any] = [] - for idx, var_name in enumerate(dyn_variable_names): - var = context.vars[var_name] - ident = f"_load_args_{sig.method_id}_dynarg{idx}" - o = make_unpacker(ident=ident, i_placeholder=i_placeholder, begin_pos=var.pos) - unpackers.append(o) - - if not unpackers: - unpackers = ["pass"] + # unreachable at runtime + stop_func = [["invalid"]] - # 0 added to complete full overarching 'seq' statement, see internal_label. - unpackers.append(0) - clampers.append( - LLLnode.from_list( - ["seq_unchecked"] + unpackers, - typ=None, - annotation="dynamic unpacker", - pos=getpos(code), - ) - ) - - # Function has default arguments. - if sig.total_default_args > 0: # Function with default parameters. - - default_sigs = sig_utils.generate_default_arg_sigs(code, context.sigs, context.global_ctx) - sig_chain: List[Any] = ["seq"] - - for default_sig in default_sigs: - sig_compare, internal_label = get_sig_statements(default_sig, getpos(code)) - - # Populate unset default variables - set_defaults = [] - for arg_name in get_default_names_to_set(sig, default_sig): - value = Expr(sig.default_values[arg_name], context).lll_node - var = context.vars[arg_name] - left = LLLnode.from_list( - var.pos, typ=var.typ, location="memory", pos=getpos(code), mutable=var.mutable - ) - set_defaults.append(make_setter(left, value, "memory", pos=getpos(code))) - current_sig_arg_names = [x.name for x in default_sig.args] - - # Load all variables in default section, if internal, - # because the stack is a linear pipe. - copier_arg_count = len(default_sig.args) - copier_arg_names = current_sig_arg_names - - # Order copier_arg_names, this is very important. - copier_arg_names = [x.name for x in default_sig.args if x.name in copier_arg_names] - - # Variables to be populated from calldata/stack. - default_copiers: List[Any] = [] - if copier_arg_count > 0: - # Get map of variables in calldata, with thier offsets - offset = 4 - calldata_offset_map = {} - for arg in default_sig.args: - calldata_offset_map[arg.name] = offset - offset += ( - 32 if isinstance(arg.typ, ByteArrayLike) else get_size_of_type(arg.typ) * 32 - ) - - # Copy set default parameters from calldata - dynamics = [] - for arg_name in copier_arg_names: - var = context.vars[arg_name] - if isinstance(var.typ, ByteArrayLike): - _size = 32 - dynamics.append(var.pos) - else: - _size = var.size * 32 - default_copiers.append( - get_internal_arg_copier(memory_dest=var.pos, total_size=_size,) - ) + body = parse_body(c, context) for c in code.body + body = ["with", "return_buffer", "pass", body] - # Unpack byte array if necessary. - if dynamics: - i_placeholder = context.new_internal_variable(typ=BaseType("uint256")) - for idx, var_pos in enumerate(dynamics): - ident = f"unpack_default_sig_dyn_{default_sig.method_id}_arg{idx}" - default_copiers.append( - make_unpacker( - ident=ident, i_placeholder=i_placeholder, begin_pos=var_pos, - ) - ) - default_copiers.append(0) # for over arching seq, POP - - sig_chain.append( - [ - "if", - sig_compare, - [ - "seq", - internal_label, - LLLnode.from_list( - ["mstore", context.callback_ptr, "pass"], - annotation="pop callback pointer", - pos=getpos(code), - ), - ["seq"] + set_defaults if set_defaults else ["pass"], - ["seq_unchecked"] + default_copiers if default_copiers else ["pass"], - ["goto", _post_callback_ptr], - ], - ] - ) - - # With internal functions all variable loading occurs in the default - # function sub routine. - _clampers = [["label", _post_callback_ptr]] - - # Function with default parameters. - return LLLnode.from_list( - [ - "seq", - sig_chain, - ["seq"] - + nonreentrant_pre - + _clampers - + [parse_body(c, context) for c in code.body] - + nonreentrant_post - + stop_func, - ], - typ=None, - pos=getpos(code), - ) - - else: - # Function without default parameters. - sig_compare, internal_label = get_sig_statements(sig, getpos(code)) - return LLLnode.from_list( + return LLLnode.from_list( ["seq"] - + [internal_label] + + ["label", function_label] + nonreentrant_pre - + clampers - + [parse_body(c, context) for c in code.body] + nonreentrant_post + stop_func, typ=None, diff --git a/vyper/old_codegen/function_definitions/utils.py b/vyper/old_codegen/function_definitions/utils.py index 2ddc9afc22..e905140ed2 100644 --- a/vyper/old_codegen/function_definitions/utils.py +++ b/vyper/old_codegen/function_definitions/utils.py @@ -16,30 +16,6 @@ def get_sig_statements(sig, pos): return sig_compare, private_label -def make_unpacker(ident, i_placeholder, begin_pos): - start_label = "dyn_unpack_start_" + ident - end_label = "dyn_unpack_end_" + ident - return [ - "seq_unchecked", - ["mstore", begin_pos, "pass"], # get len - ["mstore", i_placeholder, 0], - ["label", start_label], - [ # break - "if", - ["ge", ["mload", i_placeholder], ["ceil32", ["mload", begin_pos]]], - ["goto", end_label], - ], - [ # pop into correct memory slot. - "mstore", - ["add", ["add", begin_pos, 32], ["mload", i_placeholder]], - "pass", - ], - ["mstore", i_placeholder, ["add", 32, ["mload", i_placeholder]]], # increment i - ["goto", start_label], - ["label", end_label], - ] - - def get_nonreentrant_lock(func_type): nonreentrant_pre = [["pass"]] nonreentrant_post = [["pass"]] diff --git a/vyper/old_codegen/parser.py b/vyper/old_codegen/parser.py index 540332bde5..4ba47b4e0f 100644 --- a/vyper/old_codegen/parser.py +++ b/vyper/old_codegen/parser.py @@ -109,7 +109,7 @@ def parse_other_functions(o, otherfuncs, sigs, external_interfaces, global_ctx, for func_node in otherfuncs: func_type = func_node._metadata["type"] - func_lll, frame_size = generate_lll_for_function( + func_lll, frame_start, frame_size = generate_lll_for_function( func_node, {**{"self": sigs}, **external_interfaces}, global_ctx, check_per_function ) if func_type.visibility == FunctionVisibility.INTERNAL: @@ -124,6 +124,7 @@ def parse_other_functions(o, otherfuncs, sigs, external_interfaces, global_ctx, # update sigs with metadata gathered from compiling the function for sig in sig_utils.generate_default_arg_sigs(func_node, external_interfaces, global_ctx): sig.gas = func_lll.total_gas + sig.frame_start = frame_start sig.frame_size = frame_size sigs[sig.sig] = sig diff --git a/vyper/old_codegen/return_.py b/vyper/old_codegen/return_.py index 27da3ee6ba..8f958e2c91 100644 --- a/vyper/old_codegen/return_.py +++ b/vyper/old_codegen/return_.py @@ -14,13 +14,12 @@ class FakeType: def __init__(self, maxlen): self.size_in_bytes = maxlen -def allocate_return_buffer(context: Context) -> int: +def _allocate_return_buffer(context: Context) -> int: maxlen = abi_type_of(context.return_type).size_bound() return context.new_internal_variable(FakeType(maxlen=maxlen)) - # Generate return code for stmt -def make_return_stmt(return_buffer_ofst: int, lll_val: LLLnode, stmt: "Stmt", context: Context) -> LLLnode: +def make_return_stmt(lll_val: LLLnode, stmt: "Stmt", context: Context) -> LLLnode: _pos = getpos(stmt) # sanity typecheck @@ -31,18 +30,18 @@ def make_return_stmt(return_buffer_ofst: int, lll_val: LLLnode, stmt: "Stmt", co _pre, nonreentrant_post = get_nonreentrant_lock(func_type) - if context.is_internal: os = ["seq_unchecked"] # write into the return buffer, # unlock any non-reentrant locks # and JUMP out of here - os += [abi_encode(return_buffer_ofst, lll_val, pos=_pos)] + os += [abi_encode("return_buffer", lll_val, pos=_pos)] os += nonreentrant_post - os += [["jump", ["mload", context.callback_ptr]]] + os += [["jump", "pass"]] else: + return_buffer_ofst = _allocate_return_buffer(context) # abi-encode the data into the return buffer, # unlock any non-reentrant locks # and RETURN out of here diff --git a/vyper/old_codegen/self_call.py b/vyper/old_codegen/self_call.py index 4674f33fe0..2dc70ec34a 100644 --- a/vyper/old_codegen/self_call.py +++ b/vyper/old_codegen/self_call.py @@ -20,52 +20,29 @@ ) -def _call_make_placeholder(stmt_expr, context, sig): - if sig.output_type is None: - return 0, 0, 0 - - output_placeholder = context.new_internal_variable(typ=sig.output_type) - output_size = get_size_of_type(sig.output_type) * 32 - - if isinstance(sig.output_type, BaseType): - returner = output_placeholder - elif isinstance(sig.output_type, ByteArrayLike): - returner = output_placeholder - elif isinstance(sig.output_type, TupleLike): - # incase of struct we need to decode the output and then return it - returner = ["seq"] - decoded_placeholder = context.new_internal_variable(typ=sig.output_type) - decoded_node = LLLnode(decoded_placeholder, typ=sig.output_type, location="memory") - output_node = LLLnode(output_placeholder, typ=sig.output_type, location="memory") - returner.append(abi_decode(decoded_node, output_node)) - returner.extend([decoded_placeholder]) - elif isinstance(sig.output_type, ListType): - returner = output_placeholder - else: - raise TypeCheckFailure(f"Invalid output type: {sig.output_type}") - return output_placeholder, returner, output_size +_label_counter = 0 +# TODO a more general way of doing this +def _generate_label(name: str) -> str: + _label_counter += 1 + return f"label{_label_counter}" def make_call(stmt_expr, context): + pos = getpos(stmt_expr) + # ** Internal Call ** # Steps: - # (x) push current local variables - # (x) push arguments - # (x) push jumpdest (callback ptr) + # (x) copy arguments into the soon-to-be callee + # (x) allocate return buffer + # (x) push jumpdest (callback ptr), then return buffer location # (x) jump to label - # (x) pop return values - # (x) pop local variables - pop_local_vars = [] - push_local_vars = [] - pop_return_values = [] - push_args = [] method_name = stmt_expr.func.attr - # TODO check this out - from vyper.old_codegen.expr import parse_sequence + args_lll = Expr(x, self.context).lll_node for x in stmt_expr.args + args_tuple_t = TupleType([x.typ for x in args_lll]) + args_as_tuple = LLLnode.from_list(["multi"] + [x for x in args_lll], typ=args_tuple_t) - pre_init, expr_args = parse_sequence(stmt_expr, stmt_expr.args, context) - sig = FunctionSignature.lookup_sig(context.sigs, method_name, expr_args, stmt_expr, context,) + sig = FunctionSignature.lookup_sig(context.sigs, method_name, args_lll, stmt_expr, context) # register callee to help calculate our starting frame offset context.register_callee(sig.frame_size) @@ -80,212 +57,24 @@ def make_call(stmt_expr, context): if not sig.internal: raise StructureException("Cannot call external functions via 'self'", stmt_expr) - # Push local variables. - var_slots = [(v.pos, v.size) for name, v in context.vars.items() if v.location == "memory"] - if var_slots: - var_slots.sort(key=lambda x: x[0]) - - if len(var_slots) > 10: - # if memory is large enough, push and pop it via iteration - mem_from, mem_to = var_slots[0][0], var_slots[-1][0] + var_slots[-1][1] * 32 - i_placeholder = context.new_internal_variable(BaseType("uint256")) - local_save_ident = f"_{stmt_expr.lineno}_{stmt_expr.col_offset}" - push_loop_label = "save_locals_start" + local_save_ident - pop_loop_label = "restore_locals_start" + local_save_ident - push_local_vars = [ - ["mstore", i_placeholder, mem_from], - ["label", push_loop_label], - ["mload", ["mload", i_placeholder]], - ["mstore", i_placeholder, ["add", ["mload", i_placeholder], 32]], - ["if", ["lt", ["mload", i_placeholder], mem_to], ["goto", push_loop_label]], - ] - pop_local_vars = [ - ["mstore", i_placeholder, mem_to - 32], - ["label", pop_loop_label], - ["mstore", ["mload", i_placeholder], "pass"], - ["mstore", i_placeholder, ["sub", ["mload", i_placeholder], 32]], - ["if", ["ge", ["mload", i_placeholder], mem_from], ["goto", pop_loop_label]], - ] - else: - # for smaller memory, hardcode the mload/mstore locations - push_mem_slots = [] - for pos, size in var_slots: - push_mem_slots.extend([pos + i * 32 for i in range(size)]) - - push_local_vars = [["mload", pos] for pos in push_mem_slots] - pop_local_vars = [["mstore", pos, "pass"] for pos in push_mem_slots[::-1]] - - # Push Arguments - if expr_args: - inargs, inargsize, arg_pos = pack_arguments( - sig, expr_args, context, stmt_expr, is_external_call=False - ) - push_args += [inargs] # copy arguments first, to not mess up the push/pop sequencing. - - static_arg_size = 32 * sum([get_static_size_of_type(arg.typ) for arg in expr_args]) - static_pos = int(arg_pos + static_arg_size) - needs_dyn_section = any([has_dynamic_data(arg.typ) for arg in expr_args]) - - if needs_dyn_section: - ident = f"push_args_{sig.method_id}_{stmt_expr.lineno}_{stmt_expr.col_offset}" - start_label = ident + "_start" - end_label = ident + "_end" - i_placeholder = context.new_internal_variable(BaseType("uint256")) + # allocate space for the return buffer + return_buffer = context.new_internal_variable(sig.return_type) - # Calculate copy start position. - # Given | static | dynamic | section in memory, - # copy backwards so the values are in order on the stack. - # We calculate i, the end of the whole encoded part - # (i.e. the starting index for copy) - # by taking ceil32(len) + offset + arg_pos - # for the last dynamic argument and arg_pos is the start - # the whole argument section. - idx = 0 - for arg in expr_args: - if isinstance(arg.typ, ByteArrayLike): - last_idx = idx - idx += get_static_size_of_type(arg.typ) - push_args += [ - [ - "with", - "offset", - ["mload", arg_pos + last_idx * 32], - [ - "with", - "len_pos", - ["add", arg_pos, "offset"], - [ - "with", - "len_value", - ["mload", "len_pos"], - ["mstore", i_placeholder, ["add", "len_pos", ["ceil32", "len_value"]]], - ], - ], - ] - ] - # loop from end of dynamic section to start of dynamic section, - # pushing each element onto the stack. - push_args += [ - ["label", start_label], - ["if", ["lt", ["mload", i_placeholder], static_pos], ["goto", end_label]], - ["mload", ["mload", i_placeholder]], - ["mstore", i_placeholder, ["sub", ["mload", i_placeholder], 32]], # decrease i - ["goto", start_label], - ["label", end_label], - ] - - # push static section - push_args += [["mload", pos] for pos in reversed(range(arg_pos, static_pos, 32))] - elif sig.args: - raise StructureException( - f"Wrong number of args for: {sig.name} (0 args given, expected {len(sig.args)})", - stmt_expr, - ) - - # Jump to function label. - jump_to_func = [ - ["add", ["pc"], 6], # set callback pointer. - ["goto", f"priv_{sig.method_id}"], + call_sequence = [ + "seq_unchecked", + make_setter(sig.frame_start, args_as_tuple, "memory", pos), + return_buffer, + ["goto", sig.internal_function_label], + ["label", _generate_label(f"{sig.internal_function_label}_exit")] ["jumpdest"], + return_buffer, ] - # Pop return values. - returner = [0] - if sig.output_type: - output_placeholder, returner, output_size = _call_make_placeholder(stmt_expr, context, sig) - if output_size > 0: - dynamic_offsets = [] - if isinstance(sig.output_type, (BaseType, ListType)): - pop_return_values = [ - ["mstore", ["add", output_placeholder, pos], "pass"] - for pos in range(0, output_size, 32) - ] - elif isinstance(sig.output_type, ByteArrayLike): - dynamic_offsets = [(0, sig.output_type)] - pop_return_values = [ - ["pop", "pass"], - ] - elif isinstance(sig.output_type, TupleLike): - static_offset = 0 - pop_return_values = [] - for name, typ in sig.output_type.tuple_items(): - if isinstance(typ, ByteArrayLike): - pop_return_values.append( - ["mstore", ["add", output_placeholder, static_offset], "pass"] - ) - dynamic_offsets.append( - (["mload", ["add", output_placeholder, static_offset]], name) - ) - static_offset += 32 - else: - member_output_size = get_size_of_type(typ) * 32 - pop_return_values.extend( - [ - ["mstore", ["add", output_placeholder, pos], "pass"] - for pos in range( - static_offset, static_offset + member_output_size, 32 - ) - ] - ) - static_offset += member_output_size - - # append dynamic unpacker. - dyn_idx = 0 - for in_memory_offset, _out_type in dynamic_offsets: - ident = f"{stmt_expr.lineno}_{stmt_expr.col_offset}_arg_{dyn_idx}" - dyn_idx += 1 - start_label = "dyn_unpack_start_" + ident - end_label = "dyn_unpack_end_" + ident - i_placeholder = context.new_internal_variable(typ=BaseType("uint256")) - begin_pos = ["add", output_placeholder, in_memory_offset] - # loop until length. - o = LLLnode.from_list( - [ - "seq_unchecked", - ["mstore", begin_pos, "pass"], # get len - ["mstore", i_placeholder, 0], - ["label", start_label], - [ # break - "if", - ["ge", ["mload", i_placeholder], ["ceil32", ["mload", begin_pos]]], - ["goto", end_label], - ], - [ # pop into correct memory slot. - "mstore", - ["add", ["add", begin_pos, 32], ["mload", i_placeholder]], - "pass", - ], - # increment i - ["mstore", i_placeholder, ["add", 32, ["mload", i_placeholder]]], - ["goto", start_label], - ["label", end_label], - ], - typ=None, - annotation="dynamic unpacker", - pos=getpos(stmt_expr), - ) - pop_return_values.append(o) - - call_body = list( - itertools.chain( - ["seq_unchecked"], - pre_init, - push_local_vars, - push_args, - jump_to_func, - pop_return_values, - pop_local_vars, - [returner], - ) - ) - # If we have no return, we need to pop off - pop_returner_call_body = ["pop", call_body] if sig.output_type is None else call_body - o = LLLnode.from_list( - pop_returner_call_body, + call_sequence, typ=sig.output_type, location="memory", - pos=getpos(stmt_expr), + pos=pos, annotation=f"Internal Call: {method_name}", add_gas_estimate=sig.gas, ) diff --git a/vyper/old_codegen/stmt.py b/vyper/old_codegen/stmt.py index 207790f89b..dc7ec04747 100644 --- a/vyper/old_codegen/stmt.py +++ b/vyper/old_codegen/stmt.py @@ -388,10 +388,6 @@ def parse_Break(self): return LLLnode.from_list("break", typ=None, pos=getpos(self.stmt)) def parse_Return(self): - # allocate the return buffer BEFORE parsing the return value, - # (registering it with the local context variable dictionary) - # otherwise any private calls may clobber the buffer. - return_buf = allocate_return_buffer(self.context) lll_val = Expr(self.stmt.value, self.context).lll_node return make_return_stmt(return_buf, lll_val, self.stmt, self.context) diff --git a/vyper/semantics/types/function.py b/vyper/semantics/types/function.py index 262eb7c70e..ffc288a538 100644 --- a/vyper/semantics/types/function.py +++ b/vyper/semantics/types/function.py @@ -407,6 +407,20 @@ def method_ids(self) -> Dict[str, int]: method_ids.update(_generate_method_id(self.name, arg_types[:i])) return method_ids + # for caller-fills-args calling convention + def get_args_buffer_offset(self) -> int: + """ + Get the location of the args buffer in the function frame (caller sets) + """ + return 0 + + # TODO is this needed? + def get_args_buffer_len(self) -> int: + """ + Get the length of the argument buffer in the function frame + """ + return sum(arg_t.size_in_bytes() for arg_t in self.arguments.values()) + @property def is_constructor(self) -> bool: return self.name == "__init__" diff --git a/vyper/semantics/types/value/array_value.py b/vyper/semantics/types/value/array_value.py index 60c8b2cd85..a8029269b6 100644 --- a/vyper/semantics/types/value/array_value.py +++ b/vyper/semantics/types/value/array_value.py @@ -61,8 +61,7 @@ def size_in_bytes(self): # because this data type is single-bytes, we make it so it takes the max 32 byte # boundary as it's size, instead of giving it a size that is not cleanly divisble by 32 - # TODO adding 64 here instead of 32 to be compatible with parser - fix this! - return 64 + math.ceil(self.length / 32) * 32 + return 32 + ceil32(self.length) @property def canonical_type(self) -> str: From 69d2f6dcd614099e1b6149d47b8a27b6adc3bf6d Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 30 Aug 2021 08:11:01 -0700 Subject: [PATCH 013/163] wip cleanup internal calls --- .../function_definitions/internal_function.py | 18 ++++++++---- vyper/old_codegen/lll_node.py | 5 ++++ vyper/old_codegen/return_.py | 2 +- vyper/old_codegen/self_call.py | 29 ++++++++++++------- 4 files changed, 37 insertions(+), 17 deletions(-) diff --git a/vyper/old_codegen/function_definitions/internal_function.py b/vyper/old_codegen/function_definitions/internal_function.py index 1f7ff84547..bd555bbd8c 100644 --- a/vyper/old_codegen/function_definitions/internal_function.py +++ b/vyper/old_codegen/function_definitions/internal_function.py @@ -67,7 +67,9 @@ def generate_lll_for_internal_function( # pseudocode def mkidentifier(s): "".join(c if c.isalnumeric() else "_" for c in s) - function_label = func_type.internal_function_label + + function_entry_label = func_type.internal_function_label + cleanup_label = func_type.exit_sequence_label # internal functions without return types need to jump back to the calling # function, as there is no guarantee there is a user-provided return @@ -78,15 +80,21 @@ def mkidentifier(s): # unreachable at runtime stop_func = [["invalid"]] + enter = nonreentrant_pre + body = parse_body(c, context) for c in code.body - body = ["with", "return_buffer", "pass", body] + + if sig.return_type is not None: + body = ["with", "return_buffer", "pass", body] + + exit = nonreentrant_post + stop_func return LLLnode.from_list( ["seq"] + ["label", function_label] - + nonreentrant_pre - + nonreentrant_post - + stop_func, + + enter + + body + + exit, typ=None, pos=getpos(code), ) diff --git a/vyper/old_codegen/lll_node.py b/vyper/old_codegen/lll_node.py index ed6499c9f4..bf0ae6c239 100644 --- a/vyper/old_codegen/lll_node.py +++ b/vyper/old_codegen/lll_node.py @@ -35,6 +35,11 @@ def __repr__(self) -> str: __mul__ = __add__ +def push_label_to_stack(labelname: str) -> str: + # items prefixed with `_sym_` are ignored until asm phase + return "_sym_" + labelname + + # Data structure for LLL parse tree class LLLnode: repr_show_gas = False diff --git a/vyper/old_codegen/return_.py b/vyper/old_codegen/return_.py index 8f958e2c91..995d79374c 100644 --- a/vyper/old_codegen/return_.py +++ b/vyper/old_codegen/return_.py @@ -23,7 +23,7 @@ def make_return_stmt(lll_val: LLLnode, stmt: "Stmt", context: Context) -> LLLnod _pos = getpos(stmt) # sanity typecheck - _tmp = LLLnode(0, location="memory", typ=context.return_type) + _tmp = LLLnode(-1, location="memory", typ=context.return_type) check_assign(_tmp, lll_val, _pos) func_type = stmt.get_ancestor(vy_ast.FunctionDef)._metadata["type"] diff --git a/vyper/old_codegen/self_call.py b/vyper/old_codegen/self_call.py index 2dc70ec34a..dbff80b03c 100644 --- a/vyper/old_codegen/self_call.py +++ b/vyper/old_codegen/self_call.py @@ -7,7 +7,7 @@ TypeCheckFailure, ) from vyper.old_codegen.abi import abi_decode -from vyper.old_codegen.lll_node import LLLnode +from vyper.old_codegen.lll_node import LLLnode, push_label_to_stack from vyper.old_codegen.parser_utils import getpos, pack_arguments from vyper.old_codegen.types import ( BaseType, @@ -39,6 +39,7 @@ def make_call(stmt_expr, context): method_name = stmt_expr.func.attr args_lll = Expr(x, self.context).lll_node for x in stmt_expr.args + # TODO handle default args args_tuple_t = TupleType([x.typ for x in args_lll]) args_as_tuple = LLLnode.from_list(["multi"] + [x for x in args_lll], typ=args_tuple_t) @@ -57,18 +58,25 @@ def make_call(stmt_expr, context): if not sig.internal: raise StructureException("Cannot call external functions via 'self'", stmt_expr) + return_label = _generate_label(f"{sig.internal_function_label}_call") + # allocate space for the return buffer - return_buffer = context.new_internal_variable(sig.return_type) + # TODO allocate in stmt and/or expr.py + return_buffer = context.new_internal_variable(sig.return_type) if sig.return_type is not None else "pass" + return_buffer = LLLnode.from_list([return_buffer], annotation=f"{return_label}_return_buf") + + copy_args = make_setter(sig.frame_start, args_as_tuple, "memory", pos) call_sequence = [ - "seq_unchecked", - make_setter(sig.frame_start, args_as_tuple, "memory", pos), - return_buffer, - ["goto", sig.internal_function_label], - ["label", _generate_label(f"{sig.internal_function_label}_exit")] - ["jumpdest"], - return_buffer, - ] + "seq_unchecked", + copy_args, + push_label_to_stack(return_label), # pass return label to subroutine + return_buffer, # pass return buffer to subroutine + ["goto", sig.internal_function_label] + return_label, + "jumpdest", + return_buffer, # push return buffer location to stack + ] o = LLLnode.from_list( call_sequence, @@ -78,5 +86,4 @@ def make_call(stmt_expr, context): annotation=f"Internal Call: {method_name}", add_gas_estimate=sig.gas, ) - o.gas += sig.gas return o From 5273abf4033a320985dae967997291ee461871fb Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 30 Aug 2021 10:15:07 -0700 Subject: [PATCH 014/163] wip handle return None cases use a single exit sequence for every function --- vyper/ast/signatures/function_signature.py | 9 +-- .../function_definitions/internal_function.py | 5 +- .../old_codegen/function_definitions/utils.py | 12 ++-- vyper/old_codegen/return_.py | 55 ++++++++++--------- vyper/utils.py | 5 ++ 5 files changed, 49 insertions(+), 37 deletions(-) diff --git a/vyper/ast/signatures/function_signature.py b/vyper/ast/signatures/function_signature.py index 44d1fbdf7e..ced283e333 100644 --- a/vyper/ast/signatures/function_signature.py +++ b/vyper/ast/signatures/function_signature.py @@ -12,7 +12,7 @@ get_size_of_type, parse_type, ) -from vyper.utils import fourbytes_to_int, keccak256 +from vyper.utils import fourbytes_to_int, keccak256, mkalphanum # Function argument @@ -91,12 +91,13 @@ def __str__(self): def internal_function_label(self): assert self.internal, "why are you doing this" - def mkalphanum(s): - return "".join([c if c.isalnumeric() else "_"] for c in s) - # we could do a bit better than this but it just needs to be unique return mkalphanum(str(self)) + @property + def exit_sequence_label(self): + return mkalphanum(str(self)) + "_cleanup" + def calculate_arg_totals(self): """ Calculate base arguments, and totals. """ diff --git a/vyper/old_codegen/function_definitions/internal_function.py b/vyper/old_codegen/function_definitions/internal_function.py index bd555bbd8c..c95ce87016 100644 --- a/vyper/old_codegen/function_definitions/internal_function.py +++ b/vyper/old_codegen/function_definitions/internal_function.py @@ -75,7 +75,7 @@ def mkidentifier(s): # function, as there is no guarantee there is a user-provided return # statement (which would generate the jump) if sig.output_type is None: - stop_func = [["jump", "pass"]] + stop_func = [["jump", "pass"]] # was passed in via stack else: # unreachable at runtime stop_func = [["invalid"]] @@ -85,9 +85,10 @@ def mkidentifier(s): body = parse_body(c, context) for c in code.body if sig.return_type is not None: + # name the variable that was passed via stack body = ["with", "return_buffer", "pass", body] - exit = nonreentrant_post + stop_func + exit = cleanup_label + nonreentrant_post + stop_func return LLLnode.from_list( ["seq"] diff --git a/vyper/old_codegen/function_definitions/utils.py b/vyper/old_codegen/function_definitions/utils.py index e905140ed2..a4beb8c3c7 100644 --- a/vyper/old_codegen/function_definitions/utils.py +++ b/vyper/old_codegen/function_definitions/utils.py @@ -17,12 +17,12 @@ def get_sig_statements(sig, pos): def get_nonreentrant_lock(func_type): - nonreentrant_pre = [["pass"]] - nonreentrant_post = [["pass"]] - if func_type.nonreentrant: - nkey = func_type.reentrancy_key_position.position - nonreentrant_pre = [["seq", ["assert", ["iszero", ["sload", nkey]]], ["sstore", nkey, 1]]] - nonreentrant_post = [["sstore", nkey, 0]] + if not func_type.nonreentrant: + return [], [] + + nkey = func_type.reentrancy_key_position.position + nonreentrant_pre = [["seq", ["assert", ["iszero", ["sload", nkey]]], ["sstore", nkey, 1]]] + nonreentrant_post = [["sstore", nkey, 0]] return nonreentrant_pre, nonreentrant_post diff --git a/vyper/old_codegen/return_.py b/vyper/old_codegen/return_.py index 995d79374c..5343c93a12 100644 --- a/vyper/old_codegen/return_.py +++ b/vyper/old_codegen/return_.py @@ -14,37 +14,48 @@ class FakeType: def __init__(self, maxlen): self.size_in_bytes = maxlen + def _allocate_return_buffer(context: Context) -> int: maxlen = abi_type_of(context.return_type).size_bound() return context.new_internal_variable(FakeType(maxlen=maxlen)) -# Generate return code for stmt + +# Generate code for return stmt def make_return_stmt(lll_val: LLLnode, stmt: "Stmt", context: Context) -> LLLnode: + + func_type = stmt.get_ancestor(vy_ast.FunctionDef)._metadata["type"] + jump_to_exit = ["goto", func_type.exit_sequence_label] + _pos = getpos(stmt) - # sanity typecheck - _tmp = LLLnode(-1, location="memory", typ=context.return_type) - check_assign(_tmp, lll_val, _pos) + if context.return_type is None: + if stmt.value is not None: + return None # triggers an exception - func_type = stmt.get_ancestor(vy_ast.FunctionDef)._metadata["type"] + else: + # sanity typecheck + _tmp = LLLnode(-1, location="memory", typ=context.return_type) + check_assign(_tmp, lll_val, _pos) - _pre, nonreentrant_post = get_nonreentrant_lock(func_type) + # helper function + def finalize(fill_return_buffer): + return LLLnode.from_list( + ["seq_unchecked", fill_return_buffer, jump_to_exit], typ=None, pos=_pos, valency=0 + ) if context.is_internal: - - os = ["seq_unchecked"] - # write into the return buffer, - # unlock any non-reentrant locks - # and JUMP out of here - os += [abi_encode("return_buffer", lll_val, pos=_pos)] - os += nonreentrant_post - os += [["jump", "pass"]] + if context.return_type is None: + return finalize([]) + return finalize(make_setter("return_buffer", lll_val, pos=_pos)) else: + # we are in an external function. + if context.return_type is None: + # push arguments onto the stack for RETURN opcode + return finalize(["seq_unchecked", 0, 0]) + return_buffer_ofst = _allocate_return_buffer(context) - # abi-encode the data into the return buffer, - # unlock any non-reentrant locks - # and RETURN out of here + # abi-encode the data into the return buffer and jump to the function cleanup code # according to the ABI spec, return types are ALWAYS tuples even # if only one element is being returned. @@ -71,11 +82,5 @@ def make_return_stmt(lll_val: LLLnode, stmt: "Stmt", context: Context) -> LLLnod # also returns the length of the output as a stack element encode_out = abi_encode(return_buffer_ofst, lll_val, pos=_pos, returns_len=True) - # run encode_out before nonreentrant post, in case there are side-effects in encode_out - os = ["with", "return_buffer_len", encode_out, - ["seq"] + - nonreentrant_post + - [["return", return_buffer_ofst, "return_buffer_len"]] - ] - - return LLLnode.from_list(os, typ=None, pos=_pos, valency=0) + # fill the return buffer and push the location and length onto the stack + return finalize(["seq_unchecked", encode_out, return_buffer_offset]) diff --git a/vyper/utils.py b/vyper/utils.py index 837e7506e7..912feacb91 100644 --- a/vyper/utils.py +++ b/vyper/utils.py @@ -25,6 +25,11 @@ def abi_method_id(method_sig): return fourbytes_to_int(keccak256(bytes(method_sig, "utf-8"))[:4]) +# map a string to only-alphanumeric chars +def mkalphanum(s): + return "".join([c if c.isalnumeric() else "_"] for c in s) + + # Converts string to bytes def string_to_bytes(str): bytez = b"" From ccb167d8d706657f141562f96837655569cc81c2 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 31 Aug 2021 11:27:24 -0700 Subject: [PATCH 015/163] feat: add "code" location to LLL this will make it easier to generate LLL for __init__ functions. --- vyper/old_codegen/parser_utils.py | 43 ++++++++++++++----------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 849fa1d2d3..97e932733e 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -165,15 +165,10 @@ def make_byte_slice_copier(destination, source, length, max_length, pos=None): else: loader = 0 # Copy over data - elif source.location == "memory": - loader = ["mload", ["add", "_pos", ["mul", 32, ["mload", MemoryPositions.FREE_LOOP_INDEX]]]] + elif source.location in ("memory", "calldata", "code"): + loader = [load_op(source.location), ["add", "_pos", ["mul", 32, ["mload", MemoryPositions.FREE_LOOP_INDEX]]]] elif source.location == "storage": loader = ["sload", ["add", "_pos", ["mload", MemoryPositions.FREE_LOOP_INDEX]]] - elif source.location == "calldata": - loader = [ - "calldataload", - ["add", "_pos", ["mul", 32, ["mload", MemoryPositions.FREE_LOOP_INDEX]]], - ] else: raise CompilerPanic(f"Unsupported location: {source.location}") # Where to paste it? @@ -255,13 +250,7 @@ def byte_array_to_num( def get_bytearray_length(arg): typ = BaseType("uint256") - if arg.location == "memory": - return LLLnode.from_list(["mload", arg], typ=typ) - elif arg.location == "storage": - return LLLnode.from_list(["sload", arg], typ=typ) - elif arg.location == "calldata": - return LLLnode.from_list(["calldataload", arg], typ=typ) - raise CompilerPanic("unreachable", arg) # pragma: no test + return LLLnode.from_list(load_op(arg.location), typ=BaseType("uint256")) def getpos(node): @@ -300,7 +289,7 @@ def add_variable_offset(parent, key, pos, array_bounds_check=True): for i in range(index): offset += get_size_of_type(typ.members[attrs[i]]) return LLLnode.from_list(["add", parent, offset], typ=subtype, location="storage",) - elif location in ("calldata", "memory"): + elif location in ("calldata", "memory", "code"): offset = 0 for i in range(index): offset += 32 * get_size_of_type(typ.members[attrs[i]]) @@ -370,21 +359,29 @@ def add_variable_offset(parent, key, pos, array_bounds_check=True): return LLLnode.from_list( ["add", parent, ["mul", sub, offset]], typ=subtype, location="storage", pos=pos ) - elif location in ("calldata", "memory"): + elif location in ("calldata", "memory", "code"): offset = 32 * get_size_of_type(subtype) return LLLnode.from_list( ["add", ["mul", offset, sub], parent], typ=subtype, location=location, pos=pos ) +def load_op(location): + if location == "memory": + return "mload" + if location == "storage": + return "sload" + if location == "calldata": + return "calldataload" + if location == "code": + return "codeload" + raise CompilerPanic("unreachable", arg) # pragma: no test + + # Unwrap location def unwrap_location(orig): - if orig.location == "memory": - return LLLnode.from_list(["mload", orig], typ=orig.typ) - elif orig.location == "storage": - return LLLnode.from_list(["sload", orig], typ=orig.typ) - elif orig.location == "calldata": - return LLLnode.from_list(["calldataload", orig], typ=orig.typ) + if orig.location in ("memory", "storage", "calldata", "code"): + return LLLnode.from_list([load_op(orig.location), typ=orig.typ) else: # handle None value inserted by `empty` if orig.value is None: @@ -609,7 +606,7 @@ def make_setter(left, right, location, pos): elif isinstance(left.typ, TupleType) and isinstance(right.typ, TupleType): subs = [] for var_arg in left.args: - if var_arg.location == "calldata": + if var_arg.location in ("calldata", "code"): return right_token = LLLnode.from_list("_R", typ=right.typ, location=right.location) From 295fe951517d1b7608dfadc4ebe957f318e92630 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 31 Aug 2021 13:28:03 -0700 Subject: [PATCH 016/163] wip: external functions rewrite --- .../function_definitions/external_function.py | 52 +++++++++++++++---- .../function_definitions/internal_function.py | 6 +-- vyper/old_codegen/parser_utils.py | 2 + 3 files changed, 44 insertions(+), 16 deletions(-) diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index fd1d95b999..4d4385dfe3 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -18,18 +18,48 @@ from vyper.utils import MemoryPositions -def get_external_arg_copier( - total_size: int, memory_dest: int, offset: Union[int, List[Any]] = 4 -) -> List[Any]: - """ - Generate argument copier. +def generate_lll_for_external_function( + code: vy_ast.FunctionDef, sig: FunctionSignature, context: Context, check_nonpayable: bool, +) -> LLLnode: + func_type = code._metadata["type"] - :param total_size: total memory size to copy - :param memory_dest: base memory address to start from - :param offset: starting offset, used for ByteArrays - """ - copier = ["calldatacopy", memory_dest, offset, total_size] - return copier + # generate kwarg handlers + for arg in tbd_kwargs: + v = context.new_variable(argname, argtype, is_mutable=False) + + if len(base_args) > 0: + # tuple with the abi_encoded args + base_args = LLLnode(4, location="calldata", typ=tbd_base_args_type) + base_args = lazy_abi_decode(base_args) + + assert base_args.value == "multi" + for (argname, arg_lll) in zip(tbd, base_args.args): # the actual values + # register the record in the local namespace + context.vars[argname] = LLLnode(arg_lll, location="calldata") + + nonreentrant_pre, nonreentrant_post = get_nonreentrant_lock(func_type) + + # once kwargs have been handled + entrance = [["label", f"{sig.base_method_id}_entry"]] + + if check_nonpayable and sig.mutability != "payable": + # if the contract contains payable functions, but this is not one of them + # add an assertion that the value of the call is zero + entrance.append(["assert", ["iszero", "callvalue"]]) + + body = parse_body(c, context) for c in code.body + + exit = [["label", func_type.exit_sequence_label]] + + [nonreentrant_post] + + [["return", "pass", "pass"]] # passed by + + return LLLnode.from_list( + ["seq"] + + kwarg_handlers + + entrance + + body + + exit, + pos=getpos(code) def generate_lll_for_external_function( diff --git a/vyper/old_codegen/function_definitions/internal_function.py b/vyper/old_codegen/function_definitions/internal_function.py index c95ce87016..18fc1f9e14 100644 --- a/vyper/old_codegen/function_definitions/internal_function.py +++ b/vyper/old_codegen/function_definitions/internal_function.py @@ -74,11 +74,7 @@ def mkidentifier(s): # internal functions without return types need to jump back to the calling # function, as there is no guarantee there is a user-provided return # statement (which would generate the jump) - if sig.output_type is None: - stop_func = [["jump", "pass"]] # was passed in via stack - else: - # unreachable at runtime - stop_func = [["invalid"]] + stop_func = [["jump", "pass"]] # was passed in via stack enter = nonreentrant_pre diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 97e932733e..dec97eef71 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -85,6 +85,7 @@ def make_byte_array_copier(destination, source, pos=None): ) # Special case: memory to memory + # TODO: this should be handled by make_byte_slice_copier. if source.location == "memory" and destination.location == "memory": o = LLLnode.from_list( [ @@ -155,6 +156,7 @@ def make_byte_slice_copier(destination, source, length, max_length, pos=None): annotation=f"copy byte slice dest: {str(destination)}", add_gas_estimate=_identity_gas_bound(max_length), ) + # TODO optimize: calldatacopy for calldata; codecopy for code # special case: rhs is zero if source.value is None: From 1d7d488818badd582078447f7e74f2ecd5ca5284 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 31 Aug 2021 13:28:19 -0700 Subject: [PATCH 017/163] wip add clampers to abi_decode --- vyper/old_codegen/abi.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index cc411767e1..8b0a71a0b3 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -501,9 +501,7 @@ def _add_ofst(loc, ofst): # (multi # (mload 320/*int128*/) # (mload (add 320/*buf start*/ (mload 352/*ofst loc*/)))) -# thought of the day: it might be nice to have an argument like `sanitize` -# which will add well-formedness checks (clamps) for all inputs. -def lazy_abi_decode(typ, src, pos=None): +def lazy_abi_decode(typ, src, clamp=True, pos=None): if isinstance(typ, (ListType, TupleLike)): if isinstance(typ, TupleLike): ts = typ.tuple_members() From 6eac3f6ccfc33c8704c50d34deed065e15f9a7f8 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 1 Sep 2021 10:21:16 -0700 Subject: [PATCH 018/163] wip external functions rewrite --- vyper/ast/signatures/function_signature.py | 109 ++++++++---------- vyper/ast/signatures/sig_utils.py | 2 +- .../function_definitions/common.py | 7 +- .../function_definitions/external_function.py | 48 ++++++-- .../old_codegen/function_definitions/utils.py | 1 + vyper/semantics/types/function.py | 2 +- 6 files changed, 90 insertions(+), 79 deletions(-) diff --git a/vyper/ast/signatures/function_signature.py b/vyper/ast/signatures/function_signature.py index ced283e333..204546df92 100644 --- a/vyper/ast/signatures/function_signature.py +++ b/vyper/ast/signatures/function_signature.py @@ -6,6 +6,7 @@ from vyper.old_codegen.lll_node import LLLnode from vyper.old_codegen.parser_utils import check_single_exit, getpos from vyper.old_codegen.types import ( + NodeType, ByteArrayLike, TupleType, canonicalize_type, @@ -53,6 +54,12 @@ def __init__(self, *args): super(ContractRecord, self).__init__(*args) +@dataclass +class FunctionArg: + name: str + typ: NodeType + + # Function signature object class FunctionSignature: def __init__( @@ -64,7 +71,6 @@ def __init__( internal, nonreentrant_key, sig, - method_id, func_ast_code, is_from_json, ): @@ -74,12 +80,12 @@ def __init__( self.mutability = mutability self.internal = internal self.sig = sig - self.method_id = method_id self.gas = None self.nonreentrant_key = nonreentrant_key self.func_ast_code = func_ast_code self.is_from_json = is_from_json - self.calculate_arg_totals() + + self.set_default_args() def __str__(self): input_name = "def " + self.name + "(" + ",".join([str(arg.typ) for arg in self.args]) + ")" @@ -87,6 +93,12 @@ def __str__(self): return input_name + " -> " + str(self.output_type) + ":" return input_name + ":" + @property + def external_method_ids(self): + assert not self.internal, "method_ids only make sense for external functions" + + return + @property def internal_function_label(self): assert self.internal, "why are you doing this" @@ -98,42 +110,20 @@ def internal_function_label(self): def exit_sequence_label(self): return mkalphanum(str(self)) + "_cleanup" - def calculate_arg_totals(self): - """ Calculate base arguments, and totals. """ - - code = self.func_ast_code - self.base_args = [] - self.total_default_args = 0 - - if hasattr(code.args, "defaults"): - self.total_default_args = len(code.args.defaults) - if self.total_default_args > 0: - # all argument w/o defaults - self.base_args = self.args[: -self.total_default_args] - else: - # No default args, so base_args = args. - self.base_args = self.args - # All default argument name/type definitions. - self.default_args = code.args.args[-self.total_default_args :] # noqa: E203 - # Keep all the value to assign to default parameters. - self.default_values = dict( - zip([arg.arg for arg in self.default_args], code.args.defaults) - ) + def set_default_args(self): + """Split base from kwargs and set member data structures""" - # Calculate the total sizes in memory the function arguments will take use. - # Total memory size of all arguments (base + default together). - self.max_copy_size = sum( - [ - 32 if isinstance(arg.typ, ByteArrayLike) else get_size_of_type(arg.typ) * 32 - for arg in self.args - ] - ) - # Total memory size of base arguments (arguments exclude default parameters). - self.base_copy_size = sum( - [ - 32 if isinstance(arg.typ, ByteArrayLike) else get_size_of_type(arg.typ) * 32 - for arg in self.base_args - ] + args = self.func_ast_code.args + + defaults = getattr(args, "defaults", []) + num_base_args = len(args) - len(defaults) + + self.base_args = self.args[:num_base_args] + self.default_args = self.args[num_base_args:] + + # Keep all the value to assign to default parameters. + self.default_values = dict( + zip([arg.name for arg in self.default_args], args.defaults) ) # Get the canonical function signature @@ -153,34 +143,24 @@ def get_type(arg): @classmethod def from_definition( cls, - code, + func_ast, sigs=None, custom_structs=None, interface_def=False, constant_override=False, is_from_json=False, ): - if not custom_structs: + if custom_structs is None: custom_structs = {} name = code.name - mem_pos = 0 - # Determine the arguments, expects something of the form def foo(arg1: - # int128, arg2: int128 ... args = [] - for arg in code.args.args: - # Each arg needs a type specified. - typ = arg.annotation - parsed_type = parse_type(typ, None, sigs, custom_structs=custom_structs,) - args.append( - VariableRecord(arg.arg, mem_pos, parsed_type, False, defined_at=getpos(arg),) - ) + for arg in func_ast.args.args: + argname = arg.arg + argtyp = parse_type(arg.annotation, None, sigs, custom_structs=custom_structs,) - if isinstance(parsed_type, ByteArrayLike): - mem_pos += 32 - else: - mem_pos += get_size_of_type(parsed_type) * 32 + args.append(FunctionArg(argname, argtyp)) mutability = "nonpayable" # Assume nonpayable by default nonreentrant_key = "" @@ -188,7 +168,7 @@ def from_definition( # Update function properties from decorators # NOTE: Can't import enums here because of circular import - for dec in code.decorator_list: + for dec in func_ast.decorator_list: if isinstance(dec, vy_ast.Name) and dec.id in ("payable", "view", "pure"): mutability = dec.id elif isinstance(dec, vy_ast.Name) and dec.id == "internal": @@ -210,26 +190,27 @@ def from_definition( # def foo() -> int128: ... # If there is no return type, ie. it's of the form def foo(): ... # and NOT def foo() -> type: ..., then it's null - output_type = None - if code.returns: - output_type = parse_type(code.returns, None, sigs, custom_structs=custom_structs,) - # Output type must be canonicalizable - assert isinstance(output_type, TupleType) or canonicalize_type(output_type) + retutn_type = None + if func_ast.returns: + return_type = parse_type(func_ast.returns, None, sigs, custom_structs=custom_structs,) + # sanity check: Output type must be canonicalizable + assert canonicalize_type(return_type) + # Get the canonical function signature - sig = cls.get_full_sig(name, code.args.args, sigs, custom_structs) + sig = cls.get_full_sig(name, func_ast.args.args, sigs, custom_structs) # Take the first 4 bytes of the hash of the sig to get the method ID method_id = fourbytes_to_int(keccak256(bytes(sig, "utf-8"))[:4]) return cls( name, args, - output_type, + return_type, mutability, is_internal, nonreentrant_key, sig, method_id, - code, + func_ast, is_from_json, ) @@ -281,7 +262,7 @@ def synonymise(s): def is_default_func(self): return self.name == "__default__" - def is_initializer(self): + def is_init_func(self): return self.name == "__init__" def validate_return_statement_balance(self): diff --git a/vyper/ast/signatures/sig_utils.py b/vyper/ast/signatures/sig_utils.py index 011970bc77..eb48dd763b 100644 --- a/vyper/ast/signatures/sig_utils.py +++ b/vyper/ast/signatures/sig_utils.py @@ -2,8 +2,8 @@ from vyper.ast.signatures.function_signature import FunctionSignature - # Generate default argument function signatures. +# TODO dead code def generate_default_arg_sigs(code, interfaces, global_ctx): # generate all sigs, and attach. total_default_args = len(code.args.defaults) diff --git a/vyper/old_codegen/function_definitions/common.py b/vyper/old_codegen/function_definitions/common.py index e4f01390f2..4fd2c0cfd8 100644 --- a/vyper/old_codegen/function_definitions/common.py +++ b/vyper/old_codegen/function_definitions/common.py @@ -77,7 +77,12 @@ def _run_pass(memory_allocator=None): frame_size = context.memory_allocator.size_of_mem + if sig.external: + # frame_size of external function includes all private functions called + o.total_gas = o.gas + calc_mem_gas(frame_size) + else: + o.total_gas = o.gas + o.context = context - o.total_gas = o.gas + calc_mem_gas(frame_size) o.func_name = sig.name return o, allocate_start, frame_size diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index 4d4385dfe3..d2ac546e0b 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -17,28 +17,50 @@ from vyper.old_codegen.types.types import ByteArrayLike, get_size_of_type from vyper.utils import MemoryPositions - -def generate_lll_for_external_function( - code: vy_ast.FunctionDef, sig: FunctionSignature, context: Context, check_nonpayable: bool, -) -> LLLnode: - func_type = code._metadata["type"] - - # generate kwarg handlers +# register function args with the local calling context. +# also allocate the ones that live in memory (i.e. kwargs) +def _register_function_args(context: Context, sig: FunctionSignature): + + # generate kwarg handlers. + # since they might come in thru calldata or be default, + # allocate them in memory and then fill it in based on calldata or default, + # depending on the signature + #kwarg_locations = {} for arg in tbd_kwargs: v = context.new_variable(argname, argtype, is_mutable=False) + #kwarg_locations[argname] = v - if len(base_args) > 0: + if len(args) > 0: # tuple with the abi_encoded args - base_args = LLLnode(4, location="calldata", typ=tbd_base_args_type) - base_args = lazy_abi_decode(base_args) + base_args_location = LLLnode(4, location="calldata", typ=tbd_base_args_type) + base_args = lazy_abi_decode(tbd_base_args_type, base_args_location) assert base_args.value == "multi" - for (argname, arg_lll) in zip(tbd, base_args.args): # the actual values + for (argname, arg_lll) in zip(tbd_argnames, base_args.args): # the actual values # register the record in the local namespace context.vars[argname] = LLLnode(arg_lll, location="calldata") + +def _generate_all_signatures(context: Context, function_def: vy_ast.FunctionDef): + for kwarg in function_def.args.defaults: + + +def _generate_kwarg_handlers(context: Context, sig): + pass + + + +def generate_lll_for_external_function( + code: vy_ast.FunctionDef, sig: FunctionSignature, context: Context, check_nonpayable: bool, +) -> LLLnode: + func_type = code._metadata["type"] + + _register_function_args(context, sig) + nonreentrant_pre, nonreentrant_post = get_nonreentrant_lock(func_type) + kwarg_handlers = _generate_kwarg_handlers(context, sig) + # once kwargs have been handled entrance = [["label", f"{sig.base_method_id}_entry"]] @@ -47,7 +69,9 @@ def generate_lll_for_external_function( # add an assertion that the value of the call is zero entrance.append(["assert", ["iszero", "callvalue"]]) - body = parse_body(c, context) for c in code.body + # TODO: handle __init__ and default functions? + + body = [parse_body(c, context) for c in code.body] exit = [["label", func_type.exit_sequence_label]] + [nonreentrant_post] diff --git a/vyper/old_codegen/function_definitions/utils.py b/vyper/old_codegen/function_definitions/utils.py index a4beb8c3c7..4f27af11a6 100644 --- a/vyper/old_codegen/function_definitions/utils.py +++ b/vyper/old_codegen/function_definitions/utils.py @@ -1,6 +1,7 @@ from vyper.old_codegen.lll_node import LLLnode +# TODO dead code def get_sig_statements(sig, pos): method_id_node = LLLnode.from_list(sig.method_id, pos=pos, annotation=f"{sig.sig}") diff --git a/vyper/semantics/types/function.py b/vyper/semantics/types/function.py index ffc288a538..13f0f8c31c 100644 --- a/vyper/semantics/types/function.py +++ b/vyper/semantics/types/function.py @@ -395,7 +395,7 @@ def method_ids(self) -> Dict[str, int]: * For functions without default arguments the dict contains one item. * For functions with default arguments, there is one key for each - function signfature. + function signature. """ arg_types = [i.canonical_type for i in self.arguments.values()] From 54ce49c5be08b3ff25f1c8bb517caee23c45e73b Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 1 Sep 2021 10:49:44 -0700 Subject: [PATCH 019/163] wip: add notes on parse_type --- vyper/old_codegen/global_context.py | 2 +- vyper/old_codegen/types/types.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/vyper/old_codegen/global_context.py b/vyper/old_codegen/global_context.py index efd20f001f..09a5419d0a 100644 --- a/vyper/old_codegen/global_context.py +++ b/vyper/old_codegen/global_context.py @@ -219,5 +219,5 @@ def add_globals_and_events(self, item): else: raise InvalidType("Invalid global type specified", item) - def parse_type(self, ast_node, location): + def parse_type(self, ast_node, location=None): return parse_type(ast_node, location, sigs=self._contracts, custom_structs=self._structs,) diff --git a/vyper/old_codegen/types/types.py b/vyper/old_codegen/types/types.py index fdb89d26c2..8f58eebc32 100644 --- a/vyper/old_codegen/types/types.py +++ b/vyper/old_codegen/types/types.py @@ -181,6 +181,7 @@ def canonicalize_type(t, is_indexed=False): raise InvalidType(f"Invalid or unsupported type: {repr(t)}") +# TODO location is unused def make_struct_type(name, location, sigs, members, custom_structs): o = OrderedDict() @@ -196,7 +197,9 @@ def make_struct_type(name, location, sigs, members, custom_structs): # Parses an expression representing a type. Annotation refers to whether # the type is to be located in memory or storage -def parse_type(item, location, sigs=None, custom_structs=None): +# TODO: location is unused +# TODO: rename me to "lll_type_from_annotation" +def parse_type(item, location=None, sigs=None, custom_structs=None): # Base and custom types, e.g. num if isinstance(item, vy_ast.Name): if item.id in BASE_TYPES: From 2c6bc8a0edaff67ad9e5b0b1dba776bf590b09fb Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 2 Sep 2021 11:57:20 -0700 Subject: [PATCH 020/163] remove validate_return_statement_balance method now FunctionSignature does not need to store the function ast code --- vyper/ast/signatures/function_signature.py | 1 + vyper/old_codegen/function_definitions/common.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/vyper/ast/signatures/function_signature.py b/vyper/ast/signatures/function_signature.py index 204546df92..85427d08ef 100644 --- a/vyper/ast/signatures/function_signature.py +++ b/vyper/ast/signatures/function_signature.py @@ -265,6 +265,7 @@ def is_default_func(self): def is_init_func(self): return self.name == "__init__" + # TODO dead def validate_return_statement_balance(self): # Run balanced return statement check. check_single_exit(self.func_ast_code) diff --git a/vyper/old_codegen/function_definitions/common.py b/vyper/old_codegen/function_definitions/common.py index 4fd2c0cfd8..fc5e322e9a 100644 --- a/vyper/old_codegen/function_definitions/common.py +++ b/vyper/old_codegen/function_definitions/common.py @@ -35,7 +35,7 @@ def generate_lll_for_function(code, sigs, global_ctx, check_nonpayable, _vars=No sig = FunctionSignature.from_definition(code, sigs=sigs, custom_structs=global_ctx._structs,) # Validate return statements. - sig.validate_return_statement_balance() + check_single_exit(code) # in order to statically allocate function frames, # we codegen functions in two passes. From eceb7030ecde318094afdff5f27a518768914cbe Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 2 Sep 2021 12:24:52 -0700 Subject: [PATCH 021/163] clarify ContractFunction.from_AnnAssign --- vyper/semantics/types/function.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vyper/semantics/types/function.py b/vyper/semantics/types/function.py index 13f0f8c31c..95f74fb698 100644 --- a/vyper/semantics/types/function.py +++ b/vyper/semantics/types/function.py @@ -360,7 +360,7 @@ def from_AnnAssign(cls, node: vy_ast.AnnAssign) -> "ContractFunction": """ Generate a `ContractFunction` object from an `AnnAssign` node. - Used to create function definitions for public variables. + Used to create getter functions for public variables. Arguments --------- From d701ed761020def1a783b60ecc2fdde729167050 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 2 Sep 2021 13:03:34 -0700 Subject: [PATCH 022/163] wip --- vyper/ast/signatures/function_signature.py | 23 +++++++++++----------- vyper/ast/signatures/sig_utils.py | 4 ---- vyper/old_codegen/parser_utils.py | 2 ++ vyper/semantics/types/function.py | 1 + 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/vyper/ast/signatures/function_signature.py b/vyper/ast/signatures/function_signature.py index 85427d08ef..8a4123390d 100644 --- a/vyper/ast/signatures/function_signature.py +++ b/vyper/ast/signatures/function_signature.py @@ -70,7 +70,6 @@ def __init__( mutability, internal, nonreentrant_key, - sig, func_ast_code, is_from_json, ): @@ -79,7 +78,6 @@ def __init__( self.output_type = output_type self.mutability = mutability self.internal = internal - self.sig = sig self.gas = None self.nonreentrant_key = nonreentrant_key self.func_ast_code = func_ast_code @@ -93,11 +91,20 @@ def __str__(self): return input_name + " -> " + str(self.output_type) + ":" return input_name + ":" + #@property + #def abi_signature(self): + # assert not self.internal, "abi_signatures only make sense for external functions" +# +# return self.func_name + "(" + ",".join([canonicalize_type(arg.typ) for arg in self.args]) + ")" + @property - def external_method_ids(self): - assert not self.internal, "method_ids only make sense for external functions" + def method_id(self): + return util.method_id(self.abi_signature) - return + def addl_kwarg_sigs(self) -> tbd_List[Tuple(str, sig)]: + ret = [] + for x in self.default_args: + ret.append() @property def internal_function_label(self): @@ -196,11 +203,7 @@ def from_definition( # sanity check: Output type must be canonicalizable assert canonicalize_type(return_type) - # Get the canonical function signature - sig = cls.get_full_sig(name, func_ast.args.args, sigs, custom_structs) - # Take the first 4 bytes of the hash of the sig to get the method ID - method_id = fourbytes_to_int(keccak256(bytes(sig, "utf-8"))[:4]) return cls( name, args, @@ -208,8 +211,6 @@ def from_definition( mutability, is_internal, nonreentrant_key, - sig, - method_id, func_ast, is_from_json, ) diff --git a/vyper/ast/signatures/sig_utils.py b/vyper/ast/signatures/sig_utils.py index eb48dd763b..83c67ca9d9 100644 --- a/vyper/ast/signatures/sig_utils.py +++ b/vyper/ast/signatures/sig_utils.py @@ -3,18 +3,14 @@ from vyper.ast.signatures.function_signature import FunctionSignature # Generate default argument function signatures. -# TODO dead code def generate_default_arg_sigs(code, interfaces, global_ctx): # generate all sigs, and attach. - total_default_args = len(code.args.defaults) if total_default_args == 0: return [ FunctionSignature.from_definition( code, sigs=interfaces, custom_structs=global_ctx._structs, ) ] - base_args = code.args.args[:-total_default_args] - default_args = code.args.args[-total_default_args:] # Generate a list of default function combinations. row = [False] * (total_default_args) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index dec97eef71..fc8953a197 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -640,6 +640,7 @@ def make_setter(left, right, location, pos): ) +# TODO move return checks to vyper/semantics/validation def is_return_from_function(node): if isinstance(node, vy_ast.Expr) and node.get("value.func.id") == "selfdestruct": return True @@ -675,6 +676,7 @@ def _check_return_body(node, node_list): ) +# TODO dead def _return_check(node): if is_return_from_function(node): return True diff --git a/vyper/semantics/types/function.py b/vyper/semantics/types/function.py index 95f74fb698..0279c73a17 100644 --- a/vyper/semantics/types/function.py +++ b/vyper/semantics/types/function.py @@ -96,6 +96,7 @@ def __init__( self, name: str, arguments: OrderedDict, + # TODO rename to something like positional_args, keyword_args min_arg_count: int, max_arg_count: int, return_type: Optional[BaseTypeDefinition], From 36f2e096a063a29a49e890c0e537841621cb2180 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 2 Sep 2021 13:46:54 -0700 Subject: [PATCH 023/163] wip external kwargs --- vyper/ast/signatures/function_signature.py | 27 ++++--- .../function_definitions/external_function.py | 73 +++++++++++++++---- 2 files changed, 72 insertions(+), 28 deletions(-) diff --git a/vyper/ast/signatures/function_signature.py b/vyper/ast/signatures/function_signature.py index 8a4123390d..aa3f6d47d1 100644 --- a/vyper/ast/signatures/function_signature.py +++ b/vyper/ast/signatures/function_signature.py @@ -14,6 +14,7 @@ parse_type, ) from vyper.utils import fourbytes_to_int, keccak256, mkalphanum +from functools import cached_property # Function argument @@ -91,20 +92,22 @@ def __str__(self): return input_name + " -> " + str(self.output_type) + ":" return input_name + ":" - #@property - #def abi_signature(self): - # assert not self.internal, "abi_signatures only make sense for external functions" -# -# return self.func_name + "(" + ",".join([canonicalize_type(arg.typ) for arg in self.args]) + ")" + def _abi_signature(self, args): + return self.func_name + "(" + ",".join([canonicalize_type(arg.typ) for arg in args]) + ")" - @property - def method_id(self): - return util.method_id(self.abi_signature) - - def addl_kwarg_sigs(self) -> tbd_List[Tuple(str, sig)]: + @cached_property + def all_kwarg_sigs(self) -> List[str]: + assert not self.internal, "abi_signatures only make sense for external functions" ret = [] - for x in self.default_args: - ret.append() + argz = self.base_args.copy() + + ret.append(self._abi_signature(argz)) + + for arg in self.default_args: + argz.append(arg) + ret.append(self._abi_signature(argz)) + + return ret @property def internal_function_label(self): diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index d2ac546e0b..dbb7bcd7bd 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -21,15 +21,6 @@ # also allocate the ones that live in memory (i.e. kwargs) def _register_function_args(context: Context, sig: FunctionSignature): - # generate kwarg handlers. - # since they might come in thru calldata or be default, - # allocate them in memory and then fill it in based on calldata or default, - # depending on the signature - #kwarg_locations = {} - for arg in tbd_kwargs: - v = context.new_variable(argname, argtype, is_mutable=False) - #kwarg_locations[argname] = v - if len(args) > 0: # tuple with the abi_encoded args base_args_location = LLLnode(4, location="calldata", typ=tbd_base_args_type) @@ -42,12 +33,56 @@ def _register_function_args(context: Context, sig: FunctionSignature): def _generate_all_signatures(context: Context, function_def: vy_ast.FunctionDef): - for kwarg in function_def.args.defaults: + for sig, kwarg_value in zip(all_sigs, kwarg_values): + yield pass def _generate_kwarg_handlers(context: Context, sig): - pass + # generate kwarg handlers. + # since they might come in thru calldata or be default, + # allocate them in memory and then fill it in based on calldata or default, + # depending on the signature + + ret = [] + + def handler_for(calldata_kwargs, default_kwargs): + calldata_args = base_args + calldata_kwargs + calldata_args_t = TupleType(list(arg.typ for arg in calldata_args)) + + sig = func_name + canonicalize_type(calldata_args_t) + method_id = util.method_id(sig) + calldata_args_location = LLLnode(4, location="calldata", typ=calldata_args_t) + calldata_args = lazy_abi_decode(calldata_args_t, base_args_location) + + assert calldata_args.value == "multi" # sanity check + # extract just the kwargs from the ABI payload + calldata_args = calldata.args[len(base_args):] + + # a sequence of statements to strictify kwargs into memory + ret = ["seq"] + + for x in calldata_args: + context.new_variable(argname, argtype, mutable=False) + ret.append(make_setter(context.lookup_var(x.name), x, "memory")) + for x in default_kwargs: + context.new_variable(argname, argtype, mutable=False) + ret.append(make_setter(context.lookup_var(x.name), Expr(x, context).lll_node, "memory")) + + ret.append(["goto", tbd_entry_point]) + + ret = ["if", ["eq", tbd_mload_method_id, method_id], ret] + return ret + + for i, kwarg in enumerate(keyword_args): + calldata_kwargs = keyword_args[:i] + default_kwargs = keyword_args[i:] + + sig = tbd_sig + + ret.append(handler_for(calldata_kwargs, default_kwargs)) + + return ret def generate_lll_for_external_function( @@ -61,8 +96,11 @@ def generate_lll_for_external_function( kwarg_handlers = _generate_kwarg_handlers(context, sig) + entrance = [] + # once kwargs have been handled - entrance = [["label", f"{sig.base_method_id}_entry"]] + if len(kwarg_handlers) > 0: + entrance.append(["label", f"{sig.base_method_id}_entry"]) if check_nonpayable and sig.mutability != "payable": # if the contract contains payable functions, but this is not one of them @@ -77,13 +115,16 @@ def generate_lll_for_external_function( + [nonreentrant_post] + [["return", "pass", "pass"]] # passed by - return LLLnode.from_list( - ["seq"] + ret = (["seq"] + kwarg_handlers + entrance + body - + exit, - pos=getpos(code) + + exit + ) + if len(kwarg_handlers) > 0: + ret = ["if", ["eq", tbd_mload_method_id, sig.method_id], ret] + + return LLLnode.from_list(ret, pos=getpos(code)) def generate_lll_for_external_function( From aef37f7c94f5029c45796853f0e9d000c916b210 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 7 Sep 2021 09:51:53 -0700 Subject: [PATCH 024/163] wip external kwargs --- .../old_codegen/function_definitions/external_function.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index dbb7bcd7bd..22c8046aa8 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -46,6 +46,8 @@ def _generate_kwarg_handlers(context: Context, sig): ret = [] def handler_for(calldata_kwargs, default_kwargs): + default_kwargs = [Expr(x, context).lll_node for x in default_kwargs] + calldata_args = base_args + calldata_kwargs calldata_args_t = TupleType(list(arg.typ for arg in calldata_args)) @@ -53,6 +55,7 @@ def handler_for(calldata_kwargs, default_kwargs): method_id = util.method_id(sig) calldata_args_location = LLLnode(4, location="calldata", typ=calldata_args_t) + calldata_args = lazy_abi_decode(calldata_args_t, base_args_location) assert calldata_args.value == "multi" # sanity check @@ -62,6 +65,10 @@ def handler_for(calldata_kwargs, default_kwargs): # a sequence of statements to strictify kwargs into memory ret = ["seq"] + all_kwargs_t = TupleType(list(arg.typ for arg in sig_kwargs)) + + all_kwargs_src = LLLnode.from_list(["multi"] + calldata_args + default_args, typ=all_kwargs_t) + for x in calldata_args: context.new_variable(argname, argtype, mutable=False) ret.append(make_setter(context.lookup_var(x.name), x, "memory")) From bc3ca37e60ae1e38724223787a13b5188396839d Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 7 Sep 2021 10:05:53 -0700 Subject: [PATCH 025/163] rewrite FunctionSignature --- vyper/ast/signatures/function_signature.py | 100 +++++++-------------- 1 file changed, 34 insertions(+), 66 deletions(-) diff --git a/vyper/ast/signatures/function_signature.py b/vyper/ast/signatures/function_signature.py index aa3f6d47d1..0675759b3f 100644 --- a/vyper/ast/signatures/function_signature.py +++ b/vyper/ast/signatures/function_signature.py @@ -67,7 +67,7 @@ def __init__( self, name, args, - output_type, + return_type, mutability, internal, nonreentrant_key, @@ -76,7 +76,7 @@ def __init__( ): self.name = name self.args = args - self.output_type = output_type + self.return_type = return_type self.mutability = mutability self.internal = internal self.gas = None @@ -136,28 +136,15 @@ def set_default_args(self): zip([arg.name for arg in self.default_args], args.defaults) ) - # Get the canonical function signature - @staticmethod - def get_full_sig(func_name, args, sigs, custom_structs): - def get_type(arg): - if isinstance(arg, LLLnode): - return canonicalize_type(arg.typ) - elif hasattr(arg, "annotation"): - return canonicalize_type( - parse_type(arg.annotation, None, sigs, custom_structs=custom_structs,) - ) - - return func_name + "(" + ",".join([get_type(arg) for arg in args]) + ")" - # Get a signature from a function definition @classmethod def from_definition( cls, - func_ast, - sigs=None, + func_ast, # vy_ast.FunctionDef + sigs=None, # TODO replace sigs and custom_structs with GlobalContext? custom_structs=None, interface_def=False, - constant_override=False, + constant_override=False, # CMC 20210907 what does this do? is_from_json=False, ): if custom_structs is None: @@ -174,7 +161,7 @@ def from_definition( mutability = "nonpayable" # Assume nonpayable by default nonreentrant_key = "" - is_internal = False + is_internal = None # Update function properties from decorators # NOTE: Can't import enums here because of circular import @@ -200,13 +187,12 @@ def from_definition( # def foo() -> int128: ... # If there is no return type, ie. it's of the form def foo(): ... # and NOT def foo() -> type: ..., then it's null - retutn_type = None + return_type = None if func_ast.returns: return_type = parse_type(func_ast.returns, None, sigs, custom_structs=custom_structs,) # sanity check: Output type must be canonicalizable assert canonicalize_type(return_type) - # Take the first 4 bytes of the hash of the sig to get the method ID return cls( name, args, @@ -219,57 +205,39 @@ def from_definition( ) @classmethod - def lookup_sig(cls, sigs, method_name, expr_args, stmt_or_expr, context): + def lookup_internal_function(cls, method_name, args_lll, context): """ - Using a list of args, determine the most accurate signature to use from - the given context + Using a list of args, find the internal method to use, and + the kwargs which need to be filled in by the compiler """ + def _check(cond, s = "Unreachable"): + if not cond: + raise CompilerPanic(s) + + for sig in context.sigs['self']: + if sig.name != method_name: + continue + + _check(sig.internal) # sanity check + # should have been caught during type checking, sanity check anyway + _check(len(sig.base_args) <= len(args_lll) <= len(sig.args)) + + # more sanity check, that the types match + _check(all(l.typ == r.typ for l, r in zip(args_lll, sig.args))) - def synonymise(s): - return s.replace("int128", "num").replace("uint256", "num") - - # for sig in sigs['self'] - full_sig = cls.get_full_sig(stmt_or_expr.func.attr, expr_args, None, context.structs,) - method_names_dict = dict(Counter([x.split("(")[0] for x in context.sigs["self"]])) - if method_name not in method_names_dict: - raise FunctionDeclarationException( - "Function not declared yet (reminder: functions cannot " - f"call functions later in code than themselves): {method_name}" - ) - - if method_names_dict[method_name] == 1: - return next( - sig - for name, sig in context.sigs["self"].items() - if name.split("(")[0] == method_name - ) - if full_sig in context.sigs["self"]: - return context.sigs["self"][full_sig] - else: - synonym_sig = synonymise(full_sig) - syn_sigs_test = [synonymise(k) for k in context.sigs.keys()] - if len(syn_sigs_test) != len(set(syn_sigs_test)): - raise Exception( - "Incompatible default parameter signature," - "can not tell the number type of literal", - stmt_or_expr, - ) - synonym_sigs = [(synonymise(k), v) for k, v in context.sigs["self"].items()] - ssig = [s[1] for s in synonym_sigs if s[0] == synonym_sig] - if len(ssig) == 0: - raise FunctionDeclarationException( - "Function not declared yet (reminder: functions cannot " - f"call functions later in code than themselves): {method_name}" - ) - return ssig[0] + num_provided_args = len(args_lll) + total_args = len(sig.args) + num_kwargs = len(sig.kwargs) + args_needed = total_args - num_provided + + kw_vals = sig.kwarg_values.items()[:num_kwargs - args_needed] + + return sig, kw_vals + + _check(False) def is_default_func(self): return self.name == "__default__" def is_init_func(self): return self.name == "__init__" - - # TODO dead - def validate_return_statement_balance(self): - # Run balanced return statement check. - check_single_exit(self.func_ast_code) From ec47c629f3fc2ffed54e4980370cbaec251037e7 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 7 Sep 2021 11:13:42 -0700 Subject: [PATCH 026/163] wip self call kwargs --- vyper/ast/signatures/function_signature.py | 3 ++- vyper/old_codegen/self_call.py | 20 ++++++++++++-------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/vyper/ast/signatures/function_signature.py b/vyper/ast/signatures/function_signature.py index 0675759b3f..04e122fd54 100644 --- a/vyper/ast/signatures/function_signature.py +++ b/vyper/ast/signatures/function_signature.py @@ -17,7 +17,8 @@ from functools import cached_property -# Function argument +# Function variable +# TODO move to context.py class VariableRecord: def __init__( self, diff --git a/vyper/old_codegen/self_call.py b/vyper/old_codegen/self_call.py index dbff80b03c..7ee5dc0b46 100644 --- a/vyper/old_codegen/self_call.py +++ b/vyper/old_codegen/self_call.py @@ -31,20 +31,23 @@ def make_call(stmt_expr, context): # ** Internal Call ** # Steps: - # (x) copy arguments into the soon-to-be callee - # (x) allocate return buffer - # (x) push jumpdest (callback ptr), then return buffer location - # (x) jump to label + # - copy arguments into the soon-to-be callee + # - allocate return buffer + # - push jumpdest (callback ptr) and return buffer location + # - jump to label + # - (private function will fill return buffer and jump back) method_name = stmt_expr.func.attr - args_lll = Expr(x, self.context).lll_node for x in stmt_expr.args - # TODO handle default args + sig, kw_vals = FunctionSignature.lookup_sig(method_name, args_lll, context) + + pos_args_lll = Expr(x, self.context).lll_node for x in stmt_expr.args + kw_args_lll = Expr(x, self.context).lll_node for x in kw_vals + args_lll = pos_args_lll + kw_args_lll + args_tuple_t = TupleType([x.typ for x in args_lll]) args_as_tuple = LLLnode.from_list(["multi"] + [x for x in args_lll], typ=args_tuple_t) - sig = FunctionSignature.lookup_sig(context.sigs, method_name, args_lll, stmt_expr, context) - # register callee to help calculate our starting frame offset context.register_callee(sig.frame_size) @@ -55,6 +58,7 @@ def make_call(stmt_expr, context): getpos(stmt_expr), ) + # TODO move me to type checker phase if not sig.internal: raise StructureException("Cannot call external functions via 'self'", stmt_expr) From 4f3a4cbea9342f3cd82c1799f9c1d28083d479ad Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 7 Sep 2021 12:50:04 -0700 Subject: [PATCH 027/163] move pack_arguments to external_call --- vyper/old_codegen/external_call.py | 52 +++++++++++++++++++++++++++- vyper/old_codegen/parser_utils.py | 54 ------------------------------ 2 files changed, 51 insertions(+), 55 deletions(-) diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index 5cea5e7d36..89f6aa627d 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -4,7 +4,7 @@ StructureException, TypeCheckFailure, ) -from vyper.old_codegen.abi import abi_decode +from vyper.old_codegen.abi import abi_encode, abi_decode, abi_type_of from vyper.old_codegen.lll_node import LLLnode from vyper.old_codegen.parser_utils import ( getpos, @@ -21,6 +21,56 @@ has_dynamic_data, ) +@type_check_wrapper +def pack_arguments(signature, args, context, stmt_expr, is_external_call): + # NOTE cyclic dep + from vyper.old_codegen.abi import abi_encode, abi_type_of + + pos = getpos(stmt_expr) + + # abi encoding just treats all args as a big tuple + args_tuple_t = TupleType([x.typ for x in args]) + args_as_tuple = LLLnode.from_list(["multi"] + [x for x in args], typ=args_tuple_t) + args_abi_t = abi_type_of(args_tuple_t) + + maxlen = args_abi_t.dynamic_size_bound() + args_abi_t.static_size() + if is_external_call: + maxlen += 32 # padding for the method id + + buf_t = ByteArrayType(maxlen=maxlen) + buf = context.new_internal_variable(buf_t) + + mstore_method_id = [] + if is_external_call: + # layout: + # 32 bytes | args + # 0x..00 | args + # the reason for the left padding is just so the alignment is easier. + # if we were only targeting constantinople, we could align + # to buf (and also keep code size small) by using + # (mstore buf (shl signature.method_id 224)) + mstore_method_id.append(["mstore", buf, signature.method_id]) + buf += 32 + + if len(signature.args) != len(args): + return + + encode_args = abi_encode(buf, args_as_tuple, pos) + + if is_external_call: + returner = [[buf - 4]] + inargsize = buf_t.maxlen - 28 + else: + return + + return ( + LLLnode.from_list( + ["seq"] + mstore_method_id + [encode_args] + returner, typ=buf_t, location="memory" + ), + inargsize, + buf, + ) + def external_call(node, context, interface_name, contract_address, pos, value=None, gas=None): from vyper.old_codegen.expr import Expr diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index fc8953a197..0152b83f5e 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -391,60 +391,6 @@ def unwrap_location(orig): return orig -# Pack function arguments for a call -@type_check_wrapper -def pack_arguments(signature, args, context, stmt_expr, is_external_call): - # NOTE cyclic dep - from vyper.old_codegen.abi import abi_encode, abi_type_of - - pos = getpos(stmt_expr) - - # abi encoding just treats all args as a big tuple - args_tuple_t = TupleType([x.typ for x in args]) - args_as_tuple = LLLnode.from_list(["multi"] + [x for x in args], typ=args_tuple_t) - args_abi_t = abi_type_of(args_tuple_t) - - maxlen = args_abi_t.dynamic_size_bound() + args_abi_t.static_size() - if is_external_call: - maxlen += 32 # padding for the method id - - buf_t = ByteArrayType(maxlen=maxlen) - buf = context.new_internal_variable(buf_t) - - mstore_method_id = [] - if is_external_call: - # layout: - # 32 bytes | args - # 0x..00 | args - # the reason for the left padding is just so the alignment is easier. - # if we were only targeting constantinople, we could align - # to buf (and also keep code size small) by using - # (mstore buf (shl signature.method_id 224)) - mstore_method_id.append(["mstore", buf, signature.method_id]) - buf += 32 - - if len(signature.args) != len(args): - return - - encode_args = abi_encode(buf, args_as_tuple, pos) - - if is_external_call: - returner = [[buf - 4]] - inargsize = buf_t.maxlen - 28 - else: - # internal call does not use a returner or adjust max length for signature - returner = [] - inargsize = buf_t.maxlen - - return ( - LLLnode.from_list( - ["seq"] + mstore_method_id + [encode_args] + returner, typ=buf_t, location="memory" - ), - inargsize, - buf, - ) - - def _make_array_index_setter(target, target_token, pos, location, offset): if location == "memory" and isinstance(target.value, int): offset = target.value + 32 * get_size_of_type(target.typ.subtype) * offset From b4dd43204f3574f5a6d848b70437053e8dfc7b27 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 7 Sep 2021 12:52:49 -0700 Subject: [PATCH 028/163] wip remove external_call_output function --- vyper/old_codegen/external_call.py | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index 89f6aa627d..5e824ed46d 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -182,30 +182,6 @@ def external_call(node, context, interface_name, contract_address, pos, value=No return LLLnode.from_list(sub, typ=sig.output_type, location="memory", pos=getpos(node)) -def get_external_call_output(sig, context): - if not sig.output_type: - return 0, 0, [] - output_placeholder = context.new_internal_variable(typ=sig.output_type) - output_size = get_size_of_type(sig.output_type) * 32 - if isinstance(sig.output_type, BaseType): - returner = [0, output_placeholder] - elif isinstance(sig.output_type, ByteArrayLike): - returner = [0, output_placeholder + 32] - elif isinstance(sig.output_type, TupleLike): - # incase of struct we need to decode the output and then return it - returner = ["seq"] - decoded_placeholder = context.new_internal_variable(typ=sig.output_type) - decoded_node = LLLnode(decoded_placeholder, typ=sig.output_type, location="memory") - output_node = LLLnode(output_placeholder, typ=sig.output_type, location="memory") - returner.append(abi_decode(decoded_node, output_node)) - returner.extend([0, decoded_placeholder]) - elif isinstance(sig.output_type, ListType): - returner = [0, output_placeholder] - else: - raise TypeCheckFailure(f"Invalid output type: {sig.output_type}") - return output_placeholder, output_size, returner - - def get_external_interface_keywords(stmt_expr, context): # circular import! from vyper.old_codegen.expr import Expr From 80bae2ee724e042feacfb5da5496bf1607c3cd3b Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 7 Sep 2021 13:01:49 -0700 Subject: [PATCH 029/163] remove a dead branch --- vyper/old_codegen/external_call.py | 30 +++--------------------------- 1 file changed, 3 insertions(+), 27 deletions(-) diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index 5e824ed46d..4b130df551 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -206,6 +206,8 @@ def make_external_call(stmt_expr, context): if isinstance(stmt_expr.func, vy_ast.Attribute) and isinstance( stmt_expr.func.value, vy_ast.Call ): + # e.g. `Foo(address).bar()` + contract_name = stmt_expr.func.value.func.id contract_address = Expr.parse_value_expr(stmt_expr.func.value.args[0], context) @@ -219,38 +221,12 @@ def make_external_call(stmt_expr, context): gas=gas, ) - elif ( - isinstance(stmt_expr.func.value, vy_ast.Attribute) - and stmt_expr.func.value.attr in context.sigs - ): # noqa: E501 - contract_name = stmt_expr.func.value.attr - type_ = stmt_expr.func.value._metadata["type"] - var = context.globals[stmt_expr.func.value.attr] - contract_address = unwrap_location( - LLLnode.from_list( - type_.position.position, - typ=var.typ, - location="storage", - pos=getpos(stmt_expr), - annotation="self." + stmt_expr.func.value.attr, - ) - ) - - return external_call( - stmt_expr, - context, - contract_name, - contract_address, - pos=getpos(stmt_expr), - value=value, - gas=gas, - ) - elif ( isinstance(stmt_expr.func.value, vy_ast.Attribute) and stmt_expr.func.value.attr in context.globals and hasattr(context.globals[stmt_expr.func.value.attr].typ, "name") ): + # e.g. `self.foo.bar()` contract_name = context.globals[stmt_expr.func.value.attr].typ.name type_ = stmt_expr.func.value._metadata["type"] From b911f7df70f00b67007a1cc74f4fc292e35028f3 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 7 Sep 2021 13:21:03 -0700 Subject: [PATCH 030/163] wip external call rewrite --- vyper/old_codegen/external_call.py | 242 ++++++++++------------------- 1 file changed, 81 insertions(+), 161 deletions(-) diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index 4b130df551..9cb254abee 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -21,168 +21,105 @@ has_dynamic_data, ) -@type_check_wrapper -def pack_arguments(signature, args, context, stmt_expr, is_external_call): - # NOTE cyclic dep - from vyper.old_codegen.abi import abi_encode, abi_type_of - - pos = getpos(stmt_expr) - +def pack_arguments(sig, args, buf): # abi encoding just treats all args as a big tuple args_tuple_t = TupleType([x.typ for x in args]) args_as_tuple = LLLnode.from_list(["multi"] + [x for x in args], typ=args_tuple_t) args_abi_t = abi_type_of(args_tuple_t) - maxlen = args_abi_t.dynamic_size_bound() + args_abi_t.static_size() - if is_external_call: - maxlen += 32 # padding for the method id - - buf_t = ByteArrayType(maxlen=maxlen) - buf = context.new_internal_variable(buf_t) - - mstore_method_id = [] - if is_external_call: - # layout: - # 32 bytes | args - # 0x..00 | args - # the reason for the left padding is just so the alignment is easier. - # if we were only targeting constantinople, we could align - # to buf (and also keep code size small) by using - # (mstore buf (shl signature.method_id 224)) - mstore_method_id.append(["mstore", buf, signature.method_id]) - buf += 32 + maxlen = args_abi_t.size_bound() + maxlen += 32 # padding for the method id - if len(signature.args) != len(args): - return + buf_t = ByteArrayType(maxlen=maxlen) + buf = context.new_internal_variable(buf_t) + + args_ofst = buf + 28 + args_len = maxlen - 28 + + # layout: + # 32 bytes | args + # 0x..00 | args + # the reason for the left padding is just so the alignment is easier. + # if we were only targeting constantinople, we could align + # to buf (and also keep code size small) by using + # (mstore buf (shl signature.method_id 224)) + mstore_method_id = [["mstore", buf, sig.method_id]] encode_args = abi_encode(buf, args_as_tuple, pos) - if is_external_call: - returner = [[buf - 4]] - inargsize = buf_t.maxlen - 28 - else: - return + return mstore_method_id + [encode_args], args_ofst, args_len - return ( - LLLnode.from_list( - ["seq"] + mstore_method_id + [encode_args] + returner, typ=buf_t, location="memory" - ), - inargsize, - buf, - ) +def unpack_returndata(sig, context): + return_t = abi_type_of(sig.output_type) + min_return_size = return_t.static_size() + maxlen = return_t.size_bound() + buf_t = ByteArrayType(maxlen=maxlen) + buf = context.new_internal_variable(buf_t) + ret_ofst = buf + ret_len = maxlen + + # when return data is expected, revert when the length of `returndatasize` is insufficient + ret = [["assert", ["gt", "returndatasize", min_return_size - 1]]] + # TODO assert returndatasize <= maxlen + + # abi_decode has appropriate clampers for the individual members of the return type + ret += [abi_decode_lazy(buf)] + + return ret, ret_ofst, ret_len -def external_call(node, context, interface_name, contract_address, pos, value=None, gas=None): - from vyper.old_codegen.expr import Expr + +@type_check_wrapper +def external_call(node, context, interface_name, contract_address, value=None, gas=None): + method_name = node.func.attr + sig = context.sigs[interface_name][method_name] if value is None: value = 0 if gas is None: gas = "gas" - method_name = node.func.attr - sig = context.sigs[interface_name][method_name] - inargs, inargsize, _ = pack_arguments( - sig, - [Expr(arg, context).lll_node for arg in node.args], - context, - node.func, - is_external_call=True, - ) - output_placeholder, output_size, returner = get_external_call_output(sig, context) - sub = ["seq"] - if not output_size: - # if we do not expect return data, check that a contract exists at the target address - # we can omit this when we _do_ expect return data because we later check `returndatasize` - sub.append(["assert", ["extcodesize", contract_address]]) + # sanity check + if len(signature.args) != len(args): + return + if context.is_constant() and sig.mutability not in ("view", "pure"): - # TODO this can probably go + # TODO is this already done in type checker? raise StateAccessViolation( f"May not call state modifying function '{method_name}' " f"within {context.pp_constancy()}.", node, ) + sub = ["seq"] + + arg_packer, args_ofst, args_len = pack_arguments(sig, args, context) + ret_unpacker, ret_ofst, ret_len = unpack_arguments(sig, context) + + sub += arg_packer + + if sig.return_type is None: + # if we do not expect return data, check that a contract exists at the target address + # we can omit this when we _do_ expect return data because we later check `returndatasize` + # CMC 20210907 do we need to check this before the call, or can we defer until after? + # if we can defer, this code can be pushed down into unpack_returndata + sub.append(["assert", ["extcodesize", contract_address]]) + if context.is_constant() or sig.mutability in ("view", "pure"): - sub.append( - [ - "assert", - [ - "staticcall", - gas, - contract_address, - inargs, - inargsize, - output_placeholder, - output_size, - ], - ] - ) + call_op = ["staticcall", gas, contract_address, args_ofst, args_len, ret_ofst, ret_len] else: - sub.append( - [ - "assert", - [ - "call", - gas, - contract_address, - value, - inargs, - inargsize, - output_placeholder, - output_size, - ], - ] - ) - if output_size: - # when return data is expected, revert when the length of `returndatasize` is insufficient - output_type = sig.output_type - if not has_dynamic_data(output_type): - static_output_size = get_static_size_of_type(output_type) * 32 - sub.append(["assert", ["gt", "returndatasize", static_output_size - 1]]) - else: - if isinstance(output_type, ByteArrayLike): - types_list = (output_type,) - elif isinstance(output_type, TupleLike): - types_list = output_type.tuple_members() - else: - raise - - dynamic_checks = [] - static_offset = output_placeholder - static_output_size = 0 - for typ in types_list: - # ensure length of bytes does not exceed max allowable length for type - if isinstance(typ, ByteArrayLike): - static_output_size += 32 - # do not perform this check on calls to a JSON interface - we don't know - # for certain how long the expected data is - if not sig.is_from_json: - dynamic_checks.append( - [ - "assert", - [ - "lt", - [ - "mload", - ["add", ["mload", static_offset], output_placeholder], - ], - typ.maxlen + 1, - ], - ] - ) - static_offset += get_static_size_of_type(typ) * 32 - static_output_size += get_static_size_of_type(typ) * 32 - - sub.append(["assert", ["gt", "returndatasize", static_output_size - 1]]) - sub.extend(dynamic_checks) - - sub.extend(returner) - - return LLLnode.from_list(sub, typ=sig.output_type, location="memory", pos=getpos(node)) - - -def get_external_interface_keywords(stmt_expr, context): + call_op = ["call", gas, contract_address, value, args_ofst, args_len, ret_ofst, ret_len] + + sub.append(["assert", call_op]) + + if sig.return_type is not None: + sub += ret_unpacker + + return LLLnode.from_list(sub, typ=sig.return_type, location="memory", pos=getpos(node)) + + +# TODO push me up to expr.py +def get_gas_and_value(stmt_expr, context): # circular import! from vyper.old_codegen.expr import Expr @@ -198,10 +135,7 @@ def get_external_interface_keywords(stmt_expr, context): def make_external_call(stmt_expr, context): - # circular import! - from vyper.old_codegen.expr import Expr - - value, gas = get_external_interface_keywords(stmt_expr, context) + value, gas = get_gas_and_value(stmt_expr, context) if isinstance(stmt_expr.func, vy_ast.Attribute) and isinstance( stmt_expr.func.value, vy_ast.Call @@ -211,16 +145,6 @@ def make_external_call(stmt_expr, context): contract_name = stmt_expr.func.value.func.id contract_address = Expr.parse_value_expr(stmt_expr.func.value.args[0], context) - return external_call( - stmt_expr, - context, - contract_name, - contract_address, - pos=getpos(stmt_expr), - value=value, - gas=gas, - ) - elif ( isinstance(stmt_expr.func.value, vy_ast.Attribute) and stmt_expr.func.value.attr in context.globals @@ -241,15 +165,11 @@ def make_external_call(stmt_expr, context): ) ) - return external_call( - stmt_expr, - context, - contract_name, - contract_address, - pos=getpos(stmt_expr), - value=value, - gas=gas, - ) - - else: - raise StructureException("Unsupported operator.", stmt_expr) + return external_call( + stmt_expr, + context, + contract_name, + contract_address, + value=value, + gas=gas, + ) From e035bef58cf83318c13accb03be261805341ad31 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 8 Sep 2021 09:10:31 -0700 Subject: [PATCH 031/163] add clamping to ABI decoder --- vyper/old_codegen/abi.py | 19 +++++++++-- vyper/old_codegen/arg_clamps.py | 9 +++++ vyper/old_codegen/parser_utils.py | 55 ++++++++++++++++++++++++++++++- 3 files changed, 80 insertions(+), 3 deletions(-) diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index 8b0a71a0b3..26bc01b9ff 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -396,6 +396,8 @@ def abi_encode(dst, lll_node, pos=None, bufsz=None, returns_len=False): abi_t = abi_type_of(o.typ) if parent_abi_t.is_complex_type(): + # TODO optimize: special case where there is only one dynamic + # member, the location is statically known. if abi_t.is_dynamic(): lll_ret.append(["mstore", dst_loc, dyn_ofst]) # recurse @@ -455,7 +457,7 @@ def abi_encode(dst, lll_node, pos=None, bufsz=None, returns_len=False): # recursively copy the buffer items into lll_node, based on its type. # src: pointer to beginning of buffer # src_loc: pointer to read location in static section -def abi_decode(lll_node, src, pos=None): +def abi_decode(lll_node, src, clamp=True, pos=None): os = o_list(lll_node, pos=pos) lll_ret = ["seq"] parent_abi_t = abi_type_of(lll_node.typ) @@ -464,13 +466,20 @@ def abi_decode(lll_node, src, pos=None): src_loc = LLLnode("src_loc", typ=o.typ, location=src.location) if parent_abi_t.is_complex_type(): if abi_t.is_dynamic(): + # TODO optimize: special case where there is only one dynamic + # member, the location is statically known. child_loc = ["add", "src", unwrap_location(src_loc)] child_loc = LLLnode.from_list(child_loc, typ=o.typ, location=src.location) else: child_loc = src_loc # descend into the child tuple lll_ret.append(abi_decode(o, child_loc, pos=pos)) + else: + + if clamp: + lll_ret.append(clamp_basetype(unwrap_location(src_loc))) + lll_ret.append(make_setter(o, src_loc, location=o.location, pos=pos)) if i + 1 == len(os): @@ -512,6 +521,8 @@ def lazy_abi_decode(typ, src, clamp=True, pos=None): for t in ts: child_abi_t = abi_type_of(t) loc = _add_ofst(src, ofst) + # TODO optimize: special case where there is only one dynamic + # member, the location is statically known. if child_abi_t.is_dynamic(): # load the offset word, which is the # (location-independent) offset from the start of the @@ -524,6 +535,10 @@ def lazy_abi_decode(typ, src, clamp=True, pos=None): return LLLnode.from_list(["multi"] + os, typ=typ, pos=pos) elif isinstance(typ, (BaseType, ByteArrayLike)): - return unwrap_location(src) + if clamp: + x = LLLnode.from_list(["x"], typ=typ) + return ["with", x, unwrap_location(src), ["seq", clamp_basetype(x), x]] + else: + return unwrap_location(src) else: raise CompilerPanic(f"unknown type for lazy_abi_decode {typ}") diff --git a/vyper/old_codegen/arg_clamps.py b/vyper/old_codegen/arg_clamps.py index 61765053ff..0a99d15fba 100644 --- a/vyper/old_codegen/arg_clamps.py +++ b/vyper/old_codegen/arg_clamps.py @@ -20,6 +20,7 @@ def _mk_codecopy_copier(pos, sz, mempos): return ["codecopy", mempos, ["add", "~codelen", pos], sz] +# TODO dead code def make_arg_clamper(datapos, mempos, typ, is_init=False): """ Clamps argument to type limits. @@ -121,6 +122,14 @@ def make_arg_clamper(datapos, mempos, typ, is_init=False): else: return LLLnode.from_list("pass") +def _shr(x, bits): + if version_check(begin="constantinople"): + return ["shr", x, bits] + return ["div", x, ["pow", 2, bits]] +def _sar(x, bits): + if version_check(begin="constantinople"): + return ["sar", x, bits] + return ["sdiv", x, ["pow", 2, bits]] def address_clamp(lll_node): if version_check(begin="constantinople"): diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 0152b83f5e..4e3c6029d4 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -252,7 +252,7 @@ def byte_array_to_num( def get_bytearray_length(arg): typ = BaseType("uint256") - return LLLnode.from_list(load_op(arg.location), typ=BaseType("uint256")) + return LLLnode.from_list([load_op(arg.location), arg], typ=BaseType("uint256")) def getpos(node): @@ -382,9 +382,12 @@ def load_op(location): # Unwrap location def unwrap_location(orig): + if not isinstance(orig.typ, BaseType): + raise CompilerPanic("unwrap location only for base types") if orig.location in ("memory", "storage", "calldata", "code"): return LLLnode.from_list([load_op(orig.location), typ=orig.typ) else: + # CMC 20210909 TODO double check if this branch can be removed # handle None value inserted by `empty` if orig.value is None: return LLLnode.from_list(0, typ=orig.typ) @@ -663,3 +666,53 @@ def zero_pad(bytez_placeholder): ["with", "len", len_, ["with", "dst", dst, mzero("dst", num_zero_bytes)]], annotation="Zero pad", ) + + +# convenience rewrites for shr/sar/shl +def _shr(x, bits): + if version_check(begin="constantinople"): + return ["shr", x, bits] + return ["div", x, ["exp", 2, bits]] +def _sar(x, bits): + if version_check(begin="constantinople"): + return ["sar", x, bits] + return ["sdiv", x, ["exp", 2, bits]] + + +# clampers for basetype +@typecheck_wrapper +def clamp_basetype(lll_node): + t = lll_node.typ + if isinstance(t, ByteArrayLike): + b = LLLnode.from_list(["b"], typ=lll_node.typ, location=lll_node.location) + return ["assert", ["le", get_bytearray_length(b), t.maxlen]] + if isinstance(t, BaseType): + if t.typ in ("int128", "decimal"): + return int_clamp(t, 128, signed=True) + if t.typ in ("address",): + return int_clamp(t, 160) + if t.typ in ("bool",): + return int_clamp(t, 1) + if t.typ in ("int256", "uint256"): + return [] # special case, no clamp + return # raises + +def int_clamp(lll_node, bits, signed=False): + """Generalized clamper for integer types. Takes the number of bits, + whether it's signed, and returns an LLL node which checks it is + in bounds. + """ + if bits >= 256: + raise CompilerPanic("shouldn't clamp", lll_node) + if signed: + # example for bits==128: + # if _val is in bounds, + # _val >>> 127 == 0 for positive _val + # _val >>> 127 == -1 for negative _val + # -1 and 0 are the only numbers which are unchanged by sar, + # so sar'ing (_val>>>127) one more bit should leave it unchanged. + ret = ["with", "x", lll_node, ["assert", ["eq", _sar("x", bits-1), _sar("x", bits)]]] + else: + ret = ["assert", ["iszero", _shr(lll_node, bits)]] + + return ret From 155f16612f8e7cdf7bac7e0a8deaab28a161a5c8 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 9 Sep 2021 19:34:28 -0700 Subject: [PATCH 032/163] polish work on new calling convention --- vyper/ast/signatures/function_signature.py | 2 + .../function_definitions/common.py | 2 + .../function_definitions/external_function.py | 43 ++++++++++--------- .../old_codegen/function_definitions/utils.py | 1 + vyper/old_codegen/return_.py | 2 + vyper/old_codegen/self_call.py | 2 +- 6 files changed, 31 insertions(+), 21 deletions(-) diff --git a/vyper/ast/signatures/function_signature.py b/vyper/ast/signatures/function_signature.py index 04e122fd54..1bc59672f4 100644 --- a/vyper/ast/signatures/function_signature.py +++ b/vyper/ast/signatures/function_signature.py @@ -237,8 +237,10 @@ def _check(cond, s = "Unreachable"): _check(False) + @property def is_default_func(self): return self.name == "__default__" + @property def is_init_func(self): return self.name == "__init__" diff --git a/vyper/old_codegen/function_definitions/common.py b/vyper/old_codegen/function_definitions/common.py index fc5e322e9a..f4cd3c79cc 100644 --- a/vyper/old_codegen/function_definitions/common.py +++ b/vyper/old_codegen/function_definitions/common.py @@ -12,11 +12,13 @@ from vyper.utils import calc_mem_gas +# TODO dead code - replaced by FunctionSignature method # Is a function the initializer? def is_initializer(code): return code.name == "__init__" +# TODO dead code - replaced by FunctionSignature method # Is a function the default function? def is_default_func(code): return code.name == "__default__" diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index 22c8046aa8..6d96368730 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -21,20 +21,22 @@ # also allocate the ones that live in memory (i.e. kwargs) def _register_function_args(context: Context, sig: FunctionSignature): - if len(args) > 0: - # tuple with the abi_encoded args + if len(args) == 0: + return + + # tuple with the abi_encoded args + if sig.is_init_func(): + base_args_location = LLLnode("~codelen", location="code", typ=tbd_base_args_type) + else: base_args_location = LLLnode(4, location="calldata", typ=tbd_base_args_type) - base_args = lazy_abi_decode(tbd_base_args_type, base_args_location) - assert base_args.value == "multi" - for (argname, arg_lll) in zip(tbd_argnames, base_args.args): # the actual values - # register the record in the local namespace - context.vars[argname] = LLLnode(arg_lll, location="calldata") + base_args = lazy_abi_decode(tbd_base_args_type, base_args_location) + assert base_args.value == "multi", "you've been bad" -def _generate_all_signatures(context: Context, function_def: vy_ast.FunctionDef): - for sig, kwarg_value in zip(all_sigs, kwarg_values): - yield pass + for (argname, arg_lll) in zip(tbd_argnames, base_args.args): # the actual values + # register the record in the local namespace + context.vars[argname] = LLLnode(arg_lll, location=location) def _generate_kwarg_handlers(context: Context, sig): @@ -60,16 +62,15 @@ def handler_for(calldata_kwargs, default_kwargs): assert calldata_args.value == "multi" # sanity check # extract just the kwargs from the ABI payload - calldata_args = calldata.args[len(base_args):] + # TODO come up with a better name for these variables + calldata_kwargs = calldata_args.args[:len(base_args)] # a sequence of statements to strictify kwargs into memory ret = ["seq"] all_kwargs_t = TupleType(list(arg.typ for arg in sig_kwargs)) - all_kwargs_src = LLLnode.from_list(["multi"] + calldata_args + default_args, typ=all_kwargs_t) - - for x in calldata_args: + for x in calldata_kwargs: context.new_variable(argname, argtype, mutable=False) ret.append(make_setter(context.lookup_var(x.name), x, "memory")) for x in default_kwargs: @@ -105,35 +106,37 @@ def generate_lll_for_external_function( entrance = [] - # once kwargs have been handled + # once args have been handled if len(kwarg_handlers) > 0: entrance.append(["label", f"{sig.base_method_id}_entry"]) + # TODO need a case for no kwargs? if check_nonpayable and sig.mutability != "payable": # if the contract contains payable functions, but this is not one of them # add an assertion that the value of the call is zero entrance.append(["assert", ["iszero", "callvalue"]]) - # TODO: handle __init__ and default functions? - body = [parse_body(c, context) for c in code.body] exit = [["label", func_type.exit_sequence_label]] + [nonreentrant_post] - + [["return", "pass", "pass"]] # passed by + # TODO optimize case where return_type is None: use STOP + + [["return", "pass", "pass"]] # ret_ofst and ret_len stack items passed by function body ret = (["seq"] - + kwarg_handlers + + arg_handlers + entrance + body + exit ) - if len(kwarg_handlers) > 0: + # TODO special handling for default function + if len(kwarg_handlers) == 0: # TODO is this check correct? ret = ["if", ["eq", tbd_mload_method_id, sig.method_id], ret] return LLLnode.from_list(ret, pos=getpos(code)) +# TODO dead code def generate_lll_for_external_function( code: vy_ast.FunctionDef, sig: FunctionSignature, context: Context, check_nonpayable: bool, ) -> LLLnode: diff --git a/vyper/old_codegen/function_definitions/utils.py b/vyper/old_codegen/function_definitions/utils.py index 4f27af11a6..d7204b9e06 100644 --- a/vyper/old_codegen/function_definitions/utils.py +++ b/vyper/old_codegen/function_definitions/utils.py @@ -27,6 +27,7 @@ def get_nonreentrant_lock(func_type): return nonreentrant_pre, nonreentrant_post +# TODO dead code def get_default_names_to_set(primary_sig, default_sig): """ Get names for default parameters that require a default value to be assigned. diff --git a/vyper/old_codegen/return_.py b/vyper/old_codegen/return_.py index 5343c93a12..8ec6f3ac4f 100644 --- a/vyper/old_codegen/return_.py +++ b/vyper/old_codegen/return_.py @@ -50,8 +50,10 @@ def finalize(fill_return_buffer): else: # we are in an external function. + if context.return_type is None: # push arguments onto the stack for RETURN opcode + # TODO optimize this case: just use STOP return finalize(["seq_unchecked", 0, 0]) return_buffer_ofst = _allocate_return_buffer(context) diff --git a/vyper/old_codegen/self_call.py b/vyper/old_codegen/self_call.py index 7ee5dc0b46..bfb8d8e1f1 100644 --- a/vyper/old_codegen/self_call.py +++ b/vyper/old_codegen/self_call.py @@ -39,7 +39,7 @@ def make_call(stmt_expr, context): method_name = stmt_expr.func.attr - sig, kw_vals = FunctionSignature.lookup_sig(method_name, args_lll, context) + sig, kw_vals = FunctionSignature.lookup_internal_function(method_name, args_lll, context) pos_args_lll = Expr(x, self.context).lll_node for x in stmt_expr.args kw_args_lll = Expr(x, self.context).lll_node for x in kw_vals From d791ffe9cbeb1ae91da730b9a08264c6f5e5ed8a Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 9 Sep 2021 19:45:40 -0700 Subject: [PATCH 033/163] fix some lint/surface errors --- vyper/ast/signatures/function_signature.py | 21 ++++++------- vyper/old_codegen/abi.py | 4 +-- vyper/old_codegen/arg_clamps.py | 4 +++ vyper/old_codegen/context.py | 8 +++-- vyper/old_codegen/external_call.py | 30 +++++++------------ .../function_definitions/common.py | 8 +++-- .../function_definitions/external_function.py | 3 +- vyper/old_codegen/parser.py | 4 +-- vyper/old_codegen/parser_utils.py | 16 ++++++---- vyper/old_codegen/self_call.py | 29 ++++++++++-------- vyper/old_codegen/stmt.py | 2 +- 11 files changed, 70 insertions(+), 59 deletions(-) diff --git a/vyper/ast/signatures/function_signature.py b/vyper/ast/signatures/function_signature.py index 1bc59672f4..e23b96c236 100644 --- a/vyper/ast/signatures/function_signature.py +++ b/vyper/ast/signatures/function_signature.py @@ -1,5 +1,7 @@ import math from collections import Counter +from dataclasses import dataclass +from typing import List from vyper import ast as vy_ast from vyper.exceptions import FunctionDeclarationException, StructureException @@ -133,19 +135,17 @@ def set_default_args(self): self.default_args = self.args[num_base_args:] # Keep all the value to assign to default parameters. - self.default_values = dict( - zip([arg.name for arg in self.default_args], args.defaults) - ) + self.default_values = dict(zip([arg.name for arg in self.default_args], args.defaults)) # Get a signature from a function definition @classmethod def from_definition( cls, - func_ast, # vy_ast.FunctionDef - sigs=None, # TODO replace sigs and custom_structs with GlobalContext? + func_ast, # vy_ast.FunctionDef + sigs=None, # TODO replace sigs and custom_structs with GlobalContext? custom_structs=None, interface_def=False, - constant_override=False, # CMC 20210907 what does this do? + constant_override=False, # CMC 20210907 what does this do? is_from_json=False, ): if custom_structs is None: @@ -211,15 +211,16 @@ def lookup_internal_function(cls, method_name, args_lll, context): Using a list of args, find the internal method to use, and the kwargs which need to be filled in by the compiler """ - def _check(cond, s = "Unreachable"): + + def _check(cond, s="Unreachable"): if not cond: raise CompilerPanic(s) - for sig in context.sigs['self']: + for sig in context.sigs["self"]: if sig.name != method_name: continue - _check(sig.internal) # sanity check + _check(sig.internal) # sanity check # should have been caught during type checking, sanity check anyway _check(len(sig.base_args) <= len(args_lll) <= len(sig.args)) @@ -231,7 +232,7 @@ def _check(cond, s = "Unreachable"): num_kwargs = len(sig.kwargs) args_needed = total_args - num_provided - kw_vals = sig.kwarg_values.items()[:num_kwargs - args_needed] + kw_vals = sig.kwarg_values.items()[: num_kwargs - args_needed] return sig, kw_vals diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index 26bc01b9ff..b896bd72e9 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -51,6 +51,7 @@ def selector_name(self): def is_complex_type(self): raise NotImplementedError("ABIType.is_complex_type") + # uint: unsigned integer type of M bits, 0 < M <= 256, M % 8 == 0. e.g. uint32, uint8, uint256. # int: two’s complement signed integer type of M bits, 0 < M <= 256, M % 8 == 0. class ABI_GIntM(ABIType): @@ -373,7 +374,6 @@ def abi_encode(dst, lll_node, pos=None, bufsz=None, returns_len=False): if bufsz is not None and bufsz < 32 * size_bound: raise CompilerPanic("buffer provided to abi_encode not large enough") - # fastpath: if there is no dynamic data, we can optimize the # encoding by using make_setter, since our memory encoding happens # to be identical to the ABI encoding. @@ -384,14 +384,12 @@ def abi_encode(dst, lll_node, pos=None, bufsz=None, returns_len=False): lll_ret.append(parent_abi_t.embedded_static_size()) return LLLnode.from_list(lll_ret, pos=pos, annotation=f"abi_encode {lll_node.typ}") - lll_ret = ["seq"] dyn_ofst = "dyn_ofst" # current offset in the dynamic section dst_begin = "dst" # pointer to beginning of buffer dst_loc = "dst_loc" # pointer to write location in static section os = o_list(lll_node, pos=pos) - for i, o in enumerate(os): abi_t = abi_type_of(o.typ) diff --git a/vyper/old_codegen/arg_clamps.py b/vyper/old_codegen/arg_clamps.py index 0a99d15fba..dc2383ec9b 100644 --- a/vyper/old_codegen/arg_clamps.py +++ b/vyper/old_codegen/arg_clamps.py @@ -122,15 +122,19 @@ def make_arg_clamper(datapos, mempos, typ, is_init=False): else: return LLLnode.from_list("pass") + def _shr(x, bits): if version_check(begin="constantinople"): return ["shr", x, bits] return ["div", x, ["pow", 2, bits]] + + def _sar(x, bits): if version_check(begin="constantinople"): return ["sar", x, bits] return ["sdiv", x, ["pow", 2, bits]] + def address_clamp(lll_node): if version_check(begin="constantinople"): return ["assert", ["iszero", ["shr", 160, lll_node]]] diff --git a/vyper/old_codegen/context.py b/vyper/old_codegen/context.py index 7449cc46e1..1a4712c491 100644 --- a/vyper/old_codegen/context.py +++ b/vyper/old_codegen/context.py @@ -133,7 +133,9 @@ def block_scope(self): # Remove block scopes self._scopes.remove(scope_id) - def _new_variable(self, name: str, typ: NodeType, var_size: int, is_internal: bool, is_mutable: bool True) -> int: + def _new_variable( + self, name: str, typ: NodeType, var_size: int, is_internal: bool, is_mutable: bool = True + ) -> int: if is_internal: var_pos = self.memory_allocator.expand_memory(var_size) else: @@ -148,7 +150,9 @@ def _new_variable(self, name: str, typ: NodeType, var_size: int, is_internal: bo ) return var_pos - def new_variable(self, name: str, typ: NodeType, pos: VyperNode = None, is_mutable: bool = True) -> int: + def new_variable( + self, name: str, typ: NodeType, pos: VyperNode = None, is_mutable: bool = True + ) -> int: """ Allocate memory for a user-defined variable. diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index 9cb254abee..1f578ca77c 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -8,7 +8,6 @@ from vyper.old_codegen.lll_node import LLLnode from vyper.old_codegen.parser_utils import ( getpos, - pack_arguments, unwrap_location, ) from vyper.old_codegen.types import ( @@ -21,6 +20,7 @@ has_dynamic_data, ) + def pack_arguments(sig, args, buf): # abi encoding just treats all args as a big tuple args_tuple_t = TupleType([x.typ for x in args]) @@ -30,11 +30,11 @@ def pack_arguments(sig, args, buf): maxlen = args_abi_t.size_bound() maxlen += 32 # padding for the method id - buf_t = ByteArrayType(maxlen=maxlen) - buf = context.new_internal_variable(buf_t) + buf_t = ByteArrayType(maxlen=maxlen) + buf = context.new_internal_variable(buf_t) - args_ofst = buf + 28 - args_len = maxlen - 28 + args_ofst = buf + 28 + args_len = maxlen - 28 # layout: # 32 bytes | args @@ -49,15 +49,16 @@ def pack_arguments(sig, args, buf): return mstore_method_id + [encode_args], args_ofst, args_len + def unpack_returndata(sig, context): return_t = abi_type_of(sig.output_type) min_return_size = return_t.static_size() maxlen = return_t.size_bound() - buf_t = ByteArrayType(maxlen=maxlen) - buf = context.new_internal_variable(buf_t) + buf_t = ByteArrayType(maxlen=maxlen) + buf = context.new_internal_variable(buf_t) ret_ofst = buf - ret_len = maxlen + ret_len = maxlen # when return data is expected, revert when the length of `returndatasize` is insufficient ret = [["assert", ["gt", "returndatasize", min_return_size - 1]]] @@ -69,7 +70,6 @@ def unpack_returndata(sig, context): return ret, ret_ofst, ret_len -@type_check_wrapper def external_call(node, context, interface_name, contract_address, value=None, gas=None): method_name = node.func.attr sig = context.sigs[interface_name][method_name] @@ -80,8 +80,7 @@ def external_call(node, context, interface_name, contract_address, value=None, g gas = "gas" # sanity check - if len(signature.args) != len(args): - return + assert len(signature.args) == len(args) if context.is_constant() and sig.mutability not in ("view", "pure"): # TODO is this already done in type checker? @@ -165,11 +164,4 @@ def make_external_call(stmt_expr, context): ) ) - return external_call( - stmt_expr, - context, - contract_name, - contract_address, - value=value, - gas=gas, - ) + return external_call(stmt_expr, context, contract_name, contract_address, value=value, gas=gas,) diff --git a/vyper/old_codegen/function_definitions/common.py b/vyper/old_codegen/function_definitions/common.py index f4cd3c79cc..4f93ac3d1e 100644 --- a/vyper/old_codegen/function_definitions/common.py +++ b/vyper/old_codegen/function_definitions/common.py @@ -50,15 +50,17 @@ def _run_pass(memory_allocator=None): # Create a local (per function) context. if memory_allocator is None: memory_allocator = MemoryAllocator() - _vars = _vars.copy() # these will get clobbered in produce_* functions - sig = copy.deepcopy(sig) # just in case + _vars = _vars.copy() # these will get clobbered in produce_* functions + sig = copy.deepcopy(sig) # just in case context = ctx.Context( vars=_vars, global_ctx=global_ctx, sigs=sigs, memory_allocator=memory_allocator, return_type=sig.output_type, - constancy=Constancy.Constant if sig.mutability in ("view", "pure") else Constancy.Mutable, + constancy=Constancy.Constant + if sig.mutability in ("view", "pure") + else Constancy.Mutable, is_payable=sig.mutability == "payable", is_internal=sig.internal, method_id=sig.method_id, diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index 6d96368730..9d264a3663 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -118,10 +118,11 @@ def generate_lll_for_external_function( body = [parse_body(c, context) for c in code.body] - exit = [["label", func_type.exit_sequence_label]] + exit = ([["label", func_type.exit_sequence_label]] + [nonreentrant_post] # TODO optimize case where return_type is None: use STOP + [["return", "pass", "pass"]] # ret_ofst and ret_len stack items passed by function body + ) ret = (["seq"] + arg_handlers diff --git a/vyper/old_codegen/parser.py b/vyper/old_codegen/parser.py index 4ba47b4e0f..f7b536dab6 100644 --- a/vyper/old_codegen/parser.py +++ b/vyper/old_codegen/parser.py @@ -200,8 +200,8 @@ def parse_tree_to_lll(global_ctx: GlobalContext) -> Tuple[LLLnode, LLLnode]: if initfunc: o.append(init_func_init_lll()) init_func_lll, _frame_size = generate_lll_for_function( - initfunc[0], {**{"self": sigs}, **external_interfaces}, global_ctx, False, - ) + initfunc[0], {**{"self": sigs}, **external_interfaces}, global_ctx, False, + ) o.append(init_func_lll) # If there are regular functions... diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 4e3c6029d4..2fe59e74e6 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -168,7 +168,10 @@ def make_byte_slice_copier(destination, source, length, max_length, pos=None): loader = 0 # Copy over data elif source.location in ("memory", "calldata", "code"): - loader = [load_op(source.location), ["add", "_pos", ["mul", 32, ["mload", MemoryPositions.FREE_LOOP_INDEX]]]] + loader = [ + load_op(source.location), + ["add", "_pos", ["mul", 32, ["mload", MemoryPositions.FREE_LOOP_INDEX]]], + ] elif source.location == "storage": loader = ["sload", ["add", "_pos", ["mload", MemoryPositions.FREE_LOOP_INDEX]]] else: @@ -385,7 +388,7 @@ def unwrap_location(orig): if not isinstance(orig.typ, BaseType): raise CompilerPanic("unwrap location only for base types") if orig.location in ("memory", "storage", "calldata", "code"): - return LLLnode.from_list([load_op(orig.location), typ=orig.typ) + return LLLnode.from_list([load_op(orig.location)], typ=orig.typ) else: # CMC 20210909 TODO double check if this branch can be removed # handle None value inserted by `empty` @@ -673,6 +676,8 @@ def _shr(x, bits): if version_check(begin="constantinople"): return ["shr", x, bits] return ["div", x, ["exp", 2, bits]] + + def _sar(x, bits): if version_check(begin="constantinople"): return ["sar", x, bits] @@ -680,7 +685,7 @@ def _sar(x, bits): # clampers for basetype -@typecheck_wrapper +@type_check_wrapper def clamp_basetype(lll_node): t = lll_node.typ if isinstance(t, ByteArrayLike): @@ -694,9 +699,10 @@ def clamp_basetype(lll_node): if t.typ in ("bool",): return int_clamp(t, 1) if t.typ in ("int256", "uint256"): - return [] # special case, no clamp + return [] # special case, no clamp return # raises + def int_clamp(lll_node, bits, signed=False): """Generalized clamper for integer types. Takes the number of bits, whether it's signed, and returns an LLL node which checks it is @@ -711,7 +717,7 @@ def int_clamp(lll_node, bits, signed=False): # _val >>> 127 == -1 for negative _val # -1 and 0 are the only numbers which are unchanged by sar, # so sar'ing (_val>>>127) one more bit should leave it unchanged. - ret = ["with", "x", lll_node, ["assert", ["eq", _sar("x", bits-1), _sar("x", bits)]]] + ret = ["with", "x", lll_node, ["assert", ["eq", _sar("x", bits - 1), _sar("x", bits)]]] else: ret = ["assert", ["iszero", _shr(lll_node, bits)]] diff --git a/vyper/old_codegen/self_call.py b/vyper/old_codegen/self_call.py index bfb8d8e1f1..8a394a8f97 100644 --- a/vyper/old_codegen/self_call.py +++ b/vyper/old_codegen/self_call.py @@ -8,7 +8,7 @@ ) from vyper.old_codegen.abi import abi_decode from vyper.old_codegen.lll_node import LLLnode, push_label_to_stack -from vyper.old_codegen.parser_utils import getpos, pack_arguments +from vyper.old_codegen.parser_utils import getpos from vyper.old_codegen.types import ( BaseType, ByteArrayLike, @@ -26,6 +26,7 @@ def _generate_label(name: str) -> str: _label_counter += 1 return f"label{_label_counter}" + def make_call(stmt_expr, context): pos = getpos(stmt_expr) @@ -41,8 +42,8 @@ def make_call(stmt_expr, context): sig, kw_vals = FunctionSignature.lookup_internal_function(method_name, args_lll, context) - pos_args_lll = Expr(x, self.context).lll_node for x in stmt_expr.args - kw_args_lll = Expr(x, self.context).lll_node for x in kw_vals + pos_args_lll = [Expr(x, self.context).lll_node for x in stmt_expr.args] + kw_args_lll = [Expr(x, self.context).lll_node for x in kw_vals] args_lll = pos_args_lll + kw_args_lll args_tuple_t = TupleType([x.typ for x in args_lll]) @@ -66,21 +67,23 @@ def make_call(stmt_expr, context): # allocate space for the return buffer # TODO allocate in stmt and/or expr.py - return_buffer = context.new_internal_variable(sig.return_type) if sig.return_type is not None else "pass" + return_buffer = ( + context.new_internal_variable(sig.return_type) if sig.return_type is not None else "pass" + ) return_buffer = LLLnode.from_list([return_buffer], annotation=f"{return_label}_return_buf") copy_args = make_setter(sig.frame_start, args_as_tuple, "memory", pos) call_sequence = [ - "seq_unchecked", - copy_args, - push_label_to_stack(return_label), # pass return label to subroutine - return_buffer, # pass return buffer to subroutine - ["goto", sig.internal_function_label] - return_label, - "jumpdest", - return_buffer, # push return buffer location to stack - ] + "seq_unchecked", + copy_args, + push_label_to_stack(return_label), # pass return label to subroutine + return_buffer, # pass return buffer to subroutine + ["goto", sig.internal_function_label], + return_label, + "jumpdest", + return_buffer, # push return buffer location to stack + ] o = LLLnode.from_list( call_sequence, diff --git a/vyper/old_codegen/stmt.py b/vyper/old_codegen/stmt.py index dc7ec04747..9bcf2fa64b 100644 --- a/vyper/old_codegen/stmt.py +++ b/vyper/old_codegen/stmt.py @@ -13,7 +13,7 @@ unwrap_location, zero_pad, ) -from vyper.old_codegen.return_ import make_return_stmt, allocate_return_buffer +from vyper.old_codegen.return_ import make_return_stmt from vyper.old_codegen.types import ( BaseType, ByteArrayLike, From 1dbd7096b62fd96b5b84cb78a65240a2dffb50dc Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 9 Sep 2021 19:48:02 -0700 Subject: [PATCH 034/163] kill parse_sequence --- vyper/old_codegen/expr.py | 76 +++------------------------------------ 1 file changed, 5 insertions(+), 71 deletions(-) diff --git a/vyper/old_codegen/expr.py b/vyper/old_codegen/expr.py index df5862d8e8..ea6b7dc270 100644 --- a/vyper/old_codegen/expr.py +++ b/vyper/old_codegen/expr.py @@ -983,26 +983,20 @@ def parse_Call(self): return external_call.make_external_call(self.expr, self.context) def parse_List(self): - call_lll, multi_lll = parse_sequence(self.expr, self.expr.elements, self.context) + multi_lll = [Expr(x, self.context).lll_node for x in self.expr.elements] + # TODO this type inference is wrong. instead should use + # parse_type(canonical_type_of(self.expr._metadata["type"])) out_type = next((i.typ for i in multi_lll if not i.typ.is_literal), multi_lll[0].typ) typ = ListType(out_type, len(self.expr.elements), is_literal=True) multi_lll = LLLnode.from_list(["multi"] + multi_lll, typ=typ, pos=getpos(self.expr)) - if not call_lll: - return multi_lll - - lll_node = ["seq_unchecked"] + call_lll + [multi_lll] - return LLLnode.from_list(lll_node, typ=typ, pos=getpos(self.expr)) + return multi_lll def parse_Tuple(self): - #call_lll, multi_lll = parse_sequence(self.expr, self.expr.elements, self.context) tuple_elements = [Expr(x, self.context).lll_node for x in self.expr.elements] typ = TupleType([x.typ for x in tuple_elements], is_literal=True) multi_lll = LLLnode.from_list(["multi"] + tuple_elements, typ=typ, pos=getpos(self.expr)) - if True: # if not call_lll: - return multi_lll + return multi_lll - lll_node = ["seq_unchecked"] + call_lll + [multi_lll] - return LLLnode.from_list(lll_node, typ=typ, pos=getpos(self.expr)) @staticmethod def struct_literals(expr, name, context): @@ -1034,63 +1028,3 @@ def parse_variable_location(cls, expr, context): if not o.location: raise StructureException("Looking for a variable location, instead got a value", expr) return o - - -def parse_sequence(base_node, elements, context): - """ - Generate an LLL node from a sequence of Vyper AST nodes, such as values inside a - list/tuple or arguments inside a call. - - Arguments - --------- - base_node : VyperNode - Parent node which contains the sequence being parsed. - elements : List[VyperNode] - A list of nodes within the sequence. - context : Context - Currently active local context. - - Returns - ------- - List[LLLNode] - LLL nodes that must execute prior to generating the actual sequence in order to - avoid memory corruption issues. This list may be empty, depending on the values - within `elements`. - List[LLLNode] - LLL nodes which collectively represent `elements`. - """ - init_lll = [] - sequence_lll = [] - for node in elements: - if isinstance(node, vy_ast.List): - # for nested lists, ensure the init LLL is also processed before the values - init, seq = parse_sequence(node, node.elements, context) - init_lll.extend(init) - out_type = next((i.typ for i in seq if not i.typ.is_literal), seq[0].typ) - typ = ListType(out_type, len(node.elements), is_literal=True) - multi_lll = LLLnode.from_list(["multi"] + seq, typ=typ, pos=getpos(node)) - sequence_lll.append(multi_lll) - continue - - lll_node = Expr(node, context).lll_node - if isinstance(node, vy_ast.Call) or ( - isinstance(node, vy_ast.Subscript) and isinstance(node.value, vy_ast.Call) - ): - # nodes which potentially create their own internal memory variables, and so must - # be parsed prior to generating the final sequence to avoid memory corruption - target = LLLnode.from_list( - context.new_internal_variable(lll_node.typ), - typ=lll_node.typ, - location="memory", - pos=getpos(base_node), - ) - init_lll.append(make_setter(target, lll_node, "memory", pos=getpos(base_node))) - sequence_lll.append( - LLLnode.from_list( - target, typ=lll_node.typ, pos=getpos(base_node), location="memory" - ), - ) - else: - sequence_lll.append(lll_node) - - return init_lll, sequence_lll From d3177d23e280b0ab9d94e3d252ec380447cfac07 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 9 Sep 2021 19:48:18 -0700 Subject: [PATCH 035/163] lint a test --- tests/functional/codegen/test_struct_return.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/functional/codegen/test_struct_return.py b/tests/functional/codegen/test_struct_return.py index 300f6e60c0..8bdc731de0 100644 --- a/tests/functional/codegen/test_struct_return.py +++ b/tests/functional/codegen/test_struct_return.py @@ -23,8 +23,11 @@ def modify_nested_tuple(_human: Human) -> Human: c = get_contract(code) addr1 = "0x1234567890123456789012345678901234567890" addr2 = "0x1234567890123456789012345678900000000000" - #assert c.modify_nested_tuple([addr1, 123], [addr2, 456]) == [[addr1, 124], [addr2, 457]] - assert c.modify_nested_tuple({"location": addr1, "animal": {"location": addr2, "fur": "wool"}}) == [(addr1, (addr2, "wool is great"))] + # assert c.modify_nested_tuple([addr1, 123], [addr2, 456]) == [[addr1, 124], [addr2, 457]] + assert c.modify_nested_tuple( + {"location": addr1, "animal": {"location": addr2, "fur": "wool"}} + ) == [(addr1, (addr2, "wool is great"))] + @pytest.mark.parametrize("string", ["a", "abc", "abcde", "potato"]) def test_string_inside_tuple(get_contract, string): From 9e7f6e687edaa411b04dfb3bc60bd610538cac8f Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 9 Sep 2021 19:50:24 -0700 Subject: [PATCH 036/163] remove dead code --- vyper/old_codegen/arg_clamps.py | 111 ---------- .../function_definitions/external_function.py | 205 ------------------ .../old_codegen/function_definitions/utils.py | 31 --- vyper/old_codegen/parser_utils.py | 16 -- 4 files changed, 363 deletions(-) diff --git a/vyper/old_codegen/arg_clamps.py b/vyper/old_codegen/arg_clamps.py index dc2383ec9b..af2a7bd11f 100644 --- a/vyper/old_codegen/arg_clamps.py +++ b/vyper/old_codegen/arg_clamps.py @@ -12,117 +12,6 @@ from vyper.utils import MemoryPositions -def _mk_calldatacopy_copier(pos, sz, mempos): - return ["calldatacopy", mempos, ["add", 4, pos], sz] - - -def _mk_codecopy_copier(pos, sz, mempos): - return ["codecopy", mempos, ["add", "~codelen", pos], sz] - - -# TODO dead code -def make_arg_clamper(datapos, mempos, typ, is_init=False): - """ - Clamps argument to type limits. - - Arguments - --------- - datapos : int | LLLnode - Calldata offset of the value being clamped - mempos : int | LLLnode - Memory offset that the value is stored at during clamping - typ : vyper.types.types.BaseType - Type of the value - is_init : bool, optional - Boolean indicating if we are generating init bytecode - - Returns - ------- - LLLnode - Arg clamper LLL - """ - - if not is_init: - data_decl = ["calldataload", ["add", 4, datapos]] - copier = functools.partial(_mk_calldatacopy_copier, mempos=mempos) - else: - data_decl = ["codeload", ["add", "~codelen", datapos]] - copier = functools.partial(_mk_codecopy_copier, mempos=mempos) - # Numbers: make sure they're in range - if is_base_type(typ, "int128"): - return LLLnode.from_list( - int128_clamp(data_decl), typ=typ, annotation="checking int128 input" - ) - # Booleans: make sure they're zero or one - elif is_base_type(typ, "bool"): - if version_check(begin="constantinople"): - lll = ["assert", ["iszero", ["shr", 1, data_decl]]] - else: - lll = ["uclamplt", data_decl, 2] - return LLLnode.from_list(lll, typ=typ, annotation="checking bool input") - # Addresses: make sure they're in range - elif is_base_type(typ, "address"): - return LLLnode.from_list( - address_clamp(data_decl), typ=typ, annotation="checking address input" - ) - # Bytes: make sure they have the right size - elif isinstance(typ, ByteArrayLike): - return LLLnode.from_list( - [ - "seq", - copier(data_decl, 32 + typ.maxlen), - ["assert", ["le", ["calldataload", ["add", 4, data_decl]], typ.maxlen]], - ], - typ=None, - annotation="checking bytearray input", - ) - # Lists: recurse - elif isinstance(typ, ListType): - if typ.count > 5 or (type(datapos) is list and type(mempos) is list): - # find ultimate base type - subtype = typ.subtype - while hasattr(subtype, "subtype"): - subtype = subtype.subtype - - # make arg clamper for the base type - offset = MemoryPositions.FREE_LOOP_INDEX - clamper = make_arg_clamper( - ["add", datapos, ["mload", offset]], - ["add", mempos, ["mload", offset]], - subtype, - is_init, - ) - if clamper.value == "pass": - # no point looping if the base type doesn't require clamping - return clamper - - # loop the entire array at once, even if it's multidimensional - type_size = get_size_of_type(typ) - i_incr = get_size_of_type(subtype) * 32 - - mem_to = type_size * 32 - loop_label = f"_check_list_loop_{str(uuid.uuid4())}" - - lll_node = [ - ["mstore", offset, 0], # init loop - ["label", loop_label], - clamper, - ["mstore", offset, ["add", ["mload", offset], i_incr]], - ["if", ["lt", ["mload", offset], mem_to], ["goto", loop_label]], - ] - else: - lll_node = [] - for i in range(typ.count): - offset = get_size_of_type(typ.subtype) * 32 * i - lll_node.append( - make_arg_clamper(datapos + offset, mempos + offset, typ.subtype, is_init) - ) - return LLLnode.from_list(["seq"] + lll_node, typ=None, annotation="checking list input") - # Otherwise don't make any checks - else: - return LLLnode.from_list("pass") - - def _shr(x, bits): if version_check(begin="constantinople"): return ["shr", x, bits] diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index 9d264a3663..c93e4d5f31 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -135,208 +135,3 @@ def generate_lll_for_external_function( ret = ["if", ["eq", tbd_mload_method_id, sig.method_id], ret] return LLLnode.from_list(ret, pos=getpos(code)) - - -# TODO dead code -def generate_lll_for_external_function( - code: vy_ast.FunctionDef, sig: FunctionSignature, context: Context, check_nonpayable: bool, -) -> LLLnode: - """ - Parse a external function (FuncDef), and produce full function body. - - :param sig: the FuntionSignature - :param code: ast of function - :param check_nonpayable: if True, include a check that `msg.value == 0` - at the beginning of the function - :return: full sig compare & function body - """ - - func_type = code._metadata["type"] - - # Get nonreentrant lock - nonreentrant_pre, nonreentrant_post = get_nonreentrant_lock(func_type) - - clampers = [] - - # Generate copiers - copier: List[Any] = ["pass"] - if not len(sig.base_args): - copier = ["pass"] - elif sig.name == "__init__": - copier = ["codecopy", MemoryPositions.RESERVED_MEMORY, "~codelen", sig.base_copy_size] - context.memory_allocator.expand_memory(sig.max_copy_size) - clampers.append(copier) - - if check_nonpayable and sig.mutability != "payable": - # if the contract contains payable functions, but this is not one of them - # add an assertion that the value of the call is zero - clampers.append(["assert", ["iszero", "callvalue"]]) - - # Fill variable positions - default_args_start_pos = len(sig.base_args) - for i, arg in enumerate(sig.args): - if i < len(sig.base_args): - clampers.append( - make_arg_clamper( - arg.pos, - context.memory_allocator.get_next_memory_position(), - arg.typ, - sig.name == "__init__", - ) - ) - if isinstance(arg.typ, ByteArrayLike): - mem_pos = context.memory_allocator.expand_memory(32 * get_size_of_type(arg.typ)) - context.vars[arg.name] = VariableRecord(arg.name, mem_pos, arg.typ, False) - else: - if sig.name == "__init__": - context.vars[arg.name] = VariableRecord( - arg.name, MemoryPositions.RESERVED_MEMORY + arg.pos, arg.typ, False, - ) - elif i >= default_args_start_pos: # default args need to be allocated in memory. - type_size = get_size_of_type(arg.typ) * 32 - default_arg_pos = context.memory_allocator.expand_memory(type_size) - context.vars[arg.name] = VariableRecord( - name=arg.name, pos=default_arg_pos, typ=arg.typ, mutable=False, - ) - else: - context.vars[arg.name] = VariableRecord( - name=arg.name, pos=4 + arg.pos, typ=arg.typ, mutable=False, location="calldata" - ) - - # Create "clampers" (input well-formedness checkers) - # Return function body - if sig.name == "__init__": - o = LLLnode.from_list( - ["seq"] + clampers + [parse_body(code.body, context)], # type: ignore - pos=getpos(code), - ) - # Is default function. - elif sig.is_default_func(): - o = LLLnode.from_list( - ["seq"] + clampers + [parse_body(code.body, context)] + [["stop"]], # type: ignore - pos=getpos(code), - ) - # Is a normal function. - else: - # Function with default parameters. - if sig.total_default_args > 0: - function_routine = f"{sig.name}_{sig.method_id}" - default_sigs = sig_utils.generate_default_arg_sigs( - code, context.sigs, context.global_ctx - ) - sig_chain: List[Any] = ["seq"] - - for default_sig in default_sigs: - sig_compare, _ = get_sig_statements(default_sig, getpos(code)) - - # Populate unset default variables - set_defaults = [] - for arg_name in get_default_names_to_set(sig, default_sig): - value = Expr(sig.default_values[arg_name], context).lll_node - var = context.vars[arg_name] - left = LLLnode.from_list( - var.pos, - typ=var.typ, - location="memory", - pos=getpos(code), - mutable=var.mutable, - ) - set_defaults.append(make_setter(left, value, "memory", pos=getpos(code))) - - current_sig_arg_names = {x.name for x in default_sig.args} - base_arg_names = {arg.name for arg in sig.base_args} - copier_arg_count = len(default_sig.args) - len(sig.base_args) - copier_arg_names = list(current_sig_arg_names - base_arg_names) - - # Order copier_arg_names, this is very important. - copier_arg_names = [x.name for x in default_sig.args if x.name in copier_arg_names] - - # Variables to be populated from calldata/stack. - default_copiers: List[Any] = [] - if copier_arg_count > 0: - # Get map of variables in calldata, with thier offsets - offset = 4 - calldata_offset_map = {} - for arg in default_sig.args: - calldata_offset_map[arg.name] = offset - offset += ( - 32 - if isinstance(arg.typ, ByteArrayLike) - else get_size_of_type(arg.typ) * 32 - ) - - # Copy default parameters from calldata. - for arg_name in copier_arg_names: - var = context.vars[arg_name] - calldata_offset = calldata_offset_map[arg_name] - - # Add clampers. - default_copiers.append( - make_arg_clamper(calldata_offset - 4, var.pos, var.typ,) - ) - # Add copying code. - _offset: Union[int, List[Any]] = calldata_offset - if isinstance(var.typ, ByteArrayLike): - _offset = ["add", 4, ["calldataload", calldata_offset]] - default_copiers.append( - get_external_arg_copier( - memory_dest=var.pos, total_size=var.size * 32, offset=_offset, - ) - ) - - default_copiers.append(0) # for over arching seq, POP - - sig_chain.append( - [ - "if", - sig_compare, - [ - "seq", - ["seq"] + set_defaults if set_defaults else ["pass"], - ["seq_unchecked"] + default_copiers if default_copiers else ["pass"], - ["goto", function_routine], - ], - ] - ) - - # Function with default parameters. - function_jump_label = f"{sig.name}_{sig.method_id}_skip" - o = LLLnode.from_list( - [ - "seq", - sig_chain, - [ - "seq", - ["goto", function_jump_label], - ["label", function_routine], - ["seq"] - + nonreentrant_pre - + clampers - + [parse_body(c, context) for c in code.body] - + nonreentrant_post - + [["stop"]], - ["label", function_jump_label], - ], - ], - typ=None, - pos=getpos(code), - ) - - else: - # Function without default parameters. - sig_compare, _ = get_sig_statements(sig, getpos(code)) - o = LLLnode.from_list( - [ - "if", - sig_compare, - ["seq"] - + nonreentrant_pre - + clampers - + [parse_body(c, context) for c in code.body] - + nonreentrant_post - + [["stop"]], - ], - typ=None, - pos=getpos(code), - ) - return o diff --git a/vyper/old_codegen/function_definitions/utils.py b/vyper/old_codegen/function_definitions/utils.py index d7204b9e06..87ddd8eb82 100644 --- a/vyper/old_codegen/function_definitions/utils.py +++ b/vyper/old_codegen/function_definitions/utils.py @@ -1,22 +1,3 @@ -from vyper.old_codegen.lll_node import LLLnode - - -# TODO dead code -def get_sig_statements(sig, pos): - method_id_node = LLLnode.from_list(sig.method_id, pos=pos, annotation=f"{sig.sig}") - - if sig.internal: - sig_compare = 0 - private_label = LLLnode.from_list( - ["label", f"priv_{sig.method_id}"], pos=pos, annotation=f"{sig.sig}" - ) - else: - sig_compare = ["eq", "_func_sig", method_id_node] - private_label = ["pass"] - - return sig_compare, private_label - - def get_nonreentrant_lock(func_type): if not func_type.nonreentrant: return [], [] @@ -25,15 +6,3 @@ def get_nonreentrant_lock(func_type): nonreentrant_pre = [["seq", ["assert", ["iszero", ["sload", nkey]]], ["sstore", nkey, 1]]] nonreentrant_post = [["sstore", nkey, 0]] return nonreentrant_pre, nonreentrant_post - - -# TODO dead code -def get_default_names_to_set(primary_sig, default_sig): - """ - Get names for default parameters that require a default value to be assigned. - """ - - current_sig_arg_names = [x.name for x in default_sig.args] - for arg in primary_sig.default_args: - if arg.arg not in current_sig_arg_names: - yield arg.arg diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 2fe59e74e6..6c6fda000c 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -628,22 +628,6 @@ def _check_return_body(node, node_list): ) -# TODO dead -def _return_check(node): - if is_return_from_function(node): - return True - elif isinstance(node, list): - return any(_return_check(stmt) for stmt in node) - elif isinstance(node, vy_ast.If): - if_body_check = _return_check(node.body) - else_body_check = _return_check(node.orelse) - if if_body_check and else_body_check: # both side need to match. - return True - else: - return False - return False - - def mzero(dst, nbytes): # calldatacopy from past-the-end gives zero bytes. # cf. YP H.2 (ops section) with CALLDATACOPY spec. From c6bd13bc8b5aac92dc5e83653537d14838a517fb Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sat, 11 Sep 2021 08:28:24 -0700 Subject: [PATCH 037/163] only return len if it's asked for --- vyper/old_codegen/abi.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index b896bd72e9..cbafb8fcf1 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -381,7 +381,8 @@ def abi_encode(dst, lll_node, pos=None, bufsz=None, returns_len=False): # cast the output buffer to something that make_setter accepts dst = LLLnode(dst, typ=lll_node.typ, location="memory") lll_ret = ["seq_unchecked", make_setter(dst, lll_node, "memory", pos)] - lll_ret.append(parent_abi_t.embedded_static_size()) + if returns_len: + lll_ret.append(parent_abi_t.embedded_static_size()) return LLLnode.from_list(lll_ret, pos=pos, annotation=f"abi_encode {lll_node.typ}") lll_ret = ["seq"] From adc7338b9b989f215f1f604344aff4cc9709eb15 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sat, 11 Sep 2021 09:31:34 -0700 Subject: [PATCH 038/163] wip rewrite jump table --- vyper/ast/signatures/function_signature.py | 2 +- .../function_definitions/common.py | 6 +- .../function_definitions/external_function.py | 19 +++-- vyper/old_codegen/parser.py | 61 +++++++++------ vyper/old_codegen/return_.py | 74 +++++++++---------- 5 files changed, 89 insertions(+), 73 deletions(-) diff --git a/vyper/ast/signatures/function_signature.py b/vyper/ast/signatures/function_signature.py index e23b96c236..b660ac5ee2 100644 --- a/vyper/ast/signatures/function_signature.py +++ b/vyper/ast/signatures/function_signature.py @@ -161,7 +161,7 @@ def from_definition( args.append(FunctionArg(argname, argtyp)) mutability = "nonpayable" # Assume nonpayable by default - nonreentrant_key = "" + nonreentrant_key = None is_internal = None # Update function properties from decorators diff --git a/vyper/old_codegen/function_definitions/common.py b/vyper/old_codegen/function_definitions/common.py index 4f93ac3d1e..f85b66e1fb 100644 --- a/vyper/old_codegen/function_definitions/common.py +++ b/vyper/old_codegen/function_definitions/common.py @@ -12,15 +12,13 @@ from vyper.utils import calc_mem_gas -# TODO dead code - replaced by FunctionSignature method # Is a function the initializer? -def is_initializer(code): +def is_initializer(code: vy_ast.FunctionDef): return code.name == "__init__" -# TODO dead code - replaced by FunctionSignature method # Is a function the default function? -def is_default_func(code): +def is_default_func(code: vy_ast.FunctionDef): return code.name == "__default__" diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index c93e4d5f31..db8f3f65be 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -93,9 +93,15 @@ def handler_for(calldata_kwargs, default_kwargs): return ret +# TODO it would be nice if this returned a data structure which were +# amenable to generating a jump table instead of the linear search for +# method_id we have now. def generate_lll_for_external_function( code: vy_ast.FunctionDef, sig: FunctionSignature, context: Context, check_nonpayable: bool, ) -> LLLnode: + """Return the LLL for an external function. Includes code to inspect the method_id, + enter the function (nonpayable and reentrancy checks) and handle kwargs. + """ func_type = code._metadata["type"] _register_function_args(context, sig) @@ -118,11 +124,13 @@ def generate_lll_for_external_function( body = [parse_body(c, context) for c in code.body] - exit = ([["label", func_type.exit_sequence_label]] - + [nonreentrant_post] - # TODO optimize case where return_type is None: use STOP - + [["return", "pass", "pass"]] # ret_ofst and ret_len stack items passed by function body - ) + + exit = [["label", func_type.exit_sequence_label]] + [nonreentrant_post] + if context.return_type is None: + exit += [["stop"]] + else: + # ret_ofst and ret_len stack items passed by function body; consume using 'pass' + exit += [["return", "pass", "pass"]] ret = (["seq"] + arg_handlers @@ -130,6 +138,7 @@ def generate_lll_for_external_function( + body + exit ) + # TODO special handling for default function if len(kwarg_handlers) == 0: # TODO is this check correct? ret = ["if", ["eq", tbd_mload_method_id, sig.method_id], ret] diff --git a/vyper/old_codegen/parser.py b/vyper/old_codegen/parser.py index f7b536dab6..9e60d0cee3 100644 --- a/vyper/old_codegen/parser.py +++ b/vyper/old_codegen/parser.py @@ -29,7 +29,7 @@ # check that calldatasize is at least 4, otherwise # calldataload will load zeros (cf. yellow paper). ["if", ["lt", "calldatasize", 4], ["goto", "fallback"]], - ["mstore", 28, ["calldataload", 0]], + ["calldatacopy", 28, 0, 4], ] # Store limit constants at fixed addresses in memory. LIMIT_MEMORY_SET: List[Any] = [ @@ -47,6 +47,7 @@ def init_func_init_lll(): def parse_external_interfaces(external_interfaces, global_ctx): for _interfacename in global_ctx._contracts: + # TODO factor me into helper function _interface_defs = global_ctx._contracts[_interfacename] _defnames = [_def.name for _def in _interface_defs] interface = {} @@ -87,16 +88,20 @@ def parse_external_interfaces(external_interfaces, global_ctx): return external_interfaces -def parse_other_functions(o, otherfuncs, sigs, external_interfaces, global_ctx, default_function): +def parse_regular_functions(o, otherfuncs, sigs, external_interfaces, global_ctx, default_function): # check for payable/nonpayable external functions to optimize nonpayable assertions func_types = [i._metadata["type"] for i in global_ctx._defs] mutabilities = [i.mutability for i in func_types if i.visibility == FunctionVisibility.EXTERNAL] - has_payable = next((True for i in mutabilities if i == StateMutability.PAYABLE), False) - has_nonpayable = next((True for i in mutabilities if i != StateMutability.PAYABLE), False) + has_payable = any(i == StateMutability.PAYABLE for i in mutabilities) + has_nonpayable = any(i != StateMutability.PAYABLE for i in mutabilities) + is_default_payable = ( default_function is not None and default_function._metadata["type"].mutability == StateMutability.PAYABLE ) + + # TODO streamline the nonpayable check logic + # when a contract has a payable default function and at least one nonpayable # external function, we must perform the nonpayable check on every function check_per_function = is_default_payable and has_nonpayable @@ -112,21 +117,29 @@ def parse_other_functions(o, otherfuncs, sigs, external_interfaces, global_ctx, func_lll, frame_start, frame_size = generate_lll_for_function( func_node, {**{"self": sigs}, **external_interfaces}, global_ctx, check_per_function ) + if func_type.visibility == FunctionVisibility.INTERNAL: internal_func_sub.append(func_lll) + elif func_type.mutability == StateMutability.PAYABLE: - add_gas += 30 + add_gas += 30 # CMC 20210910 why? payable_func_sub.append(func_lll) + else: external_func_sub.append(func_lll) - add_gas += 30 + add_gas += 30 # CMC 20210910 why? + func_lll.total_gas += add_gas - # update sigs with metadata gathered from compiling the function - for sig in sig_utils.generate_default_arg_sigs(func_node, external_interfaces, global_ctx): - sig.gas = func_lll.total_gas - sig.frame_start = frame_start - sig.frame_size = frame_size - sigs[sig.sig] = sig + + # update sigs with metadata gathered from compiling the function so that + # we can handle calls to self + # TODO we only need to do this for internal functions; external functions + # cannot be called via `self` + sig = FunctionSignature.from_definition(func_node, external_interfaces, global_ctx._custom_structs) + sig.gas = func_lll.total_gas + sig.frame_start = frame_start + sig.frame_size = frame_size + sigs[sig.sig] = sig # generate LLL for fallback function if default_function: @@ -142,6 +155,7 @@ def parse_other_functions(o, otherfuncs, sigs, external_interfaces, global_ctx, if check_per_function: external_seq = ["seq", payable_func_sub, external_func_sub] + else: # payable functions are placed prior to nonpayable functions # and seperated by a nonpayable assertion @@ -182,11 +196,11 @@ def parse_tree_to_lll(global_ctx: GlobalContext) -> Tuple[LLLnode, LLLnode]: {[name for name in _names_events if _names_events.count(name) > 1][0]}""" ) # Initialization function - initfunc = [_def for _def in global_ctx._defs if is_initializer(_def)] + init_function = next((_def for _def in global_ctx._defs if is_initializer(_def)), None) # Default function - defaultfunc = next((i for i in global_ctx._defs if is_default_func(i)), None) - # Regular functions - otherfuncs = [ + default_function = next((i for i in global_ctx._defs if is_default_func(i)), None) + + regular_functions = [ _def for _def in global_ctx._defs if not is_initializer(_def) and not is_default_func(_def) ] @@ -196,23 +210,22 @@ def parse_tree_to_lll(global_ctx: GlobalContext) -> Tuple[LLLnode, LLLnode]: o = ["seq"] if global_ctx._contracts or global_ctx._interfaces: external_interfaces = parse_external_interfaces(external_interfaces, global_ctx) - # If there is an init func... - if initfunc: + + if init_function: o.append(init_func_init_lll()) init_func_lll, _frame_size = generate_lll_for_function( - initfunc[0], {**{"self": sigs}, **external_interfaces}, global_ctx, False, + init_function, {**{"self": sigs}, **external_interfaces}, global_ctx, False, ) o.append(init_func_lll) - # If there are regular functions... - if otherfuncs or defaultfunc: - o, runtime = parse_other_functions( - o, otherfuncs, sigs, external_interfaces, global_ctx, defaultfunc, + if regular_function or defaultfunc: + o, runtime = parse_regular_functions( + o, regular_functions, sigs, external_interfaces, global_ctx, defaultfunc, ) else: runtime = o.copy() - return LLLnode.from_list(o, typ=None), LLLnode.from_list(runtime, typ=None) + return LLLnode.from_list(o), LLLnode.from_list(runtime) def parse_to_lll( diff --git a/vyper/old_codegen/return_.py b/vyper/old_codegen/return_.py index 8ec6f3ac4f..dc1f87424b 100644 --- a/vyper/old_codegen/return_.py +++ b/vyper/old_codegen/return_.py @@ -39,50 +39,46 @@ def make_return_stmt(lll_val: LLLnode, stmt: "Stmt", context: Context) -> LLLnod # helper function def finalize(fill_return_buffer): + # do NOT bypass this. the exit label may do important function cleanup. return LLLnode.from_list( ["seq_unchecked", fill_return_buffer, jump_to_exit], typ=None, pos=_pos, valency=0 ) + if context.return_type is None: + return finalize([]) + if context.is_internal: - if context.return_type is None: - return finalize([]) return finalize(make_setter("return_buffer", lll_val, pos=_pos)) + # we are in an external function. + + return_buffer_ofst = _allocate_return_buffer(context) + # abi-encode the data into the return buffer and jump to the function cleanup code + + # according to the ABI spec, return types are ALWAYS tuples even + # if only one element is being returned. + # https://solidity.readthedocs.io/en/latest/abi-spec.html#function-selector-and-argument-encoding + # "and the return values v_1, ..., v_k of f are encoded as + # + # enc((v_1, ..., v_k)) + # i.e. the values are combined into a tuple and encoded. + # " + # therefore, wrap it in a tuple if it's not already a tuple. + # for example, `bytes` is returned as abi-encoded (bytes,) + # and `(bytes,)` is returned as abi-encoded ((bytes,),) + + if isinstance(lll_val.typ, TupleType) and len(lll_val.typ.members) > 1: + # returning something like (int, bytes, string) + pass else: - # we are in an external function. - - if context.return_type is None: - # push arguments onto the stack for RETURN opcode - # TODO optimize this case: just use STOP - return finalize(["seq_unchecked", 0, 0]) - - return_buffer_ofst = _allocate_return_buffer(context) - # abi-encode the data into the return buffer and jump to the function cleanup code - - # according to the ABI spec, return types are ALWAYS tuples even - # if only one element is being returned. - # https://solidity.readthedocs.io/en/latest/abi-spec.html#function-selector-and-argument-encoding - # "and the return values v_1, ..., v_k of f are encoded as - # - # enc((v_1, ..., v_k)) - # i.e. the values are combined into a tuple and encoded. - # " - # therefore, wrap it in a tuple if it's not already a tuple. - # for example, `bytes` is returned as abi-encoded (bytes,) - # and `(bytes,)` is returned as abi-encoded ((bytes,),) - - if isinstance(lll_val.typ, TupleType) and len(lll_val.typ.members) > 1: - # returning something like (int, bytes, string) - pass - else: - # `-> (bytes,)` gets returned as ((bytes,),) - # In general `-> X` gets returned as (X,) - # (Sorry this is so confusing. I didn't make these rules.) - lll_val = lll_tuple_from_args([lll_val]) - - # encode_out is cleverly a sequence which does the abi-encoding and - # also returns the length of the output as a stack element - encode_out = abi_encode(return_buffer_ofst, lll_val, pos=_pos, returns_len=True) - - # fill the return buffer and push the location and length onto the stack - return finalize(["seq_unchecked", encode_out, return_buffer_offset]) + # `-> (bytes,)` gets returned as ((bytes,),) + # In general `-> X` gets returned as (X,) + # (Sorry this is so confusing. I didn't make these rules.) + lll_val = lll_tuple_from_args([lll_val]) + + # encode_out is cleverly a sequence which does the abi-encoding and + # also returns the length of the output as a stack element + encode_out = abi_encode(return_buffer_ofst, lll_val, pos=_pos, returns_len=True) + + # fill the return buffer and push the location and length onto the stack + return finalize(["seq_unchecked", encode_out, return_buffer_offset]) From 7a816e379779a0a0e5a33dea2a97208be52a4cb8 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sat, 11 Sep 2021 10:02:20 -0700 Subject: [PATCH 039/163] wip polish --- vyper/ast/signatures/function_signature.py | 17 +++--- vyper/ast/signatures/interface.py | 15 +++++- vyper/ast/signatures/sig_utils.py | 54 ------------------- vyper/old_codegen/expr.py | 1 - .../function_definitions/common.py | 1 + .../function_definitions/external_function.py | 22 ++------ .../function_definitions/internal_function.py | 23 +++----- vyper/old_codegen/parser.py | 24 +++++---- vyper/semantics/validation/local.py | 1 + 9 files changed, 47 insertions(+), 111 deletions(-) delete mode 100644 vyper/ast/signatures/sig_utils.py diff --git a/vyper/ast/signatures/function_signature.py b/vyper/ast/signatures/function_signature.py index b660ac5ee2..2307052b41 100644 --- a/vyper/ast/signatures/function_signature.py +++ b/vyper/ast/signatures/function_signature.py @@ -1,26 +1,23 @@ import math -from collections import Counter from dataclasses import dataclass from typing import List from vyper import ast as vy_ast -from vyper.exceptions import FunctionDeclarationException, StructureException -from vyper.old_codegen.lll_node import LLLnode -from vyper.old_codegen.parser_utils import check_single_exit, getpos +from vyper.exceptions import StructureException, CompilerPanic +from vyper.old_codegen.parser_utils import check_single_exit from vyper.old_codegen.types import ( NodeType, - ByteArrayLike, - TupleType, canonicalize_type, get_size_of_type, parse_type, ) -from vyper.utils import fourbytes_to_int, keccak256, mkalphanum +from vyper.utils import mkalphanum from functools import cached_property # Function variable # TODO move to context.py +# TODO use dataclass class VariableRecord: def __init__( self, @@ -151,7 +148,7 @@ def from_definition( if custom_structs is None: custom_structs = {} - name = code.name + name = func_ast.name args = [] for arg in func_ast.args.args: @@ -225,12 +222,12 @@ def _check(cond, s="Unreachable"): _check(len(sig.base_args) <= len(args_lll) <= len(sig.args)) # more sanity check, that the types match - _check(all(l.typ == r.typ for l, r in zip(args_lll, sig.args))) + _check(all(l.typ == r.typ for (l, r) in zip(args_lll, sig.args))) num_provided_args = len(args_lll) total_args = len(sig.args) num_kwargs = len(sig.kwargs) - args_needed = total_args - num_provided + args_needed = total_args - num_provided_args kw_vals = sig.kwarg_values.items()[: num_kwargs - args_needed] diff --git a/vyper/ast/signatures/interface.py b/vyper/ast/signatures/interface.py index fbb2e41f1e..d15554e801 100644 --- a/vyper/ast/signatures/interface.py +++ b/vyper/ast/signatures/interface.py @@ -5,7 +5,6 @@ import vyper.builtin_interfaces from vyper import ast as vy_ast -from vyper.ast.signatures import sig_utils from vyper.ast.signatures.function_signature import FunctionSignature from vyper.exceptions import StructureException from vyper.old_codegen.global_context import GlobalContext @@ -94,6 +93,18 @@ def mk_full_signature_from_json(abi): return sigs +def _get_external_signatures(global_ctx, sig_formatter=lambda x: x): + ret = [] + + for code in global_ctx._defs: + sig = FunctionSignature.from_definition( + code, sigs=global_ctx._contracts, custom_structs=global_ctx._structs, + ) + if not sig.internal: + ret.append(sig_formatter(sig)) + return ret + + def extract_sigs(sig_code, interface_name=None): if sig_code["type"] == "vyper": interface_ast = [ @@ -116,7 +127,7 @@ def extract_sigs(sig_code, interface_name=None): or (isinstance(i, vy_ast.AnnAssign) and i.target.id != "implements") ] global_ctx = GlobalContext.get_global_context(interface_ast) - return sig_utils.mk_full_signature(global_ctx, sig_formatter=lambda x: x) + return _get_external_signatures(global_ctx) elif sig_code["type"] == "json": return mk_full_signature_from_json(sig_code["code"]) else: diff --git a/vyper/ast/signatures/sig_utils.py b/vyper/ast/signatures/sig_utils.py deleted file mode 100644 index 83c67ca9d9..0000000000 --- a/vyper/ast/signatures/sig_utils.py +++ /dev/null @@ -1,54 +0,0 @@ -import copy - -from vyper.ast.signatures.function_signature import FunctionSignature - -# Generate default argument function signatures. -def generate_default_arg_sigs(code, interfaces, global_ctx): - # generate all sigs, and attach. - if total_default_args == 0: - return [ - FunctionSignature.from_definition( - code, sigs=interfaces, custom_structs=global_ctx._structs, - ) - ] - - # Generate a list of default function combinations. - row = [False] * (total_default_args) - table = [row.copy()] - for i in range(total_default_args): - row[i] = True - table.append(row.copy()) - - default_sig_strs = [] - sig_fun_defs = [] - for truth_row in table: - new_code = copy.deepcopy(code) - new_code.args.args = copy.deepcopy(base_args) - new_code.args.default = [] - # Add necessary default args. - for idx, val in enumerate(truth_row): - if val is True: - new_code.args.args.append(default_args[idx]) - sig = FunctionSignature.from_definition( - new_code, sigs=interfaces, custom_structs=global_ctx._structs, - ) - default_sig_strs.append(sig.sig) - sig_fun_defs.append(sig) - - return sig_fun_defs - - -# Get ABI signature -def mk_full_signature(global_ctx, sig_formatter): - o = [] - - # Produce function signatures. - for code in global_ctx._defs: - sig = FunctionSignature.from_definition( - code, sigs=global_ctx._contracts, custom_structs=global_ctx._structs, - ) - if not sig.internal: - default_sigs = generate_default_arg_sigs(code, global_ctx._contracts, global_ctx) - for s in default_sigs: - o.append(sig_formatter(s)) - return o diff --git a/vyper/old_codegen/expr.py b/vyper/old_codegen/expr.py index ea6b7dc270..a97ccbc4db 100644 --- a/vyper/old_codegen/expr.py +++ b/vyper/old_codegen/expr.py @@ -997,7 +997,6 @@ def parse_Tuple(self): multi_lll = LLLnode.from_list(["multi"] + tuple_elements, typ=typ, pos=getpos(self.expr)) return multi_lll - @staticmethod def struct_literals(expr, name, context): member_subs = {} diff --git a/vyper/old_codegen/function_definitions/common.py b/vyper/old_codegen/function_definitions/common.py index f85b66e1fb..99abce6a5d 100644 --- a/vyper/old_codegen/function_definitions/common.py +++ b/vyper/old_codegen/function_definitions/common.py @@ -1,4 +1,5 @@ # can't use from [module] import [object] because it breaks mocks in testing +import vyper.ast as vy_ast from vyper.ast.signatures import FunctionSignature from vyper.old_codegen import context as ctx from vyper.old_codegen.context import Constancy diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index db8f3f65be..0635f6c04a 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -1,16 +1,10 @@ from typing import Any, List, Union from vyper import ast as vy_ast -from vyper.ast.signatures import sig_utils from vyper.ast.signatures.function_signature import FunctionSignature -from vyper.old_codegen.arg_clamps import make_arg_clamper from vyper.old_codegen.context import Context, VariableRecord from vyper.old_codegen.expr import Expr -from vyper.old_codegen.function_definitions.utils import ( - get_default_names_to_set, - get_nonreentrant_lock, - get_sig_statements, -) +from vyper.old_codegen.function_definitions.utils import get_nonreentrant_lock from vyper.old_codegen.lll_node import LLLnode from vyper.old_codegen.parser_utils import getpos, make_setter from vyper.old_codegen.stmt import parse_body @@ -60,10 +54,10 @@ def handler_for(calldata_kwargs, default_kwargs): calldata_args = lazy_abi_decode(calldata_args_t, base_args_location) - assert calldata_args.value == "multi" # sanity check + assert calldata_args.value == "multi" # sanity check # extract just the kwargs from the ABI payload # TODO come up with a better name for these variables - calldata_kwargs = calldata_args.args[:len(base_args)] + calldata_kwargs = calldata_args.args[: len(base_args)] # a sequence of statements to strictify kwargs into memory ret = ["seq"] @@ -124,7 +118,6 @@ def generate_lll_for_external_function( body = [parse_body(c, context) for c in code.body] - exit = [["label", func_type.exit_sequence_label]] + [nonreentrant_post] if context.return_type is None: exit += [["stop"]] @@ -132,15 +125,10 @@ def generate_lll_for_external_function( # ret_ofst and ret_len stack items passed by function body; consume using 'pass' exit += [["return", "pass", "pass"]] - ret = (["seq"] - + arg_handlers - + entrance - + body - + exit - ) + ret = ["seq"] + arg_handlers + entrance + body + exit # TODO special handling for default function - if len(kwarg_handlers) == 0: # TODO is this check correct? + if len(kwarg_handlers) == 0: # TODO is this check correct? ret = ["if", ["eq", tbd_mload_method_id, sig.method_id], ret] return LLLnode.from_list(ret, pos=getpos(code)) diff --git a/vyper/old_codegen/function_definitions/internal_function.py b/vyper/old_codegen/function_definitions/internal_function.py index 18fc1f9e14..e36bdff13c 100644 --- a/vyper/old_codegen/function_definitions/internal_function.py +++ b/vyper/old_codegen/function_definitions/internal_function.py @@ -1,16 +1,11 @@ from typing import Any, List from vyper import ast as vy_ast -from vyper.ast.signatures import FunctionSignature, sig_utils +from vyper.ast.signatures import FunctionSignature from vyper.ast.signatures.function_signature import VariableRecord from vyper.old_codegen.context import Context from vyper.old_codegen.expr import Expr -from vyper.old_codegen.function_definitions.utils import ( - get_default_names_to_set, - get_nonreentrant_lock, - get_sig_statements, - make_unpacker, -) +from vyper.old_codegen.function_definitions.utils import get_nonreentrant_lock from vyper.old_codegen.lll_node import LLLnode from vyper.old_codegen.parser_utils import getpos, make_setter from vyper.old_codegen.stmt import parse_body @@ -74,11 +69,11 @@ def mkidentifier(s): # internal functions without return types need to jump back to the calling # function, as there is no guarantee there is a user-provided return # statement (which would generate the jump) - stop_func = [["jump", "pass"]] # was passed in via stack + stop_func = [["jump", "pass"]] # was passed in via stack enter = nonreentrant_pre - body = parse_body(c, context) for c in code.body + body = [parse_body(c, context) for c in code.body] if sig.return_type is not None: # name the variable that was passed via stack @@ -87,11 +82,5 @@ def mkidentifier(s): exit = cleanup_label + nonreentrant_post + stop_func return LLLnode.from_list( - ["seq"] - + ["label", function_label] - + enter - + body - + exit, - typ=None, - pos=getpos(code), - ) + ["seq"] + ["label", function_label] + enter + body + exit, typ=None, pos=getpos(code), + ) diff --git a/vyper/old_codegen/parser.py b/vyper/old_codegen/parser.py index 9e60d0cee3..ee6064f292 100644 --- a/vyper/old_codegen/parser.py +++ b/vyper/old_codegen/parser.py @@ -1,7 +1,6 @@ from typing import Any, List, Optional, Tuple from vyper import ast as vy_ast -from vyper.ast.signatures import sig_utils from vyper.ast.signatures.function_signature import FunctionSignature from vyper.exceptions import ( EventDeclarationException, @@ -88,7 +87,9 @@ def parse_external_interfaces(external_interfaces, global_ctx): return external_interfaces -def parse_regular_functions(o, otherfuncs, sigs, external_interfaces, global_ctx, default_function): +def parse_regular_functions( + o, regular_functions, sigs, external_interfaces, global_ctx, default_function +): # check for payable/nonpayable external functions to optimize nonpayable assertions func_types = [i._metadata["type"] for i in global_ctx._defs] mutabilities = [i.mutability for i in func_types if i.visibility == FunctionVisibility.EXTERNAL] @@ -112,7 +113,7 @@ def parse_regular_functions(o, otherfuncs, sigs, external_interfaces, global_ctx internal_func_sub = ["seq"] add_gas = func_init_lll().gas - for func_node in otherfuncs: + for func_node in regular_functions: func_type = func_node._metadata["type"] func_lll, frame_start, frame_size = generate_lll_for_function( func_node, {**{"self": sigs}, **external_interfaces}, global_ctx, check_per_function @@ -122,12 +123,12 @@ def parse_regular_functions(o, otherfuncs, sigs, external_interfaces, global_ctx internal_func_sub.append(func_lll) elif func_type.mutability == StateMutability.PAYABLE: - add_gas += 30 # CMC 20210910 why? + add_gas += 30 # CMC 20210910 why? payable_func_sub.append(func_lll) else: external_func_sub.append(func_lll) - add_gas += 30 # CMC 20210910 why? + add_gas += 30 # CMC 20210910 why? func_lll.total_gas += add_gas @@ -135,7 +136,9 @@ def parse_regular_functions(o, otherfuncs, sigs, external_interfaces, global_ctx # we can handle calls to self # TODO we only need to do this for internal functions; external functions # cannot be called via `self` - sig = FunctionSignature.from_definition(func_node, external_interfaces, global_ctx._custom_structs) + sig = FunctionSignature.from_definition( + func_node, external_interfaces, global_ctx._custom_structs + ) sig.gas = func_lll.total_gas sig.frame_start = frame_start sig.frame_size = frame_size @@ -167,7 +170,7 @@ def parse_regular_functions(o, otherfuncs, sigs, external_interfaces, global_ctx # bytecode is organized by: external functions, fallback fn, internal functions # this way we save gas and reduce bytecode by not jumping over internal functions - main_seq = [ + runtime = [ "seq", func_init_lll(), ["with", "_func_sig", ["mload", 0], external_seq], @@ -175,8 +178,9 @@ def parse_regular_functions(o, otherfuncs, sigs, external_interfaces, global_ctx internal_func_sub, ] - o.append(["return", 0, ["lll", main_seq, 0]]) - return o, main_seq + # TODO CMC 20210911 why does the lll have a trailing 0 + o.append(["return", 0, ["lll", runtime, 0]]) + return o, runtime # Main python parse tree => LLL method @@ -220,7 +224,7 @@ def parse_tree_to_lll(global_ctx: GlobalContext) -> Tuple[LLLnode, LLLnode]: if regular_function or defaultfunc: o, runtime = parse_regular_functions( - o, regular_functions, sigs, external_interfaces, global_ctx, defaultfunc, + o, regular_functions, sigs, external_interfaces, global_ctx, default_function, ) else: runtime = o.copy() diff --git a/vyper/semantics/validation/local.py b/vyper/semantics/validation/local.py index 861517add6..b02c7ff585 100644 --- a/vyper/semantics/validation/local.py +++ b/vyper/semantics/validation/local.py @@ -19,6 +19,7 @@ VariableDeclarationException, VyperException, ) + # TODO consolidate some of these imports from vyper.semantics.environment import ( CONSTANT_ENVIRONMENT_VARS, From e58050331612bce67f61eb03993bf4084b308e7c Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sat, 11 Sep 2021 10:06:02 -0700 Subject: [PATCH 040/163] add back a missing exception --- vyper/ast/signatures/function_signature.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vyper/ast/signatures/function_signature.py b/vyper/ast/signatures/function_signature.py index 2307052b41..cb1e03455f 100644 --- a/vyper/ast/signatures/function_signature.py +++ b/vyper/ast/signatures/function_signature.py @@ -233,7 +233,10 @@ def _check(cond, s="Unreachable"): return sig, kw_vals - _check(False) + raise FunctionDeclarationException( + "Function not declared yet (reminder: functions cannot " + f"call functions later in code than themselves): {method_name}" + ) @property def is_default_func(self): From 9d1a455dbd403a3009f95a5f8f272d89d0ff6881 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sat, 11 Sep 2021 10:41:55 -0700 Subject: [PATCH 041/163] wip polish --- vyper/ast/signatures/function_signature.py | 41 +-------- vyper/old_codegen/abi.py | 1 + vyper/old_codegen/arg_clamps.py | 62 +++++-------- vyper/old_codegen/context.py | 39 +++++++- vyper/old_codegen/external_call.py | 75 +++++++++------- .../function_definitions/__init__.py | 2 +- .../function_definitions/common.py | 15 ++-- .../function_definitions/external_function.py | 90 +++++++++++-------- .../function_definitions/internal_function.py | 18 +--- vyper/old_codegen/parser.py | 9 +- vyper/old_codegen/parser_utils.py | 19 ++-- vyper/old_codegen/return_.py | 22 ++--- vyper/old_codegen/self_call.py | 41 ++++----- vyper/old_codegen/stmt.py | 13 +-- vyper/old_codegen/types/types.py | 16 +--- vyper/semantics/types/value/array_value.py | 2 +- vyper/semantics/validation/local.py | 1 - 17 files changed, 218 insertions(+), 248 deletions(-) diff --git a/vyper/ast/signatures/function_signature.py b/vyper/ast/signatures/function_signature.py index cb1e03455f..2e11da9646 100644 --- a/vyper/ast/signatures/function_signature.py +++ b/vyper/ast/signatures/function_signature.py @@ -1,10 +1,10 @@ import math from dataclasses import dataclass +from functools import cached_property from typing import List from vyper import ast as vy_ast -from vyper.exceptions import StructureException, CompilerPanic -from vyper.old_codegen.parser_utils import check_single_exit +from vyper.exceptions import StructureException from vyper.old_codegen.types import ( NodeType, canonicalize_type, @@ -12,7 +12,6 @@ parse_type, ) from vyper.utils import mkalphanum -from functools import cached_property # Function variable @@ -202,42 +201,6 @@ def from_definition( is_from_json, ) - @classmethod - def lookup_internal_function(cls, method_name, args_lll, context): - """ - Using a list of args, find the internal method to use, and - the kwargs which need to be filled in by the compiler - """ - - def _check(cond, s="Unreachable"): - if not cond: - raise CompilerPanic(s) - - for sig in context.sigs["self"]: - if sig.name != method_name: - continue - - _check(sig.internal) # sanity check - # should have been caught during type checking, sanity check anyway - _check(len(sig.base_args) <= len(args_lll) <= len(sig.args)) - - # more sanity check, that the types match - _check(all(l.typ == r.typ for (l, r) in zip(args_lll, sig.args))) - - num_provided_args = len(args_lll) - total_args = len(sig.args) - num_kwargs = len(sig.kwargs) - args_needed = total_args - num_provided_args - - kw_vals = sig.kwarg_values.items()[: num_kwargs - args_needed] - - return sig, kw_vals - - raise FunctionDeclarationException( - "Function not declared yet (reminder: functions cannot " - f"call functions later in code than themselves): {method_name}" - ) - @property def is_default_func(self): return self.name == "__default__" diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index cbafb8fcf1..52011509de 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -3,6 +3,7 @@ from vyper.old_codegen.lll_node import LLLnode from vyper.old_codegen.parser_utils import ( add_variable_offset, + clamp_basetype, make_setter, unwrap_location, zero_pad, diff --git a/vyper/old_codegen/arg_clamps.py b/vyper/old_codegen/arg_clamps.py index af2a7bd11f..67e3caa7fd 100644 --- a/vyper/old_codegen/arg_clamps.py +++ b/vyper/old_codegen/arg_clamps.py @@ -1,57 +1,37 @@ -import functools -import uuid - from vyper.evm.opcodes import version_check -from vyper.old_codegen.lll_node import LLLnode -from vyper.old_codegen.types.types import ( - ByteArrayLike, - ListType, - get_size_of_type, - is_base_type, -) -from vyper.utils import MemoryPositions + +# TODO this whole module should be replaced with parser_utils.clamp_basetype def _shr(x, bits): if version_check(begin="constantinople"): - return ["shr", x, bits] - return ["div", x, ["pow", 2, bits]] + return ["shr", bits, x] + return ["div", x, ["exp", 2, bits]] def _sar(x, bits): if version_check(begin="constantinople"): - return ["sar", x, bits] - return ["sdiv", x, ["pow", 2, bits]] + return ["sar", bits, x] + return ["sdiv", x, ["exp", 2, bits]] def address_clamp(lll_node): - if version_check(begin="constantinople"): - return ["assert", ["iszero", ["shr", 160, lll_node]]] - else: - return ["uclamplt", lll_node, ["mload", MemoryPositions.ADDRSIZE]] + return ["assert", ["iszero", _shr(lll_node, 160)]] def int128_clamp(lll_node): - if version_check(begin="constantinople"): - return [ - "with", + return [ + "with", + "_val", + lll_node, + [ + "seq", + # if _val is in bounds, + # _val >>> 127 == 0 for positive _val + # _val >>> 127 == -1 for negative _val + # -1 and 0 are the only numbers which are unchanged by sar, + # so sar'ing (_val>>>127) one more bit should leave it unchanged. + ["assert", ["eq", _sar("_val", 128), _sar("_val", 127)]], "_val", - lll_node, - [ - "seq", - # if _val is in bounds, - # _val >>> 127 == 0 for positive _val - # _val >>> 127 == -1 for negative _val - # -1 and 0 are the only numbers which are unchanged by sar, - # so sar'ing (_val>>>127) one more bit should leave it unchanged. - ["assert", ["eq", ["sar", 128, "_val"], ["sar", 127, "_val"]]], - "_val", - ], - ] - else: - return [ - "clamp", - ["mload", MemoryPositions.MIN_INT128], - lll_node, - ["mload", MemoryPositions.MAX_INT128], - ] + ], + ] diff --git a/vyper/old_codegen/context.py b/vyper/old_codegen/context.py index 1a4712c491..6ba8219fb9 100644 --- a/vyper/old_codegen/context.py +++ b/vyper/old_codegen/context.py @@ -3,7 +3,7 @@ from vyper.ast import VyperNode from vyper.ast.signatures.function_signature import VariableRecord -from vyper.exceptions import CompilerPanic +from vyper.exceptions import CompilerPanic, FunctionDeclarationException from vyper.old_codegen.types import NodeType, get_size_of_type @@ -73,7 +73,7 @@ def __init__( def is_constant(self): return self.constancy is Constancy.Constant or self.in_assertion or self.in_range_expr - def register_callee(frame_size): + def register_callee(self, frame_size): self._callee_frame_sizes.append(frame_size) # @@ -209,6 +209,41 @@ def new_internal_variable(self, typ: NodeType) -> int: def parse_type(self, ast_node, location): return self.global_ctx.parse_type(ast_node, location) + def lookup_internal_function(self, method_name, args_lll): + # TODO is this the right module for me? + """ + Using a list of args, find the internal method to use, and + the kwargs which need to be filled in by the compiler + """ + + def _check(cond, s="Unreachable"): + if not cond: + raise CompilerPanic(s) + + sig = next((sig for sig in self.sigs["self"] if sig.name == method_name), None) + if sig is None: + raise FunctionDeclarationException( + "Function does not exist or has not been declared yet " + "(reminder: functions cannot call functions later in code " + f"than themselves): {method_name}" + ) + + _check(sig.internal) # sanity check + # should have been caught during type checking, sanity check anyway + _check(len(sig.base_args) <= len(args_lll) <= len(sig.args)) + + # more sanity check, that the types match + _check(all(l.typ == r.typ for (l, r) in zip(args_lll, sig.args))) + + num_provided_args = len(args_lll) + total_args = len(sig.args) + num_kwargs = len(sig.kwargs) + args_needed = total_args - num_provided_args + + kw_vals = sig.kwarg_values.items()[: num_kwargs - args_needed] + + return sig, kw_vals + # Pretty print constancy for error messages def pp_constancy(self): if self.in_assertion: diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index 1f578ca77c..5622b90921 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -4,24 +4,13 @@ StructureException, TypeCheckFailure, ) -from vyper.old_codegen.abi import abi_encode, abi_decode, abi_type_of +from vyper.old_codegen.abi import abi_encode, abi_type_of, lazy_abi_decode from vyper.old_codegen.lll_node import LLLnode -from vyper.old_codegen.parser_utils import ( - getpos, - unwrap_location, -) -from vyper.old_codegen.types import ( - BaseType, - ByteArrayLike, - ListType, - TupleLike, - get_size_of_type, - get_static_size_of_type, - has_dynamic_data, -) +from vyper.old_codegen.parser_utils import getpos, unwrap_location +from vyper.old_codegen.types import TupleType, get_type_for_exact_size -def pack_arguments(sig, args, buf): +def _pack_arguments(sig, args, context, pos): # abi encoding just treats all args as a big tuple args_tuple_t = TupleType([x.typ for x in args]) args_as_tuple = LLLnode.from_list(["multi"] + [x for x in args], typ=args_tuple_t) @@ -30,7 +19,7 @@ def pack_arguments(sig, args, buf): maxlen = args_abi_t.size_bound() maxlen += 32 # padding for the method id - buf_t = ByteArrayType(maxlen=maxlen) + buf_t = get_type_for_exact_size(maxlen) buf = context.new_internal_variable(buf_t) args_ofst = buf + 28 @@ -50,12 +39,12 @@ def pack_arguments(sig, args, buf): return mstore_method_id + [encode_args], args_ofst, args_len -def unpack_returndata(sig, context): +def _unpack_returndata(sig, context): return_t = abi_type_of(sig.output_type) min_return_size = return_t.static_size() maxlen = return_t.size_bound() - buf_t = ByteArrayType(maxlen=maxlen) + buf_t = get_type_for_exact_size(maxlen) buf = context.new_internal_variable(buf_t) ret_ofst = buf ret_len = maxlen @@ -65,14 +54,12 @@ def unpack_returndata(sig, context): # TODO assert returndatasize <= maxlen # abi_decode has appropriate clampers for the individual members of the return type - ret += [abi_decode_lazy(buf)] + ret += [lazy_abi_decode(buf)] return ret, ret_ofst, ret_len -def external_call(node, context, interface_name, contract_address, value=None, gas=None): - method_name = node.func.attr - sig = context.sigs[interface_name][method_name] +def _external_call_helper(contract_address, sig, args_lll, context, pos=None, value=None, gas=None): if value is None: value = 0 @@ -80,20 +67,20 @@ def external_call(node, context, interface_name, contract_address, value=None, g gas = "gas" # sanity check - assert len(signature.args) == len(args) + assert len(sig.args) == len(args_lll) if context.is_constant() and sig.mutability not in ("view", "pure"): # TODO is this already done in type checker? raise StateAccessViolation( - f"May not call state modifying function '{method_name}' " + f"May not call state modifying function '{sig.name}' " f"within {context.pp_constancy()}.", - node, + pos, ) sub = ["seq"] - arg_packer, args_ofst, args_len = pack_arguments(sig, args, context) - ret_unpacker, ret_ofst, ret_len = unpack_arguments(sig, context) + arg_packer, args_ofst, args_len = _pack_arguments(sig, args_lll, context, pos) + ret_unpacker, ret_ofst, ret_len = _unpack_returndata(sig, context, pos) sub += arg_packer @@ -114,13 +101,14 @@ def external_call(node, context, interface_name, contract_address, value=None, g if sig.return_type is not None: sub += ret_unpacker - return LLLnode.from_list(sub, typ=sig.return_type, location="memory", pos=getpos(node)) + return LLLnode.from_list(sub, typ=sig.return_type, location="memory", pos=pos) # TODO push me up to expr.py def get_gas_and_value(stmt_expr, context): - # circular import! - from vyper.old_codegen.expr import Expr + from vyper.old_codegen.expr import ( + Expr, # TODO rethink this circular import + ) value, gas = None, None for kw in stmt_expr.keywords: @@ -133,24 +121,36 @@ def get_gas_and_value(stmt_expr, context): return value, gas -def make_external_call(stmt_expr, context): +def lll_for_external_call(stmt_expr, context): + from vyper.old_codegen.expr import ( + Expr, # TODO rethink this circular import + ) + + pos = getpos(stmt_expr) value, gas = get_gas_and_value(stmt_expr, context) + args_lll = [Expr(x, context).lll_node for x in stmt_expr.args] if isinstance(stmt_expr.func, vy_ast.Attribute) and isinstance( stmt_expr.func.value, vy_ast.Call ): # e.g. `Foo(address).bar()` + # sanity check + assert len(stmt_expr.func.value.args) == 1 contract_name = stmt_expr.func.value.func.id contract_address = Expr.parse_value_expr(stmt_expr.func.value.args[0], context) elif ( isinstance(stmt_expr.func.value, vy_ast.Attribute) and stmt_expr.func.value.attr in context.globals + # TODO check for self? and hasattr(context.globals[stmt_expr.func.value.attr].typ, "name") ): # e.g. `self.foo.bar()` + # sanity check + assert stmt_expr.func.value.id == "self" + contract_name = context.globals[stmt_expr.func.value.attr].typ.name type_ = stmt_expr.func.value._metadata["type"] var = context.globals[stmt_expr.func.value.attr] @@ -159,9 +159,16 @@ def make_external_call(stmt_expr, context): type_.position.position, typ=var.typ, location="storage", - pos=getpos(stmt_expr), + pos=pos, annotation="self." + stmt_expr.func.value.attr, ) ) - - return external_call(stmt_expr, context, contract_name, contract_address, value=value, gas=gas,) + else: + # TODO catch this during type checking + raise StructureException("Unsupported operator.", stmt_expr) + + method_name = stmt_expr.func.attr + sig = context.sigs[contract_name][method_name] + return _external_call_helper( + contract_address, sig, args_lll, context, pos, value=value, gas=gas, + ) diff --git a/vyper/old_codegen/function_definitions/__init__.py b/vyper/old_codegen/function_definitions/__init__.py index b78d58a9f3..bc757c9f68 100644 --- a/vyper/old_codegen/function_definitions/__init__.py +++ b/vyper/old_codegen/function_definitions/__init__.py @@ -1,5 +1,5 @@ from .common import ( # noqa + generate_lll_for_function, is_default_func, is_initializer, - generate_lll_for_function, ) diff --git a/vyper/old_codegen/function_definitions/common.py b/vyper/old_codegen/function_definitions/common.py index 99abce6a5d..86b15d2b88 100644 --- a/vyper/old_codegen/function_definitions/common.py +++ b/vyper/old_codegen/function_definitions/common.py @@ -1,4 +1,6 @@ # can't use from [module] import [object] because it breaks mocks in testing +import copy + import vyper.ast as vy_ast from vyper.ast.signatures import FunctionSignature from vyper.old_codegen import context as ctx @@ -10,6 +12,7 @@ generate_lll_for_internal_function, ) from vyper.old_codegen.memory_allocator import MemoryAllocator +from vyper.old_codegen.parser_utils import check_single_exit from vyper.utils import calc_mem_gas @@ -45,11 +48,13 @@ def generate_lll_for_function(code, sigs, global_ctx, check_nonpayable, _vars=No # to see what the max frame size of any callee in the function was, # then we run the codegen again with the max frame size as # the start of the frame for this function. - def _run_pass(memory_allocator=None): + def _run_pass(code, memory_allocator=None): # Create a local (per function) context. if memory_allocator is None: memory_allocator = MemoryAllocator() - _vars = _vars.copy() # these will get clobbered in produce_* functions + nonlocal _vars + _vars = _vars.copy() # these will get clobbered in called functions + nonlocal sig sig = copy.deepcopy(sig) # just in case context = ctx.Context( vars=_vars, @@ -67,11 +72,9 @@ def _run_pass(memory_allocator=None): ) if sig.internal: - o = parse_internal_function(code=code, sig=sig, context=context,) + o = generate_lll_for_internal_function(code, sig, context) else: - o = parse_external_function( - code=code, sig=sig, context=context, check_nonpayable=check_nonpayable - ) + o = generate_lll_for_external_function(code, sig, context, check_nonpayable) return o, context _, context = _run_pass(None) diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index 0635f6c04a..d995ca89c3 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -1,87 +1,104 @@ -from typing import Any, List, Union +from typing import Any +import vyper.utils as util from vyper import ast as vy_ast from vyper.ast.signatures.function_signature import FunctionSignature -from vyper.old_codegen.context import Context, VariableRecord +from vyper.old_codegen.abi import lazy_abi_decode +from vyper.old_codegen.context import Context from vyper.old_codegen.expr import Expr from vyper.old_codegen.function_definitions.utils import get_nonreentrant_lock from vyper.old_codegen.lll_node import LLLnode from vyper.old_codegen.parser_utils import getpos, make_setter from vyper.old_codegen.stmt import parse_body -from vyper.old_codegen.types.types import ByteArrayLike, get_size_of_type -from vyper.utils import MemoryPositions +from vyper.old_codegen.types.types import TupleType, canonicalize_type + # register function args with the local calling context. # also allocate the ones that live in memory (i.e. kwargs) def _register_function_args(context: Context, sig: FunctionSignature): - if len(args) == 0: + if len(sig.args) == 0: return + base_args_t = TupleType([arg.typ for arg in sig.base_args]) + # tuple with the abi_encoded args if sig.is_init_func(): - base_args_location = LLLnode("~codelen", location="code", typ=tbd_base_args_type) + base_args_location = LLLnode("~codelen", location="code", typ=base_args_t) else: - base_args_location = LLLnode(4, location="calldata", typ=tbd_base_args_type) + base_args_location = LLLnode(4, location="calldata", typ=base_args_t) - base_args = lazy_abi_decode(tbd_base_args_type, base_args_location) + base_args = lazy_abi_decode(base_args_t, base_args_location) - assert base_args.value == "multi", "you've been bad" + assert base_args.value == "multi", "lazy_abi_decode did not return multi" + base_args = base_args.args # the (lazily) decoded values - for (argname, arg_lll) in zip(tbd_argnames, base_args.args): # the actual values + assert len(base_args) == len(sig.base_args) + for (arg, arg_lll) in zip(sig.base_args, base_args.args): # the actual values + assert arg.typ == arg_lll.typ # register the record in the local namespace - context.vars[argname] = LLLnode(arg_lll, location=location) + context.vars[arg.name] = LLLnode(arg_lll, location=base_args_location) + + +def _base_entry_point(sig): + return f"{sig.base_method_id}_entry" -def _generate_kwarg_handlers(context: Context, sig): +def _generate_kwarg_handlers(context: Context, sig: FunctionSignature, pos: Any): # generate kwarg handlers. # since they might come in thru calldata or be default, # allocate them in memory and then fill it in based on calldata or default, # depending on the signature - ret = [] - def handler_for(calldata_kwargs, default_kwargs): default_kwargs = [Expr(x, context).lll_node for x in default_kwargs] - calldata_args = base_args + calldata_kwargs + calldata_args = sig.base_args + calldata_kwargs calldata_args_t = TupleType(list(arg.typ for arg in calldata_args)) - sig = func_name + canonicalize_type(calldata_args_t) - method_id = util.method_id(sig) + abi_sig = sig.name + canonicalize_type(calldata_args_t) + method_id = util.method_id(abi_sig) calldata_args_location = LLLnode(4, location="calldata", typ=calldata_args_t) - calldata_args = lazy_abi_decode(calldata_args_t, base_args_location) + calldata_args = lazy_abi_decode(calldata_args_t, calldata_args_location) assert calldata_args.value == "multi" # sanity check # extract just the kwargs from the ABI payload # TODO come up with a better name for these variables - calldata_kwargs = calldata_args.args[: len(base_args)] + calldata_kwargs = calldata_args.args[: len(sig.base_args)] # a sequence of statements to strictify kwargs into memory ret = ["seq"] - all_kwargs_t = TupleType(list(arg.typ for arg in sig_kwargs)) + # TODO optimize make_setter by using + # TupleType(list(arg.typ for arg in calldata_kwargs + default_kwargs)) + lhs_location = "memory" for x in calldata_kwargs: - context.new_variable(argname, argtype, mutable=False) - ret.append(make_setter(context.lookup_var(x.name), x, "memory")) + context.new_variable(x.name, x.typ, mutable=False) + lhs = context.lookup_var(x.name) + rhs = x + ret.append(make_setter(lhs, rhs, lhs_location, pos)) for x in default_kwargs: - context.new_variable(argname, argtype, mutable=False) - ret.append(make_setter(context.lookup_var(x.name), Expr(x, context).lll_node, "memory")) + context.new_variable(x.name, x.typ, mutable=False) + lhs = context.lookup_var(x.name) + rhs = Expr(x, context).lll_node + ret.append(make_setter(lhs, rhs, lhs_location, pos)) - ret.append(["goto", tbd_entry_point]) + ret.append(["goto", _base_entry_point(sig)]) - ret = ["if", ["eq", tbd_mload_method_id, method_id], ret] + ret = ["if", ["eq", "_calldata_method_id", method_id], ret] return ret - for i, kwarg in enumerate(keyword_args): + ret = ["seq"] + + keyword_args = sig.default_args + + for i, _ in enumerate(keyword_args): calldata_kwargs = keyword_args[:i] default_kwargs = keyword_args[i:] - sig = tbd_sig - ret.append(handler_for(calldata_kwargs, default_kwargs)) return ret @@ -94,22 +111,23 @@ def generate_lll_for_external_function( code: vy_ast.FunctionDef, sig: FunctionSignature, context: Context, check_nonpayable: bool, ) -> LLLnode: """Return the LLL for an external function. Includes code to inspect the method_id, - enter the function (nonpayable and reentrancy checks) and handle kwargs. + enter the function (nonpayable and reentrancy checks), handle kwargs and exit + the function (clean up reentrancy storage variables) """ func_type = code._metadata["type"] + pos = getpos(code) _register_function_args(context, sig) nonreentrant_pre, nonreentrant_post = get_nonreentrant_lock(func_type) - kwarg_handlers = _generate_kwarg_handlers(context, sig) + kwarg_handlers = _generate_kwarg_handlers(context, sig, pos) entrance = [] # once args have been handled if len(kwarg_handlers) > 0: - entrance.append(["label", f"{sig.base_method_id}_entry"]) - # TODO need a case for no kwargs? + entrance.append(["label", _base_entry_point(sig)]) if check_nonpayable and sig.mutability != "payable": # if the contract contains payable functions, but this is not one of them @@ -125,10 +143,10 @@ def generate_lll_for_external_function( # ret_ofst and ret_len stack items passed by function body; consume using 'pass' exit += [["return", "pass", "pass"]] - ret = ["seq"] + arg_handlers + entrance + body + exit + ret = ["seq"] + kwarg_handlers + entrance + body + exit # TODO special handling for default function - if len(kwarg_handlers) == 0: # TODO is this check correct? - ret = ["if", ["eq", tbd_mload_method_id, sig.method_id], ret] + if len(kwarg_handlers) == 0: + ret = ["if", ["eq", "_calldata_method_id", sig.method_id], ret] return LLLnode.from_list(ret, pos=getpos(code)) diff --git a/vyper/old_codegen/function_definitions/internal_function.py b/vyper/old_codegen/function_definitions/internal_function.py index e36bdff13c..432c508b1b 100644 --- a/vyper/old_codegen/function_definitions/internal_function.py +++ b/vyper/old_codegen/function_definitions/internal_function.py @@ -1,20 +1,10 @@ -from typing import Any, List - from vyper import ast as vy_ast from vyper.ast.signatures import FunctionSignature -from vyper.ast.signatures.function_signature import VariableRecord from vyper.old_codegen.context import Context -from vyper.old_codegen.expr import Expr from vyper.old_codegen.function_definitions.utils import get_nonreentrant_lock from vyper.old_codegen.lll_node import LLLnode -from vyper.old_codegen.parser_utils import getpos, make_setter +from vyper.old_codegen.parser_utils import getpos from vyper.old_codegen.stmt import parse_body -from vyper.old_codegen.types.types import ( - BaseType, - ByteArrayLike, - get_size_of_type, -) -from vyper.utils import MemoryPositions def generate_lll_for_internal_function( @@ -71,7 +61,7 @@ def mkidentifier(s): # statement (which would generate the jump) stop_func = [["jump", "pass"]] # was passed in via stack - enter = nonreentrant_pre + enter = [["label", function_entry_label], nonreentrant_pre] body = [parse_body(c, context) for c in code.body] @@ -81,6 +71,4 @@ def mkidentifier(s): exit = cleanup_label + nonreentrant_post + stop_func - return LLLnode.from_list( - ["seq"] + ["label", function_label] + enter + body + exit, typ=None, pos=getpos(code), - ) + return LLLnode.from_list(["seq"] + enter + body + exit, typ=None, pos=getpos(code),) diff --git a/vyper/old_codegen/parser.py b/vyper/old_codegen/parser.py index ee6064f292..e152d01a9d 100644 --- a/vyper/old_codegen/parser.py +++ b/vyper/old_codegen/parser.py @@ -8,9 +8,9 @@ StructureException, ) from vyper.old_codegen.function_definitions import ( + generate_lll_for_function, is_default_func, is_initializer, - generate_lll_for_function, ) from vyper.old_codegen.global_context import GlobalContext from vyper.old_codegen.lll_node import LLLnode @@ -151,7 +151,7 @@ def parse_regular_functions( {**{"self": sigs}, **external_interfaces}, global_ctx, # include a nonpayble check here if the contract only has a default function - check_per_function or not otherfuncs, + check_per_function or not regular_functions, ) else: fallback_lll = LLLnode.from_list(["revert", 0, 0], typ=None, annotation="Default function") @@ -173,7 +173,7 @@ def parse_regular_functions( runtime = [ "seq", func_init_lll(), - ["with", "_func_sig", ["mload", 0], external_seq], + ["with", "_calldata_method_id", ["mload", 0], external_seq], ["seq_unchecked", ["label", "fallback"], fallback_lll], internal_func_sub, ] @@ -215,6 +215,7 @@ def parse_tree_to_lll(global_ctx: GlobalContext) -> Tuple[LLLnode, LLLnode]: if global_ctx._contracts or global_ctx._interfaces: external_interfaces = parse_external_interfaces(external_interfaces, global_ctx) + # TODO: fix for #2251 is to move this after parse_regular_functions if init_function: o.append(init_func_init_lll()) init_func_lll, _frame_size = generate_lll_for_function( @@ -222,7 +223,7 @@ def parse_tree_to_lll(global_ctx: GlobalContext) -> Tuple[LLLnode, LLLnode]: ) o.append(init_func_lll) - if regular_function or defaultfunc: + if regular_functions or default_function: o, runtime = parse_regular_functions( o, regular_functions, sigs, external_interfaces, global_ctx, default_function, ) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 6c6fda000c..b633eac0fc 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -1,6 +1,7 @@ from decimal import Decimal, getcontext from vyper import ast as vy_ast +from vyper.evm.opcodes import version_check from vyper.exceptions import ( CompilerPanic, InvalidLiteral, @@ -255,7 +256,7 @@ def byte_array_to_num( def get_bytearray_length(arg): typ = BaseType("uint256") - return LLLnode.from_list([load_op(arg.location), arg], typ=BaseType("uint256")) + return LLLnode.from_list([load_op(arg.location), arg], typ=typ) def getpos(node): @@ -380,7 +381,7 @@ def load_op(location): return "calldataload" if location == "code": return "codeload" - raise CompilerPanic("unreachable", arg) # pragma: no test + raise CompilerPanic("unreachable", location) # pragma: no test # Unwrap location @@ -658,13 +659,13 @@ def zero_pad(bytez_placeholder): # convenience rewrites for shr/sar/shl def _shr(x, bits): if version_check(begin="constantinople"): - return ["shr", x, bits] + return ["shr", bits, x] return ["div", x, ["exp", 2, bits]] def _sar(x, bits): if version_check(begin="constantinople"): - return ["sar", x, bits] + return ["sar", bits, x] return ["sdiv", x, ["exp", 2, bits]] @@ -676,8 +677,16 @@ def clamp_basetype(lll_node): b = LLLnode.from_list(["b"], typ=lll_node.typ, location=lll_node.location) return ["assert", ["le", get_bytearray_length(b), t.maxlen]] if isinstance(t, BaseType): - if t.typ in ("int128", "decimal"): + if t.typ in ("int128"): return int_clamp(t, 128, signed=True) + if t.typ in ("decimal"): + return [ + "clamp", + ["mload", MemoryPositions.MINDECIMAL], + lll_node, + ["mload", MemoryPositions.MAXDECIMAL], + ] + if t.typ in ("address",): return int_clamp(t, 160) if t.typ in ("bool",): diff --git a/vyper/old_codegen/return_.py b/vyper/old_codegen/return_.py index dc1f87424b..7c69f2fc6c 100644 --- a/vyper/old_codegen/return_.py +++ b/vyper/old_codegen/return_.py @@ -1,27 +1,19 @@ from vyper import ast as vy_ast -from vyper.old_codegen.function_definitions.utils import get_nonreentrant_lock +from vyper.old_codegen.abi import abi_encode, abi_type_of, lll_tuple_from_args +from vyper.old_codegen.context import Context from vyper.old_codegen.lll_node import LLLnode from vyper.old_codegen.parser_utils import getpos, make_setter -from vyper.old_codegen.types import ByteArrayType, TupleType, get_size_of_type +from vyper.old_codegen.types import TupleType, get_type_for_exact_size from vyper.old_codegen.types.check import check_assign -from vyper.old_codegen.context import Context -from vyper.utils import MemoryPositions - -from vyper.old_codegen.abi import lll_tuple_from_args, abi_encode, abi_type_of - -# something that's compatible with new_internal_variable -class FakeType: - def __init__(self, maxlen): - self.size_in_bytes = maxlen def _allocate_return_buffer(context: Context) -> int: maxlen = abi_type_of(context.return_type).size_bound() - return context.new_internal_variable(FakeType(maxlen=maxlen)) + return context.new_internal_variable(get_type_for_exact_size(maxlen)) # Generate code for return stmt -def make_return_stmt(lll_val: LLLnode, stmt: "Stmt", context: Context) -> LLLnode: +def make_return_stmt(lll_val: LLLnode, stmt, context: Context) -> LLLnode: func_type = stmt.get_ancestor(vy_ast.FunctionDef)._metadata["type"] jump_to_exit = ["goto", func_type.exit_sequence_label] @@ -39,7 +31,7 @@ def make_return_stmt(lll_val: LLLnode, stmt: "Stmt", context: Context) -> LLLnod # helper function def finalize(fill_return_buffer): - # do NOT bypass this. the exit label may do important function cleanup. + # do NOT bypass this. jump_to_exit may do important function cleanup. return LLLnode.from_list( ["seq_unchecked", fill_return_buffer, jump_to_exit], typ=None, pos=_pos, valency=0 ) @@ -81,4 +73,4 @@ def finalize(fill_return_buffer): encode_out = abi_encode(return_buffer_ofst, lll_val, pos=_pos, returns_len=True) # fill the return buffer and push the location and length onto the stack - return finalize(["seq_unchecked", encode_out, return_buffer_offset]) + return finalize(["seq_unchecked", encode_out, return_buffer_ofst]) diff --git a/vyper/old_codegen/self_call.py b/vyper/old_codegen/self_call.py index 8a394a8f97..9591ab16c1 100644 --- a/vyper/old_codegen/self_call.py +++ b/vyper/old_codegen/self_call.py @@ -1,33 +1,24 @@ -import itertools - -from vyper.ast.signatures.function_signature import FunctionSignature -from vyper.exceptions import ( - StateAccessViolation, - StructureException, - TypeCheckFailure, -) -from vyper.old_codegen.abi import abi_decode +from vyper.exceptions import StateAccessViolation, StructureException +from vyper.old_codegen.context import Context from vyper.old_codegen.lll_node import LLLnode, push_label_to_stack -from vyper.old_codegen.parser_utils import getpos -from vyper.old_codegen.types import ( - BaseType, - ByteArrayLike, - ListType, - TupleLike, - get_size_of_type, - get_static_size_of_type, - has_dynamic_data, -) - +from vyper.old_codegen.parser_utils import getpos, make_setter +from vyper.old_codegen.types import TupleType _label_counter = 0 + + # TODO a more general way of doing this def _generate_label(name: str) -> str: + global _label_counter _label_counter += 1 return f"label{_label_counter}" -def make_call(stmt_expr, context): +def lll_for_self_call(stmt_expr, context: Context) -> LLLnode: + from vyper.old_codegen.expr import ( + Expr, # TODO rethink this circular import + ) + pos = getpos(stmt_expr) # ** Internal Call ** @@ -40,10 +31,12 @@ def make_call(stmt_expr, context): method_name = stmt_expr.func.attr - sig, kw_vals = FunctionSignature.lookup_internal_function(method_name, args_lll, context) + pos_args_lll = [Expr(x, context).lll_node for x in stmt_expr.args] + + sig, kw_vals = context.lookup_internal_function(method_name, pos_args_lll) + + kw_args_lll = [Expr(x, context).lll_node for x in kw_vals] - pos_args_lll = [Expr(x, self.context).lll_node for x in stmt_expr.args] - kw_args_lll = [Expr(x, self.context).lll_node for x in kw_vals] args_lll = pos_args_lll + kw_args_lll args_tuple_t = TupleType([x.typ for x in args_lll]) diff --git a/vyper/old_codegen/stmt.py b/vyper/old_codegen/stmt.py index 9bcf2fa64b..f6394731ac 100644 --- a/vyper/old_codegen/stmt.py +++ b/vyper/old_codegen/stmt.py @@ -8,24 +8,18 @@ from vyper.old_codegen.parser_utils import ( LLLnode, getpos, - make_byte_array_copier, make_setter, unwrap_location, - zero_pad, ) from vyper.old_codegen.return_ import make_return_stmt from vyper.old_codegen.types import ( BaseType, - ByteArrayLike, ByteArrayType, ListType, - NodeType, - StructType, - TupleType, get_size_of_type, parse_type, ) -from vyper.utils import SizeLimits, bytes_to_int, fourbytes_to_int, keccak256 +from vyper.utils import bytes_to_int, fourbytes_to_int, keccak256 class Stmt: @@ -136,8 +130,9 @@ def parse_Call(self): if isinstance(self.stmt.func, vy_ast.Name): funcname = self.stmt.func.id return STMT_DISPATCH_TABLE[funcname].build_LLL(self.stmt, self.context) + elif is_self_function: - return self_call.make_call(self.stmt, self.context) + return self_call.lll_for_self_call(self.stmt, self.context) else: return external_call.make_external_call(self.stmt, self.context) @@ -389,7 +384,7 @@ def parse_Break(self): def parse_Return(self): lll_val = Expr(self.stmt.value, self.context).lll_node - return make_return_stmt(return_buf, lll_val, self.stmt, self.context) + return make_return_stmt(lll_val, self.stmt, self.context) def _get_target(self, target): if isinstance(target, vy_ast.Name) and target.id in self.context.forvars: diff --git a/vyper/old_codegen/types/types.py b/vyper/old_codegen/types/types.py index 8f58eebc32..0aa00bc979 100644 --- a/vyper/old_codegen/types/types.py +++ b/vyper/old_codegen/types/types.py @@ -307,7 +307,7 @@ def get_type_for_exact_size(n_bytes): return ByteArrayType(n_bytes - 32 * BYTE_ARRAY_OVERHEAD) -# amount of space a type takes in the static section of its ABI encoding +# TODO dead code def get_static_size_of_type(typ): if isinstance(typ, BaseType): return 1 @@ -323,20 +323,6 @@ def get_static_size_of_type(typ): raise InvalidType(f"Can not get size of type, Unexpected type: {repr(typ)}") -# could be rewritten as get_static_size_of_type == get_size_of_type? -def has_dynamic_data(typ): - if isinstance(typ, BaseType): - return False - elif isinstance(typ, ByteArrayLike): - return True - elif isinstance(typ, ListType): - return has_dynamic_data(typ.subtype) - elif isinstance(typ, TupleLike): - return any([has_dynamic_data(v) for v in typ.tuple_members()]) - else: - raise InvalidType(f"Unexpected type: {repr(typ)}") - - def get_type(input): if not hasattr(input, "typ"): typ, len = "num_literal", 32 diff --git a/vyper/semantics/types/value/array_value.py b/vyper/semantics/types/value/array_value.py index a8029269b6..56575daf4b 100644 --- a/vyper/semantics/types/value/array_value.py +++ b/vyper/semantics/types/value/array_value.py @@ -1,9 +1,9 @@ -import math from typing import Type from vyper import ast as vy_ast from vyper.exceptions import CompilerPanic, StructureException, UnexpectedValue from vyper.semantics import validation +from vyper.utils import ceil32 from ..abstract import ArrayValueAbstractType, BytesAbstractType from ..bases import BasePrimitive, DataLocation, ValueTypeDefinition diff --git a/vyper/semantics/validation/local.py b/vyper/semantics/validation/local.py index b02c7ff585..861517add6 100644 --- a/vyper/semantics/validation/local.py +++ b/vyper/semantics/validation/local.py @@ -19,7 +19,6 @@ VariableDeclarationException, VyperException, ) - # TODO consolidate some of these imports from vyper.semantics.environment import ( CONSTANT_ENVIRONMENT_VARS, From 7c6689c425725192e36bc7c1a7653374f22d556c Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sat, 11 Sep 2021 13:53:31 -0700 Subject: [PATCH 042/163] wip lint --- vyper/ast/signatures/function_signature.py | 6 ++--- vyper/old_codegen/context.py | 4 +-- vyper/old_codegen/external_call.py | 8 ++---- .../function_definitions/common.py | 26 +++++++++++-------- .../function_definitions/external_function.py | 17 +++++++----- .../function_definitions/internal_function.py | 4 +-- vyper/old_codegen/return_.py | 2 +- vyper/old_codegen/self_call.py | 4 +-- vyper/semantics/validation/local.py | 1 + vyper/utils.py | 2 +- 10 files changed, 39 insertions(+), 35 deletions(-) diff --git a/vyper/ast/signatures/function_signature.py b/vyper/ast/signatures/function_signature.py index 2e11da9646..95c9410429 100644 --- a/vyper/ast/signatures/function_signature.py +++ b/vyper/ast/signatures/function_signature.py @@ -87,8 +87,8 @@ def __init__( def __str__(self): input_name = "def " + self.name + "(" + ",".join([str(arg.typ) for arg in self.args]) + ")" - if self.output_type: - return input_name + " -> " + str(self.output_type) + ":" + if self.return_type: + return input_name + " -> " + str(self.return_type) + ":" return input_name + ":" def _abi_signature(self, args): @@ -125,7 +125,7 @@ def set_default_args(self): args = self.func_ast_code.args defaults = getattr(args, "defaults", []) - num_base_args = len(args) - len(defaults) + num_base_args = len(args.args) - len(defaults) self.base_args = self.args[:num_base_args] self.default_args = self.args[num_base_args:] diff --git a/vyper/old_codegen/context.py b/vyper/old_codegen/context.py index 6ba8219fb9..8cc3cd95d0 100644 --- a/vyper/old_codegen/context.py +++ b/vyper/old_codegen/context.py @@ -25,7 +25,7 @@ def __init__( constancy=Constancy.Mutable, is_internal=False, is_payable=False, - method_id="", + # method_id="", sig=None, ): # In-memory variables, in the form (name, memory location, type) @@ -52,7 +52,7 @@ def __init__( self.callback_ptr = None self.is_internal = is_internal # method_id of current function - self.method_id = method_id + # self.method_id = method_id # store global context self.global_ctx = global_ctx # full function signature diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index 5622b90921..16fbce8f55 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -106,9 +106,7 @@ def _external_call_helper(contract_address, sig, args_lll, context, pos=None, va # TODO push me up to expr.py def get_gas_and_value(stmt_expr, context): - from vyper.old_codegen.expr import ( - Expr, # TODO rethink this circular import - ) + from vyper.old_codegen.expr import Expr # TODO rethink this circular import value, gas = None, None for kw in stmt_expr.keywords: @@ -122,9 +120,7 @@ def get_gas_and_value(stmt_expr, context): def lll_for_external_call(stmt_expr, context): - from vyper.old_codegen.expr import ( - Expr, # TODO rethink this circular import - ) + from vyper.old_codegen.expr import Expr # TODO rethink this circular import pos = getpos(stmt_expr) value, gas = get_gas_and_value(stmt_expr, context) diff --git a/vyper/old_codegen/function_definitions/common.py b/vyper/old_codegen/function_definitions/common.py index 86b15d2b88..7589343d90 100644 --- a/vyper/old_codegen/function_definitions/common.py +++ b/vyper/old_codegen/function_definitions/common.py @@ -1,9 +1,13 @@ # can't use from [module] import [object] because it breaks mocks in testing import copy +from typing import Dict, Optional, Tuple + import vyper.ast as vy_ast -from vyper.ast.signatures import FunctionSignature -from vyper.old_codegen import context as ctx +from vyper.ast.signatures import FunctionSignature, VariableRecord +from vyper.old_codegen.lll_node import LLLnode +from vyper.old_codegen.context import Context +from vyper.old_codegen.global_context import GlobalContext from vyper.old_codegen.context import Constancy from vyper.old_codegen.function_definitions.external_function import ( generate_lll_for_external_function, @@ -17,15 +21,16 @@ # Is a function the initializer? -def is_initializer(code: vy_ast.FunctionDef): +def is_initializer(code: vy_ast.FunctionDef) -> bool: return code.name == "__init__" # Is a function the default function? -def is_default_func(code: vy_ast.FunctionDef): +def is_default_func(code: vy_ast.FunctionDef) -> bool: return code.name == "__default__" +# def generate_lll_for_function(code: vy_ast.FunctionDef, sigs: Dict[str, FunctionSignature], global_ctx: GlobalContext, check_nonpayable: bool, _vars: Optional[Dict[str, VariableRecord]]) -> Tuple[LLLnode, int]: def generate_lll_for_function(code, sigs, global_ctx, check_nonpayable, _vars=None): """ Parse a function and produce LLL code for the function, includes: @@ -48,7 +53,7 @@ def generate_lll_for_function(code, sigs, global_ctx, check_nonpayable, _vars=No # to see what the max frame size of any callee in the function was, # then we run the codegen again with the max frame size as # the start of the frame for this function. - def _run_pass(code, memory_allocator=None): + def _run_pass(memory_allocator=None): # Create a local (per function) context. if memory_allocator is None: memory_allocator = MemoryAllocator() @@ -56,18 +61,17 @@ def _run_pass(code, memory_allocator=None): _vars = _vars.copy() # these will get clobbered in called functions nonlocal sig sig = copy.deepcopy(sig) # just in case - context = ctx.Context( + context = Context( vars=_vars, global_ctx=global_ctx, sigs=sigs, memory_allocator=memory_allocator, - return_type=sig.output_type, + return_type=sig.return_type, constancy=Constancy.Constant if sig.mutability in ("view", "pure") else Constancy.Mutable, is_payable=sig.mutability == "payable", is_internal=sig.internal, - method_id=sig.method_id, sig=sig, ) @@ -77,9 +81,9 @@ def _run_pass(code, memory_allocator=None): o = generate_lll_for_external_function(code, sig, context, check_nonpayable) return o, context - _, context = _run_pass(None) - allocate_start = max(ctx.callee_frame_sizes) - o, context = _run_pass(MemoryAllocator(allocate_start)) + _, context = _run_pass(memory_allocator=None) + allocate_start = max(context.callee_frame_sizes) + o, context = _run_pass(memory_allocator=MemoryAllocator(allocate_start)) frame_size = context.memory_allocator.size_of_mem diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index d995ca89c3..5a3059aaa0 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -1,4 +1,4 @@ -from typing import Any +from typing import Any, List import vyper.utils as util from vyper import ast as vy_ast @@ -44,7 +44,7 @@ def _base_entry_point(sig): return f"{sig.base_method_id}_entry" -def _generate_kwarg_handlers(context: Context, sig: FunctionSignature, pos: Any): +def _generate_kwarg_handlers(context: Context, sig: FunctionSignature, pos: Any) -> List[Any]: # generate kwarg handlers. # since they might come in thru calldata or be default, # allocate them in memory and then fill it in based on calldata or default, @@ -107,9 +107,11 @@ def handler_for(calldata_kwargs, default_kwargs): # TODO it would be nice if this returned a data structure which were # amenable to generating a jump table instead of the linear search for # method_id we have now. -def generate_lll_for_external_function( - code: vy_ast.FunctionDef, sig: FunctionSignature, context: Context, check_nonpayable: bool, -) -> LLLnode: +def generate_lll_for_external_function(code, sig, context, check_nonpayable): + # TODO type hints: + # def generate_lll_for_external_function( + # code: vy_ast.FunctionDef, sig: FunctionSignature, context: Context, check_nonpayable: bool, + # ) -> LLLnode: """Return the LLL for an external function. Includes code to inspect the method_id, enter the function (nonpayable and reentrancy checks), handle kwargs and exit the function (clean up reentrancy storage variables) @@ -147,6 +149,9 @@ def generate_lll_for_external_function( # TODO special handling for default function if len(kwarg_handlers) == 0: - ret = ["if", ["eq", "_calldata_method_id", sig.method_id], ret] + _sigs = sig.all_kwarg_sigs + assert len(_sigs) == 1 + _method_id = util.abi_method_id(_sigs[0]) + ret = ["if", ["eq", "_calldata_method_id", _method_id], ret] return LLLnode.from_list(ret, pos=getpos(code)) diff --git a/vyper/old_codegen/function_definitions/internal_function.py b/vyper/old_codegen/function_definitions/internal_function.py index 432c508b1b..4c9dd8d61b 100644 --- a/vyper/old_codegen/function_definitions/internal_function.py +++ b/vyper/old_codegen/function_definitions/internal_function.py @@ -53,8 +53,8 @@ def generate_lll_for_internal_function( def mkidentifier(s): "".join(c if c.isalnumeric() else "_" for c in s) - function_entry_label = func_type.internal_function_label - cleanup_label = func_type.exit_sequence_label + function_entry_label = sig.internal_function_label + cleanup_label = sig.exit_sequence_label # internal functions without return types need to jump back to the calling # function, as there is no guarantee there is a user-provided return diff --git a/vyper/old_codegen/return_.py b/vyper/old_codegen/return_.py index 7c69f2fc6c..8352df35fc 100644 --- a/vyper/old_codegen/return_.py +++ b/vyper/old_codegen/return_.py @@ -16,7 +16,7 @@ def _allocate_return_buffer(context: Context) -> int: def make_return_stmt(lll_val: LLLnode, stmt, context: Context) -> LLLnode: func_type = stmt.get_ancestor(vy_ast.FunctionDef)._metadata["type"] - jump_to_exit = ["goto", func_type.exit_sequence_label] + jump_to_exit = ["goto", context.sig.exit_sequence_label] _pos = getpos(stmt) diff --git a/vyper/old_codegen/self_call.py b/vyper/old_codegen/self_call.py index 9591ab16c1..7f3aa029b3 100644 --- a/vyper/old_codegen/self_call.py +++ b/vyper/old_codegen/self_call.py @@ -15,9 +15,7 @@ def _generate_label(name: str) -> str: def lll_for_self_call(stmt_expr, context: Context) -> LLLnode: - from vyper.old_codegen.expr import ( - Expr, # TODO rethink this circular import - ) + from vyper.old_codegen.expr import Expr # TODO rethink this circular import pos = getpos(stmt_expr) diff --git a/vyper/semantics/validation/local.py b/vyper/semantics/validation/local.py index 861517add6..b02c7ff585 100644 --- a/vyper/semantics/validation/local.py +++ b/vyper/semantics/validation/local.py @@ -19,6 +19,7 @@ VariableDeclarationException, VyperException, ) + # TODO consolidate some of these imports from vyper.semantics.environment import ( CONSTANT_ENVIRONMENT_VARS, diff --git a/vyper/utils.py b/vyper/utils.py index 912feacb91..ac72ab59c0 100644 --- a/vyper/utils.py +++ b/vyper/utils.py @@ -27,7 +27,7 @@ def abi_method_id(method_sig): # map a string to only-alphanumeric chars def mkalphanum(s): - return "".join([c if c.isalnumeric() else "_"] for c in s) + return "".join([c if c.isalnum() else "_" for c in s]) # Converts string to bytes From bd1237db60a6471e45dfb09105053d3d3236d8c7 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sat, 11 Sep 2021 14:38:30 -0700 Subject: [PATCH 043/163] wip polish, has compile --- vyper/ast/signatures/function_signature.py | 19 ++++++++++++++++--- vyper/old_codegen/context.py | 12 +++++++++--- vyper/old_codegen/expr.py | 2 +- vyper/old_codegen/external_call.py | 2 +- .../function_definitions/common.py | 6 ++++-- .../function_definitions/external_function.py | 17 ++++++++++++----- .../function_definitions/internal_function.py | 4 ++-- .../old_codegen/function_definitions/utils.py | 2 +- vyper/old_codegen/lll_node.py | 4 ++-- vyper/old_codegen/parser.py | 6 +++--- vyper/old_codegen/parser_utils.py | 2 +- vyper/old_codegen/return_.py | 3 ++- vyper/old_codegen/self_call.py | 8 ++++---- 13 files changed, 58 insertions(+), 29 deletions(-) diff --git a/vyper/ast/signatures/function_signature.py b/vyper/ast/signatures/function_signature.py index 95c9410429..a071298bf4 100644 --- a/vyper/ast/signatures/function_signature.py +++ b/vyper/ast/signatures/function_signature.py @@ -91,6 +91,16 @@ def __str__(self): return input_name + " -> " + str(self.return_type) + ":" return input_name + ":" + @property + def mk_identifier(self): + # we could do a bit better than this but it just needs to be unique + visibility = "internal" if self.internal else "external" + argz = ",".join([str(arg.typ) for arg in self.args]) + ret = f"{visibility} {self.name} ({argz})" + if self.return_type: + ret += " -> " + str(self.return_type) + return mkalphanum(ret) + def _abi_signature(self, args): return self.func_name + "(" + ",".join([canonicalize_type(arg.typ) for arg in args]) + ")" @@ -108,16 +118,19 @@ def all_kwarg_sigs(self) -> List[str]: return ret + @property + def base_signature(self): + return self.all_kwarg_sigs[0] + @property def internal_function_label(self): assert self.internal, "why are you doing this" - # we could do a bit better than this but it just needs to be unique - return mkalphanum(str(self)) + return self.mk_identifier @property def exit_sequence_label(self): - return mkalphanum(str(self)) + "_cleanup" + return self.mk_identifier + "_cleanup" def set_default_args(self): """Split base from kwargs and set member data structures""" diff --git a/vyper/old_codegen/context.py b/vyper/old_codegen/context.py index 8cc3cd95d0..e76e32ab70 100644 --- a/vyper/old_codegen/context.py +++ b/vyper/old_codegen/context.py @@ -76,6 +76,12 @@ def is_constant(self): def register_callee(self, frame_size): self._callee_frame_sizes.append(frame_size) + @property + def max_callee_frame_size(self): + if len(self._callee_frame_sizes) == 0: + return 0 + return max(self._callee_frame_sizes) + # # Context Managers # - Context managers are used to ensure proper wrapping of scopes and context states. @@ -220,7 +226,7 @@ def _check(cond, s="Unreachable"): if not cond: raise CompilerPanic(s) - sig = next((sig for sig in self.sigs["self"] if sig.name == method_name), None) + sig = self.sigs["self"].get(method_name, None) if sig is None: raise FunctionDeclarationException( "Function does not exist or has not been declared yet " @@ -237,10 +243,10 @@ def _check(cond, s="Unreachable"): num_provided_args = len(args_lll) total_args = len(sig.args) - num_kwargs = len(sig.kwargs) + num_kwargs = len(sig.default_args) args_needed = total_args - num_provided_args - kw_vals = sig.kwarg_values.items()[: num_kwargs - args_needed] + kw_vals = list(sig.default_values.values())[: num_kwargs - args_needed] return sig, kw_vals diff --git a/vyper/old_codegen/expr.py b/vyper/old_codegen/expr.py index a97ccbc4db..8f0eee2510 100644 --- a/vyper/old_codegen/expr.py +++ b/vyper/old_codegen/expr.py @@ -978,7 +978,7 @@ def parse_Call(self): and isinstance(self.expr.func.value, vy_ast.Name) and self.expr.func.value.id == "self" ): # noqa: E501 - return self_call.make_call(self.expr, self.context) + return self_call.lll_for_self_call(self.expr, self.context) else: return external_call.make_external_call(self.expr, self.context) diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index 16fbce8f55..3f1b86da86 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -40,7 +40,7 @@ def _pack_arguments(sig, args, context, pos): def _unpack_returndata(sig, context): - return_t = abi_type_of(sig.output_type) + return_t = abi_type_of(sig.return_type) min_return_size = return_t.static_size() maxlen = return_t.size_bound() diff --git a/vyper/old_codegen/function_definitions/common.py b/vyper/old_codegen/function_definitions/common.py index 7589343d90..a99f19c43d 100644 --- a/vyper/old_codegen/function_definitions/common.py +++ b/vyper/old_codegen/function_definitions/common.py @@ -82,15 +82,17 @@ def _run_pass(memory_allocator=None): return o, context _, context = _run_pass(memory_allocator=None) - allocate_start = max(context.callee_frame_sizes) + allocate_start = context.max_callee_frame_size o, context = _run_pass(memory_allocator=MemoryAllocator(allocate_start)) frame_size = context.memory_allocator.size_of_mem - if sig.external: + if not sig.internal: # frame_size of external function includes all private functions called o.total_gas = o.gas + calc_mem_gas(frame_size) else: + # frame size for internal function does not need to be adjusted + # since it is already accounted for by the caller o.total_gas = o.gas o.context = context diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index 5a3059aaa0..2f9fbd16fe 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -40,8 +40,9 @@ def _register_function_args(context: Context, sig: FunctionSignature): context.vars[arg.name] = LLLnode(arg_lll, location=base_args_location) +# TODO move me to function_signature.py? def _base_entry_point(sig): - return f"{sig.base_method_id}_entry" + return f"{sig.mk_identifier}_entry" def _generate_kwarg_handlers(context: Context, sig: FunctionSignature, pos: Any) -> List[Any]: @@ -128,17 +129,23 @@ def generate_lll_for_external_function(code, sig, context, check_nonpayable): entrance = [] # once args have been handled - if len(kwarg_handlers) > 0: - entrance.append(["label", _base_entry_point(sig)]) + if len(kwarg_handlers) > 1: + entrance += [["label", _base_entry_point(sig)]] + else: + # otherwise, the label is redundant since there is only + # one control flow path into the external method + pass if check_nonpayable and sig.mutability != "payable": # if the contract contains payable functions, but this is not one of them # add an assertion that the value of the call is zero - entrance.append(["assert", ["iszero", "callvalue"]]) + entrance += [["assert", ["iszero", "callvalue"]]] + + entrance += nonreentrant_pre body = [parse_body(c, context) for c in code.body] - exit = [["label", func_type.exit_sequence_label]] + [nonreentrant_post] + exit = [["label", sig.exit_sequence_label]] + nonreentrant_post if context.return_type is None: exit += [["stop"]] else: diff --git a/vyper/old_codegen/function_definitions/internal_function.py b/vyper/old_codegen/function_definitions/internal_function.py index 4c9dd8d61b..6927164fe3 100644 --- a/vyper/old_codegen/function_definitions/internal_function.py +++ b/vyper/old_codegen/function_definitions/internal_function.py @@ -67,8 +67,8 @@ def mkidentifier(s): if sig.return_type is not None: # name the variable that was passed via stack - body = ["with", "return_buffer", "pass", body] + body = [["with", "return_buffer", "pass", ["seq"] + body]] - exit = cleanup_label + nonreentrant_post + stop_func + exit = [["label", cleanup_label]] + nonreentrant_post + stop_func return LLLnode.from_list(["seq"] + enter + body + exit, typ=None, pos=getpos(code),) diff --git a/vyper/old_codegen/function_definitions/utils.py b/vyper/old_codegen/function_definitions/utils.py index 87ddd8eb82..61fa6b71b4 100644 --- a/vyper/old_codegen/function_definitions/utils.py +++ b/vyper/old_codegen/function_definitions/utils.py @@ -1,6 +1,6 @@ def get_nonreentrant_lock(func_type): if not func_type.nonreentrant: - return [], [] + return ["pass"], ["pass"] nkey = func_type.reentrancy_key_position.position nonreentrant_pre = [["seq", ["assert", ["iszero", ["sload", nkey]]], ["sstore", nkey, 1]]] diff --git a/vyper/old_codegen/lll_node.py b/vyper/old_codegen/lll_node.py index bf0ae6c239..490922388b 100644 --- a/vyper/old_codegen/lll_node.py +++ b/vyper/old_codegen/lll_node.py @@ -141,9 +141,9 @@ def __init__( # With statements: with elif self.value == "with": if len(self.args) != 3: - raise CompilerPanic("With statement must have 3 arguments") + raise CompilerPanic(f"With statement must have 3 arguments: {self}") if len(self.args[0].args) or not isinstance(self.args[0].value, str): - raise CompilerPanic("First argument to with statement must be a variable") + raise CompilerPanic(f"First argument to with statement must be a variable: {self}") if not self.args[1].valency and self.args[1].value != "pass": raise CompilerPanic( ( diff --git a/vyper/old_codegen/parser.py b/vyper/old_codegen/parser.py index e152d01a9d..33437b0f0e 100644 --- a/vyper/old_codegen/parser.py +++ b/vyper/old_codegen/parser.py @@ -137,16 +137,16 @@ def parse_regular_functions( # TODO we only need to do this for internal functions; external functions # cannot be called via `self` sig = FunctionSignature.from_definition( - func_node, external_interfaces, global_ctx._custom_structs + func_node, external_interfaces, global_ctx._structs ) sig.gas = func_lll.total_gas sig.frame_start = frame_start sig.frame_size = frame_size - sigs[sig.sig] = sig + sigs[sig.name] = sig # generate LLL for fallback function if default_function: - fallback_lll, _frame_size = generate_lll_for_function( + fallback_lll, _frame_start, _frame_size = generate_lll_for_function( default_function, {**{"self": sigs}, **external_interfaces}, global_ctx, diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index b633eac0fc..29e1447773 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -389,7 +389,7 @@ def unwrap_location(orig): if not isinstance(orig.typ, BaseType): raise CompilerPanic("unwrap location only for base types") if orig.location in ("memory", "storage", "calldata", "code"): - return LLLnode.from_list([load_op(orig.location)], typ=orig.typ) + return LLLnode.from_list([load_op(orig.location), orig], typ=orig.typ) else: # CMC 20210909 TODO double check if this branch can be removed # handle None value inserted by `empty` diff --git a/vyper/old_codegen/return_.py b/vyper/old_codegen/return_.py index 8352df35fc..5ce34d4e1d 100644 --- a/vyper/old_codegen/return_.py +++ b/vyper/old_codegen/return_.py @@ -40,7 +40,8 @@ def finalize(fill_return_buffer): return finalize([]) if context.is_internal: - return finalize(make_setter("return_buffer", lll_val, pos=_pos)) + dst = LLLnode.from_list(["return_buffer"], typ=context.return_type) + return finalize(make_setter(dst, lll_val, location="memory", pos=_pos)) # we are in an external function. diff --git a/vyper/old_codegen/self_call.py b/vyper/old_codegen/self_call.py index 7f3aa029b3..6aad2312ca 100644 --- a/vyper/old_codegen/self_call.py +++ b/vyper/old_codegen/self_call.py @@ -63,7 +63,8 @@ def lll_for_self_call(stmt_expr, context: Context) -> LLLnode: ) return_buffer = LLLnode.from_list([return_buffer], annotation=f"{return_label}_return_buf") - copy_args = make_setter(sig.frame_start, args_as_tuple, "memory", pos) + args_dst = LLLnode(sig.frame_start, typ=args_tuple_t, location="memory") + copy_args = make_setter(args_dst, args_as_tuple, "memory", pos) call_sequence = [ "seq_unchecked", @@ -71,14 +72,13 @@ def lll_for_self_call(stmt_expr, context: Context) -> LLLnode: push_label_to_stack(return_label), # pass return label to subroutine return_buffer, # pass return buffer to subroutine ["goto", sig.internal_function_label], - return_label, - "jumpdest", + ["label", return_label], return_buffer, # push return buffer location to stack ] o = LLLnode.from_list( call_sequence, - typ=sig.output_type, + typ=sig.return_type, location="memory", pos=pos, annotation=f"Internal Call: {method_name}", From a6e3f32a545a9c6869a16d1d5b35966392bc8a64 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sat, 11 Sep 2021 14:59:38 -0700 Subject: [PATCH 044/163] wip polish, remove some dead code --- vyper/old_codegen/expr.py | 2 +- vyper/old_codegen/external_call.py | 33 ++++++++++--------- .../function_definitions/external_function.py | 2 +- vyper/old_codegen/types/types.py | 16 --------- 4 files changed, 19 insertions(+), 34 deletions(-) diff --git a/vyper/old_codegen/expr.py b/vyper/old_codegen/expr.py index 8f0eee2510..7d695dc86a 100644 --- a/vyper/old_codegen/expr.py +++ b/vyper/old_codegen/expr.py @@ -980,7 +980,7 @@ def parse_Call(self): ): # noqa: E501 return self_call.lll_for_self_call(self.expr, self.context) else: - return external_call.make_external_call(self.expr, self.context) + return external_call.lll_for_external_call(self.expr, self.context) def parse_List(self): multi_lll = [Expr(x, self.context).lll_node for x in self.expr.elements] diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index 3f1b86da86..6f83e640ff 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -10,7 +10,7 @@ from vyper.old_codegen.types import TupleType, get_type_for_exact_size -def _pack_arguments(sig, args, context, pos): +def _pack_arguments(contract_sig, args, context, pos): # abi encoding just treats all args as a big tuple args_tuple_t = TupleType([x.typ for x in args]) args_as_tuple = LLLnode.from_list(["multi"] + [x for x in args], typ=args_tuple_t) @@ -32,15 +32,15 @@ def _pack_arguments(sig, args, context, pos): # if we were only targeting constantinople, we could align # to buf (and also keep code size small) by using # (mstore buf (shl signature.method_id 224)) - mstore_method_id = [["mstore", buf, sig.method_id]] + mstore_method_id = [["mstore", buf, contract_sig.method_id]] encode_args = abi_encode(buf, args_as_tuple, pos) return mstore_method_id + [encode_args], args_ofst, args_len -def _unpack_returndata(sig, context): - return_t = abi_type_of(sig.return_type) +def _unpack_returndata(contract_sig, context): + return_t = abi_type_of(contract_sig.return_type) min_return_size = return_t.static_size() maxlen = return_t.size_bound() @@ -54,12 +54,13 @@ def _unpack_returndata(sig, context): # TODO assert returndatasize <= maxlen # abi_decode has appropriate clampers for the individual members of the return type + buf = LLLnode(buf, typ=contract_sig.return_type, location="memory") ret += [lazy_abi_decode(buf)] return ret, ret_ofst, ret_len -def _external_call_helper(contract_address, sig, args_lll, context, pos=None, value=None, gas=None): +def _external_call_helper(contract_address, contract_sig, args_lll, context, pos=None, value=None, gas=None): if value is None: value = 0 @@ -67,41 +68,41 @@ def _external_call_helper(contract_address, sig, args_lll, context, pos=None, va gas = "gas" # sanity check - assert len(sig.args) == len(args_lll) + assert len(contract_sig.args) == len(args_lll) - if context.is_constant() and sig.mutability not in ("view", "pure"): + if context.is_constant() and contract_sig.mutability not in ("view", "pure"): # TODO is this already done in type checker? raise StateAccessViolation( - f"May not call state modifying function '{sig.name}' " + f"May not call state modifying function '{contract_sig.name}' " f"within {context.pp_constancy()}.", pos, ) sub = ["seq"] - arg_packer, args_ofst, args_len = _pack_arguments(sig, args_lll, context, pos) - ret_unpacker, ret_ofst, ret_len = _unpack_returndata(sig, context, pos) + arg_packer, args_ofst, args_len = _pack_arguments(contract_sig, args_lll, context, pos) + ret_unpacker, ret_ofst, ret_len = _unpack_returndata(contract_sig, context, pos) sub += arg_packer - if sig.return_type is None: + if contract_sig.return_type is None: # if we do not expect return data, check that a contract exists at the target address # we can omit this when we _do_ expect return data because we later check `returndatasize` # CMC 20210907 do we need to check this before the call, or can we defer until after? # if we can defer, this code can be pushed down into unpack_returndata sub.append(["assert", ["extcodesize", contract_address]]) - if context.is_constant() or sig.mutability in ("view", "pure"): + if context.is_constant() or contract_sig.mutability in ("view", "pure"): call_op = ["staticcall", gas, contract_address, args_ofst, args_len, ret_ofst, ret_len] else: call_op = ["call", gas, contract_address, value, args_ofst, args_len, ret_ofst, ret_len] sub.append(["assert", call_op]) - if sig.return_type is not None: + if contract_sig.return_type is not None: sub += ret_unpacker - return LLLnode.from_list(sub, typ=sig.return_type, location="memory", pos=pos) + return LLLnode.from_list(sub, typ=contract_sig.return_type, location="memory", pos=pos) # TODO push me up to expr.py @@ -164,7 +165,7 @@ def lll_for_external_call(stmt_expr, context): raise StructureException("Unsupported operator.", stmt_expr) method_name = stmt_expr.func.attr - sig = context.sigs[contract_name][method_name] + contract_sig = context.sigs[contract_name][method_name] return _external_call_helper( - contract_address, sig, args_lll, context, pos, value=value, gas=gas, + contract_address, contract_sig, args_lll, context, pos, value=value, gas=gas, ) diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index 2f9fbd16fe..5cb09e314b 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -23,7 +23,7 @@ def _register_function_args(context: Context, sig: FunctionSignature): base_args_t = TupleType([arg.typ for arg in sig.base_args]) # tuple with the abi_encoded args - if sig.is_init_func(): + if sig.is_init_func: base_args_location = LLLnode("~codelen", location="code", typ=base_args_t) else: base_args_location = LLLnode(4, location="calldata", typ=base_args_t) diff --git a/vyper/old_codegen/types/types.py b/vyper/old_codegen/types/types.py index 0aa00bc979..f6eb58c3e4 100644 --- a/vyper/old_codegen/types/types.py +++ b/vyper/old_codegen/types/types.py @@ -307,22 +307,6 @@ def get_type_for_exact_size(n_bytes): return ByteArrayType(n_bytes - 32 * BYTE_ARRAY_OVERHEAD) -# TODO dead code -def get_static_size_of_type(typ): - if isinstance(typ, BaseType): - return 1 - elif isinstance(typ, ByteArrayLike): - return 1 - elif isinstance(typ, ListType): - return get_size_of_type(typ.subtype) * typ.count - elif isinstance(typ, MappingType): - raise InvalidType("Maps are not supported for function arguments or outputs.") - elif isinstance(typ, TupleLike): - return sum([get_size_of_type(v) for v in typ.tuple_members()]) - else: - raise InvalidType(f"Can not get size of type, Unexpected type: {repr(typ)}") - - def get_type(input): if not hasattr(input, "typ"): typ, len = "num_literal", 32 From 345fde380b9ad1e7131674a74eda9f09a198a3d2 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sat, 11 Sep 2021 14:59:49 -0700 Subject: [PATCH 045/163] update BYTE_ARRAY_OVERHEAD it is now unborked --- vyper/old_codegen/types/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vyper/old_codegen/types/types.py b/vyper/old_codegen/types/types.py index f6eb58c3e4..5f44df6b8f 100644 --- a/vyper/old_codegen/types/types.py +++ b/vyper/old_codegen/types/types.py @@ -274,7 +274,7 @@ def parse_type(item, location=None, sigs=None, custom_structs=None): # byte array overhead, in words. (it should really be 1, but there are # some places in our calling convention where the layout expects 2) -BYTE_ARRAY_OVERHEAD = 2 +BYTE_ARRAY_OVERHEAD = 1 # Gets the maximum number of memory or storage keys needed to ABI-encode From e2cb3454920c2f2a6a8e0bbda40975ba5e3c6f5a Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sat, 11 Sep 2021 15:34:52 -0700 Subject: [PATCH 046/163] wip polish more, fix external calls --- vyper/old_codegen/abi.py | 2 +- vyper/old_codegen/external_call.py | 14 +++++++++----- vyper/old_codegen/parser_utils.py | 5 +---- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index 52011509de..6a3e748527 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -536,7 +536,7 @@ def lazy_abi_decode(typ, src, clamp=True, pos=None): elif isinstance(typ, (BaseType, ByteArrayLike)): if clamp: - x = LLLnode.from_list(["x"], typ=typ) + x = LLLnode.from_list(["x"], typ=typ, location=src.location) return ["with", x, unwrap_location(src), ["seq", clamp_basetype(x), x]] else: return unwrap_location(src) diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index 6f83e640ff..f0127a613d 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -7,7 +7,8 @@ from vyper.old_codegen.abi import abi_encode, abi_type_of, lazy_abi_decode from vyper.old_codegen.lll_node import LLLnode from vyper.old_codegen.parser_utils import getpos, unwrap_location -from vyper.old_codegen.types import TupleType, get_type_for_exact_size +from vyper.old_codegen.types import TupleType, get_type_for_exact_size, canonicalize_type +import vyper.utils as util def _pack_arguments(contract_sig, args, context, pos): @@ -25,6 +26,8 @@ def _pack_arguments(contract_sig, args, context, pos): args_ofst = buf + 28 args_len = maxlen - 28 + abi_signature = contract_sig.name + canonicalize_type(args_tuple_t) + # layout: # 32 bytes | args # 0x..00 | args @@ -32,14 +35,14 @@ def _pack_arguments(contract_sig, args, context, pos): # if we were only targeting constantinople, we could align # to buf (and also keep code size small) by using # (mstore buf (shl signature.method_id 224)) - mstore_method_id = [["mstore", buf, contract_sig.method_id]] + mstore_method_id = [["mstore", buf, util.abi_method_id(abi_signature)]] encode_args = abi_encode(buf, args_as_tuple, pos) return mstore_method_id + [encode_args], args_ofst, args_len -def _unpack_returndata(contract_sig, context): +def _unpack_returndata(contract_sig, context, pos): return_t = abi_type_of(contract_sig.return_type) min_return_size = return_t.static_size() @@ -54,8 +57,8 @@ def _unpack_returndata(contract_sig, context): # TODO assert returndatasize <= maxlen # abi_decode has appropriate clampers for the individual members of the return type - buf = LLLnode(buf, typ=contract_sig.return_type, location="memory") - ret += [lazy_abi_decode(buf)] + buf = LLLnode(buf, location="memory") + ret += [lazy_abi_decode(contract_sig.return_type, buf, pos=pos)] return ret, ret_ofst, ret_len @@ -166,6 +169,7 @@ def lll_for_external_call(stmt_expr, context): method_name = stmt_expr.func.attr contract_sig = context.sigs[contract_name][method_name] + return _external_call_helper( contract_address, contract_sig, args_lll, context, pos, value=value, gas=gas, ) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 29e1447773..863ada2968 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -386,8 +386,6 @@ def load_op(location): # Unwrap location def unwrap_location(orig): - if not isinstance(orig.typ, BaseType): - raise CompilerPanic("unwrap location only for base types") if orig.location in ("memory", "storage", "calldata", "code"): return LLLnode.from_list([load_op(orig.location), orig], typ=orig.typ) else: @@ -674,8 +672,7 @@ def _sar(x, bits): def clamp_basetype(lll_node): t = lll_node.typ if isinstance(t, ByteArrayLike): - b = LLLnode.from_list(["b"], typ=lll_node.typ, location=lll_node.location) - return ["assert", ["le", get_bytearray_length(b), t.maxlen]] + return ["assert", ["le", get_bytearray_length(lll_node), t.maxlen]] if isinstance(t, BaseType): if t.typ in ("int128"): return int_clamp(t, 128, signed=True) From b91a2c818fe2c0441f34a0f9088180f47e1bef87 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sat, 11 Sep 2021 15:43:13 -0700 Subject: [PATCH 047/163] fix returndatasize negative check --- vyper/old_codegen/abi.py | 4 ++-- vyper/old_codegen/external_call.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index 6a3e748527..0627b372d1 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -473,7 +473,7 @@ def abi_decode(lll_node, src, clamp=True, pos=None): else: child_loc = src_loc # descend into the child tuple - lll_ret.append(abi_decode(o, child_loc, pos=pos)) + lll_ret.append(abi_decode(o, child_loc, clamp=clamp, pos=pos)) else: @@ -529,7 +529,7 @@ def lazy_abi_decode(typ, src, clamp=True, pos=None): # src buffer. dyn_ofst = unwrap_location(ofst) loc = _add_ofst(src, dyn_ofst) - os.append(lazy_abi_decode(t, loc, pos)) + os.append(lazy_abi_decode(t, loc, clamp=clamp, pos=pos)) ofst += child_abi_t.embedded_static_size() return LLLnode.from_list(["multi"] + os, typ=typ, pos=pos) diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index f0127a613d..cb2e834a6a 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -53,7 +53,9 @@ def _unpack_returndata(contract_sig, context, pos): ret_len = maxlen # when return data is expected, revert when the length of `returndatasize` is insufficient - ret = [["assert", ["gt", "returndatasize", min_return_size - 1]]] + ret = [] + if min_return_size > 0: + ret += [["assert", ["gt", "returndatasize", min_return_size - 1]]] # TODO assert returndatasize <= maxlen # abi_decode has appropriate clampers for the individual members of the return type From 29db5be29e886734d3361ec762d1fe036242ccb2 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sat, 11 Sep 2021 16:09:33 -0700 Subject: [PATCH 048/163] wip fix more external call issues --- vyper/old_codegen/abi.py | 13 +++++++------ .../function_definitions/external_function.py | 18 ++++++++++-------- vyper/old_codegen/parser_utils.py | 2 +- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index 0627b372d1..7420cc43c5 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -494,9 +494,9 @@ def abi_decode(lll_node, src, clamp=True, pos=None): def _add_ofst(loc, ofst): - if isinstance(loc.value, int): - return LLLnode(loc.value + ofst) - return ["add", loc, ofst] + if isinstance(loc.value, int) and isinstance(ofst, int): + return LLLnode(loc.value + ofst, location=loc.location) + return LLLnode.from_list(["add", loc, ofst], location=loc.location) # decode a buffer containing abi-encoded data structure in place @@ -527,7 +527,7 @@ def lazy_abi_decode(typ, src, clamp=True, pos=None): # load the offset word, which is the # (location-independent) offset from the start of the # src buffer. - dyn_ofst = unwrap_location(ofst) + dyn_ofst = unwrap_location(loc) loc = _add_ofst(src, dyn_ofst) os.append(lazy_abi_decode(t, loc, clamp=clamp, pos=pos)) ofst += child_abi_t.embedded_static_size() @@ -537,8 +537,9 @@ def lazy_abi_decode(typ, src, clamp=True, pos=None): elif isinstance(typ, (BaseType, ByteArrayLike)): if clamp: x = LLLnode.from_list(["x"], typ=typ, location=src.location) - return ["with", x, unwrap_location(src), ["seq", clamp_basetype(x), x]] + ret = ["with", x, unwrap_location(src), ["seq", clamp_basetype(x), x]] else: - return unwrap_location(src) + ret = unwrap_location(src) + return LLLnode.from_list(ret, typ=typ, location=src.location) else: raise CompilerPanic(f"unknown type for lazy_abi_decode {typ}") diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index 5cb09e314b..87f6ea7a01 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -2,7 +2,7 @@ import vyper.utils as util from vyper import ast as vy_ast -from vyper.ast.signatures.function_signature import FunctionSignature +from vyper.ast.signatures.function_signature import FunctionSignature, VariableRecord from vyper.old_codegen.abi import lazy_abi_decode from vyper.old_codegen.context import Context from vyper.old_codegen.expr import Expr @@ -28,16 +28,18 @@ def _register_function_args(context: Context, sig: FunctionSignature): else: base_args_location = LLLnode(4, location="calldata", typ=base_args_t) - base_args = lazy_abi_decode(base_args_t, base_args_location) + base_args_lll = lazy_abi_decode(base_args_t, base_args_location) - assert base_args.value == "multi", "lazy_abi_decode did not return multi" - base_args = base_args.args # the (lazily) decoded values + assert base_args_lll.value == "multi", "lazy_abi_decode did not return multi" + base_args_lll = base_args_lll.args # the (lazily) decoded values - assert len(base_args) == len(sig.base_args) - for (arg, arg_lll) in zip(sig.base_args, base_args.args): # the actual values - assert arg.typ == arg_lll.typ + assert len(base_args_lll) == len(sig.base_args) + for (arg, arg_lll) in zip(sig.base_args, base_args_lll): # the actual values + assert arg.typ == arg_lll.typ, (arg.typ, arg_lll.typ) # register the record in the local namespace - context.vars[arg.name] = LLLnode(arg_lll, location=base_args_location) + context.vars[arg.name] = VariableRecord( + name=arg.name, pos=arg_lll, typ=arg.typ, mutable=False, location=arg_lll.location + ) # TODO move me to function_signature.py? diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 863ada2968..5baa20b4f4 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -689,7 +689,7 @@ def clamp_basetype(lll_node): if t.typ in ("bool",): return int_clamp(t, 1) if t.typ in ("int256", "uint256"): - return [] # special case, no clamp + return ["pass"] # special case, no clamp return # raises From 9c72d0a5bea94c61180929db35f047be7941a98d Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sat, 11 Sep 2021 16:23:09 -0700 Subject: [PATCH 049/163] fix int_clamp for address and bool --- vyper/old_codegen/parser_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 5baa20b4f4..93deb913bb 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -685,9 +685,9 @@ def clamp_basetype(lll_node): ] if t.typ in ("address",): - return int_clamp(t, 160) + return int_clamp(lll_node, 160) if t.typ in ("bool",): - return int_clamp(t, 1) + return int_clamp(lll_node, 1) if t.typ in ("int256", "uint256"): return ["pass"] # special case, no clamp return # raises From cfff4593b5a4820bf8282fb736645d71fa41d9be Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sat, 11 Sep 2021 16:52:13 -0700 Subject: [PATCH 050/163] fix some bugs for list args --- vyper/old_codegen/abi.py | 7 ++++--- vyper/old_codegen/external_call.py | 4 +++- .../old_codegen/function_definitions/external_function.py | 2 +- vyper/old_codegen/lll_node.py | 4 +++- vyper/old_codegen/parser.py | 6 ++---- vyper/old_codegen/parser_utils.py | 2 ++ vyper/old_codegen/return_.py | 4 ++-- vyper/old_codegen/stmt.py | 4 +++- 8 files changed, 20 insertions(+), 13 deletions(-) diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index 7420cc43c5..7f1e4f97b3 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -515,7 +515,7 @@ def lazy_abi_decode(typ, src, clamp=True, pos=None): if isinstance(typ, TupleLike): ts = typ.tuple_members() else: - ts = [typ.subtyp for _ in range(typ.count)] + ts = [typ.subtype for _ in range(typ.count)] ofst = 0 os = [] for t in ts: @@ -532,7 +532,7 @@ def lazy_abi_decode(typ, src, clamp=True, pos=None): os.append(lazy_abi_decode(t, loc, clamp=clamp, pos=pos)) ofst += child_abi_t.embedded_static_size() - return LLLnode.from_list(["multi"] + os, typ=typ, pos=pos) + ret = ["multi"] + os elif isinstance(typ, (BaseType, ByteArrayLike)): if clamp: @@ -540,6 +540,7 @@ def lazy_abi_decode(typ, src, clamp=True, pos=None): ret = ["with", x, unwrap_location(src), ["seq", clamp_basetype(x), x]] else: ret = unwrap_location(src) - return LLLnode.from_list(ret, typ=typ, location=src.location) else: raise CompilerPanic(f"unknown type for lazy_abi_decode {typ}") + + return LLLnode.from_list(ret, typ=typ, location=src.location, pos=pos) diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index cb2e834a6a..00ca970b7e 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -65,7 +65,9 @@ def _unpack_returndata(contract_sig, context, pos): return ret, ret_ofst, ret_len -def _external_call_helper(contract_address, contract_sig, args_lll, context, pos=None, value=None, gas=None): +def _external_call_helper( + contract_address, contract_sig, args_lll, context, pos=None, value=None, gas=None +): if value is None: value = 0 diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index 87f6ea7a01..bb65c55ce4 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -39,7 +39,7 @@ def _register_function_args(context: Context, sig: FunctionSignature): # register the record in the local namespace context.vars[arg.name] = VariableRecord( name=arg.name, pos=arg_lll, typ=arg.typ, mutable=False, location=arg_lll.location - ) + ) # TODO move me to function_signature.py? diff --git a/vyper/old_codegen/lll_node.py b/vyper/old_codegen/lll_node.py index 490922388b..4dfc1b0772 100644 --- a/vyper/old_codegen/lll_node.py +++ b/vyper/old_codegen/lll_node.py @@ -143,7 +143,9 @@ def __init__( if len(self.args) != 3: raise CompilerPanic(f"With statement must have 3 arguments: {self}") if len(self.args[0].args) or not isinstance(self.args[0].value, str): - raise CompilerPanic(f"First argument to with statement must be a variable: {self}") + raise CompilerPanic( + f"First argument to with statement must be a variable: {self}" + ) if not self.args[1].valency and self.args[1].value != "pass": raise CompilerPanic( ( diff --git a/vyper/old_codegen/parser.py b/vyper/old_codegen/parser.py index 33437b0f0e..8838c652e2 100644 --- a/vyper/old_codegen/parser.py +++ b/vyper/old_codegen/parser.py @@ -136,9 +136,7 @@ def parse_regular_functions( # we can handle calls to self # TODO we only need to do this for internal functions; external functions # cannot be called via `self` - sig = FunctionSignature.from_definition( - func_node, external_interfaces, global_ctx._structs - ) + sig = FunctionSignature.from_definition(func_node, external_interfaces, global_ctx._structs) sig.gas = func_lll.total_gas sig.frame_start = frame_start sig.frame_size = frame_size @@ -218,7 +216,7 @@ def parse_tree_to_lll(global_ctx: GlobalContext) -> Tuple[LLLnode, LLLnode]: # TODO: fix for #2251 is to move this after parse_regular_functions if init_function: o.append(init_func_init_lll()) - init_func_lll, _frame_size = generate_lll_for_function( + init_func_lll, _frame_start, _frame_size = generate_lll_for_function( init_function, {**{"self": sigs}, **external_interfaces}, global_ctx, False, ) o.append(init_func_lll) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 93deb913bb..c1b5604253 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -439,6 +439,8 @@ def make_setter(left, right, location, pos): # If the right side is a literal if right.value in ["multi", "seq_unchecked"] and right.typ.is_literal: if right.value == "seq_unchecked": + # CMC 20210911 I think this branch can be removed now + # when the LLL is `seq_unchecked`, this is a literal where one or # more values must be pre-processed to avoid memory corruption subs = right.args[:-1] diff --git a/vyper/old_codegen/return_.py b/vyper/old_codegen/return_.py index 5ce34d4e1d..c2aa817883 100644 --- a/vyper/old_codegen/return_.py +++ b/vyper/old_codegen/return_.py @@ -37,10 +37,10 @@ def finalize(fill_return_buffer): ) if context.return_type is None: - return finalize([]) + return finalize(["pass"]) if context.is_internal: - dst = LLLnode.from_list(["return_buffer"], typ=context.return_type) + dst = LLLnode.from_list(["return_buffer"], typ=context.return_type, location="memory") return finalize(make_setter(dst, lll_val, location="memory", pos=_pos)) # we are in an external function. diff --git a/vyper/old_codegen/stmt.py b/vyper/old_codegen/stmt.py index f6394731ac..985e76ed08 100644 --- a/vyper/old_codegen/stmt.py +++ b/vyper/old_codegen/stmt.py @@ -383,7 +383,9 @@ def parse_Break(self): return LLLnode.from_list("break", typ=None, pos=getpos(self.stmt)) def parse_Return(self): - lll_val = Expr(self.stmt.value, self.context).lll_node + lll_val = None + if self.stmt.value is not None: + lll_val = Expr(self.stmt.value, self.context).lll_node return make_return_stmt(lll_val, self.stmt, self.context) def _get_target(self, target): From b35765a8aea520f441cadfef03641cd7e07de483 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sat, 11 Sep 2021 20:09:45 -0700 Subject: [PATCH 051/163] fix called function name --- vyper/old_codegen/stmt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vyper/old_codegen/stmt.py b/vyper/old_codegen/stmt.py index 985e76ed08..8b9bcad212 100644 --- a/vyper/old_codegen/stmt.py +++ b/vyper/old_codegen/stmt.py @@ -134,7 +134,7 @@ def parse_Call(self): elif is_self_function: return self_call.lll_for_self_call(self.stmt, self.context) else: - return external_call.make_external_call(self.stmt, self.context) + return external_call.lll_for_external_call(self.stmt, self.context) def _assert_reason(self, test_expr, msg): if isinstance(msg, vy_ast.Name) and msg.id == "UNREACHABLE": From d762ea69b1fd887e3177d82e146a3d84c5700514 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sun, 12 Sep 2021 10:32:34 -0700 Subject: [PATCH 052/163] add byte array copier cases for calldata and code --- vyper/old_codegen/parser_utils.py | 44 ++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index c1b5604253..1a365fe714 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -24,7 +24,7 @@ get_size_of_type, is_base_type, ) -from vyper.utils import GAS_IDENTITY, GAS_IDENTITYWORD, MemoryPositions +from vyper.utils import GAS_IDENTITY, GAS_IDENTITYWORD, GAS_CALLDATACOPY_WORD, GAS_CODECOPY_WORD, MemoryPositions getcontext().prec = 78 # MAX_UINT256 < 1e78 @@ -67,6 +67,14 @@ def _identity_gas_bound(num_bytes): return GAS_IDENTITY + GAS_IDENTITYWORD * (ceil32(num_bytes) // 32) +def _calldatacopy_gas_bound(num_bytes): + return GAS_CALLDATACOPY_WORD * ceil32(num_bytes) // 32 + + +def _codecopy_gas_bound(num_bytes): + return GAS_CODECOPY_WORD * ceil32(num_bytes) // 32 + + # Copy byte array word-for-word (including layout) def make_byte_array_copier(destination, source, pos=None): if not isinstance(source.typ, ByteArrayLike): @@ -87,22 +95,27 @@ def make_byte_array_copier(destination, source, pos=None): # Special case: memory to memory # TODO: this should be handled by make_byte_slice_copier. - if source.location == "memory" and destination.location == "memory": + if destination.location == "memory" and source.location in ("memory", "code", "calldata"): + if source.location == "memory": + # TODO turn this into an LLL macro: memorycopy + copy_op = ["assert", ["call", ["gas"], 4, 0, "src", "sz", destination, "sz"]] + gas_bound = _identity_gas_bound(source.typ.maxlen) + elif source.location == "calldata": + copy_op = ["calldatacopy", destination, "src", "sz"] + gas_bound = _calldatacopy_gas_bound(source.typ.maxlen) + elif source.location == "code": + copy_op = ["code", destination, "src", "sz"] + gas_bound = _code_copy_gas_bound(source.typ.maxlen) o = LLLnode.from_list( [ "with", - "_source", + "src", source, - [ - "with", - "_sz", - ["add", 32, ["mload", "_source"]], - ["assert", ["call", ["gas"], 4, 0, "_source", "_sz", destination, "_sz"]], - ], - ], # noqa: E501 + ["with", "sz", [load_op(source.location), "src"], copy_op], + ], typ=None, - add_gas_estimate=_identity_gas_bound(source.typ.maxlen), - annotation="Memory copy", + add_gas_estimate=gas_bound, + annotation="copy bytestring to memory", ) return o @@ -113,13 +126,13 @@ def make_byte_array_copier(destination, source, pos=None): # Get the length if source.value is None: length = 1 - elif source.location == "memory": - length = ["add", ["mload", "_pos"], 32] + elif source.location in ("memory", "code", "calldata"): + length = ["add", [load_op(source.location), "_pos"], 32] elif source.location == "storage": length = ["add", ["sload", "_pos"], 32] pos_node = LLLnode.from_list(pos_node, typ=source.typ, location=source.location,) else: - raise CompilerPanic(f"Unsupported location: {source.location}") + raise CompilerPanic(f"Unsupported location: {source.location} to {destination.location}") if destination.location == "storage": destination = LLLnode.from_list( destination, typ=destination.typ, location=destination.location, @@ -157,7 +170,6 @@ def make_byte_slice_copier(destination, source, length, max_length, pos=None): annotation=f"copy byte slice dest: {str(destination)}", add_gas_estimate=_identity_gas_bound(max_length), ) - # TODO optimize: calldatacopy for calldata; codecopy for code # special case: rhs is zero if source.value is None: From 2027f6b22992cc01d17fcc38cc9d58d6789ae19f Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sun, 12 Sep 2021 10:18:39 -0700 Subject: [PATCH 053/163] fix some kwarg handling issues --- vyper/ast/signatures/function_signature.py | 9 +++--- vyper/old_codegen/abi.py | 2 +- vyper/old_codegen/context.py | 3 ++ vyper/old_codegen/external_call.py | 15 ++++++---- .../function_definitions/external_function.py | 30 +++++++++---------- vyper/utils.py | 2 ++ 6 files changed, 36 insertions(+), 25 deletions(-) diff --git a/vyper/ast/signatures/function_signature.py b/vyper/ast/signatures/function_signature.py index a071298bf4..35b05eb3d2 100644 --- a/vyper/ast/signatures/function_signature.py +++ b/vyper/ast/signatures/function_signature.py @@ -101,8 +101,9 @@ def mk_identifier(self): ret += " -> " + str(self.return_type) return mkalphanum(ret) - def _abi_signature(self, args): - return self.func_name + "(" + ",".join([canonicalize_type(arg.typ) for arg in args]) + ")" + # calculate the abi signature for a given set of args + def abi_signature_for_args(self, args): + return self.name + "(" + ",".join([canonicalize_type(arg.typ) for arg in args]) + ")" @cached_property def all_kwarg_sigs(self) -> List[str]: @@ -110,11 +111,11 @@ def all_kwarg_sigs(self) -> List[str]: ret = [] argz = self.base_args.copy() - ret.append(self._abi_signature(argz)) + ret.append(self.abi_signature_for_args(argz)) for arg in self.default_args: argz.append(arg) - ret.append(self._abi_signature(argz)) + ret.append(self.abi_signature_for_args(argz)) return ret diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index 7f1e4f97b3..d120673a00 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -532,7 +532,7 @@ def lazy_abi_decode(typ, src, clamp=True, pos=None): os.append(lazy_abi_decode(t, loc, clamp=clamp, pos=pos)) ofst += child_abi_t.embedded_static_size() - ret = ["multi"] + os + ret = ["multi"] + os elif isinstance(typ, (BaseType, ByteArrayLike)): if clamp: diff --git a/vyper/old_codegen/context.py b/vyper/old_codegen/context.py index e76e32ab70..41645c70f3 100644 --- a/vyper/old_codegen/context.py +++ b/vyper/old_codegen/context.py @@ -215,6 +215,9 @@ def new_internal_variable(self, typ: NodeType) -> int: def parse_type(self, ast_node, location): return self.global_ctx.parse_type(ast_node, location) + def lookup_var(self, varname): + return self.vars[varname] + def lookup_internal_function(self, method_name, args_lll): # TODO is this the right module for me? """ diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index 00ca970b7e..70da8f641b 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -43,10 +43,14 @@ def _pack_arguments(contract_sig, args, context, pos): def _unpack_returndata(contract_sig, context, pos): - return_t = abi_type_of(contract_sig.return_type) - min_return_size = return_t.static_size() + return_t = contract_sig.return_type + if return_t is None: + return ["pass"], 0, 0 + + abi_return_t = abi_type_of(return_t) + min_return_size = abi_return_t.static_size() + maxlen = abi_return_t.size_bound() - maxlen = return_t.size_bound() buf_t = get_type_for_exact_size(maxlen) buf = context.new_internal_variable(buf_t) ret_ofst = buf @@ -60,7 +64,7 @@ def _unpack_returndata(contract_sig, context, pos): # abi_decode has appropriate clampers for the individual members of the return type buf = LLLnode(buf, location="memory") - ret += [lazy_abi_decode(contract_sig.return_type, buf, pos=pos)] + ret += [lazy_abi_decode(return_t, buf, pos=pos)] return ret, ret_ofst, ret_len @@ -88,6 +92,7 @@ def _external_call_helper( sub = ["seq"] arg_packer, args_ofst, args_len = _pack_arguments(contract_sig, args_lll, context, pos) + ret_unpacker, ret_ofst, ret_len = _unpack_returndata(contract_sig, context, pos) sub += arg_packer @@ -153,7 +158,7 @@ def lll_for_external_call(stmt_expr, context): # e.g. `self.foo.bar()` # sanity check - assert stmt_expr.func.value.id == "self" + assert stmt_expr.func.value.value.id == "self", stmt_expr contract_name = context.globals[stmt_expr.func.value.attr].typ.name type_ = stmt_expr.func.value._metadata["type"] diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index bb65c55ce4..4026db9290 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -54,22 +54,19 @@ def _generate_kwarg_handlers(context: Context, sig: FunctionSignature, pos: Any) # depending on the signature def handler_for(calldata_kwargs, default_kwargs): - default_kwargs = [Expr(x, context).lll_node for x in default_kwargs] - calldata_args = sig.base_args + calldata_kwargs calldata_args_t = TupleType(list(arg.typ for arg in calldata_args)) - abi_sig = sig.name + canonicalize_type(calldata_args_t) - method_id = util.method_id(abi_sig) + abi_sig = sig.abi_signature_for_args(calldata_args) + method_id = util.abi_method_id(abi_sig) calldata_args_location = LLLnode(4, location="calldata", typ=calldata_args_t) - calldata_args = lazy_abi_decode(calldata_args_t, calldata_args_location) + calldata_args_lll = lazy_abi_decode(calldata_args_t, calldata_args_location) - assert calldata_args.value == "multi" # sanity check + assert calldata_args_lll.value == "multi" # sanity check # extract just the kwargs from the ABI payload - # TODO come up with a better name for these variables - calldata_kwargs = calldata_args.args[: len(sig.base_args)] + calldata_kwargs_lll = calldata_args_lll.args[len(sig.base_args) :] # a sequence of statements to strictify kwargs into memory ret = ["seq"] @@ -78,15 +75,18 @@ def handler_for(calldata_kwargs, default_kwargs): # TupleType(list(arg.typ for arg in calldata_kwargs + default_kwargs)) lhs_location = "memory" - for x in calldata_kwargs: - context.new_variable(x.name, x.typ, mutable=False) - lhs = context.lookup_var(x.name) - rhs = x + assert len(calldata_kwargs_lll) == len(calldata_kwargs), calldata_kwargs + for arg_meta, arg_lll in zip(calldata_kwargs, calldata_kwargs_lll): + assert arg_meta.typ == arg_lll.typ + dst = context.new_variable(arg_meta.name, arg_meta.typ, is_mutable=False) + lhs = LLLnode(dst, location="memory", typ=arg_meta.typ) + rhs = arg_lll ret.append(make_setter(lhs, rhs, lhs_location, pos)) for x in default_kwargs: - context.new_variable(x.name, x.typ, mutable=False) - lhs = context.lookup_var(x.name) - rhs = Expr(x, context).lll_node + dst = context.new_variable(x.name, x.typ, is_mutable=False) + lhs = LLLnode(dst, location="memory", typ=x.typ) + kw_ast_val = sig.default_values[x.name] # e.g. `3` in x: int = 3 + rhs = Expr(kw_ast_val, context).lll_node ret.append(make_setter(lhs, rhs, lhs_location, pos)) ret.append(["goto", _base_entry_point(sig)]) diff --git a/vyper/utils.py b/vyper/utils.py index ac72ab59c0..e60ef66263 100644 --- a/vyper/utils.py +++ b/vyper/utils.py @@ -82,6 +82,8 @@ def calc_mem_gas(memsize): # Specific gas usage GAS_IDENTITY = 15 GAS_IDENTITYWORD = 3 +GAS_CODECOPY_WORD = 3 +GAS_CALLDATACOPY_WORD = 3 # A decimal value can store multiples of 1/DECIMAL_DIVISOR MAX_DECIMAL_PLACES = 10 From 135da22137b7f7c4ec371f48e46b4778b05dd6cf Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sun, 12 Sep 2021 11:52:50 -0700 Subject: [PATCH 054/163] fix array arguments --- vyper/ast/signatures/function_signature.py | 3 +- vyper/old_codegen/abi.py | 3 ++ .../function_definitions/external_function.py | 37 ++++++++++++++----- vyper/old_codegen/parser_utils.py | 12 +----- 4 files changed, 35 insertions(+), 20 deletions(-) diff --git a/vyper/ast/signatures/function_signature.py b/vyper/ast/signatures/function_signature.py index 35b05eb3d2..e614dc387f 100644 --- a/vyper/ast/signatures/function_signature.py +++ b/vyper/ast/signatures/function_signature.py @@ -58,6 +58,7 @@ def __init__(self, *args): class FunctionArg: name: str typ: NodeType + ast_source: vy_ast.VyperNode # Function signature object @@ -168,7 +169,7 @@ def from_definition( argname = arg.arg argtyp = parse_type(arg.annotation, None, sigs, custom_structs=custom_structs,) - args.append(FunctionArg(argname, argtyp)) + args.append(FunctionArg(argname, argtyp, arg)) mutability = "nonpayable" # Assume nonpayable by default nonreentrant_key = None diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index d120673a00..e5fc9f2b96 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -1,4 +1,5 @@ import vyper.semantics.types as vy +import copy from vyper.exceptions import CompilerPanic from vyper.old_codegen.lll_node import LLLnode from vyper.old_codegen.parser_utils import ( @@ -511,6 +512,8 @@ def _add_ofst(loc, ofst): # (mload 320/*int128*/) # (mload (add 320/*buf start*/ (mload 352/*ofst loc*/)))) def lazy_abi_decode(typ, src, clamp=True, pos=None): + typ = copy.deepcopy(typ) + typ.is_literal = True if isinstance(typ, (ListType, TupleLike)): if isinstance(typ, TupleLike): ts = typ.tuple_members() diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index 4026db9290..044ace0181 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -10,15 +10,20 @@ from vyper.old_codegen.lll_node import LLLnode from vyper.old_codegen.parser_utils import getpos, make_setter from vyper.old_codegen.stmt import parse_body -from vyper.old_codegen.types.types import TupleType, canonicalize_type +from vyper.old_codegen.types.types import ListType, TupleType, canonicalize_type # register function args with the local calling context. # also allocate the ones that live in memory (i.e. kwargs) -def _register_function_args(context: Context, sig: FunctionSignature): +# returns an LLLnode with copy operations for base args which +# need to be copied to memory (they don't play nicely with +# downstream code). +def _register_function_args(context: Context, sig: FunctionSignature) -> List[Any]: + + ret = ["seq"] if len(sig.args) == 0: - return + return ret base_args_t = TupleType([arg.typ for arg in sig.base_args]) @@ -36,10 +41,24 @@ def _register_function_args(context: Context, sig: FunctionSignature): assert len(base_args_lll) == len(sig.base_args) for (arg, arg_lll) in zip(sig.base_args, base_args_lll): # the actual values assert arg.typ == arg_lll.typ, (arg.typ, arg_lll.typ) - # register the record in the local namespace - context.vars[arg.name] = VariableRecord( - name=arg.name, pos=arg_lll, typ=arg.typ, mutable=False, location=arg_lll.location - ) + + if isinstance(arg.typ, ListType): + assert arg_lll.value == "multi" + # ListTypes might be accessed with add_variable_offset + # which doesn't work for `multi`, so instead copy them + # to memory. + # TODO nested lists are still broken! + dst = context.new_variable(arg.name, typ=arg.typ, is_mutable=False) + dst = LLLnode(dst, typ=arg.typ, location="memory") + x = make_setter(dst, arg_lll, "memory", pos=getpos(arg.ast_source)) + ret.append(x) + else: + # register the record in the local namespace, no copy needed + context.vars[arg.name] = VariableRecord( + name=arg.name, pos=arg_lll, typ=arg.typ, mutable=False, location=arg_lll.location + ) + + return ret # TODO move me to function_signature.py? @@ -122,13 +141,13 @@ def generate_lll_for_external_function(code, sig, context, check_nonpayable): func_type = code._metadata["type"] pos = getpos(code) - _register_function_args(context, sig) + base_arg_handlers = _register_function_args(context, sig) nonreentrant_pre, nonreentrant_post = get_nonreentrant_lock(func_type) kwarg_handlers = _generate_kwarg_handlers(context, sig, pos) - entrance = [] + entrance = [base_arg_handlers] # once args have been handled if len(kwarg_handlers) > 1: diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 1a365fe714..8796914339 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -449,16 +449,8 @@ def make_setter(left, right, location, pos): left_token = LLLnode.from_list("_L", typ=left.typ, location=left.location) # If the right side is a literal - if right.value in ["multi", "seq_unchecked"] and right.typ.is_literal: - if right.value == "seq_unchecked": - # CMC 20210911 I think this branch can be removed now - - # when the LLL is `seq_unchecked`, this is a literal where one or - # more values must be pre-processed to avoid memory corruption - subs = right.args[:-1] - right = right.args[-1] - else: - subs = [] + if right.value == "multi": + subs = [] for i in range(left.typ.count): lhs_setter = _make_array_index_setter(left, left_token, pos, location, i) subs.append(make_setter(lhs_setter, right.args[i], location, pos=pos,)) From b2dc1df574ce0cdd08c516e7679c5fe1929a3fc6 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sun, 12 Sep 2021 12:22:04 -0700 Subject: [PATCH 055/163] fix private frame size adjustment --- vyper/old_codegen/function_definitions/common.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/vyper/old_codegen/function_definitions/common.py b/vyper/old_codegen/function_definitions/common.py index a99f19c43d..c06e77a9da 100644 --- a/vyper/old_codegen/function_definitions/common.py +++ b/vyper/old_codegen/function_definitions/common.py @@ -17,7 +17,7 @@ ) from vyper.old_codegen.memory_allocator import MemoryAllocator from vyper.old_codegen.parser_utils import check_single_exit -from vyper.utils import calc_mem_gas +from vyper.utils import calc_mem_gas, MemoryPositions # Is a function the initializer? @@ -82,10 +82,13 @@ def _run_pass(memory_allocator=None): return o, context _, context = _run_pass(memory_allocator=None) + allocate_start = context.max_callee_frame_size + allocate_start += MemoryPositions.RESERVED_MEMORY + o, context = _run_pass(memory_allocator=MemoryAllocator(allocate_start)) - frame_size = context.memory_allocator.size_of_mem + frame_size = context.memory_allocator.size_of_mem - MemoryPositions.RESERVED_MEMORY if not sig.internal: # frame_size of external function includes all private functions called From 8acd20a7e89ada4c3c41b4f383d2ba35f120a2b3 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sun, 12 Sep 2021 12:43:56 -0700 Subject: [PATCH 056/163] fix mypy and lint issues --- vyper/old_codegen/abi.py | 3 ++- vyper/old_codegen/external_call.py | 16 ++++++++++++---- .../function_definitions/common.py | 19 +++++++++++-------- .../function_definitions/external_function.py | 14 ++++++++------ vyper/old_codegen/lll_node.py | 4 ++-- vyper/old_codegen/parser.py | 4 ++-- vyper/old_codegen/parser_utils.py | 17 +++++++++-------- vyper/old_codegen/return_.py | 9 ++++++--- vyper/old_codegen/self_call.py | 7 ++++--- vyper/semantics/validation/local.py | 1 - 10 files changed, 56 insertions(+), 38 deletions(-) diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index e5fc9f2b96..fd8998531f 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -1,5 +1,6 @@ -import vyper.semantics.types as vy import copy + +import vyper.semantics.types as vy from vyper.exceptions import CompilerPanic from vyper.old_codegen.lll_node import LLLnode from vyper.old_codegen.parser_utils import ( diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index 70da8f641b..cabfaa33c2 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -1,3 +1,4 @@ +import vyper.utils as util from vyper import ast as vy_ast from vyper.exceptions import ( StateAccessViolation, @@ -7,8 +8,11 @@ from vyper.old_codegen.abi import abi_encode, abi_type_of, lazy_abi_decode from vyper.old_codegen.lll_node import LLLnode from vyper.old_codegen.parser_utils import getpos, unwrap_location -from vyper.old_codegen.types import TupleType, get_type_for_exact_size, canonicalize_type -import vyper.utils as util +from vyper.old_codegen.types import ( + TupleType, + canonicalize_type, + get_type_for_exact_size, +) def _pack_arguments(contract_sig, args, context, pos): @@ -119,7 +123,9 @@ def _external_call_helper( # TODO push me up to expr.py def get_gas_and_value(stmt_expr, context): - from vyper.old_codegen.expr import Expr # TODO rethink this circular import + from vyper.old_codegen.expr import ( + Expr, # TODO rethink this circular import + ) value, gas = None, None for kw in stmt_expr.keywords: @@ -133,7 +139,9 @@ def get_gas_and_value(stmt_expr, context): def lll_for_external_call(stmt_expr, context): - from vyper.old_codegen.expr import Expr # TODO rethink this circular import + from vyper.old_codegen.expr import ( + Expr, # TODO rethink this circular import + ) pos = getpos(stmt_expr) value, gas = get_gas_and_value(stmt_expr, context) diff --git a/vyper/old_codegen/function_definitions/common.py b/vyper/old_codegen/function_definitions/common.py index c06e77a9da..492f545d72 100644 --- a/vyper/old_codegen/function_definitions/common.py +++ b/vyper/old_codegen/function_definitions/common.py @@ -1,23 +1,21 @@ # can't use from [module] import [object] because it breaks mocks in testing import copy - from typing import Dict, Optional, Tuple import vyper.ast as vy_ast from vyper.ast.signatures import FunctionSignature, VariableRecord -from vyper.old_codegen.lll_node import LLLnode -from vyper.old_codegen.context import Context -from vyper.old_codegen.global_context import GlobalContext -from vyper.old_codegen.context import Constancy +from vyper.old_codegen.context import Constancy, Context from vyper.old_codegen.function_definitions.external_function import ( generate_lll_for_external_function, ) from vyper.old_codegen.function_definitions.internal_function import ( generate_lll_for_internal_function, ) +from vyper.old_codegen.global_context import GlobalContext +from vyper.old_codegen.lll_node import LLLnode from vyper.old_codegen.memory_allocator import MemoryAllocator from vyper.old_codegen.parser_utils import check_single_exit -from vyper.utils import calc_mem_gas, MemoryPositions +from vyper.utils import MemoryPositions, calc_mem_gas # Is a function the initializer? @@ -30,8 +28,13 @@ def is_default_func(code: vy_ast.FunctionDef) -> bool: return code.name == "__default__" -# def generate_lll_for_function(code: vy_ast.FunctionDef, sigs: Dict[str, FunctionSignature], global_ctx: GlobalContext, check_nonpayable: bool, _vars: Optional[Dict[str, VariableRecord]]) -> Tuple[LLLnode, int]: -def generate_lll_for_function(code, sigs, global_ctx, check_nonpayable, _vars=None): +def generate_lll_for_function( + code: vy_ast.FunctionDef, + sigs: Dict[str, Dict[str, FunctionSignature]], + global_ctx: GlobalContext, + check_nonpayable: bool, + _vars: Optional[Dict[str, VariableRecord]] = None, +) -> Tuple[LLLnode, int, int]: """ Parse a function and produce LLL code for the function, includes: - Signature method if statement diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index 044ace0181..86a23c60b7 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -1,8 +1,10 @@ from typing import Any, List import vyper.utils as util -from vyper import ast as vy_ast -from vyper.ast.signatures.function_signature import FunctionSignature, VariableRecord +from vyper.ast.signatures.function_signature import ( + FunctionSignature, + VariableRecord, +) from vyper.old_codegen.abi import lazy_abi_decode from vyper.old_codegen.context import Context from vyper.old_codegen.expr import Expr @@ -10,7 +12,7 @@ from vyper.old_codegen.lll_node import LLLnode from vyper.old_codegen.parser_utils import getpos, make_setter from vyper.old_codegen.stmt import parse_body -from vyper.old_codegen.types.types import ListType, TupleType, canonicalize_type +from vyper.old_codegen.types.types import ListType, TupleType # register function args with the local calling context. @@ -48,8 +50,8 @@ def _register_function_args(context: Context, sig: FunctionSignature) -> List[An # which doesn't work for `multi`, so instead copy them # to memory. # TODO nested lists are still broken! - dst = context.new_variable(arg.name, typ=arg.typ, is_mutable=False) - dst = LLLnode(dst, typ=arg.typ, location="memory") + dst_ofst = context.new_variable(arg.name, typ=arg.typ, is_mutable=False) + dst = LLLnode(dst_ofst, typ=arg.typ, location="memory") x = make_setter(dst, arg_lll, "memory", pos=getpos(arg.ast_source)) ret.append(x) else: @@ -85,7 +87,7 @@ def handler_for(calldata_kwargs, default_kwargs): assert calldata_args_lll.value == "multi" # sanity check # extract just the kwargs from the ABI payload - calldata_kwargs_lll = calldata_args_lll.args[len(sig.base_args) :] + calldata_kwargs_lll = calldata_args_lll.args[len(sig.base_args) :] # noqa: E203 # a sequence of statements to strictify kwargs into memory ret = ["seq"] diff --git a/vyper/old_codegen/lll_node.py b/vyper/old_codegen/lll_node.py index 4dfc1b0772..e58ace6728 100644 --- a/vyper/old_codegen/lll_node.py +++ b/vyper/old_codegen/lll_node.py @@ -52,7 +52,7 @@ def __init__( self, value: Union[str, int], args: List["LLLnode"] = None, - typ: "BaseType" = None, + typ: NodeType = None, location: str = None, pos: Optional[Tuple[int, int]] = None, annotation: Optional[str] = None, @@ -335,7 +335,7 @@ def __repr__(self): def from_list( cls, obj: Any, - typ: "BaseType" = None, + typ: NodeType = None, location: str = None, pos: Tuple[int, int] = None, annotation: Optional[str] = None, diff --git a/vyper/old_codegen/parser.py b/vyper/old_codegen/parser.py index 8838c652e2..d374ebbfc8 100644 --- a/vyper/old_codegen/parser.py +++ b/vyper/old_codegen/parser.py @@ -1,4 +1,4 @@ -from typing import Any, List, Optional, Tuple +from typing import Any, List, Optional, Tuple, Union from vyper import ast as vy_ast from vyper.ast.signatures.function_signature import FunctionSignature @@ -209,7 +209,7 @@ def parse_tree_to_lll(global_ctx: GlobalContext) -> Tuple[LLLnode, LLLnode]: sigs: dict = {} external_interfaces: dict = {} # Create the main statement - o = ["seq"] + o: List[Union[str, LLLnode]] = ["seq"] if global_ctx._contracts or global_ctx._interfaces: external_interfaces = parse_external_interfaces(external_interfaces, global_ctx) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 8796914339..541ace72e0 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -24,7 +24,13 @@ get_size_of_type, is_base_type, ) -from vyper.utils import GAS_IDENTITY, GAS_IDENTITYWORD, GAS_CALLDATACOPY_WORD, GAS_CODECOPY_WORD, MemoryPositions +from vyper.utils import ( + GAS_CALLDATACOPY_WORD, + GAS_CODECOPY_WORD, + GAS_IDENTITY, + GAS_IDENTITYWORD, + MemoryPositions, +) getcontext().prec = 78 # MAX_UINT256 < 1e78 @@ -105,14 +111,9 @@ def make_byte_array_copier(destination, source, pos=None): gas_bound = _calldatacopy_gas_bound(source.typ.maxlen) elif source.location == "code": copy_op = ["code", destination, "src", "sz"] - gas_bound = _code_copy_gas_bound(source.typ.maxlen) + gas_bound = _codecopy_gas_bound(source.typ.maxlen) o = LLLnode.from_list( - [ - "with", - "src", - source, - ["with", "sz", [load_op(source.location), "src"], copy_op], - ], + ["with", "src", source, ["with", "sz", [load_op(source.location), "src"], copy_op]], typ=None, add_gas_estimate=gas_bound, annotation="copy bytestring to memory", diff --git a/vyper/old_codegen/return_.py b/vyper/old_codegen/return_.py index c2aa817883..a8735745df 100644 --- a/vyper/old_codegen/return_.py +++ b/vyper/old_codegen/return_.py @@ -1,4 +1,5 @@ -from vyper import ast as vy_ast +from typing import Any, Optional + from vyper.old_codegen.abi import abi_encode, abi_type_of, lll_tuple_from_args from vyper.old_codegen.context import Context from vyper.old_codegen.lll_node import LLLnode @@ -12,10 +13,12 @@ def _allocate_return_buffer(context: Context) -> int: return context.new_internal_variable(get_type_for_exact_size(maxlen)) +Stmt = Any # mypy kludge + + # Generate code for return stmt -def make_return_stmt(lll_val: LLLnode, stmt, context: Context) -> LLLnode: +def make_return_stmt(lll_val: LLLnode, stmt: Any, context: Context) -> Optional[LLLnode]: - func_type = stmt.get_ancestor(vy_ast.FunctionDef)._metadata["type"] jump_to_exit = ["goto", context.sig.exit_sequence_label] _pos = getpos(stmt) diff --git a/vyper/old_codegen/self_call.py b/vyper/old_codegen/self_call.py index 6aad2312ca..f36cbee8f9 100644 --- a/vyper/old_codegen/self_call.py +++ b/vyper/old_codegen/self_call.py @@ -1,5 +1,4 @@ from vyper.exceptions import StateAccessViolation, StructureException -from vyper.old_codegen.context import Context from vyper.old_codegen.lll_node import LLLnode, push_label_to_stack from vyper.old_codegen.parser_utils import getpos, make_setter from vyper.old_codegen.types import TupleType @@ -14,8 +13,10 @@ def _generate_label(name: str) -> str: return f"label{_label_counter}" -def lll_for_self_call(stmt_expr, context: Context) -> LLLnode: - from vyper.old_codegen.expr import Expr # TODO rethink this circular import +def lll_for_self_call(stmt_expr, context): + from vyper.old_codegen.expr import ( + Expr, # TODO rethink this circular import + ) pos = getpos(stmt_expr) diff --git a/vyper/semantics/validation/local.py b/vyper/semantics/validation/local.py index b02c7ff585..861517add6 100644 --- a/vyper/semantics/validation/local.py +++ b/vyper/semantics/validation/local.py @@ -19,7 +19,6 @@ VariableDeclarationException, VyperException, ) - # TODO consolidate some of these imports from vyper.semantics.environment import ( CONSTANT_ENVIRONMENT_VARS, From d3a20557838a7ec8d401cc47ec95aa0375a9d875 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sun, 12 Sep 2021 20:14:17 +0000 Subject: [PATCH 057/163] fix int_clamp argument --- vyper/old_codegen/parser_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 541ace72e0..0e712d6c12 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -682,7 +682,7 @@ def clamp_basetype(lll_node): return ["assert", ["le", get_bytearray_length(lll_node), t.maxlen]] if isinstance(t, BaseType): if t.typ in ("int128"): - return int_clamp(t, 128, signed=True) + return int_clamp(lll_node, 128, signed=True) if t.typ in ("decimal"): return [ "clamp", From ad350a827718a8cd08e97b1c5ebe3bacc328c9c2 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 13 Sep 2021 08:38:00 -0700 Subject: [PATCH 058/163] add annotation for fill return buffer sequence --- vyper/old_codegen/return_.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/vyper/old_codegen/return_.py b/vyper/old_codegen/return_.py index a8735745df..4995ea4823 100644 --- a/vyper/old_codegen/return_.py +++ b/vyper/old_codegen/return_.py @@ -19,7 +19,9 @@ def _allocate_return_buffer(context: Context) -> int: # Generate code for return stmt def make_return_stmt(lll_val: LLLnode, stmt: Any, context: Context) -> Optional[LLLnode]: - jump_to_exit = ["goto", context.sig.exit_sequence_label] + sig = context.sig + + jump_to_exit = ["goto", sig.exit_sequence_label] _pos = getpos(stmt) @@ -35,8 +37,9 @@ def make_return_stmt(lll_val: LLLnode, stmt: Any, context: Context) -> Optional[ # helper function def finalize(fill_return_buffer): # do NOT bypass this. jump_to_exit may do important function cleanup. + fill_return_buffer = LLLnode.from_list(fill_return_buffer, annotation=f"fill return buffer {sig.mk_identifier}") return LLLnode.from_list( - ["seq_unchecked", fill_return_buffer, jump_to_exit], typ=None, pos=_pos, valency=0 + ["seq_unchecked", fill_return_buffer, jump_to_exit], typ=None, pos=_pos, ) if context.return_type is None: From 034b8e12ce2fc3bbdc5b0aedea33f2299e18ae76 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 13 Sep 2021 08:39:43 -0700 Subject: [PATCH 059/163] fix lazy_abi_decode location for base types --- vyper/old_codegen/abi.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index fd8998531f..7359bac06a 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -525,7 +525,7 @@ def lazy_abi_decode(typ, src, clamp=True, pos=None): for t in ts: child_abi_t = abi_type_of(t) loc = _add_ofst(src, ofst) - # TODO optimize: special case where there is only one dynamic + # TODO optimize: special case for first dynamic # member, the location is statically known. if child_abi_t.is_dynamic(): # load the offset word, which is the @@ -534,8 +534,10 @@ def lazy_abi_decode(typ, src, clamp=True, pos=None): dyn_ofst = unwrap_location(loc) loc = _add_ofst(src, dyn_ofst) os.append(lazy_abi_decode(t, loc, clamp=clamp, pos=pos)) + ofst += child_abi_t.embedded_static_size() + location = src.location ret = ["multi"] + os elif isinstance(typ, (BaseType, ByteArrayLike)): @@ -544,7 +546,13 @@ def lazy_abi_decode(typ, src, clamp=True, pos=None): ret = ["with", x, unwrap_location(src), ["seq", clamp_basetype(x), x]] else: ret = unwrap_location(src) + + if isinstance(typ, BaseType): + location = None # unwrap_location returned a stack variable + else: + # the unwrapped value is itself a pointer to calldata/memory/code + location = src.location else: raise CompilerPanic(f"unknown type for lazy_abi_decode {typ}") - return LLLnode.from_list(ret, typ=typ, location=src.location, pos=pos) + return LLLnode.from_list(ret, typ=typ, location=location, pos=pos) From 8c303e672124924b561d91cb436f192f3d5084a4 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 13 Sep 2021 15:58:20 +0000 Subject: [PATCH 060/163] allow indexing lazy_abi_decoded structs --- vyper/old_codegen/parser_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 0e712d6c12..5c31a7e323 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -301,6 +301,9 @@ def add_variable_offset(parent, key, pos, array_bounds_check=True): if parent.value is None: return LLLnode.from_list(None, typ=subtype) + if parent.value == "multi": + return parent.args[index] + if location == "storage": # for arrays and structs, calculate the storage slot by adding an offset # of [index value being accessed] * [size of each item within the sequence] From ddc7786ae530e0118825ff2741557afc28a21867 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 14 Sep 2021 02:37:04 +0000 Subject: [PATCH 061/163] redo lazy abi decoder returning a multi from lazy_abi_decode ran into issues with dynamic indexes into lists. moving the logic into add_variable_offset allows us to handle this case. --- vyper/ast/signatures/function_signature.py | 30 ++-- vyper/old_codegen/abi.py | 63 ------- vyper/old_codegen/expr.py | 1 + vyper/old_codegen/external_call.py | 19 +-- .../function_definitions/external_function.py | 75 ++++----- vyper/old_codegen/lll_node.py | 21 ++- vyper/old_codegen/parser_utils.py | 155 +++++++++++++----- vyper/old_codegen/return_.py | 4 +- vyper/old_codegen/self_call.py | 4 +- vyper/semantics/validation/local.py | 1 + vyper/utils.py | 8 + 11 files changed, 206 insertions(+), 175 deletions(-) diff --git a/vyper/ast/signatures/function_signature.py b/vyper/ast/signatures/function_signature.py index e614dc387f..11e8a1d788 100644 --- a/vyper/ast/signatures/function_signature.py +++ b/vyper/ast/signatures/function_signature.py @@ -5,6 +5,7 @@ from vyper import ast as vy_ast from vyper.exceptions import StructureException +from vyper.old_codegen.lll_node import Encoding from vyper.old_codegen.types import ( NodeType, canonicalize_type, @@ -24,7 +25,7 @@ def __init__( pos, typ, mutable, - *, + encoding=Encoding.VYPER, location="memory", blockscopes=None, defined_at=None, @@ -35,6 +36,7 @@ def __init__( self.typ = typ self.mutable = mutable self.location = location + self.encoding = encoding self.blockscopes = [] if blockscopes is None else blockscopes self.defined_at = defined_at # source code location variable record was defined. self.is_internal = is_internal @@ -92,8 +94,8 @@ def __str__(self): return input_name + " -> " + str(self.return_type) + ":" return input_name + ":" - @property - def mk_identifier(self): + @cached_property + def lll_identifier(self) -> str: # we could do a bit better than this but it just needs to be unique visibility = "internal" if self.internal else "external" argz = ",".join([str(arg.typ) for arg in self.args]) @@ -102,37 +104,39 @@ def mk_identifier(self): ret += " -> " + str(self.return_type) return mkalphanum(ret) - # calculate the abi signature for a given set of args - def abi_signature_for_args(self, args): + # calculate the abi signature for a given set of kwargs + def abi_signature_for_kwargs(self, kwargs): + args = self.base_args + kwargs return self.name + "(" + ",".join([canonicalize_type(arg.typ) for arg in args]) + ")" @cached_property def all_kwarg_sigs(self) -> List[str]: assert not self.internal, "abi_signatures only make sense for external functions" ret = [] - argz = self.base_args.copy() - ret.append(self.abi_signature_for_args(argz)) + kwargz = [] + + ret.append(self.abi_signature_for_kwargs(kwargz)) - for arg in self.default_args: - argz.append(arg) - ret.append(self.abi_signature_for_args(argz)) + for kwarg in self.default_args: + kwargz.append(kwarg) + ret.append(self.abi_signature_for_args(kwargz)) return ret @property def base_signature(self): - return self.all_kwarg_sigs[0] + return self.abi_signature_for_kwargs([]) @property def internal_function_label(self): assert self.internal, "why are you doing this" - return self.mk_identifier + return self.lll_identifier @property def exit_sequence_label(self): - return self.mk_identifier + "_cleanup" + return self.lll_identifier + "_cleanup" def set_default_args(self): """Split base from kwargs and set member data structures""" diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index 7359bac06a..85b2dd9d1c 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -493,66 +493,3 @@ def abi_decode(lll_node, src, clamp=True, pos=None): lll_ret = ["with", "src", src, ["with", "src_loc", "src", lll_ret]] return lll_ret - - -def _add_ofst(loc, ofst): - if isinstance(loc.value, int) and isinstance(ofst, int): - return LLLnode(loc.value + ofst, location=loc.location) - return LLLnode.from_list(["add", loc, ofst], location=loc.location) - - -# decode a buffer containing abi-encoded data structure in place -# for dynamical data, a layer of indirection will be -# added for every level of nesting. for instance, -# `lazy_abi_decode(, <320>)` -# might return (mload 320), -# whereas -# `lazy_abi_decode(<(int128,bytes)>, <320>)` -# might return -# (multi -# (mload 320/*int128*/) -# (mload (add 320/*buf start*/ (mload 352/*ofst loc*/)))) -def lazy_abi_decode(typ, src, clamp=True, pos=None): - typ = copy.deepcopy(typ) - typ.is_literal = True - if isinstance(typ, (ListType, TupleLike)): - if isinstance(typ, TupleLike): - ts = typ.tuple_members() - else: - ts = [typ.subtype for _ in range(typ.count)] - ofst = 0 - os = [] - for t in ts: - child_abi_t = abi_type_of(t) - loc = _add_ofst(src, ofst) - # TODO optimize: special case for first dynamic - # member, the location is statically known. - if child_abi_t.is_dynamic(): - # load the offset word, which is the - # (location-independent) offset from the start of the - # src buffer. - dyn_ofst = unwrap_location(loc) - loc = _add_ofst(src, dyn_ofst) - os.append(lazy_abi_decode(t, loc, clamp=clamp, pos=pos)) - - ofst += child_abi_t.embedded_static_size() - - location = src.location - ret = ["multi"] + os - - elif isinstance(typ, (BaseType, ByteArrayLike)): - if clamp: - x = LLLnode.from_list(["x"], typ=typ, location=src.location) - ret = ["with", x, unwrap_location(src), ["seq", clamp_basetype(x), x]] - else: - ret = unwrap_location(src) - - if isinstance(typ, BaseType): - location = None # unwrap_location returned a stack variable - else: - # the unwrapped value is itself a pointer to calldata/memory/code - location = src.location - else: - raise CompilerPanic(f"unknown type for lazy_abi_decode {typ}") - - return LLLnode.from_list(ret, typ=typ, location=location, pos=pos) diff --git a/vyper/old_codegen/expr.py b/vyper/old_codegen/expr.py index 7d695dc86a..8e7ac185ad 100644 --- a/vyper/old_codegen/expr.py +++ b/vyper/old_codegen/expr.py @@ -298,6 +298,7 @@ def parse_Name(self): var.pos, typ=var.typ, location=var.location, # either 'memory' or 'calldata' storage is handled above. + encoding=var.encoding, pos=getpos(self.expr), annotation=self.expr.id, mutable=var.mutable, diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index cabfaa33c2..997b098247 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -5,8 +5,8 @@ StructureException, TypeCheckFailure, ) -from vyper.old_codegen.abi import abi_encode, abi_type_of, lazy_abi_decode -from vyper.old_codegen.lll_node import LLLnode +from vyper.old_codegen.abi import abi_encode, abi_type_of +from vyper.old_codegen.lll_node import LLLnode, Encoding from vyper.old_codegen.parser_utils import getpos, unwrap_location from vyper.old_codegen.types import ( TupleType, @@ -66,9 +66,10 @@ def _unpack_returndata(contract_sig, context, pos): ret += [["assert", ["gt", "returndatasize", min_return_size - 1]]] # TODO assert returndatasize <= maxlen - # abi_decode has appropriate clampers for the individual members of the return type - buf = LLLnode(buf, location="memory") - ret += [lazy_abi_decode(return_t, buf, pos=pos)] + # ABI decoder has appropriate clampers for the individual members of the return type + # cf. add_variable_offset + buf = LLLnode(buf, location="memory", encoding=Encoding.ABI) + ret += [buf] return ret, ret_ofst, ret_len @@ -123,9 +124,7 @@ def _external_call_helper( # TODO push me up to expr.py def get_gas_and_value(stmt_expr, context): - from vyper.old_codegen.expr import ( - Expr, # TODO rethink this circular import - ) + from vyper.old_codegen.expr import Expr # TODO rethink this circular import value, gas = None, None for kw in stmt_expr.keywords: @@ -139,9 +138,7 @@ def get_gas_and_value(stmt_expr, context): def lll_for_external_call(stmt_expr, context): - from vyper.old_codegen.expr import ( - Expr, # TODO rethink this circular import - ) + from vyper.old_codegen.expr import Expr # TODO rethink this circular import pos = getpos(stmt_expr) value, gas = get_gas_and_value(stmt_expr, context) diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index 86a23c60b7..3fcd737689 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -5,14 +5,13 @@ FunctionSignature, VariableRecord, ) -from vyper.old_codegen.abi import lazy_abi_decode from vyper.old_codegen.context import Context from vyper.old_codegen.expr import Expr from vyper.old_codegen.function_definitions.utils import get_nonreentrant_lock -from vyper.old_codegen.lll_node import LLLnode -from vyper.old_codegen.parser_utils import getpos, make_setter +from vyper.old_codegen.lll_node import LLLnode, Encoding +from vyper.old_codegen.parser_utils import getpos, make_setter, add_variable_offset from vyper.old_codegen.stmt import parse_body -from vyper.old_codegen.types.types import ListType, TupleType +from vyper.old_codegen.types.types import ListType, TupleType, BaseType # register function args with the local calling context. @@ -31,41 +30,28 @@ def _register_function_args(context: Context, sig: FunctionSignature) -> List[An # tuple with the abi_encoded args if sig.is_init_func: - base_args_location = LLLnode("~codelen", location="code", typ=base_args_t) + base_args_ofst = LLLnode( + "~codelen", location="code", typ=base_args_t, encoding=Encoding.ABI + ) else: - base_args_location = LLLnode(4, location="calldata", typ=base_args_t) - - base_args_lll = lazy_abi_decode(base_args_t, base_args_location) - - assert base_args_lll.value == "multi", "lazy_abi_decode did not return multi" - base_args_lll = base_args_lll.args # the (lazily) decoded values - - assert len(base_args_lll) == len(sig.base_args) - for (arg, arg_lll) in zip(sig.base_args, base_args_lll): # the actual values - assert arg.typ == arg_lll.typ, (arg.typ, arg_lll.typ) - - if isinstance(arg.typ, ListType): - assert arg_lll.value == "multi" - # ListTypes might be accessed with add_variable_offset - # which doesn't work for `multi`, so instead copy them - # to memory. - # TODO nested lists are still broken! - dst_ofst = context.new_variable(arg.name, typ=arg.typ, is_mutable=False) - dst = LLLnode(dst_ofst, typ=arg.typ, location="memory") - x = make_setter(dst, arg_lll, "memory", pos=getpos(arg.ast_source)) - ret.append(x) - else: - # register the record in the local namespace, no copy needed - context.vars[arg.name] = VariableRecord( - name=arg.name, pos=arg_lll, typ=arg.typ, mutable=False, location=arg_lll.location - ) + base_args_ofst = LLLnode(4, location="calldata", typ=base_args_t, encoding=Encoding.ABI) + + for i, arg in enumerate(sig.base_args): + # assert arg.typ == arg_lll.typ, (arg.typ, arg_lll.typ) + arg_lll = add_variable_offset(base_args_ofst, i, pos=None, array_bounds_check=False) + + # register the record in the local namespace, no copy needed + context.vars[arg.name] = VariableRecord( + name=arg.name, pos=arg_lll, typ=arg.typ, mutable=False, location=arg_lll.location, + encoding=Encoding.ABI, + ) return ret # TODO move me to function_signature.py? def _base_entry_point(sig): - return f"{sig.mk_identifier}_entry" + return f"{sig.lll_identifier}_entry" def _generate_kwarg_handlers(context: Context, sig: FunctionSignature, pos: Any) -> List[Any]: @@ -76,33 +62,34 @@ def _generate_kwarg_handlers(context: Context, sig: FunctionSignature, pos: Any) def handler_for(calldata_kwargs, default_kwargs): calldata_args = sig.base_args + calldata_kwargs + # create a fake type so that add_variable_offset works calldata_args_t = TupleType(list(arg.typ for arg in calldata_args)) - abi_sig = sig.abi_signature_for_args(calldata_args) + abi_sig = sig.abi_signature_for_kwargs(calldata_kwargs) method_id = util.abi_method_id(abi_sig) - calldata_args_location = LLLnode(4, location="calldata", typ=calldata_args_t) - - calldata_args_lll = lazy_abi_decode(calldata_args_t, calldata_args_location) - - assert calldata_args_lll.value == "multi" # sanity check - # extract just the kwargs from the ABI payload - calldata_kwargs_lll = calldata_args_lll.args[len(sig.base_args) :] # noqa: E203 + calldata_kwargs_ofst = LLLnode( + 4, location="calldata", typ=calldata_args_t, encoding=Encoding.ABI + ) # a sequence of statements to strictify kwargs into memory ret = ["seq"] # TODO optimize make_setter by using # TupleType(list(arg.typ for arg in calldata_kwargs + default_kwargs)) + # (must ensure memory area is contiguous) lhs_location = "memory" - assert len(calldata_kwargs_lll) == len(calldata_kwargs), calldata_kwargs - for arg_meta, arg_lll in zip(calldata_kwargs, calldata_kwargs_lll): - assert arg_meta.typ == arg_lll.typ + n_base_args = len(sig.base_args) + + for i, arg_meta in enumerate(calldata_kwargs): + k = n_base_args + i + dst = context.new_variable(arg_meta.name, arg_meta.typ, is_mutable=False) lhs = LLLnode(dst, location="memory", typ=arg_meta.typ) - rhs = arg_lll + rhs = add_variable_offset(calldata_kwargs_ofst, k, pos=None, array_bounds_check=False) ret.append(make_setter(lhs, rhs, lhs_location, pos)) + for x in default_kwargs: dst = context.new_variable(x.name, x.typ, is_mutable=False) lhs = LLLnode(dst, location="memory", typ=x.typ) diff --git a/vyper/old_codegen/lll_node.py b/vyper/old_codegen/lll_node.py index e58ace6728..c3d429f5fa 100644 --- a/vyper/old_codegen/lll_node.py +++ b/vyper/old_codegen/lll_node.py @@ -1,5 +1,6 @@ import re from typing import Any, List, Optional, Tuple, Union +from enum import Enum, auto from vyper.compiler.settings import VYPER_COLOR_OUTPUT from vyper.evm.opcodes import get_comb_opcodes @@ -40,6 +41,14 @@ def push_label_to_stack(labelname: str) -> str: return "_sym_" + labelname +class Encoding(Enum): + # vyper encoding, default for memory variables + VYPER = auto() + # abi encoded, default for args/return values from external funcs + ABI = auto() + # future: packed + + # Data structure for LLL parse tree class LLLnode: repr_show_gas = False @@ -59,19 +68,22 @@ def __init__( mutable: bool = True, add_gas_estimate: int = 0, valency: Optional[int] = None, + encoding: Encoding = Encoding.VYPER, ): if args is None: args = [] self.value = value self.args = args + # TODO remove this sanity check once mypy is more thorough + assert isinstance(typ, NodeType) or typ is None, repr(typ) self.typ = typ - assert isinstance(self.typ, NodeType) or self.typ is None, repr(self.typ) self.location = location self.pos = pos self.annotation = annotation self.mutable = mutable self.add_gas_estimate = add_gas_estimate + self.encoding = encoding self.as_hex = AS_HEX_DEFAULT # Optional annotation properties for gas estimation @@ -342,6 +354,7 @@ def from_list( mutable: bool = True, add_gas_estimate: int = 0, valency: Optional[int] = None, + encoding: Encoding = Encoding.VYPER, ) -> "LLLnode": if isinstance(typ, str): typ = BaseType(typ) @@ -355,6 +368,9 @@ def from_list( obj.pos = pos if obj.location is None: obj.location = location + + obj.encoding = encoding + return obj elif not isinstance(obj, list): return cls( @@ -366,6 +382,8 @@ def from_list( annotation=annotation, mutable=mutable, add_gas_estimate=add_gas_estimate, + valency=valency, + encoding=encoding, ) else: return cls( @@ -378,4 +396,5 @@ def from_list( mutable=mutable, add_gas_estimate=add_gas_estimate, valency=valency, + encoding=encoding, ) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 5c31a7e323..a8ba89f35e 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -10,7 +10,7 @@ TypeMismatch, ) from vyper.old_codegen.arg_clamps import int128_clamp -from vyper.old_codegen.lll_node import LLLnode +from vyper.old_codegen.lll_node import LLLnode, Encoding from vyper.old_codegen.types import ( BaseType, ByteArrayLike, @@ -281,11 +281,47 @@ def getpos(node): ) +def _add_ofst(loc, ofst): + if isinstance(loc.value, int) and isinstance(ofst, int): + ret = loc.value + ofst + else: + ret = ["add", loc, ofst] + return LLLnode.from_list(ret, location=loc.location, encoding=loc.encoding) + # Take a value representing a memory or storage location, and descend down to # an element or member variable +# TODO refactor / streamline this code, especially the ABI decoding @type_check_wrapper def add_variable_offset(parent, key, pos, array_bounds_check=True): + # TODO rethink this circular import + from vyper.old_codegen.abi import abi_type_of + typ, location = parent.typ, parent.location + + def _abi_helper(member_t, ofst, clamp=True): + ofst_lll = _add_ofst(parent, ofst) + + if member_abi_t.is_dynamic(): + # double dereference, according to ABI spec + # TODO optimize special case: first dynamic item + # offset is statically known. + ofst_lll = _add_ofst(parent, unwrap_location(ofst_lll)) + + x = LLLnode.from_list(["x"], typ=member_t, location=parent.location) + if isinstance(member_t, (BaseType, ByteArrayLike)) and clamp: + ret = ["with", "x", ofst_lll, ["seq", clamp_basetype(x), x]] + else: + ret = ofst_lll + + return LLLnode.from_list( + ret, + typ=member_t, + location=parent.location, + encoding=parent.encoding, + pos=pos, + # annotation=f"({parent.typ})[{key.typ}]", + ) + if isinstance(typ, TupleLike): if isinstance(typ, StructType): subtype = typ.members[key] @@ -301,8 +337,25 @@ def add_variable_offset(parent, key, pos, array_bounds_check=True): if parent.value is None: return LLLnode.from_list(None, typ=subtype) - if parent.value == "multi": - return parent.args[index] + # TODO case for parent.value == multi? + # if parent.value == "multi" + # return parent.args[index] + + if parent.encoding == Encoding.ABI: + if parent.location == "storage": + raise CompilerPanic("storage variables should not be abi encoded") + + parent_abi_t = abi_type_of(parent.typ) + member_t = typ.members[attrs[index]] + member_abi_t = abi_type_of(member_t) + + ofst = 0 # offset from parent start + + for i in range(index): + member_abi_t = abi_type_of(typ.members[attrs[i]]) + ofst += member_abi_t.embedded_static_size() + + return _abi_helper(member_t, ofst) if location == "storage": # for arrays and structs, calculate the storage slot by adding an offset @@ -310,50 +363,22 @@ def add_variable_offset(parent, key, pos, array_bounds_check=True): offset = 0 for i in range(index): offset += get_size_of_type(typ.members[attrs[i]]) - return LLLnode.from_list(["add", parent, offset], typ=subtype, location="storage",) + return LLLnode.from_list( + ["add", parent, offset], typ=subtype, location="storage", pos=pos, + ) + elif location in ("calldata", "memory", "code"): offset = 0 for i in range(index): offset += 32 * get_size_of_type(typ.members[attrs[i]]) return LLLnode.from_list( - ["add", offset, parent], + _add_ofst(parent, offset), typ=typ.members[key], location=location, annotation=annotation, + pos=pos, ) - elif isinstance(typ, MappingType): - - sub = None - if isinstance(key.typ, ByteArrayLike): - if isinstance(typ.keytype, ByteArrayLike) and (typ.keytype.maxlen >= key.typ.maxlen): - - subtype = typ.valuetype - if len(key.args[0].args) >= 3: # handle bytes literal. - sub = LLLnode.from_list( - [ - "seq", - key, - [ - "sha3", - ["add", key.args[0].args[-1], 32], - ["mload", key.args[0].args[-1]], - ], - ] - ) - else: - value = key.args[0].value - if value == "add": - # special case, key is a bytes array within a tuple/struct - value = key.args[0] - sub = LLLnode.from_list(["sha3", ["add", value, 32], key]) - else: - subtype = typ.valuetype - sub = unwrap_location(key) - - if sub is not None and location == "storage": - return LLLnode.from_list(["sha3_64", parent, sub], typ=subtype, location="storage") - elif isinstance(typ, ListType) and is_base_type(key.typ, ("int128", "int256", "uint256")): subtype = typ.subtype @@ -375,6 +400,22 @@ def add_variable_offset(parent, key, pos, array_bounds_check=True): # an array index, and the clamp will throw an error. sub = ["uclamplt", k, typ.count] + if parent.encoding == Encoding.ABI: + if parent.location == "storage": + raise CompilerPanic("storage variables should not be abi encoded") + + parent_abi_t = abi_type_of(typ) + member_t = typ.subtype + member_abi_t = abi_type_of(member_t) + + if key.typ.is_literal: + # TODO this constant folding in LLL optimizer + ofst = k.value * member_abi_t.embedded_static_size() + else: + ofst = ["mul", k, member_abi_t.embedded_static_size()] + + return _abi_helper(member_t, ofst) + if location == "storage": # storage slot determined as [initial storage slot] + [index] * [size of base type] offset = get_size_of_type(subtype) @@ -382,11 +423,46 @@ def add_variable_offset(parent, key, pos, array_bounds_check=True): ["add", parent, ["mul", sub, offset]], typ=subtype, location="storage", pos=pos ) elif location in ("calldata", "memory", "code"): + if parent.encoding == "abi": + return LLLnode.from_list() + offset = 32 * get_size_of_type(subtype) return LLLnode.from_list( ["add", ["mul", offset, sub], parent], typ=subtype, location=location, pos=pos ) + elif isinstance(typ, MappingType): + + sub = None + if isinstance(key.typ, ByteArrayLike): + if isinstance(typ.keytype, ByteArrayLike) and (typ.keytype.maxlen >= key.typ.maxlen): + + subtype = typ.valuetype + if len(key.args[0].args) >= 3: # handle bytes literal. + sub = LLLnode.from_list( + [ + "seq", + key, + [ + "sha3", + ["add", key.args[0].args[-1], 32], + ["mload", key.args[0].args[-1]], + ], + ] + ) + else: + value = key.args[0].value + if value == "add": + # special case, key is a bytes array within a tuple/struct + value = key.args[0] + sub = LLLnode.from_list(["sha3", ["add", value, 32], key]) + else: + subtype = typ.valuetype + sub = unwrap_location(key) + + if sub is not None and location == "storage": + return LLLnode.from_list(["sha3_64", parent, sub], typ=subtype, location="storage") + def load_op(location): if location == "memory": @@ -572,7 +648,7 @@ def make_setter(left, right, location, pos): if var_arg.location in ("calldata", "code"): return - right_token = LLLnode.from_list("_R", typ=right.typ, location=right.location) + right_token = LLLnode.from_list("_R", typ=right.typ, location=right.location, encoding=right.encoding) for left_arg, key, loc in zip(left.args, keyz, locations): subs.append( make_setter( @@ -586,7 +662,7 @@ def make_setter(left, right, location, pos): # If the left side is a variable i.e struct type else: subs = [] - right_token = LLLnode.from_list("_R", typ=right.typ, location=right.location) + right_token = LLLnode.from_list("_R", typ=right.typ, location=right.location, encoding=right.encoding) for typ, loc in zip(keyz, locations): subs.append( make_setter( @@ -684,6 +760,7 @@ def clamp_basetype(lll_node): if isinstance(t, ByteArrayLike): return ["assert", ["le", get_bytearray_length(lll_node), t.maxlen]] if isinstance(t, BaseType): + lll_node = unwrap_location(lll_node) if t.typ in ("int128"): return int_clamp(lll_node, 128, signed=True) if t.typ in ("decimal"): diff --git a/vyper/old_codegen/return_.py b/vyper/old_codegen/return_.py index 4995ea4823..b25e39699a 100644 --- a/vyper/old_codegen/return_.py +++ b/vyper/old_codegen/return_.py @@ -37,7 +37,9 @@ def make_return_stmt(lll_val: LLLnode, stmt: Any, context: Context) -> Optional[ # helper function def finalize(fill_return_buffer): # do NOT bypass this. jump_to_exit may do important function cleanup. - fill_return_buffer = LLLnode.from_list(fill_return_buffer, annotation=f"fill return buffer {sig.mk_identifier}") + fill_return_buffer = LLLnode.from_list( + fill_return_buffer, annotation=f"fill return buffer {sig.lll_identifier}" + ) return LLLnode.from_list( ["seq_unchecked", fill_return_buffer, jump_to_exit], typ=None, pos=_pos, ) diff --git a/vyper/old_codegen/self_call.py b/vyper/old_codegen/self_call.py index f36cbee8f9..37d5e6f9ce 100644 --- a/vyper/old_codegen/self_call.py +++ b/vyper/old_codegen/self_call.py @@ -14,9 +14,7 @@ def _generate_label(name: str) -> str: def lll_for_self_call(stmt_expr, context): - from vyper.old_codegen.expr import ( - Expr, # TODO rethink this circular import - ) + from vyper.old_codegen.expr import Expr # TODO rethink this circular import pos = getpos(stmt_expr) diff --git a/vyper/semantics/validation/local.py b/vyper/semantics/validation/local.py index 861517add6..b02c7ff585 100644 --- a/vyper/semantics/validation/local.py +++ b/vyper/semantics/validation/local.py @@ -19,6 +19,7 @@ VariableDeclarationException, VyperException, ) + # TODO consolidate some of these imports from vyper.semantics.environment import ( CONSTANT_ENVIRONMENT_VARS, diff --git a/vyper/utils.py b/vyper/utils.py index e60ef66263..246aaa8baf 100644 --- a/vyper/utils.py +++ b/vyper/utils.py @@ -1,6 +1,7 @@ import binascii import functools from typing import Dict, List, Union +import sys from vyper.exceptions import InvalidLiteral @@ -19,6 +20,13 @@ def fourbytes_to_int(inp): return (inp[0] << 24) + (inp[1] << 16) + (inp[2] << 8) + inp[3] +# utility function for debugging purposes +def trace(n=5, out=sys.stderr): + print("BEGIN TRACE", file=out) + for x in list(traceback.format_stack)[-n:]: + print(x.strip(), file=out) + print("END TRACE", file=out) + # converts a signature like Func(bool,uint256,address) to its 4 byte method ID # TODO replace manual calculations in codebase with this def abi_method_id(method_sig): From f0e3156c2b35ba6287851f73c57caf302d031348 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 14 Sep 2021 02:54:23 +0000 Subject: [PATCH 062/163] fix some minor bugs --- vyper/ast/signatures/function_signature.py | 2 +- vyper/ast/signatures/interface.py | 2 +- vyper/old_codegen/context.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/vyper/ast/signatures/function_signature.py b/vyper/ast/signatures/function_signature.py index 11e8a1d788..d96d6e44ea 100644 --- a/vyper/ast/signatures/function_signature.py +++ b/vyper/ast/signatures/function_signature.py @@ -150,7 +150,7 @@ def set_default_args(self): self.default_args = self.args[num_base_args:] # Keep all the value to assign to default parameters. - self.default_values = dict(zip([arg.name for arg in self.default_args], args.defaults)) + self.default_values = dict(zip([arg.name for arg in self.default_args], defaults)) # Get a signature from a function definition @classmethod diff --git a/vyper/ast/signatures/interface.py b/vyper/ast/signatures/interface.py index d15554e801..e7dd4a78e5 100644 --- a/vyper/ast/signatures/interface.py +++ b/vyper/ast/signatures/interface.py @@ -80,7 +80,7 @@ def mk_full_signature_from_json(abi): decorator_list.append(vy_ast.Name(id="payable")) sig = FunctionSignature.from_definition( - code=vy_ast.FunctionDef( + func_ast=vy_ast.FunctionDef( name=func["name"], args=vy_ast.arguments(args=args), decorator_list=decorator_list, diff --git a/vyper/old_codegen/context.py b/vyper/old_codegen/context.py index 41645c70f3..81ca644485 100644 --- a/vyper/old_codegen/context.py +++ b/vyper/old_codegen/context.py @@ -242,7 +242,7 @@ def _check(cond, s="Unreachable"): _check(len(sig.base_args) <= len(args_lll) <= len(sig.args)) # more sanity check, that the types match - _check(all(l.typ == r.typ for (l, r) in zip(args_lll, sig.args))) + #_check(all(l.typ == r.typ for (l, r) in zip(args_lll, sig.args)), str( (args_lll, sig.args))) num_provided_args = len(args_lll) total_args = len(sig.args) From e6038df1059a36cdfb9c55cc6d3c50c6fb9e693a Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 14 Sep 2021 03:01:01 +0000 Subject: [PATCH 063/163] fix source map for new calling convention --- vyper/lll/compile_lll.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/vyper/lll/compile_lll.py b/vyper/lll/compile_lll.py index aab847b9d6..e9746770b6 100644 --- a/vyper/lll/compile_lll.py +++ b/vyper/lll/compile_lll.py @@ -622,11 +622,15 @@ def assembly_to_evm(assembly, start_pos=0): if item == "JUMP": last = assembly[i - 1] - if last == "MLOAD": - line_number_map["pc_jump_map"][pos] = "o" - elif is_symbol(last) and "_priv_" in last: - line_number_map["pc_jump_map"][pos] = "i" + if is_symbol(last) and last.startswith("_sym_internal"): + if last.endswith("cleanup"): + # exit an internal function + line_number_map["pc_jump_map"][pos] = "o" + else: + # enter an internal function + line_number_map["pc_jump_map"][pos] = "i" else: + # everything else line_number_map["pc_jump_map"][pos] = "-" elif item in ("JUMPI", "JUMPDEST"): line_number_map["pc_jump_map"][pos] = "-" From 6eb45e51ffd910317fde4e0e0146bf46d9e2df2c Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 14 Sep 2021 03:07:11 +0000 Subject: [PATCH 064/163] add missing bytes32 case to clamp_basetype --- vyper/old_codegen/parser_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index a8ba89f35e..fc55b80220 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -775,7 +775,7 @@ def clamp_basetype(lll_node): return int_clamp(lll_node, 160) if t.typ in ("bool",): return int_clamp(lll_node, 1) - if t.typ in ("int256", "uint256"): + if t.typ in ("int256", "uint256", "bytes32"): return ["pass"] # special case, no clamp return # raises From ff125bf77fe00a8930dfe5c073349481944236ee Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 14 Sep 2021 03:15:53 +0000 Subject: [PATCH 065/163] fix base handler check --- vyper/ast/signatures/function_signature.py | 2 +- .../function_definitions/external_function.py | 19 +++++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/vyper/ast/signatures/function_signature.py b/vyper/ast/signatures/function_signature.py index d96d6e44ea..7c989efb3e 100644 --- a/vyper/ast/signatures/function_signature.py +++ b/vyper/ast/signatures/function_signature.py @@ -124,7 +124,7 @@ def all_kwarg_sigs(self) -> List[str]: return ret - @property + @cached_property def base_signature(self): return self.abi_signature_for_kwargs([]) diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index 3fcd737689..71889a4baa 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -37,8 +37,8 @@ def _register_function_args(context: Context, sig: FunctionSignature) -> List[An base_args_ofst = LLLnode(4, location="calldata", typ=base_args_t, encoding=Encoding.ABI) for i, arg in enumerate(sig.base_args): - # assert arg.typ == arg_lll.typ, (arg.typ, arg_lll.typ) arg_lll = add_variable_offset(base_args_ofst, i, pos=None, array_bounds_check=False) + assert arg.typ == arg_lll.typ, (arg.typ, arg_lll.typ) # register the record in the local namespace, no copy needed context.vars[arg.name] = VariableRecord( @@ -66,7 +66,7 @@ def handler_for(calldata_kwargs, default_kwargs): calldata_args_t = TupleType(list(arg.typ for arg in calldata_args)) abi_sig = sig.abi_signature_for_kwargs(calldata_kwargs) - method_id = util.abi_method_id(abi_sig) + method_id = LLLnode(util.abi_method_id(abi_sig), annotation=abi_sig) calldata_kwargs_ofst = LLLnode( 4, location="calldata", typ=calldata_args_t, encoding=Encoding.ABI @@ -102,7 +102,7 @@ def handler_for(calldata_kwargs, default_kwargs): ret = ["if", ["eq", "_calldata_method_id", method_id], ret] return ret - ret = ["seq"] + ret = [] keyword_args = sig.default_args @@ -139,7 +139,7 @@ def generate_lll_for_external_function(code, sig, context, check_nonpayable): entrance = [base_arg_handlers] # once args have been handled - if len(kwarg_handlers) > 1: + if len(kwarg_handlers) > 0: entrance += [["label", _base_entry_point(sig)]] else: # otherwise, the label is redundant since there is only @@ -164,11 +164,10 @@ def generate_lll_for_external_function(code, sig, context, check_nonpayable): ret = ["seq"] + kwarg_handlers + entrance + body + exit - # TODO special handling for default function - if len(kwarg_handlers) == 0: - _sigs = sig.all_kwarg_sigs - assert len(_sigs) == 1 - _method_id = util.abi_method_id(_sigs[0]) - ret = ["if", ["eq", "_calldata_method_id", _method_id], ret] + if len(kwarg_handlers) == 0 and not sig.is_default_func and not sig.is_init_func: + assert len(sig.default_args) == 0 # sanity check + abi_sig = sig.base_signature + method_id = LLLnode(util.abi_method_id(abi_sig), annotation=abi_sig) + ret = ["if", ["eq", "_calldata_method_id", method_id], ret] return LLLnode.from_list(ret, pos=getpos(code)) From fbba2ff4cc8aee25a82c2e23b343f20b576baf54 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 13 Sep 2021 23:23:00 -0700 Subject: [PATCH 066/163] fix order of base arg hanlders --- vyper/old_codegen/function_definitions/external_function.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index 71889a4baa..57e3bcd14d 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -136,7 +136,7 @@ def generate_lll_for_external_function(code, sig, context, check_nonpayable): kwarg_handlers = _generate_kwarg_handlers(context, sig, pos) - entrance = [base_arg_handlers] + entrance = [] # once args have been handled if len(kwarg_handlers) > 0: @@ -146,6 +146,8 @@ def generate_lll_for_external_function(code, sig, context, check_nonpayable): # one control flow path into the external method pass + entrance += base_arg_handlers + if check_nonpayable and sig.mutability != "payable": # if the contract contains payable functions, but this is not one of them # add an assertion that the value of the call is zero From 8bf5b985e72467450df9b647c8eeba1cd41fcadc Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 14 Sep 2021 14:52:47 +0000 Subject: [PATCH 067/163] improve annotation for internal function exit --- .../old_codegen/function_definitions/internal_function.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/vyper/old_codegen/function_definitions/internal_function.py b/vyper/old_codegen/function_definitions/internal_function.py index 6927164fe3..8d0507c2ac 100644 --- a/vyper/old_codegen/function_definitions/internal_function.py +++ b/vyper/old_codegen/function_definitions/internal_function.py @@ -56,10 +56,8 @@ def mkidentifier(s): function_entry_label = sig.internal_function_label cleanup_label = sig.exit_sequence_label - # internal functions without return types need to jump back to the calling - # function, as there is no guarantee there is a user-provided return - # statement (which would generate the jump) - stop_func = [["jump", "pass"]] # was passed in via stack + # jump to the label which was passed in via stack + stop_func = LLLnode.from_list(["jump", "pass"], annotation=f"jump to return address") enter = [["label", function_entry_label], nonreentrant_pre] @@ -69,6 +67,6 @@ def mkidentifier(s): # name the variable that was passed via stack body = [["with", "return_buffer", "pass", ["seq"] + body]] - exit = [["label", cleanup_label]] + nonreentrant_post + stop_func + exit = [["label", cleanup_label]] + nonreentrant_post + [stop_func] return LLLnode.from_list(["seq"] + enter + body + exit, typ=None, pos=getpos(code),) From ed4f545255183711d5125f32f08f9558c05e1867 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 14 Sep 2021 15:11:53 +0000 Subject: [PATCH 068/163] fix init func exit sequence --- vyper/old_codegen/function_definitions/external_function.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index 57e3bcd14d..39f9cc1e1a 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -158,7 +158,9 @@ def generate_lll_for_external_function(code, sig, context, check_nonpayable): body = [parse_body(c, context) for c in code.body] exit = [["label", sig.exit_sequence_label]] + nonreentrant_post - if context.return_type is None: + if sig.is_init_func: + pass # init func has special exit sequence generated by parser.py + elif context.return_type is None: exit += [["stop"]] else: # ret_ofst and ret_len stack items passed by function body; consume using 'pass' From 80e55f3f1a6f5b81722157b6580c9e5c2b202d8b Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 14 Sep 2021 15:36:20 +0000 Subject: [PATCH 069/163] improve annotation of method IDs --- .../function_definitions/external_function.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index 39f9cc1e1a..1baf92b82f 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -54,6 +54,12 @@ def _base_entry_point(sig): return f"{sig.lll_identifier}_entry" +def _annotated_method_id(abi_sig): + method_id = util.abi_method_id(abi_sig) + annotation = f"{hex(method_id)}: {abi_sig}" + return LLLnode(method_id, annotation=annotation) + + def _generate_kwarg_handlers(context: Context, sig: FunctionSignature, pos: Any) -> List[Any]: # generate kwarg handlers. # since they might come in thru calldata or be default, @@ -66,7 +72,7 @@ def handler_for(calldata_kwargs, default_kwargs): calldata_args_t = TupleType(list(arg.typ for arg in calldata_args)) abi_sig = sig.abi_signature_for_kwargs(calldata_kwargs) - method_id = LLLnode(util.abi_method_id(abi_sig), annotation=abi_sig) + method_id = _annotated_method_id(abi_sig) calldata_kwargs_ofst = LLLnode( 4, location="calldata", typ=calldata_args_t, encoding=Encoding.ABI @@ -171,7 +177,7 @@ def generate_lll_for_external_function(code, sig, context, check_nonpayable): if len(kwarg_handlers) == 0 and not sig.is_default_func and not sig.is_init_func: assert len(sig.default_args) == 0 # sanity check abi_sig = sig.base_signature - method_id = LLLnode(util.abi_method_id(abi_sig), annotation=abi_sig) + method_id = _annotated_method_id(abi_sig) ret = ["if", ["eq", "_calldata_method_id", method_id], ret] return LLLnode.from_list(ret, pos=getpos(code)) From d1fa8db3da8dbda620c1365d9df13ed6bf861f9d Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 14 Sep 2021 15:54:22 +0000 Subject: [PATCH 070/163] fix stack handling of return_buffer it needs to be more tightly scoped in order to get POPped --- .../function_definitions/internal_function.py | 4 ---- vyper/old_codegen/return_.py | 9 ++++++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/vyper/old_codegen/function_definitions/internal_function.py b/vyper/old_codegen/function_definitions/internal_function.py index 8d0507c2ac..93a79788e9 100644 --- a/vyper/old_codegen/function_definitions/internal_function.py +++ b/vyper/old_codegen/function_definitions/internal_function.py @@ -63,10 +63,6 @@ def mkidentifier(s): body = [parse_body(c, context) for c in code.body] - if sig.return_type is not None: - # name the variable that was passed via stack - body = [["with", "return_buffer", "pass", ["seq"] + body]] - exit = [["label", cleanup_label]] + nonreentrant_post + [stop_func] return LLLnode.from_list(["seq"] + enter + body + exit, typ=None, pos=getpos(code),) diff --git a/vyper/old_codegen/return_.py b/vyper/old_codegen/return_.py index b25e39699a..b70c9860e5 100644 --- a/vyper/old_codegen/return_.py +++ b/vyper/old_codegen/return_.py @@ -49,7 +49,14 @@ def finalize(fill_return_buffer): if context.is_internal: dst = LLLnode.from_list(["return_buffer"], typ=context.return_type, location="memory") - return finalize(make_setter(dst, lll_val, location="memory", pos=_pos)) + fill_return_buffer = [ + "with", + dst, + "pass", # return_buffer is passed on the stack by caller + make_setter(dst, lll_val, location="memory", pos=_pos), + ] + + return finalize(fill_return_buffer) # we are in an external function. From 009c3a0e36b66cb063252d3e8779c58cf67e6f90 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 14 Sep 2021 15:57:49 +0000 Subject: [PATCH 071/163] fix arg alignment for external call --- vyper/old_codegen/external_call.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index 997b098247..93e101c289 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -41,6 +41,7 @@ def _pack_arguments(contract_sig, args, context, pos): # (mstore buf (shl signature.method_id 224)) mstore_method_id = [["mstore", buf, util.abi_method_id(abi_signature)]] + buf += 32 encode_args = abi_encode(buf, args_as_tuple, pos) return mstore_method_id + [encode_args], args_ofst, args_len From 2db19b0d0d2f02b117b18b90cba49c6daf5285de Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 14 Sep 2021 16:08:34 +0000 Subject: [PATCH 072/163] fix some formatting --- vyper/old_codegen/context.py | 2 +- .../function_definitions/external_function.py | 6 +++++- vyper/old_codegen/parser_utils.py | 9 +++++++-- vyper/utils.py | 1 + 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/vyper/old_codegen/context.py b/vyper/old_codegen/context.py index 81ca644485..030b144966 100644 --- a/vyper/old_codegen/context.py +++ b/vyper/old_codegen/context.py @@ -242,7 +242,7 @@ def _check(cond, s="Unreachable"): _check(len(sig.base_args) <= len(args_lll) <= len(sig.args)) # more sanity check, that the types match - #_check(all(l.typ == r.typ for (l, r) in zip(args_lll, sig.args)), str( (args_lll, sig.args))) + # _check(all(l.typ == r.typ for (l, r) in zip(args_lll, sig.args)), str( (args_lll, sig.args))) num_provided_args = len(args_lll) total_args = len(sig.args) diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index 1baf92b82f..b985525361 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -42,7 +42,11 @@ def _register_function_args(context: Context, sig: FunctionSignature) -> List[An # register the record in the local namespace, no copy needed context.vars[arg.name] = VariableRecord( - name=arg.name, pos=arg_lll, typ=arg.typ, mutable=False, location=arg_lll.location, + name=arg.name, + pos=arg_lll, + typ=arg.typ, + mutable=False, + location=arg_lll.location, encoding=Encoding.ABI, ) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index fc55b80220..847144a9e1 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -288,6 +288,7 @@ def _add_ofst(loc, ofst): ret = ["add", loc, ofst] return LLLnode.from_list(ret, location=loc.location, encoding=loc.encoding) + # Take a value representing a memory or storage location, and descend down to # an element or member variable # TODO refactor / streamline this code, especially the ABI decoding @@ -648,7 +649,9 @@ def make_setter(left, right, location, pos): if var_arg.location in ("calldata", "code"): return - right_token = LLLnode.from_list("_R", typ=right.typ, location=right.location, encoding=right.encoding) + right_token = LLLnode.from_list( + "_R", typ=right.typ, location=right.location, encoding=right.encoding + ) for left_arg, key, loc in zip(left.args, keyz, locations): subs.append( make_setter( @@ -662,7 +665,9 @@ def make_setter(left, right, location, pos): # If the left side is a variable i.e struct type else: subs = [] - right_token = LLLnode.from_list("_R", typ=right.typ, location=right.location, encoding=right.encoding) + right_token = LLLnode.from_list( + "_R", typ=right.typ, location=right.location, encoding=right.encoding + ) for typ, loc in zip(keyz, locations): subs.append( make_setter( diff --git a/vyper/utils.py b/vyper/utils.py index 246aaa8baf..ae69ff4271 100644 --- a/vyper/utils.py +++ b/vyper/utils.py @@ -27,6 +27,7 @@ def trace(n=5, out=sys.stderr): print(x.strip(), file=out) print("END TRACE", file=out) + # converts a signature like Func(bool,uint256,address) to its 4 byte method ID # TODO replace manual calculations in codebase with this def abi_method_id(method_sig): From c0f609c8ce146d85c38861c2cc3ccb98b87155b3 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 14 Sep 2021 17:08:09 +0000 Subject: [PATCH 073/163] refactor external entry label generation --- vyper/ast/signatures/function_signature.py | 13 ++++++++++--- .../function_definitions/external_function.py | 9 ++------- vyper/old_codegen/return_.py | 2 +- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/vyper/ast/signatures/function_signature.py b/vyper/ast/signatures/function_signature.py index 7c989efb3e..60cfdfa534 100644 --- a/vyper/ast/signatures/function_signature.py +++ b/vyper/ast/signatures/function_signature.py @@ -95,7 +95,7 @@ def __str__(self): return input_name + ":" @cached_property - def lll_identifier(self) -> str: + def _lll_identifier(self) -> str: # we could do a bit better than this but it just needs to be unique visibility = "internal" if self.internal else "external" argz = ",".join([str(arg.typ) for arg in self.args]) @@ -128,15 +128,22 @@ def all_kwarg_sigs(self) -> List[str]: def base_signature(self): return self.abi_signature_for_kwargs([]) + @property + # common entry point for external function with kwargs + def external_function_base_entry_label(self): + assert not self.internal + + return self._lll_identifier + "_common" + @property def internal_function_label(self): assert self.internal, "why are you doing this" - return self.lll_identifier + return self._lll_identifier @property def exit_sequence_label(self): - return self.lll_identifier + "_cleanup" + return self._lll_identifier + "_cleanup" def set_default_args(self): """Split base from kwargs and set member data structures""" diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index b985525361..6fb0b71845 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -53,11 +53,6 @@ def _register_function_args(context: Context, sig: FunctionSignature) -> List[An return ret -# TODO move me to function_signature.py? -def _base_entry_point(sig): - return f"{sig.lll_identifier}_entry" - - def _annotated_method_id(abi_sig): method_id = util.abi_method_id(abi_sig) annotation = f"{hex(method_id)}: {abi_sig}" @@ -107,7 +102,7 @@ def handler_for(calldata_kwargs, default_kwargs): rhs = Expr(kw_ast_val, context).lll_node ret.append(make_setter(lhs, rhs, lhs_location, pos)) - ret.append(["goto", _base_entry_point(sig)]) + ret.append(["goto", sig.external_function_base_entry_label]) ret = ["if", ["eq", "_calldata_method_id", method_id], ret] return ret @@ -150,7 +145,7 @@ def generate_lll_for_external_function(code, sig, context, check_nonpayable): # once args have been handled if len(kwarg_handlers) > 0: - entrance += [["label", _base_entry_point(sig)]] + entrance += [["label", sig.external_function_base_entry_label]] else: # otherwise, the label is redundant since there is only # one control flow path into the external method diff --git a/vyper/old_codegen/return_.py b/vyper/old_codegen/return_.py index b70c9860e5..0bd0b7d6b0 100644 --- a/vyper/old_codegen/return_.py +++ b/vyper/old_codegen/return_.py @@ -38,7 +38,7 @@ def make_return_stmt(lll_val: LLLnode, stmt: Any, context: Context) -> Optional[ def finalize(fill_return_buffer): # do NOT bypass this. jump_to_exit may do important function cleanup. fill_return_buffer = LLLnode.from_list( - fill_return_buffer, annotation=f"fill return buffer {sig.lll_identifier}" + fill_return_buffer, annotation=f"fill return buffer {sig._lll_identifier}" ) return LLLnode.from_list( ["seq_unchecked", fill_return_buffer, jump_to_exit], typ=None, pos=_pos, From baf77948d5b09a1f4c7da2ccbda0691f85bcb857 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 14 Sep 2021 17:20:44 +0000 Subject: [PATCH 074/163] fix mutating of LLLnodes --- vyper/old_codegen/lll_node.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vyper/old_codegen/lll_node.py b/vyper/old_codegen/lll_node.py index c3d429f5fa..b1e1b31bff 100644 --- a/vyper/old_codegen/lll_node.py +++ b/vyper/old_codegen/lll_node.py @@ -368,8 +368,8 @@ def from_list( obj.pos = pos if obj.location is None: obj.location = location - - obj.encoding = encoding + if obj.encoding is None: + obj.encoding = encoding return obj elif not isinstance(obj, list): From f189dcc48aac969a2cc65ff4208aa85b02925b54 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 14 Sep 2021 17:31:42 +0000 Subject: [PATCH 075/163] fix variable scoping in add_variable_offset --- vyper/old_codegen/parser_utils.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 847144a9e1..903fe00768 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -300,6 +300,7 @@ def add_variable_offset(parent, key, pos, array_bounds_check=True): typ, location = parent.typ, parent.location def _abi_helper(member_t, ofst, clamp=True): + member_abi_t = abi_type_of(member_t) ofst_lll = _add_ofst(parent, ofst) if member_abi_t.is_dynamic(): @@ -348,7 +349,6 @@ def _abi_helper(member_t, ofst, clamp=True): parent_abi_t = abi_type_of(parent.typ) member_t = typ.members[attrs[index]] - member_abi_t = abi_type_of(member_t) ofst = 0 # offset from parent start @@ -424,9 +424,6 @@ def _abi_helper(member_t, ofst, clamp=True): ["add", parent, ["mul", sub, offset]], typ=subtype, location="storage", pos=pos ) elif location in ("calldata", "memory", "code"): - if parent.encoding == "abi": - return LLLnode.from_list() - offset = 32 * get_size_of_type(subtype) return LLLnode.from_list( ["add", ["mul", offset, sub], parent], typ=subtype, location=location, pos=pos From 16b3383b659778df91df33f9c67ef992e826deb0 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 14 Sep 2021 17:45:14 +0000 Subject: [PATCH 076/163] make labels more human readable return type is never used for function lookup; omit the return type since labels are easier to read if they are shorter. --- vyper/ast/signatures/function_signature.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/vyper/ast/signatures/function_signature.py b/vyper/ast/signatures/function_signature.py index 60cfdfa534..68078d4190 100644 --- a/vyper/ast/signatures/function_signature.py +++ b/vyper/ast/signatures/function_signature.py @@ -100,8 +100,6 @@ def _lll_identifier(self) -> str: visibility = "internal" if self.internal else "external" argz = ",".join([str(arg.typ) for arg in self.args]) ret = f"{visibility} {self.name} ({argz})" - if self.return_type: - ret += " -> " + str(self.return_type) return mkalphanum(ret) # calculate the abi signature for a given set of kwargs From a683b5a8555f6ffbf37ff6328767cef1d2adb7cb Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 14 Sep 2021 19:34:32 +0000 Subject: [PATCH 077/163] optimize out clamps for types that don't need it --- vyper/old_codegen/parser_utils.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 903fe00768..8f7f7e4a15 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -310,7 +310,9 @@ def _abi_helper(member_t, ofst, clamp=True): ofst_lll = _add_ofst(parent, unwrap_location(ofst_lll)) x = LLLnode.from_list(["x"], typ=member_t, location=parent.location) - if isinstance(member_t, (BaseType, ByteArrayLike)) and clamp: + + if clamp and _needs_clamp(member_t): + # special handling for args that need clamping ret = ["with", "x", ofst_lll, ["seq", clamp_basetype(x), x]] else: ret = ofst_lll @@ -755,6 +757,14 @@ def _sar(x, bits): return ["sdiv", x, ["exp", 2, bits]] +def _needs_clamp(t): + if isinstance(t, ByteArrayLike): + return True + if isinstance(t, BaseType) and t.typ not in ("int256", "uint256", "bytes32"): + return True + return False + + # clampers for basetype @type_check_wrapper def clamp_basetype(lll_node): From 52c5552e947ff42aedc25b3fb04b7af9dd861579 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 15 Sep 2021 15:40:08 +0000 Subject: [PATCH 078/163] add min_size to ABI api --- vyper/old_codegen/abi.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index 85b2dd9d1c..04b3979780 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -39,11 +39,21 @@ def static_size(self): # max size (in bytes) in the dynamic section (aka 'tail') def dynamic_size_bound(self): - return 0 + if not self.is_dynamic(): + return 0 + raise NotImplementedError("ABIType.dynamic_size_bound") def size_bound(self): return self.static_size() + self.dynamic_size_bound() + def min_size(self): + return self.static_size() + self.min_dynamic_size() + + def min_dynamic_size(self): + if not self.is_dynamic(): + return 0 + raise NotImplementedError("ABIType.min_dynamic_size") + # The canonical name of the type for calculating the function selector def selector_name(self): raise NotImplementedError("ABIType.selector_name") @@ -178,6 +188,9 @@ def static_size(self): def dynamic_size_bound(self): return self.m_elems * self.subtyp.dynamic_size_bound() + def min_dynamic_size(self): + return self.m_elems * self.subtyp.min_dynamic_size() + def selector_name(self): return f"{self.subtyp.selector_name()}[{self.m_elems}]" @@ -204,6 +217,9 @@ def dynamic_size_bound(self): # length word + data return 32 + ceil32(self.bytes_bound) + def min_dynamic_size(self): + return 32 + def selector_name(self): return "bytes" @@ -233,6 +249,9 @@ def static_size(self): def dynamic_size_bound(self): return self.subtyp.dynamic_size_bound() * self.elems_bound + def min_dynamic_size(self): + return 32 + def selector_name(self): return f"{self.subtyp.selector_name()}[]" @@ -250,9 +269,15 @@ def is_dynamic(self): def static_size(self): return sum([t.embedded_static_size() for t in self.subtyps]) + def min_dynamic_size(self): + return self.static_size() + sum([t.min_size() for t in self.subtyps]) + def dynamic_size_bound(self): return sum([t.dynamic_size_bound() for t in self.subtyps]) + def min_dynamic_size(self): + return sum([t.min_dynamic_size() for t in self.subtyps]) + def is_complex_type(self): return True From 7370c4ee0f72df215a123e338d3068b3a7071b4b Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 15 Sep 2021 15:41:04 +0000 Subject: [PATCH 079/163] fix returndatasize check --- vyper/old_codegen/external_call.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index 93e101c289..a803577f9c 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -53,7 +53,7 @@ def _unpack_returndata(contract_sig, context, pos): return ["pass"], 0, 0 abi_return_t = abi_type_of(return_t) - min_return_size = abi_return_t.static_size() + min_return_size = abi_return_t.min_size() maxlen = abi_return_t.size_bound() buf_t = get_type_for_exact_size(maxlen) From 7c1c336c5bd01e7ab4624fa9e9e7399317117bdd Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 15 Sep 2021 16:14:29 +0000 Subject: [PATCH 080/163] rewrite external call buffer allocation --- vyper/old_codegen/external_call.py | 38 +++++++++++++++++------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index a803577f9c..a7ec46aadc 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -21,14 +21,18 @@ def _pack_arguments(contract_sig, args, context, pos): args_as_tuple = LLLnode.from_list(["multi"] + [x for x in args], typ=args_tuple_t) args_abi_t = abi_type_of(args_tuple_t) - maxlen = args_abi_t.size_bound() - maxlen += 32 # padding for the method id + return_abi_t = abi_type_of(contract_sig.return_type) - buf_t = get_type_for_exact_size(maxlen) + # we use the same buffer for args and returndata, + # so allocate enough space here for the returndata too. + buflen = max(args_abi_t.size_bound(), return_abi_t.size_bound()) + buflen += 32 # padding for the method id + + buf_t = get_type_for_exact_size(buflen) buf = context.new_internal_variable(buf_t) args_ofst = buf + 28 - args_len = maxlen - 28 + args_len = args_abi_t.size_bound() + 4 abi_signature = contract_sig.name + canonicalize_type(args_tuple_t) @@ -41,25 +45,26 @@ def _pack_arguments(contract_sig, args, context, pos): # (mstore buf (shl signature.method_id 224)) mstore_method_id = [["mstore", buf, util.abi_method_id(abi_signature)]] - buf += 32 - encode_args = abi_encode(buf, args_as_tuple, pos) + #if len(args) == 0: + if False: + encode_args = ["pass"] + else: + encode_args = abi_encode(buf + 32, args_as_tuple, pos) - return mstore_method_id + [encode_args], args_ofst, args_len + return buf, mstore_method_id + [encode_args], args_ofst, args_len -def _unpack_returndata(contract_sig, context, pos): +def _unpack_returndata(buf, contract_sig, context, pos): return_t = contract_sig.return_type if return_t is None: return ["pass"], 0, 0 abi_return_t = abi_type_of(return_t) - min_return_size = abi_return_t.min_size() - maxlen = abi_return_t.size_bound() - buf_t = get_type_for_exact_size(maxlen) - buf = context.new_internal_variable(buf_t) ret_ofst = buf - ret_len = maxlen + ret_len = abi_return_t.size_bound() + + min_return_size = abi_return_t.min_size() # when return data is expected, revert when the length of `returndatasize` is insufficient ret = [] @@ -68,8 +73,7 @@ def _unpack_returndata(contract_sig, context, pos): # TODO assert returndatasize <= maxlen # ABI decoder has appropriate clampers for the individual members of the return type - # cf. add_variable_offset - buf = LLLnode(buf, location="memory", encoding=Encoding.ABI) + buf = LLLnode(buf, location="memory", ) ret += [buf] return ret, ret_ofst, ret_len @@ -97,9 +101,9 @@ def _external_call_helper( sub = ["seq"] - arg_packer, args_ofst, args_len = _pack_arguments(contract_sig, args_lll, context, pos) + buf, arg_packer, args_ofst, args_len = _pack_arguments(contract_sig, args_lll, context, pos) - ret_unpacker, ret_ofst, ret_len = _unpack_returndata(contract_sig, context, pos) + ret_unpacker, ret_ofst, ret_len = _unpack_returndata(buf, contract_sig, context, pos) sub += arg_packer From 8f6ca4dc163e4ca5c6e6106829a6a107f299f789 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 15 Sep 2021 18:29:55 +0000 Subject: [PATCH 081/163] fix abi_encode for expressions with side effects --- vyper/old_codegen/abi.py | 13 ++++++++++++- vyper/old_codegen/external_call.py | 4 ++-- vyper/old_codegen/parser_utils.py | 5 ++--- vyper/old_codegen/return_.py | 3 +-- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index 04b3979780..16ae1a27b2 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -414,10 +414,18 @@ def abi_encode(dst, lll_node, pos=None, bufsz=None, returns_len=False): return LLLnode.from_list(lll_ret, pos=pos, annotation=f"abi_encode {lll_node.typ}") lll_ret = ["seq"] + + # contains some computation, we need to only do it once. + is_complex_lll = lll_node.value in ("seq", "seq_unchecked") + if is_complex_lll: + to_encode = LLLnode.from_list("to_encode", typ=lll_node.typ, location=lll_node.location, encoding=lll_node.encoding) + else: + to_encode = lll_node + dyn_ofst = "dyn_ofst" # current offset in the dynamic section dst_begin = "dst" # pointer to beginning of buffer dst_loc = "dst_loc" # pointer to write location in static section - os = o_list(lll_node, pos=pos) + os = o_list(to_encode, pos=pos) for i, o in enumerate(os): abi_t = abi_type_of(o.typ) @@ -477,6 +485,9 @@ def abi_encode(dst, lll_node, pos=None, bufsz=None, returns_len=False): lll_ret = ["with", dst_begin, dst, ["with", dst_loc, dst_begin, lll_ret]] + if is_complex_lll: + lll_ret = ["with", to_encode, lll_node, lll_ret] + return LLLnode.from_list(lll_ret, pos=pos, annotation=f"abi_encode {lll_node.typ}") diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index a7ec46aadc..f9f0c2bec7 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -73,7 +73,7 @@ def _unpack_returndata(buf, contract_sig, context, pos): # TODO assert returndatasize <= maxlen # ABI decoder has appropriate clampers for the individual members of the return type - buf = LLLnode(buf, location="memory", ) + #buf = LLLnode(buf, location="memory", encoding=Encoding.ABI) ret += [buf] return ret, ret_ofst, ret_len @@ -124,7 +124,7 @@ def _external_call_helper( if contract_sig.return_type is not None: sub += ret_unpacker - return LLLnode.from_list(sub, typ=contract_sig.return_type, location="memory", pos=pos) + return LLLnode.from_list(sub, typ=contract_sig.return_type, location="memory", encoding=Encoding.ABI, pos=pos) # TODO push me up to expr.py diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 8f7f7e4a15..0c3b37ef3b 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -341,9 +341,8 @@ def _abi_helper(member_t, ofst, clamp=True): if parent.value is None: return LLLnode.from_list(None, typ=subtype) - # TODO case for parent.value == multi? - # if parent.value == "multi" - # return parent.args[index] + if parent.value == "multi": + return parent.args[index] if parent.encoding == Encoding.ABI: if parent.location == "storage": diff --git a/vyper/old_codegen/return_.py b/vyper/old_codegen/return_.py index 0bd0b7d6b0..799047e151 100644 --- a/vyper/old_codegen/return_.py +++ b/vyper/old_codegen/return_.py @@ -59,8 +59,6 @@ def finalize(fill_return_buffer): return finalize(fill_return_buffer) # we are in an external function. - - return_buffer_ofst = _allocate_return_buffer(context) # abi-encode the data into the return buffer and jump to the function cleanup code # according to the ABI spec, return types are ALWAYS tuples even @@ -84,6 +82,7 @@ def finalize(fill_return_buffer): # (Sorry this is so confusing. I didn't make these rules.) lll_val = lll_tuple_from_args([lll_val]) + return_buffer_ofst = _allocate_return_buffer(context) # encode_out is cleverly a sequence which does the abi-encoding and # also returns the length of the output as a stack element encode_out = abi_encode(return_buffer_ofst, lll_val, pos=_pos, returns_len=True) From 5150ae1bae144457081f034384a267f6e9c1ac9e Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 15 Sep 2021 19:57:56 +0000 Subject: [PATCH 082/163] improve some LLL annotations --- vyper/old_codegen/external_call.py | 4 +++- vyper/old_codegen/self_call.py | 2 +- vyper/old_codegen/stmt.py | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index f9f0c2bec7..69f3133b95 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -189,6 +189,8 @@ def lll_for_external_call(stmt_expr, context): method_name = stmt_expr.func.attr contract_sig = context.sigs[contract_name][method_name] - return _external_call_helper( + ret = _external_call_helper( contract_address, contract_sig, args_lll, context, pos, value=value, gas=gas, ) + ret.annotation = stmt_expr.get("node_source_code") + return ret diff --git a/vyper/old_codegen/self_call.py b/vyper/old_codegen/self_call.py index 37d5e6f9ce..793c394a1d 100644 --- a/vyper/old_codegen/self_call.py +++ b/vyper/old_codegen/self_call.py @@ -80,7 +80,7 @@ def lll_for_self_call(stmt_expr, context): typ=sig.return_type, location="memory", pos=pos, - annotation=f"Internal Call: {method_name}", + annotation=stmt_expr.get("node_source_code"), add_gas_estimate=sig.gas, ) return o diff --git a/vyper/old_codegen/stmt.py b/vyper/old_codegen/stmt.py index 8b9bcad212..346e3a3d02 100644 --- a/vyper/old_codegen/stmt.py +++ b/vyper/old_codegen/stmt.py @@ -75,6 +75,7 @@ def parse_AnnAssign(self): variable_loc = LLLnode.from_list(pos, typ=typ, location="memory", pos=getpos(self.stmt),) lll_node = make_setter(variable_loc, sub, "memory", pos=getpos(self.stmt)) + lll_node.annotation = self.stmt.get("node_source_code") return lll_node @@ -84,6 +85,7 @@ def parse_Assign(self): target = self._get_target(self.stmt.target) lll_node = make_setter(target, sub, target.location, pos=getpos(self.stmt)) lll_node.pos = getpos(self.stmt) + lll_node.annotation = self.stmt.get("node_source_code") return lll_node def parse_If(self): From a0972c8719a047fee83fe3eab0bed28eb9f39083 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 15 Sep 2021 20:56:51 +0000 Subject: [PATCH 083/163] fix debug function --- vyper/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vyper/utils.py b/vyper/utils.py index ae69ff4271..a34a1b6207 100644 --- a/vyper/utils.py +++ b/vyper/utils.py @@ -2,6 +2,7 @@ import functools from typing import Dict, List, Union import sys +import traceback from vyper.exceptions import InvalidLiteral @@ -23,7 +24,7 @@ def fourbytes_to_int(inp): # utility function for debugging purposes def trace(n=5, out=sys.stderr): print("BEGIN TRACE", file=out) - for x in list(traceback.format_stack)[-n:]: + for x in list(traceback.format_stack())[-n:]: print(x.strip(), file=out) print("END TRACE", file=out) From 0ccd33919e7c87d4fdff5a5142fdedb96c331a90 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 15 Sep 2021 21:13:45 +0000 Subject: [PATCH 084/163] wrap ABI values in tuples as needed --- vyper/builtin_functions/functions.py | 2 +- vyper/old_codegen/abi.py | 8 +---- vyper/old_codegen/events.py | 3 +- vyper/old_codegen/external_call.py | 15 +++++--- vyper/old_codegen/parser_utils.py | 54 +++++++++++++++++++++++----- vyper/old_codegen/return_.py | 28 ++------------- vyper/old_codegen/stmt.py | 3 ++ 7 files changed, 66 insertions(+), 47 deletions(-) diff --git a/vyper/builtin_functions/functions.py b/vyper/builtin_functions/functions.py index 6cc138e0d0..22f240fc6a 100644 --- a/vyper/builtin_functions/functions.py +++ b/vyper/builtin_functions/functions.py @@ -26,7 +26,6 @@ abi_encode, abi_type_of, abi_type_of2, - lll_tuple_from_args, ) from vyper.old_codegen.arg_clamps import int128_clamp from vyper.old_codegen.expr import Expr @@ -36,6 +35,7 @@ add_variable_offset, get_bytearray_length, getpos, + lll_tuple_from_args, make_byte_array_copier, make_byte_slice_copier, unwrap_location, diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index 16ae1a27b2..d56bc604c4 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -313,12 +313,6 @@ def abi_type_of(lll_typ): raise CompilerPanic(f"Unrecognized type {lll_typ}") -# utility function, constructs an LLL tuple out of a list of LLL nodes -def lll_tuple_from_args(args): - typ = TupleType([x.typ for x in args]) - return LLLnode.from_list(["multi"] + [x for x in args], typ=typ) - - # the new type system # TODO consider moving these into properties of the type itself def abi_type_of2(t: vy.BasePrimitive) -> ABIType: @@ -408,7 +402,7 @@ def abi_encode(dst, lll_node, pos=None, bufsz=None, returns_len=False): if not parent_abi_t.is_dynamic(): # cast the output buffer to something that make_setter accepts dst = LLLnode(dst, typ=lll_node.typ, location="memory") - lll_ret = ["seq_unchecked", make_setter(dst, lll_node, "memory", pos)] + lll_ret = make_setter(dst, lll_node, "memory", pos) if returns_len: lll_ret.append(parent_abi_t.embedded_static_size()) return LLLnode.from_list(lll_ret, pos=pos, annotation=f"abi_encode {lll_node.typ}") diff --git a/vyper/old_codegen/events.py b/vyper/old_codegen/events.py index d4c9140151..64421b80f8 100644 --- a/vyper/old_codegen/events.py +++ b/vyper/old_codegen/events.py @@ -6,12 +6,11 @@ abi_encode, abi_type_of, abi_type_of2, - lll_tuple_from_args, ) from vyper.old_codegen.context import Context from vyper.old_codegen.keccak256_helper import keccak256_helper from vyper.old_codegen.lll_node import LLLnode -from vyper.old_codegen.parser_utils import getpos, unwrap_location +from vyper.old_codegen.parser_utils import getpos, unwrap_location, lll_tuple_from_args from vyper.old_codegen.types.types import ( BaseType, ByteArrayLike, diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index 69f3133b95..8d1cb4340f 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -7,7 +7,7 @@ ) from vyper.old_codegen.abi import abi_encode, abi_type_of from vyper.old_codegen.lll_node import LLLnode, Encoding -from vyper.old_codegen.parser_utils import getpos, unwrap_location +from vyper.old_codegen.parser_utils import getpos, unwrap_location, set_type_for_external_return from vyper.old_codegen.types import ( TupleType, canonicalize_type, @@ -45,8 +45,7 @@ def _pack_arguments(contract_sig, args, context, pos): # (mstore buf (shl signature.method_id 224)) mstore_method_id = [["mstore", buf, util.abi_method_id(abi_signature)]] - #if len(args) == 0: - if False: + if len(args) == 0: encode_args = ["pass"] else: encode_args = abi_encode(buf + 32, args_as_tuple, pos) @@ -124,7 +123,11 @@ def _external_call_helper( if contract_sig.return_type is not None: sub += ret_unpacker - return LLLnode.from_list(sub, typ=contract_sig.return_type, location="memory", encoding=Encoding.ABI, pos=pos) + ret = LLLnode.from_list(sub, typ=contract_sig.return_type, location="memory", encoding=Encoding.ABI, pos=pos) + # the return type has been wrapped by the calling contract; set the type so that + # we unwrap it correctly when it comes to abi decoding time. + set_type_for_external_return(ret) + return ret # TODO push me up to expr.py @@ -193,4 +196,8 @@ def lll_for_external_call(stmt_expr, context): contract_address, contract_sig, args_lll, context, pos, value=value, gas=gas, ) ret.annotation = stmt_expr.get("node_source_code") + + # kludge: to let upstream code know to use abi.wrap_value_for_return + # TODO: assign/ann_assign just pass in the buffer + ret.is_external_call_returndata = True return ret diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 0c3b37ef3b..4bd546d52e 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -1,4 +1,5 @@ from decimal import Decimal, getcontext +import copy from vyper import ast as vy_ast from vyper.evm.opcodes import version_check @@ -39,7 +40,7 @@ def type_check_wrapper(fn): def _wrapped(*args, **kwargs): return_value = fn(*args, **kwargs) if return_value is None: - raise TypeCheckFailure(f"{fn.__name__} did not return a value") + raise TypeCheckFailure(f"{fn.__name__} {args} did not return a value") return return_value return _wrapped @@ -500,9 +501,46 @@ def _make_array_index_setter(target, target_token, pos, location, offset): ) +# utility function, constructs an LLL tuple out of a list of LLL nodes +def lll_tuple_from_args(args): + typ = TupleType([x.typ for x in args]) + return LLLnode.from_list(["multi"] + [x for x in args], typ=typ) + + +def set_type_for_external_return(lll_val): + # for calls to ABI conforming contracts. + # according to the ABI spec, return types are ALWAYS tuples even + # if only one element is being returned. + # https://solidity.readthedocs.io/en/latest/abi-spec.html#function-selector-and-argument-encoding + # "and the return values v_1, ..., v_k of f are encoded as + # + # enc((v_1, ..., v_k)) + # i.e. the values are combined into a tuple and encoded. + # " + # therefore, wrap it in a tuple if it's not already a tuple. + # for example, `bytes` is returned as abi-encoded (bytes,) + # and `(bytes,)` is returned as abi-encoded ((bytes,),) + # similarly, MyStruct is returned as abi-encoded (MyStruct,). + + t = lll_val.typ + if isinstance(t, TupleType) and len(t.members) > 1: + return + else: + # `-> (bytes,)` gets returned as ((bytes,),) + # In general `-> X` gets returned as (X,) + # (Sorry this is so confusing. I didn't make these rules.) + lll_val.typ = TupleType([t]) + + # Create an x=y statement, where the types may be compound @type_check_wrapper def make_setter(left, right, location, pos): + if getattr(right, "is_external_call_returndata", False): + # the rhs is the result of some external call + # set the type so that type checking works + left = copy.copy(left) + set_type_for_external_return(left) + # Basic types if isinstance(left.typ, BaseType): right = unwrap_location(right) @@ -596,7 +634,7 @@ def make_setter(left, right, location, pos): left_token = LLLnode.from_list("_L", typ=left.typ, location=left.location) keyz = left.typ.tuple_keys() - # If the left side is a literal + # If the left side is complex if left.value == "multi": locations = [arg.location for arg in left.args] else: @@ -606,6 +644,8 @@ def make_setter(left, right, location, pos): if right.value == "multi": if len(right.args) != len(keyz): return + if left.value == "multi": + left_token = left # get the RHS arguments into a dict because # they are not guaranteed to be in the same order # the LHS keys. @@ -640,8 +680,8 @@ def make_setter(left, right, location, pos): ) ) return LLLnode.from_list(["with", "_L", left, ["seq"] + subs], typ=None) - # If tuple assign. - elif isinstance(left.typ, TupleType) and isinstance(right.typ, TupleType): + # literal tuple assign. + elif isinstance(left.typ, TupleType) and left.value == "multi" and isinstance(right.typ, TupleType): subs = [] for var_arg in left.args: if var_arg.location in ("calldata", "code"): @@ -657,10 +697,8 @@ def make_setter(left, right, location, pos): ) ) - return LLLnode.from_list( - ["with", "_R", right, ["seq"] + subs], typ=None, annotation="Tuple assignment", - ) - # If the left side is a variable i.e struct type + return LLLnode.from_list(["with", "_R", right, ["seq"] + subs], typ=None) + # general case else: subs = [] right_token = LLLnode.from_list( diff --git a/vyper/old_codegen/return_.py b/vyper/old_codegen/return_.py index 799047e151..8615409473 100644 --- a/vyper/old_codegen/return_.py +++ b/vyper/old_codegen/return_.py @@ -1,9 +1,9 @@ from typing import Any, Optional -from vyper.old_codegen.abi import abi_encode, abi_type_of, lll_tuple_from_args +from vyper.old_codegen.abi import abi_encode, abi_type_of from vyper.old_codegen.context import Context from vyper.old_codegen.lll_node import LLLnode -from vyper.old_codegen.parser_utils import getpos, make_setter +from vyper.old_codegen.parser_utils import getpos, make_setter, set_type_for_external_return from vyper.old_codegen.types import TupleType, get_type_for_exact_size from vyper.old_codegen.types.check import check_assign @@ -58,29 +58,7 @@ def finalize(fill_return_buffer): return finalize(fill_return_buffer) - # we are in an external function. - # abi-encode the data into the return buffer and jump to the function cleanup code - - # according to the ABI spec, return types are ALWAYS tuples even - # if only one element is being returned. - # https://solidity.readthedocs.io/en/latest/abi-spec.html#function-selector-and-argument-encoding - # "and the return values v_1, ..., v_k of f are encoded as - # - # enc((v_1, ..., v_k)) - # i.e. the values are combined into a tuple and encoded. - # " - # therefore, wrap it in a tuple if it's not already a tuple. - # for example, `bytes` is returned as abi-encoded (bytes,) - # and `(bytes,)` is returned as abi-encoded ((bytes,),) - - if isinstance(lll_val.typ, TupleType) and len(lll_val.typ.members) > 1: - # returning something like (int, bytes, string) - pass - else: - # `-> (bytes,)` gets returned as ((bytes,),) - # In general `-> X` gets returned as (X,) - # (Sorry this is so confusing. I didn't make these rules.) - lll_val = lll_tuple_from_args([lll_val]) + set_type_for_external_return(lll_val) return_buffer_ofst = _allocate_return_buffer(context) # encode_out is cleverly a sequence which does the abi-encoding and diff --git a/vyper/old_codegen/stmt.py b/vyper/old_codegen/stmt.py index 346e3a3d02..807380b4b5 100644 --- a/vyper/old_codegen/stmt.py +++ b/vyper/old_codegen/stmt.py @@ -11,6 +11,7 @@ make_setter, unwrap_location, ) +import vyper.old_codegen.abi as abi from vyper.old_codegen.return_ import make_return_stmt from vyper.old_codegen.types import ( BaseType, @@ -74,6 +75,7 @@ def parse_AnnAssign(self): ) variable_loc = LLLnode.from_list(pos, typ=typ, location="memory", pos=getpos(self.stmt),) + lll_node = make_setter(variable_loc, sub, "memory", pos=getpos(self.stmt)) lll_node.annotation = self.stmt.get("node_source_code") @@ -83,6 +85,7 @@ def parse_Assign(self): # Assignment (e.g. x[4] = y) sub = Expr(self.stmt.value, self.context).lll_node target = self._get_target(self.stmt.target) + lll_node = make_setter(target, sub, target.location, pos=getpos(self.stmt)) lll_node.pos = getpos(self.stmt) lll_node.annotation = self.stmt.get("node_source_code") From 15bc34281f036739e43388bcdfb0cc9521e8e402 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 15 Sep 2021 22:52:16 +0000 Subject: [PATCH 085/163] clean up return.py a bit --- vyper/old_codegen/return_.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/vyper/old_codegen/return_.py b/vyper/old_codegen/return_.py index 8615409473..1841e0250a 100644 --- a/vyper/old_codegen/return_.py +++ b/vyper/old_codegen/return_.py @@ -31,7 +31,7 @@ def make_return_stmt(lll_val: LLLnode, stmt: Any, context: Context) -> Optional[ else: # sanity typecheck - _tmp = LLLnode(-1, location="memory", typ=context.return_type) + _tmp = LLLnode("fake node", location="memory", typ=context.return_type) check_assign(_tmp, lll_val, _pos) # helper function @@ -58,12 +58,14 @@ def finalize(fill_return_buffer): return finalize(fill_return_buffer) - set_type_for_external_return(lll_val) + else: + # return from external function + set_type_for_external_return(lll_val) - return_buffer_ofst = _allocate_return_buffer(context) - # encode_out is cleverly a sequence which does the abi-encoding and - # also returns the length of the output as a stack element - encode_out = abi_encode(return_buffer_ofst, lll_val, pos=_pos, returns_len=True) + return_buffer_ofst = _allocate_return_buffer(context) + # encode_out is cleverly a sequence which does the abi-encoding and + # also returns the length of the output as a stack element + encode_out = abi_encode(return_buffer_ofst, lll_val, pos=_pos, returns_len=True) - # fill the return buffer and push the location and length onto the stack - return finalize(["seq_unchecked", encode_out, return_buffer_ofst]) + # fill the return buffer and push the location and length onto the stack + return finalize(["seq_unchecked", encode_out, return_buffer_ofst]) From f4fd13426e24088a5d0fb58429227f5289f4d74d Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 16 Sep 2021 02:29:39 +0000 Subject: [PATCH 086/163] fix abi tuple wrapping --- vyper/old_codegen/parser_utils.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 4bd546d52e..176d4d4ae1 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -506,8 +506,7 @@ def lll_tuple_from_args(args): typ = TupleType([x.typ for x in args]) return LLLnode.from_list(["multi"] + [x for x in args], typ=typ) - -def set_type_for_external_return(lll_val): +def _needs_external_call_wrap(lll_typ): # for calls to ABI conforming contracts. # according to the ABI spec, return types are ALWAYS tuples even # if only one element is being returned. @@ -522,15 +521,24 @@ def set_type_for_external_return(lll_val): # and `(bytes,)` is returned as abi-encoded ((bytes,),) # similarly, MyStruct is returned as abi-encoded (MyStruct,). - t = lll_val.typ - if isinstance(t, TupleType) and len(t.members) > 1: - return - else: + return isinstance(lll_typ, TupleType) and len(lll_typ.members) > 1 + + +def wrap_value_for_external_return(lll_val): + # used for LHS promotion + if _needs_external_call_wrap(lll_val.typ): # `-> (bytes,)` gets returned as ((bytes,),) # In general `-> X` gets returned as (X,) # (Sorry this is so confusing. I didn't make these rules.) - lll_val.typ = TupleType([t]) + return lll_tuple_from_args([lll_val]) + else: + return lll_val +def set_type_for_external_return(lll_val): + # used for RHS promotion + t = lll_val.typ + if _needs_external_call_wrap(t): + lll_val.typ = TupleType([t]) # Create an x=y statement, where the types may be compound @type_check_wrapper @@ -538,8 +546,7 @@ def make_setter(left, right, location, pos): if getattr(right, "is_external_call_returndata", False): # the rhs is the result of some external call # set the type so that type checking works - left = copy.copy(left) - set_type_for_external_return(left) + left = wrap_value_for_external_return(left) # Basic types if isinstance(left.typ, BaseType): From 0f4fc8f96ece4e30ef12e4454f76da5551f61b99 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 16 Sep 2021 15:42:26 +0000 Subject: [PATCH 087/163] fix special case in ABI for single struct return issue 2457 --- vyper/semantics/types/function.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/vyper/semantics/types/function.py b/vyper/semantics/types/function.py index 0279c73a17..6532040b0a 100644 --- a/vyper/semantics/types/function.py +++ b/vyper/semantics/types/function.py @@ -483,8 +483,6 @@ def to_abi_dict(self) -> List[Dict]: abi_dict["outputs"] = [] elif isinstance(typ, TupleDefinition): abi_dict["outputs"] = [_generate_abi_type(i) for i in typ.value_type] # type: ignore - elif isinstance(typ, StructDefinition): - abi_dict["outputs"] = [_generate_abi_type(v, k) for k, v in typ.members.items()] else: abi_dict["outputs"] = [_generate_abi_type(typ)] From eef47669d1cbc094bd414854c108be08b12cd2a4 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 16 Sep 2021 17:07:37 +0000 Subject: [PATCH 088/163] fix external call return wrapping --- vyper/old_codegen/parser_utils.py | 12 ++++++++++-- vyper/old_codegen/return_.py | 10 +++++++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 176d4d4ae1..a041ad7a59 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -343,6 +343,7 @@ def _abi_helper(member_t, ofst, clamp=True): return LLLnode.from_list(None, typ=subtype) if parent.value == "multi": + assert parent.encoding != Encoding.ABI, "no abi-encoded literals" return parent.args[index] if parent.encoding == Encoding.ABI: @@ -506,6 +507,7 @@ def lll_tuple_from_args(args): typ = TupleType([x.typ for x in args]) return LLLnode.from_list(["multi"] + [x for x in args], typ=typ) + def _needs_external_call_wrap(lll_typ): # for calls to ABI conforming contracts. # according to the ABI spec, return types are ALWAYS tuples even @@ -521,7 +523,7 @@ def _needs_external_call_wrap(lll_typ): # and `(bytes,)` is returned as abi-encoded ((bytes,),) # similarly, MyStruct is returned as abi-encoded (MyStruct,). - return isinstance(lll_typ, TupleType) and len(lll_typ.members) > 1 + return not (isinstance(lll_typ, TupleType) and len(lll_typ.members) > 1) def wrap_value_for_external_return(lll_val): @@ -534,12 +536,14 @@ def wrap_value_for_external_return(lll_val): else: return lll_val + def set_type_for_external_return(lll_val): # used for RHS promotion t = lll_val.typ if _needs_external_call_wrap(t): lll_val.typ = TupleType([t]) + # Create an x=y statement, where the types may be compound @type_check_wrapper def make_setter(left, right, location, pos): @@ -688,7 +692,11 @@ def make_setter(left, right, location, pos): ) return LLLnode.from_list(["with", "_L", left, ["seq"] + subs], typ=None) # literal tuple assign. - elif isinstance(left.typ, TupleType) and left.value == "multi" and isinstance(right.typ, TupleType): + elif ( + isinstance(left.typ, TupleType) + and left.value == "multi" + and isinstance(right.typ, TupleType) + ): subs = [] for var_arg in left.args: if var_arg.location in ("calldata", "code"): diff --git a/vyper/old_codegen/return_.py b/vyper/old_codegen/return_.py index 1841e0250a..787d0029af 100644 --- a/vyper/old_codegen/return_.py +++ b/vyper/old_codegen/return_.py @@ -58,9 +58,13 @@ def finalize(fill_return_buffer): return finalize(fill_return_buffer) - else: - # return from external function - set_type_for_external_return(lll_val) + else: # return from external function + + if getattr(lll_val, "is_external_call_returndata", False): + # lll_val is already wrapped to the correct type, cf. external_call.py + pass + else: + set_type_for_external_return(lll_val) return_buffer_ofst = _allocate_return_buffer(context) # encode_out is cleverly a sequence which does the abi-encoding and From a1e9842bbddcc2aa24d296876a53b19af57adf5f Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 16 Sep 2021 17:11:11 +0000 Subject: [PATCH 089/163] fix lint --- vyper/old_codegen/abi.py | 10 +++------ vyper/old_codegen/context.py | 2 +- vyper/old_codegen/events.py | 6 ++++- vyper/old_codegen/external_call.py | 22 ++++++++++++++----- .../function_definitions/external_function.py | 10 ++++++--- .../function_definitions/internal_function.py | 2 +- vyper/old_codegen/lll_node.py | 2 +- vyper/old_codegen/parser_utils.py | 6 ++--- vyper/old_codegen/return_.py | 8 +++++-- vyper/old_codegen/self_call.py | 4 +++- vyper/old_codegen/stmt.py | 1 - vyper/semantics/validation/local.py | 1 - vyper/utils.py | 2 +- 13 files changed, 46 insertions(+), 30 deletions(-) diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index d56bc604c4..15aed6456f 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -1,5 +1,3 @@ -import copy - import vyper.semantics.types as vy from vyper.exceptions import CompilerPanic from vyper.old_codegen.lll_node import LLLnode @@ -17,7 +15,6 @@ ListType, StringType, TupleLike, - TupleType, ) from vyper.utils import ceil32 @@ -269,9 +266,6 @@ def is_dynamic(self): def static_size(self): return sum([t.embedded_static_size() for t in self.subtyps]) - def min_dynamic_size(self): - return self.static_size() + sum([t.min_size() for t in self.subtyps]) - def dynamic_size_bound(self): return sum([t.dynamic_size_bound() for t in self.subtyps]) @@ -412,7 +406,9 @@ def abi_encode(dst, lll_node, pos=None, bufsz=None, returns_len=False): # contains some computation, we need to only do it once. is_complex_lll = lll_node.value in ("seq", "seq_unchecked") if is_complex_lll: - to_encode = LLLnode.from_list("to_encode", typ=lll_node.typ, location=lll_node.location, encoding=lll_node.encoding) + to_encode = LLLnode.from_list( + "to_encode", typ=lll_node.typ, location=lll_node.location, encoding=lll_node.encoding + ) else: to_encode = lll_node diff --git a/vyper/old_codegen/context.py b/vyper/old_codegen/context.py index 030b144966..7ca85cd747 100644 --- a/vyper/old_codegen/context.py +++ b/vyper/old_codegen/context.py @@ -242,7 +242,7 @@ def _check(cond, s="Unreachable"): _check(len(sig.base_args) <= len(args_lll) <= len(sig.args)) # more sanity check, that the types match - # _check(all(l.typ == r.typ for (l, r) in zip(args_lll, sig.args)), str( (args_lll, sig.args))) + # _check(all(l.typ == r.typ for (l, r) in zip(args_lll, sig.args)) num_provided_args = len(args_lll) total_args = len(sig.args) diff --git a/vyper/old_codegen/events.py b/vyper/old_codegen/events.py index 64421b80f8..dfff06d2e0 100644 --- a/vyper/old_codegen/events.py +++ b/vyper/old_codegen/events.py @@ -10,7 +10,11 @@ from vyper.old_codegen.context import Context from vyper.old_codegen.keccak256_helper import keccak256_helper from vyper.old_codegen.lll_node import LLLnode -from vyper.old_codegen.parser_utils import getpos, unwrap_location, lll_tuple_from_args +from vyper.old_codegen.parser_utils import ( + getpos, + lll_tuple_from_args, + unwrap_location, +) from vyper.old_codegen.types.types import ( BaseType, ByteArrayLike, diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index 8d1cb4340f..957d3ec6b2 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -6,8 +6,12 @@ TypeCheckFailure, ) from vyper.old_codegen.abi import abi_encode, abi_type_of -from vyper.old_codegen.lll_node import LLLnode, Encoding -from vyper.old_codegen.parser_utils import getpos, unwrap_location, set_type_for_external_return +from vyper.old_codegen.lll_node import Encoding, LLLnode +from vyper.old_codegen.parser_utils import ( + getpos, + set_type_for_external_return, + unwrap_location, +) from vyper.old_codegen.types import ( TupleType, canonicalize_type, @@ -72,7 +76,7 @@ def _unpack_returndata(buf, contract_sig, context, pos): # TODO assert returndatasize <= maxlen # ABI decoder has appropriate clampers for the individual members of the return type - #buf = LLLnode(buf, location="memory", encoding=Encoding.ABI) + # buf = LLLnode(buf, location="memory", encoding=Encoding.ABI) ret += [buf] return ret, ret_ofst, ret_len @@ -123,7 +127,9 @@ def _external_call_helper( if contract_sig.return_type is not None: sub += ret_unpacker - ret = LLLnode.from_list(sub, typ=contract_sig.return_type, location="memory", encoding=Encoding.ABI, pos=pos) + ret = LLLnode.from_list( + sub, typ=contract_sig.return_type, location="memory", encoding=Encoding.ABI, pos=pos + ) # the return type has been wrapped by the calling contract; set the type so that # we unwrap it correctly when it comes to abi decoding time. set_type_for_external_return(ret) @@ -132,7 +138,9 @@ def _external_call_helper( # TODO push me up to expr.py def get_gas_and_value(stmt_expr, context): - from vyper.old_codegen.expr import Expr # TODO rethink this circular import + from vyper.old_codegen.expr import ( + Expr, # TODO rethink this circular import + ) value, gas = None, None for kw in stmt_expr.keywords: @@ -146,7 +154,9 @@ def get_gas_and_value(stmt_expr, context): def lll_for_external_call(stmt_expr, context): - from vyper.old_codegen.expr import Expr # TODO rethink this circular import + from vyper.old_codegen.expr import ( + Expr, # TODO rethink this circular import + ) pos = getpos(stmt_expr) value, gas = get_gas_and_value(stmt_expr, context) diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index 6fb0b71845..cf735bf5a9 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -8,10 +8,14 @@ from vyper.old_codegen.context import Context from vyper.old_codegen.expr import Expr from vyper.old_codegen.function_definitions.utils import get_nonreentrant_lock -from vyper.old_codegen.lll_node import LLLnode, Encoding -from vyper.old_codegen.parser_utils import getpos, make_setter, add_variable_offset +from vyper.old_codegen.lll_node import Encoding, LLLnode +from vyper.old_codegen.parser_utils import ( + add_variable_offset, + getpos, + make_setter, +) from vyper.old_codegen.stmt import parse_body -from vyper.old_codegen.types.types import ListType, TupleType, BaseType +from vyper.old_codegen.types.types import TupleType # register function args with the local calling context. diff --git a/vyper/old_codegen/function_definitions/internal_function.py b/vyper/old_codegen/function_definitions/internal_function.py index 93a79788e9..9036dfc136 100644 --- a/vyper/old_codegen/function_definitions/internal_function.py +++ b/vyper/old_codegen/function_definitions/internal_function.py @@ -57,7 +57,7 @@ def mkidentifier(s): cleanup_label = sig.exit_sequence_label # jump to the label which was passed in via stack - stop_func = LLLnode.from_list(["jump", "pass"], annotation=f"jump to return address") + stop_func = LLLnode.from_list(["jump", "pass"], annotation="jump to return address") enter = [["label", function_entry_label], nonreentrant_pre] diff --git a/vyper/old_codegen/lll_node.py b/vyper/old_codegen/lll_node.py index b1e1b31bff..ed379cf982 100644 --- a/vyper/old_codegen/lll_node.py +++ b/vyper/old_codegen/lll_node.py @@ -1,6 +1,6 @@ import re -from typing import Any, List, Optional, Tuple, Union from enum import Enum, auto +from typing import Any, List, Optional, Tuple, Union from vyper.compiler.settings import VYPER_COLOR_OUTPUT from vyper.evm.opcodes import get_comb_opcodes diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index a041ad7a59..c8817b6cbb 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -1,5 +1,4 @@ from decimal import Decimal, getcontext -import copy from vyper import ast as vy_ast from vyper.evm.opcodes import version_check @@ -11,7 +10,7 @@ TypeMismatch, ) from vyper.old_codegen.arg_clamps import int128_clamp -from vyper.old_codegen.lll_node import LLLnode, Encoding +from vyper.old_codegen.lll_node import Encoding, LLLnode from vyper.old_codegen.types import ( BaseType, ByteArrayLike, @@ -350,7 +349,7 @@ def _abi_helper(member_t, ofst, clamp=True): if parent.location == "storage": raise CompilerPanic("storage variables should not be abi encoded") - parent_abi_t = abi_type_of(parent.typ) + # parent_abi_t = abi_type_of(parent.typ) member_t = typ.members[attrs[index]] ofst = 0 # offset from parent start @@ -408,7 +407,6 @@ def _abi_helper(member_t, ofst, clamp=True): if parent.location == "storage": raise CompilerPanic("storage variables should not be abi encoded") - parent_abi_t = abi_type_of(typ) member_t = typ.subtype member_abi_t = abi_type_of(member_t) diff --git a/vyper/old_codegen/return_.py b/vyper/old_codegen/return_.py index 787d0029af..eb505a77f2 100644 --- a/vyper/old_codegen/return_.py +++ b/vyper/old_codegen/return_.py @@ -3,8 +3,12 @@ from vyper.old_codegen.abi import abi_encode, abi_type_of from vyper.old_codegen.context import Context from vyper.old_codegen.lll_node import LLLnode -from vyper.old_codegen.parser_utils import getpos, make_setter, set_type_for_external_return -from vyper.old_codegen.types import TupleType, get_type_for_exact_size +from vyper.old_codegen.parser_utils import ( + getpos, + make_setter, + set_type_for_external_return, +) +from vyper.old_codegen.types import get_type_for_exact_size from vyper.old_codegen.types.check import check_assign diff --git a/vyper/old_codegen/self_call.py b/vyper/old_codegen/self_call.py index 793c394a1d..1779c654ad 100644 --- a/vyper/old_codegen/self_call.py +++ b/vyper/old_codegen/self_call.py @@ -14,7 +14,9 @@ def _generate_label(name: str) -> str: def lll_for_self_call(stmt_expr, context): - from vyper.old_codegen.expr import Expr # TODO rethink this circular import + from vyper.old_codegen.expr import ( + Expr, # TODO rethink this circular import + ) pos = getpos(stmt_expr) diff --git a/vyper/old_codegen/stmt.py b/vyper/old_codegen/stmt.py index 807380b4b5..516cb29bd8 100644 --- a/vyper/old_codegen/stmt.py +++ b/vyper/old_codegen/stmt.py @@ -11,7 +11,6 @@ make_setter, unwrap_location, ) -import vyper.old_codegen.abi as abi from vyper.old_codegen.return_ import make_return_stmt from vyper.old_codegen.types import ( BaseType, diff --git a/vyper/semantics/validation/local.py b/vyper/semantics/validation/local.py index b02c7ff585..861517add6 100644 --- a/vyper/semantics/validation/local.py +++ b/vyper/semantics/validation/local.py @@ -19,7 +19,6 @@ VariableDeclarationException, VyperException, ) - # TODO consolidate some of these imports from vyper.semantics.environment import ( CONSTANT_ENVIRONMENT_VARS, diff --git a/vyper/utils.py b/vyper/utils.py index a34a1b6207..940679c359 100644 --- a/vyper/utils.py +++ b/vyper/utils.py @@ -1,8 +1,8 @@ import binascii import functools -from typing import Dict, List, Union import sys import traceback +from typing import Dict, List, Union from vyper.exceptions import InvalidLiteral From e332a25e8725cf8b306577cc725799ae55da1d23 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 16 Sep 2021 17:15:09 +0000 Subject: [PATCH 090/163] fix size calculation for bytestring copy --- vyper/old_codegen/parser_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index c8817b6cbb..b617a4c2f2 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -112,8 +112,9 @@ def make_byte_array_copier(destination, source, pos=None): elif source.location == "code": copy_op = ["code", destination, "src", "sz"] gas_bound = _codecopy_gas_bound(source.typ.maxlen) + _sz_lll = ["add", 32, [load_op(source.location), "src"]] o = LLLnode.from_list( - ["with", "src", source, ["with", "sz", [load_op(source.location), "src"], copy_op]], + ["with", "src", source, ["with", "sz", _sz_lll, copy_op]], typ=None, add_gas_estimate=gas_bound, annotation="copy bytestring to memory", From a50ea5b36d175d3c1b11606a9124dc26af323542 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 16 Sep 2021 18:20:20 +0000 Subject: [PATCH 091/163] fix some lint --- vyper/old_codegen/function_definitions/common.py | 2 +- vyper/old_codegen/function_definitions/internal_function.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/vyper/old_codegen/function_definitions/common.py b/vyper/old_codegen/function_definitions/common.py index 492f545d72..a62cd10a26 100644 --- a/vyper/old_codegen/function_definitions/common.py +++ b/vyper/old_codegen/function_definitions/common.py @@ -43,7 +43,7 @@ def generate_lll_for_function( - Function body """ if _vars is None: - _vars = {} + _vars = {} # noqa: F841 sig = FunctionSignature.from_definition(code, sigs=sigs, custom_structs=global_ctx._structs,) # Validate return statements. diff --git a/vyper/old_codegen/function_definitions/internal_function.py b/vyper/old_codegen/function_definitions/internal_function.py index 9036dfc136..a3cb9031ed 100644 --- a/vyper/old_codegen/function_definitions/internal_function.py +++ b/vyper/old_codegen/function_definitions/internal_function.py @@ -49,10 +49,6 @@ def generate_lll_for_internal_function( nonreentrant_pre, nonreentrant_post = get_nonreentrant_lock(func_type) - # pseudocode - def mkidentifier(s): - "".join(c if c.isalnumeric() else "_" for c in s) - function_entry_label = sig.internal_function_label cleanup_label = sig.exit_sequence_label From 53f64919820ad008c7ddb0d552ce28a6161d94f9 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 16 Sep 2021 20:00:40 +0000 Subject: [PATCH 092/163] fix size bounds for nested dynamic types fixes issue 2458 --- vyper/old_codegen/abi.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index 15aed6456f..7d521b3506 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -26,10 +26,22 @@ def is_dynamic(self): raise NotImplementedError("ABIType.is_dynamic") # size (in bytes) in the static section (aka 'head') - # when embedded in a tuple. + # when embedded in a complex type. def embedded_static_size(self): return 32 if self.is_dynamic() else self.static_size() + # size bound in the dynamic section (aka 'tail') + # when embedded in a complex type. + def embedded_dynamic_size_bound(self): + if not self.is_dynamic(): + return 0 + return self.size_bound() + + def embedded_min_dynamic_size(self): + if not self.is_dynamic(): + return 0 + return self.min_size() + # size (in bytes) of the static section def static_size(self): raise NotImplementedError("ABIType.static_size") @@ -61,6 +73,9 @@ def selector_name(self): def is_complex_type(self): raise NotImplementedError("ABIType.is_complex_type") + def __repr__(self): + return str({type(self).__name__: vars(self)}) + # uint: unsigned integer type of M bits, 0 < M <= 256, M % 8 == 0. e.g. uint32, uint8, uint256. # int: two’s complement signed integer type of M bits, 0 < M <= 256, M % 8 == 0. @@ -183,10 +198,10 @@ def static_size(self): return self.m_elems * self.subtyp.static_size() def dynamic_size_bound(self): - return self.m_elems * self.subtyp.dynamic_size_bound() + return self.m_elems * self.subtyp.embedded_dynamic_size_bound() def min_dynamic_size(self): - return self.m_elems * self.subtyp.min_dynamic_size() + return self.m_elems * self.subtyp.embedded_min_dynamic_size() def selector_name(self): return f"{self.subtyp.selector_name()}[{self.m_elems}]" @@ -244,9 +259,11 @@ def static_size(self): return 32 def dynamic_size_bound(self): - return self.subtyp.dynamic_size_bound() * self.elems_bound + # TODO double check me + return self.subtyp.embedded_dynamic_size_bound() * self.elems_bound def min_dynamic_size(self): + # TODO double check me return 32 def selector_name(self): @@ -267,10 +284,10 @@ def static_size(self): return sum([t.embedded_static_size() for t in self.subtyps]) def dynamic_size_bound(self): - return sum([t.dynamic_size_bound() for t in self.subtyps]) + return sum([t.embedded_dynamic_size_bound() for t in self.subtyps]) def min_dynamic_size(self): - return sum([t.min_dynamic_size() for t in self.subtyps]) + return sum([t.embedded_min_dynamic_size() for t in self.subtyps]) def is_complex_type(self): return True From 93540d86c4e54e28472e072030d3f46db5029897 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 16 Sep 2021 20:31:20 +0000 Subject: [PATCH 093/163] fix return type wrapping again --- .../functional/codegen/test_struct_return.py | 4 +-- vyper/old_codegen/external_call.py | 36 +++++++++++++------ vyper/old_codegen/parser_utils.py | 17 +++++---- vyper/old_codegen/return_.py | 4 +-- vyper/old_codegen/self_call.py | 4 +-- vyper/semantics/types/function.py | 9 +++-- vyper/semantics/validation/local.py | 1 + 7 files changed, 48 insertions(+), 27 deletions(-) diff --git a/tests/functional/codegen/test_struct_return.py b/tests/functional/codegen/test_struct_return.py index 8bdc731de0..dbec5ed2b6 100644 --- a/tests/functional/codegen/test_struct_return.py +++ b/tests/functional/codegen/test_struct_return.py @@ -26,7 +26,7 @@ def modify_nested_tuple(_human: Human) -> Human: # assert c.modify_nested_tuple([addr1, 123], [addr2, 456]) == [[addr1, 124], [addr2, 457]] assert c.modify_nested_tuple( {"location": addr1, "animal": {"location": addr2, "fur": "wool"}} - ) == [(addr1, (addr2, "wool is great"))] + ) == ((addr1, (addr2, "wool is great"),),) @pytest.mark.parametrize("string", ["a", "abc", "abcde", "potato"]) @@ -56,4 +56,4 @@ def test_values(a: address) -> Person: """ c2 = get_contract(code) - assert c2.test_values(c1.address) == [string, 42] + assert c2.test_values(c1.address) == ((string, 42),) diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index 957d3ec6b2..325627e1f8 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -10,6 +10,7 @@ from vyper.old_codegen.parser_utils import ( getpos, set_type_for_external_return, + calculate_type_for_external_return, unwrap_location, ) from vyper.old_codegen.types import ( @@ -62,18 +63,35 @@ def _unpack_returndata(buf, contract_sig, context, pos): if return_t is None: return ["pass"], 0, 0 + return_t = calculate_type_for_external_return(return_t) abi_return_t = abi_type_of(return_t) + min_return_size = abi_return_t.min_size() + max_return_size = abi_return_t.size_bound() + ret_ofst = buf ret_len = abi_return_t.size_bound() - min_return_size = abi_return_t.min_size() - - # when return data is expected, revert when the length of `returndatasize` is insufficient + # revert when returndatasize is not in bounds ret = [] - if min_return_size > 0: + assert 0 < min_return_size <= max_return_size + if contract_sig.is_from_json: + # we don't have max_size information for json abis + # runtime: min_return_size <= returndatasize ret += [["assert", ["gt", "returndatasize", min_return_size - 1]]] - # TODO assert returndatasize <= maxlen + else: + # runtime: min_return_size <= returndatasize <= max_return_size + # TODO move the +-1 optimization to LLL optimizer + ret += [ + [ + "assert", + [ + "and", + ["gt", "returndatasize", min_return_size - 1], + ["lt", "returndatasize", max_return_size + 1], + ], + ] + ] # ABI decoder has appropriate clampers for the individual members of the return type # buf = LLLnode(buf, location="memory", encoding=Encoding.ABI) @@ -138,9 +156,7 @@ def _external_call_helper( # TODO push me up to expr.py def get_gas_and_value(stmt_expr, context): - from vyper.old_codegen.expr import ( - Expr, # TODO rethink this circular import - ) + from vyper.old_codegen.expr import Expr # TODO rethink this circular import value, gas = None, None for kw in stmt_expr.keywords: @@ -154,9 +170,7 @@ def get_gas_and_value(stmt_expr, context): def lll_for_external_call(stmt_expr, context): - from vyper.old_codegen.expr import ( - Expr, # TODO rethink this circular import - ) + from vyper.old_codegen.expr import Expr # TODO rethink this circular import pos = getpos(stmt_expr) value, gas = get_gas_and_value(stmt_expr, context) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index b617a4c2f2..7b7a60a9f9 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -520,17 +520,22 @@ def _needs_external_call_wrap(lll_typ): # therefore, wrap it in a tuple if it's not already a tuple. # for example, `bytes` is returned as abi-encoded (bytes,) # and `(bytes,)` is returned as abi-encoded ((bytes,),) - # similarly, MyStruct is returned as abi-encoded (MyStruct,). + # In general `-> X` gets returned as (X,) + # including structs. MyStruct is returned as abi-encoded (MyStruct,). + # (Sorry this is so confusing. I didn't make these rules.) return not (isinstance(lll_typ, TupleType) and len(lll_typ.members) > 1) +def calculate_type_for_external_return(lll_typ): + if _needs_external_call_wrap(lll_typ): + return TupleType([lll_typ]) + return lll_typ + + def wrap_value_for_external_return(lll_val): # used for LHS promotion if _needs_external_call_wrap(lll_val.typ): - # `-> (bytes,)` gets returned as ((bytes,),) - # In general `-> X` gets returned as (X,) - # (Sorry this is so confusing. I didn't make these rules.) return lll_tuple_from_args([lll_val]) else: return lll_val @@ -538,9 +543,7 @@ def wrap_value_for_external_return(lll_val): def set_type_for_external_return(lll_val): # used for RHS promotion - t = lll_val.typ - if _needs_external_call_wrap(t): - lll_val.typ = TupleType([t]) + lll_val.typ = calculate_type_for_external_return(lll_val.typ) # Create an x=y statement, where the types may be compound diff --git a/vyper/old_codegen/return_.py b/vyper/old_codegen/return_.py index eb505a77f2..6f8e025338 100644 --- a/vyper/old_codegen/return_.py +++ b/vyper/old_codegen/return_.py @@ -6,7 +6,7 @@ from vyper.old_codegen.parser_utils import ( getpos, make_setter, - set_type_for_external_return, + wrap_value_for_external_return, ) from vyper.old_codegen.types import get_type_for_exact_size from vyper.old_codegen.types.check import check_assign @@ -68,7 +68,7 @@ def finalize(fill_return_buffer): # lll_val is already wrapped to the correct type, cf. external_call.py pass else: - set_type_for_external_return(lll_val) + lll_val = wrap_value_for_external_return(lll_val) return_buffer_ofst = _allocate_return_buffer(context) # encode_out is cleverly a sequence which does the abi-encoding and diff --git a/vyper/old_codegen/self_call.py b/vyper/old_codegen/self_call.py index 1779c654ad..793c394a1d 100644 --- a/vyper/old_codegen/self_call.py +++ b/vyper/old_codegen/self_call.py @@ -14,9 +14,7 @@ def _generate_label(name: str) -> str: def lll_for_self_call(stmt_expr, context): - from vyper.old_codegen.expr import ( - Expr, # TODO rethink this circular import - ) + from vyper.old_codegen.expr import Expr # TODO rethink this circular import pos = getpos(stmt_expr) diff --git a/vyper/semantics/types/function.py b/vyper/semantics/types/function.py index 6532040b0a..fb39b8cec5 100644 --- a/vyper/semantics/types/function.py +++ b/vyper/semantics/types/function.py @@ -481,10 +481,10 @@ def to_abi_dict(self) -> List[Dict]: typ = self.return_type if typ is None: abi_dict["outputs"] = [] - elif isinstance(typ, TupleDefinition): + elif isinstance(typ, TupleDefinition) and len(typ.value_type) > 1: abi_dict["outputs"] = [_generate_abi_type(i) for i in typ.value_type] # type: ignore else: - abi_dict["outputs"] = [_generate_abi_type(typ)] + abi_dict["outputs"] = [_generate_abi_type(TupleDefinition([typ]))] if self.has_default_args: # for functions with default args, return a dict for each possible arg count @@ -504,6 +504,11 @@ def _generate_abi_type(type_definition, name=""): "type": "tuple", "components": [_generate_abi_type(v, k) for k, v in type_definition.members.items()], } + if isinstance(type_definition, TupleDefinition): + return { + "type": "tuple", + "components": [_generate_abi_type(i) for i in type_definition.value_type], + } return {"name": name, "type": type_definition.canonical_type} diff --git a/vyper/semantics/validation/local.py b/vyper/semantics/validation/local.py index 861517add6..b02c7ff585 100644 --- a/vyper/semantics/validation/local.py +++ b/vyper/semantics/validation/local.py @@ -19,6 +19,7 @@ VariableDeclarationException, VyperException, ) + # TODO consolidate some of these imports from vyper.semantics.environment import ( CONSTANT_ENVIRONMENT_VARS, From 267a98dfc4a33c5ded4b86641e18d5ffcea03618 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 16 Sep 2021 20:56:19 +0000 Subject: [PATCH 094/163] fix bug in abi.py --- quicktest.sh | 2 +- vyper/old_codegen/abi.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/quicktest.sh b/quicktest.sh index 8ecb322df9..f94d928157 100755 --- a/quicktest.sh +++ b/quicktest.sh @@ -6,4 +6,4 @@ # run pytest but bail out on first error and suppress coverage. # useful for dev workflow -pytest -q --no-cov -s --instafail -x --disable-warnings "$@" +pytest -q --no-cov -s --instafail --disable-warnings "$@" diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index 7d521b3506..7e14732f28 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -413,7 +413,7 @@ def abi_encode(dst, lll_node, pos=None, bufsz=None, returns_len=False): if not parent_abi_t.is_dynamic(): # cast the output buffer to something that make_setter accepts dst = LLLnode(dst, typ=lll_node.typ, location="memory") - lll_ret = make_setter(dst, lll_node, "memory", pos) + lll_ret = ["seq", make_setter(dst, lll_node, "memory", pos)] if returns_len: lll_ret.append(parent_abi_t.embedded_static_size()) return LLLnode.from_list(lll_ret, pos=pos, annotation=f"abi_encode {lll_node.typ}") From e112c2cc7d350532966d88556b451cec079c0277 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 16 Sep 2021 20:59:56 +0000 Subject: [PATCH 095/163] improve an error message --- quicktest.sh | 2 +- vyper/old_codegen/expr.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/quicktest.sh b/quicktest.sh index f94d928157..8ecb322df9 100755 --- a/quicktest.sh +++ b/quicktest.sh @@ -6,4 +6,4 @@ # run pytest but bail out on first error and suppress coverage. # useful for dev workflow -pytest -q --no-cov -s --instafail --disable-warnings "$@" +pytest -q --no-cov -s --instafail -x --disable-warnings "$@" diff --git a/vyper/old_codegen/expr.py b/vyper/old_codegen/expr.py index 8e7ac185ad..9de925b11c 100644 --- a/vyper/old_codegen/expr.py +++ b/vyper/old_codegen/expr.py @@ -202,7 +202,7 @@ def __init__(self, node, context): self.lll_node = fn() if self.lll_node is None: - raise TypeCheckFailure(f"{type(node).__name__} node did not produce LLL") + raise TypeCheckFailure(f"{type(node).__name__} node did not produce LLL. {self.expr}") def parse_Int(self): # Literal (mostly likely) becomes int256 From 1b9cb686dd146ac365b1c2f4f917f8bfa43716aa Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 16 Sep 2021 23:03:00 +0000 Subject: [PATCH 096/163] fix return wrapping again --- .../functional/codegen/test_struct_return.py | 4 ++-- vyper/old_codegen/external_call.py | 21 ++++++++++++------- vyper/old_codegen/parser_utils.py | 5 ----- vyper/old_codegen/return_.py | 15 ++++--------- vyper/semantics/types/function.py | 2 +- 5 files changed, 21 insertions(+), 26 deletions(-) diff --git a/tests/functional/codegen/test_struct_return.py b/tests/functional/codegen/test_struct_return.py index dbec5ed2b6..0cfa8ab600 100644 --- a/tests/functional/codegen/test_struct_return.py +++ b/tests/functional/codegen/test_struct_return.py @@ -26,7 +26,7 @@ def modify_nested_tuple(_human: Human) -> Human: # assert c.modify_nested_tuple([addr1, 123], [addr2, 456]) == [[addr1, 124], [addr2, 457]] assert c.modify_nested_tuple( {"location": addr1, "animal": {"location": addr2, "fur": "wool"}} - ) == ((addr1, (addr2, "wool is great"),),) + ) == (addr1, (addr2, "wool is great"),) @pytest.mark.parametrize("string", ["a", "abc", "abcde", "potato"]) @@ -56,4 +56,4 @@ def test_values(a: address) -> Person: """ c2 = get_contract(code) - assert c2.test_values(c1.address) == ((string, 42),) + assert c2.test_values(c1.address) == (string, 42) diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index 325627e1f8..0a2a5d4b0c 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -11,6 +11,7 @@ getpos, set_type_for_external_return, calculate_type_for_external_return, + add_variable_offset, unwrap_location, ) from vyper.old_codegen.types import ( @@ -64,8 +65,11 @@ def _unpack_returndata(buf, contract_sig, context, pos): return ["pass"], 0, 0 return_t = calculate_type_for_external_return(return_t) + abi_return_t = abi_type_of(return_t) + needs_offset_adjustment = contract_sig.return_type != return_t and abi_return_t.is_dynamic() + min_return_size = abi_return_t.min_size() max_return_size = abi_return_t.size_bound() @@ -93,8 +97,7 @@ def _unpack_returndata(buf, contract_sig, context, pos): ] ] - # ABI decoder has appropriate clampers for the individual members of the return type - # buf = LLLnode(buf, location="memory", encoding=Encoding.ABI) + # add as the last LLLnode a pointer to the return data structure ret += [buf] return ret, ret_ofst, ret_len @@ -146,11 +149,18 @@ def _external_call_helper( sub += ret_unpacker ret = LLLnode.from_list( + # set the encoding to ABI here, downstream code will decode and add clampers. sub, typ=contract_sig.return_type, location="memory", encoding=Encoding.ABI, pos=pos ) - # the return type has been wrapped by the calling contract; set the type so that - # we unwrap it correctly when it comes to abi decoding time. + + # the return type has been wrapped by the calling contract; + # unwrap it so downstream code isn't confused. + # basically this expands to ret+32 if the return type has been wrapped + # in a tuple AND its ABI type is dynamic. OTHERWISE (most cases), + # it will evaluate to ret. set_type_for_external_return(ret) + ret = add_variable_offset(ret, 0, pos=None, array_bounds_check=False) + return ret @@ -221,7 +231,4 @@ def lll_for_external_call(stmt_expr, context): ) ret.annotation = stmt_expr.get("node_source_code") - # kludge: to let upstream code know to use abi.wrap_value_for_return - # TODO: assign/ann_assign just pass in the buffer - ret.is_external_call_returndata = True return ret diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 7b7a60a9f9..f64155c4b3 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -549,11 +549,6 @@ def set_type_for_external_return(lll_val): # Create an x=y statement, where the types may be compound @type_check_wrapper def make_setter(left, right, location, pos): - if getattr(right, "is_external_call_returndata", False): - # the rhs is the result of some external call - # set the type so that type checking works - left = wrap_value_for_external_return(left) - # Basic types if isinstance(left.typ, BaseType): right = unwrap_location(right) diff --git a/vyper/old_codegen/return_.py b/vyper/old_codegen/return_.py index 6f8e025338..5077184eb7 100644 --- a/vyper/old_codegen/return_.py +++ b/vyper/old_codegen/return_.py @@ -12,11 +12,6 @@ from vyper.old_codegen.types.check import check_assign -def _allocate_return_buffer(context: Context) -> int: - maxlen = abi_type_of(context.return_type).size_bound() - return context.new_internal_variable(get_type_for_exact_size(maxlen)) - - Stmt = Any # mypy kludge @@ -64,13 +59,11 @@ def finalize(fill_return_buffer): else: # return from external function - if getattr(lll_val, "is_external_call_returndata", False): - # lll_val is already wrapped to the correct type, cf. external_call.py - pass - else: - lll_val = wrap_value_for_external_return(lll_val) + lll_val = wrap_value_for_external_return(lll_val) + + maxlen = abi_type_of(context.return_type).size_bound() + return_buffer_ofst = context.new_internal_variable(get_type_for_exact_size(maxlen)) - return_buffer_ofst = _allocate_return_buffer(context) # encode_out is cleverly a sequence which does the abi-encoding and # also returns the length of the output as a stack element encode_out = abi_encode(return_buffer_ofst, lll_val, pos=_pos, returns_len=True) diff --git a/vyper/semantics/types/function.py b/vyper/semantics/types/function.py index fb39b8cec5..6c9926cb38 100644 --- a/vyper/semantics/types/function.py +++ b/vyper/semantics/types/function.py @@ -484,7 +484,7 @@ def to_abi_dict(self) -> List[Dict]: elif isinstance(typ, TupleDefinition) and len(typ.value_type) > 1: abi_dict["outputs"] = [_generate_abi_type(i) for i in typ.value_type] # type: ignore else: - abi_dict["outputs"] = [_generate_abi_type(TupleDefinition([typ]))] + abi_dict["outputs"] = [_generate_abi_type(typ)] if self.has_default_args: # for functions with default args, return a dict for each possible arg count From c7b53ee5022e94e6ba8df5a4ae86b118c30ba0ce Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 16 Sep 2021 23:31:20 +0000 Subject: [PATCH 097/163] move unwrapping code deeper into external call --- vyper/old_codegen/external_call.py | 31 ++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index 0a2a5d4b0c..cd4254361c 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -27,11 +27,15 @@ def _pack_arguments(contract_sig, args, context, pos): args_as_tuple = LLLnode.from_list(["multi"] + [x for x in args], typ=args_tuple_t) args_abi_t = abi_type_of(args_tuple_t) - return_abi_t = abi_type_of(contract_sig.return_type) + if contract_sig.return_type is not None: + return_abi_t = abi_type_of(calculate_type_for_external_return(contract_sig.return_type)) + + # we use the same buffer for args and returndata, + # so allocate enough space here for the returndata too. + buflen = max(args_abi_t.size_bound(), return_abi_t.size_bound()) + else: + buflen = args_abi_t.size_bound() - # we use the same buffer for args and returndata, - # so allocate enough space here for the returndata too. - buflen = max(args_abi_t.size_bound(), return_abi_t.size_bound()) buflen += 32 # padding for the method id buf_t = get_type_for_exact_size(buflen) @@ -98,6 +102,17 @@ def _unpack_returndata(buf, contract_sig, context, pos): ] # add as the last LLLnode a pointer to the return data structure + + # the return type has been wrapped by the calling contract; + # unwrap it so downstream code isn't confused. + # basically this expands to buf+32 if the return type has been wrapped + # in a tuple AND its ABI type is dynamic. + # in most cases, this simply will evaluate to ret. + # in the special case where the return type has been wrapped + # in a tuple AND its ABI type is dynamic, it expands to buf+32. + buf = LLLnode(buf, typ=return_t, encoding=Encoding.ABI, location="memory") + buf = add_variable_offset(buf, 0, pos=None, array_bounds_check=False) + ret += [buf] return ret, ret_ofst, ret_len @@ -153,14 +168,6 @@ def _external_call_helper( sub, typ=contract_sig.return_type, location="memory", encoding=Encoding.ABI, pos=pos ) - # the return type has been wrapped by the calling contract; - # unwrap it so downstream code isn't confused. - # basically this expands to ret+32 if the return type has been wrapped - # in a tuple AND its ABI type is dynamic. OTHERWISE (most cases), - # it will evaluate to ret. - set_type_for_external_return(ret) - ret = add_variable_offset(ret, 0, pos=None, array_bounds_check=False) - return ret From 145d52951c5e615ad252d16f4568be5467eaf93c Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 17 Sep 2021 01:19:04 +0000 Subject: [PATCH 098/163] generalize key calculation to handle calldata by simplifying the code --- vyper/old_codegen/expr.py | 35 +++++++++------------------ vyper/old_codegen/keccak256_helper.py | 1 + vyper/old_codegen/parser_utils.py | 3 +-- 3 files changed, 13 insertions(+), 26 deletions(-) diff --git a/vyper/old_codegen/expr.py b/vyper/old_codegen/expr.py index 9de925b11c..8f6f56f5b0 100644 --- a/vyper/old_codegen/expr.py +++ b/vyper/old_codegen/expr.py @@ -416,31 +416,18 @@ def parse_Attribute(self): def parse_Subscript(self): sub = Expr.parse_variable_location(self.expr.value, self.context) - if isinstance(sub.typ, (MappingType, ListType)): + + if isinstance(sub.typ, MappingType): + # TODO sanity check we are in a self.my_map[i] situation index = Expr.parse_value_expr(self.expr.slice.value, self.context) - if isinstance(index.typ, ByteArrayLike) and index.args[0].location == "storage": - # Special case - if the key value is a bytes-array type located in - # storage, we have to copy it to memory prior to calculating the hash - placeholder = self.context.new_internal_variable(index.typ) - placeholder_node = LLLnode.from_list(placeholder, typ=index.typ, location="memory") - copier = make_byte_array_copier( - placeholder_node, - LLLnode.from_list(index.args[0], typ=index.typ, location="storage"), - ) - return LLLnode.from_list( - [ - "seq", - copier, - [ - "sha3_64", - sub, - ["sha3", ["add", placeholder, 32], ["mload", placeholder]], - ], - ], - typ=sub.typ.valuetype, - pos=getpos(self.expr), - location="storage", - ) + if isinstance(index.typ, ByteArrayLike): + # special case, + # we have to hash the key to get a storage location + index = keccak256_helper(self.expr.slice.value, index.args, None, self.context) + + elif isinstance(sub.typ, ListType): + index = Expr.parse_value_expr(self.expr.slice.value, self.context) + elif isinstance(sub.typ, TupleType): index = self.expr.slice.value.n if not 0 <= index < len(sub.typ.members): diff --git a/vyper/old_codegen/keccak256_helper.py b/vyper/old_codegen/keccak256_helper.py index 6c7765f504..81c958ed19 100644 --- a/vyper/old_codegen/keccak256_helper.py +++ b/vyper/old_codegen/keccak256_helper.py @@ -19,6 +19,7 @@ def _gas_bound(num_words): return SHA3_BASE + num_words * SHA3_PER_WORD +# TODO kwargs is dead argument def keccak256_helper(expr, lll_args, kwargs, context): if len(lll_args) != 1: # NOTE this may be checked at a higher level, but just be safe diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index f64155c4b3..dd368f8167 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -432,11 +432,10 @@ def _abi_helper(member_t, ofst, clamp=True): ) elif isinstance(typ, MappingType): - sub = None if isinstance(key.typ, ByteArrayLike): + # CMC 20210916 pretty sure this is dead code. TODO double check if isinstance(typ.keytype, ByteArrayLike) and (typ.keytype.maxlen >= key.typ.maxlen): - subtype = typ.valuetype if len(key.args[0].args) >= 3: # handle bytes literal. sub = LLLnode.from_list( From 143d31bf17f2f670508ce6b62e6ce0744db8fe44 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 17 Sep 2021 01:32:33 +0000 Subject: [PATCH 099/163] generalize concat to more locations --- vyper/builtin_functions/functions.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vyper/builtin_functions/functions.py b/vyper/builtin_functions/functions.py index 22f240fc6a..d89426ae2b 100644 --- a/vyper/builtin_functions/functions.py +++ b/vyper/builtin_functions/functions.py @@ -35,6 +35,7 @@ add_variable_offset, get_bytearray_length, getpos, + load_op, lll_tuple_from_args, make_byte_array_copier, make_byte_slice_copier, @@ -465,8 +466,8 @@ def build_LLL(self, expr, context): if arg.typ.maxlen == 0: continue # Get the length of the current argument - if arg.location == "memory": - length = LLLnode.from_list(["mload", "_arg"], typ=BaseType("int128")) + if arg.location in ("memory", "calldata", "code"): + length = LLLnode.from_list([load_op(arg.location), "_arg"], typ=BaseType("int128")) argstart = LLLnode.from_list( ["add", "_arg", 32], typ=arg.typ, location=arg.location, ) From 149eca41ac8ebc6d563de574f032b89651ac00ed Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 17 Sep 2021 01:35:00 +0000 Subject: [PATCH 100/163] fix test for old bytestring layout --- tests/functional/context/types/test_size_in_bytes.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/functional/context/types/test_size_in_bytes.py b/tests/functional/context/types/test_size_in_bytes.py index c5ffdbee57..acad0345f3 100644 --- a/tests/functional/context/types/test_size_in_bytes.py +++ b/tests/functional/context/types/test_size_in_bytes.py @@ -24,8 +24,7 @@ def test_array_value_types(build_node, type_str, location, length, size): node = build_node(f"{type_str}[{length}]") type_definition = get_type_from_annotation(node, location) - # TODO once storage of bytes is optimized, remove the +32 - assert type_definition.size_in_bytes == size + 32 + assert type_definition.size_in_bytes == size @pytest.mark.parametrize("type_str", BASE_TYPES) From 233c92ea991648da27ecc7c156cce2d1cb3ed467 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 17 Sep 2021 01:36:46 +0000 Subject: [PATCH 101/163] fix storage layout for new bytestring layout --- tests/cli/outputs/test_storage_layout.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cli/outputs/test_storage_layout.py b/tests/cli/outputs/test_storage_layout.py index c1ac43c6f9..aaf9e42446 100644 --- a/tests/cli/outputs/test_storage_layout.py +++ b/tests/cli/outputs/test_storage_layout.py @@ -34,5 +34,5 @@ def public_bar(): "slot": 3, }, "baz": {"type": "Bytes[65]", "location": "storage", "slot": 4}, - "bar": {"type": "uint256", "location": "storage", "slot": 9}, + "bar": {"type": "uint256", "location": "storage", "slot": 8}, } From 81f8f9737f91452782958e0dcab43df15af765a1 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 17 Sep 2021 17:38:01 +0000 Subject: [PATCH 102/163] fix slice builtin for location=calldata --- vyper/builtin_functions/functions.py | 37 +++++++++++++++++----------- vyper/old_codegen/parser_utils.py | 3 ++- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/vyper/builtin_functions/functions.py b/vyper/builtin_functions/functions.py index d89426ae2b..34be233b30 100644 --- a/vyper/builtin_functions/functions.py +++ b/vyper/builtin_functions/functions.py @@ -255,6 +255,7 @@ def fetch_call_return(self, node): def build_LLL(self, expr, args, kwargs, context): sub, start, length = args + if is_base_type(sub.typ, "bytes32"): if (start.typ.is_literal and length.typ.is_literal) and not ( 0 <= start.value + length.value <= 32 @@ -263,12 +264,6 @@ def build_LLL(self, expr, args, kwargs, context): "Invalid start / length values needs to be between 0 and 32.", expr, ) sub_typ_maxlen = 32 - elif sub.location == "calldata": - # if we are slicing msg.data, the length should - # be a constant, since msg.data can be of dynamic length - # we can't use it's length as the maxlen - assert isinstance(length.value, int) # sanity check - sub_typ_maxlen = length.value else: sub_typ_maxlen = sub.typ.maxlen @@ -279,15 +274,19 @@ def build_LLL(self, expr, args, kwargs, context): ReturnType = OldStringType # Node representing the position of the output in memory + # CMC 20210917 shouldn't this be a variable with newmaxlen? np = context.new_internal_variable(ReturnType(maxlen=sub_typ_maxlen + 32)) + # TODO deallocate np placeholder_node = LLLnode.from_list(np, typ=sub.typ, location="memory") placeholder_plus_32_node = LLLnode.from_list(np + 32, typ=sub.typ, location="memory") - # Copies over bytearray data - if sub.location == "storage": - adj_sub = LLLnode.from_list( - ["add", sub, ["add", ["div", "_start", 32], 1]], typ=sub.typ, location=sub.location, - ) - elif sub.location == "calldata": + + # special handling for slice(msg.data) + if sub.location == "calldata" and sub.value == 0: + assert expr.value.id == "msg" and expr.attr == "data" + # if we are slicing msg.data, the length should + # be a constant, since msg.data can be of dynamic length + # we can't use its length as the maxlen + assert isinstance(length.value, int) # sanity check node = [ "seq", ["assert", ["le", ["add", start, length], "calldatasize"]], # runtime bounds check @@ -296,6 +295,15 @@ def build_LLL(self, expr, args, kwargs, context): np, ] return LLLnode.from_list(node, typ=ByteArrayType(length.value), location="memory") + + + # Copy over bytearray data + # CMC 20210917 how does this routine work? + + if sub.location == "storage": + adj_sub = LLLnode.from_list( + ["add", sub, ["add", ["div", "_start", 32], 1]], typ=sub.typ, location=sub.location, + ) else: adj_sub = LLLnode.from_list( ["add", sub, ["add", ["sub", "_start", ["mod", "_start", 32]], 32]], @@ -309,16 +317,17 @@ def build_LLL(self, expr, args, kwargs, context): copier = make_byte_slice_copier( placeholder_plus_32_node, adj_sub, - ["add", "_length", 32], + ["add", "_length", 32], # CMC 20210917 shouldn't this just be _length sub_typ_maxlen, pos=getpos(expr), ) + # New maximum length in the type of the result newmaxlen = length.value if not len(length.args) else sub_typ_maxlen if is_base_type(sub.typ, "bytes32"): maxlen = 32 else: - maxlen = ["mload", Expr(sub, context=context).lll_node] # Retrieve length of the bytes. + maxlen = get_bytearray_length(sub) out = [ "with", diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index dd368f8167..6f53dc175d 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -165,7 +165,7 @@ def make_byte_slice_copier(destination, source, length, max_length, pos=None): [ "with", "_l", - max_length, + max_length, # CMC 20210917 shouldn't this just be length ["pop", ["call", ["gas"], 4, 0, source, "_l", destination, "_l"]], ], typ=None, @@ -177,6 +177,7 @@ def make_byte_slice_copier(destination, source, length, max_length, pos=None): if source.value is None: if destination.location == "memory": + # CMC 20210917 shouldn't this just be length return mzero(destination, max_length) else: From aee96fd2b2dc2fb36924d1df96f8b7fe34dd2787 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 17 Sep 2021 19:06:16 +0000 Subject: [PATCH 103/163] fix abi_encode test for needing to cache result --- vyper/old_codegen/abi.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index 7e14732f28..ea9068d9c3 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -1,5 +1,6 @@ import vyper.semantics.types as vy from vyper.exceptions import CompilerPanic +from vyper.evm.opcodes import get_comb_opcodes from vyper.old_codegen.lll_node import LLLnode from vyper.old_codegen.parser_utils import ( add_variable_offset, @@ -421,7 +422,10 @@ def abi_encode(dst, lll_node, pos=None, bufsz=None, returns_len=False): lll_ret = ["seq"] # contains some computation, we need to only do it once. - is_complex_lll = lll_node.value in ("seq", "seq_unchecked") + # TODO move this to parser_utils or something since it is + # generally useful + is_complex_lll = lll_node.value in get_comb_opcodes() + if is_complex_lll: to_encode = LLLnode.from_list( "to_encode", typ=lll_node.typ, location=lll_node.location, encoding=lll_node.encoding From 71dd34c0ff540b33ab3021905a52c118de5878c5 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 17 Sep 2021 19:38:48 +0000 Subject: [PATCH 104/163] fix slice length in a new test --- tests/functional/codegen/test_struct_return.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/functional/codegen/test_struct_return.py b/tests/functional/codegen/test_struct_return.py index 0cfa8ab600..5c7fa080fe 100644 --- a/tests/functional/codegen/test_struct_return.py +++ b/tests/functional/codegen/test_struct_return.py @@ -16,7 +16,8 @@ def modify_nested_tuple(_human: Human) -> Human: human: Human = _human # do stuff, edit the structs - human.animal.fur = slice(concat(human.animal.fur, " is great"), 0, 32) + # (13 is the length of the result) + human.animal.fur = slice(concat(human.animal.fur, " is great"), 0, 13) return human """ From ba8494435ec27b96d3dc5af89437cb91d0b6cc86 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 17 Sep 2021 19:55:05 +0000 Subject: [PATCH 105/163] improve annotation of add_variable_ofst LLL --- vyper/old_codegen/parser_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 6f53dc175d..42df7b5a46 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -311,11 +311,11 @@ def _abi_helper(member_t, ofst, clamp=True): # offset is statically known. ofst_lll = _add_ofst(parent, unwrap_location(ofst_lll)) - x = LLLnode.from_list(["x"], typ=member_t, location=parent.location) + x = LLLnode.from_list(["ofst"], typ=member_t, location=parent.location, annotation=f"{typ}->{member_t}") if clamp and _needs_clamp(member_t): # special handling for args that need clamping - ret = ["with", "x", ofst_lll, ["seq", clamp_basetype(x), x]] + ret = ["with", x, ofst_lll, ["seq", clamp_basetype(x), x]] else: ret = ofst_lll From f5f3433d7a238c9c828a3b66133e1ae9cf487943 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 17 Sep 2021 20:16:33 +0000 Subject: [PATCH 106/163] fix abi_encode test, again move complex_lll test to lll_node.py --- vyper/old_codegen/abi.py | 8 ++------ vyper/old_codegen/lll_node.py | 7 +++++++ vyper/utils.py | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index ea9068d9c3..48f3e33e0a 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -422,11 +422,7 @@ def abi_encode(dst, lll_node, pos=None, bufsz=None, returns_len=False): lll_ret = ["seq"] # contains some computation, we need to only do it once. - # TODO move this to parser_utils or something since it is - # generally useful - is_complex_lll = lll_node.value in get_comb_opcodes() - - if is_complex_lll: + if lll_node.is_complex_lll: to_encode = LLLnode.from_list( "to_encode", typ=lll_node.typ, location=lll_node.location, encoding=lll_node.encoding ) @@ -496,7 +492,7 @@ def abi_encode(dst, lll_node, pos=None, bufsz=None, returns_len=False): lll_ret = ["with", dst_begin, dst, ["with", dst_loc, dst_begin, lll_ret]] - if is_complex_lll: + if lll_node.is_complex_lll: lll_ret = ["with", to_encode, lll_node, lll_ret] return LLLnode.from_list(lll_ret, pos=pos, annotation=f"abi_encode {lll_node.typ}") diff --git a/vyper/old_codegen/lll_node.py b/vyper/old_codegen/lll_node.py index ed379cf982..819fcf065d 100644 --- a/vyper/old_codegen/lll_node.py +++ b/vyper/old_codegen/lll_node.py @@ -257,6 +257,13 @@ def __init__( self.gas += self.add_gas_estimate + # may contain some side effects + @property + def is_complex_lll(self): + return isinstance(self.value, str) and ( + self.value.lower() in VALID_LLL_MACROS or self.value.upper() in get_comb_opcodes() + ) + def __getitem__(self, i): return self.to_list()[i] diff --git a/vyper/utils.py b/vyper/utils.py index 940679c359..d4ab3c9348 100644 --- a/vyper/utils.py +++ b/vyper/utils.py @@ -156,7 +156,7 @@ def in_bounds(cls, type_str, value): "send", } -# List of valid LLL macros. Used for colorising LLL output +# List of valid LLL macros. VALID_LLL_MACROS = { "assert", "break", From ae6d3034c4f0328faf2d488e18a82ed074a11afb Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 17 Sep 2021 20:18:55 +0000 Subject: [PATCH 107/163] fix some lint --- vyper/builtin_functions/functions.py | 5 +++-- vyper/old_codegen/abi.py | 1 - vyper/old_codegen/expr.py | 1 - vyper/old_codegen/external_call.py | 9 +++++---- vyper/old_codegen/parser_utils.py | 6 ++++-- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/vyper/builtin_functions/functions.py b/vyper/builtin_functions/functions.py index 34be233b30..111a748d9a 100644 --- a/vyper/builtin_functions/functions.py +++ b/vyper/builtin_functions/functions.py @@ -296,7 +296,6 @@ def build_LLL(self, expr, args, kwargs, context): ] return LLLnode.from_list(node, typ=ByteArrayType(length.value), location="memory") - # Copy over bytearray data # CMC 20210917 how does this routine work? @@ -476,7 +475,9 @@ def build_LLL(self, expr, context): continue # Get the length of the current argument if arg.location in ("memory", "calldata", "code"): - length = LLLnode.from_list([load_op(arg.location), "_arg"], typ=BaseType("int128")) + length = LLLnode.from_list( + [load_op(arg.location), "_arg"], typ=BaseType("int128") + ) argstart = LLLnode.from_list( ["add", "_arg", 32], typ=arg.typ, location=arg.location, ) diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index 48f3e33e0a..e41cbde541 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -1,6 +1,5 @@ import vyper.semantics.types as vy from vyper.exceptions import CompilerPanic -from vyper.evm.opcodes import get_comb_opcodes from vyper.old_codegen.lll_node import LLLnode from vyper.old_codegen.parser_utils import ( add_variable_offset, diff --git a/vyper/old_codegen/expr.py b/vyper/old_codegen/expr.py index 8f6f56f5b0..d623c394a0 100644 --- a/vyper/old_codegen/expr.py +++ b/vyper/old_codegen/expr.py @@ -18,7 +18,6 @@ add_variable_offset, get_number_as_fraction, getpos, - make_byte_array_copier, make_setter, unwrap_location, ) diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index cd4254361c..4de2084db7 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -9,7 +9,6 @@ from vyper.old_codegen.lll_node import Encoding, LLLnode from vyper.old_codegen.parser_utils import ( getpos, - set_type_for_external_return, calculate_type_for_external_return, add_variable_offset, unwrap_location, @@ -72,8 +71,6 @@ def _unpack_returndata(buf, contract_sig, context, pos): abi_return_t = abi_type_of(return_t) - needs_offset_adjustment = contract_sig.return_type != return_t and abi_return_t.is_dynamic() - min_return_size = abi_return_t.min_size() max_return_size = abi_return_t.size_bound() @@ -165,7 +162,11 @@ def _external_call_helper( ret = LLLnode.from_list( # set the encoding to ABI here, downstream code will decode and add clampers. - sub, typ=contract_sig.return_type, location="memory", encoding=Encoding.ABI, pos=pos + sub, + typ=contract_sig.return_type, + location="memory", + encoding=Encoding.ABI, + pos=pos, ) return ret diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 42df7b5a46..3b79bade48 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -165,7 +165,7 @@ def make_byte_slice_copier(destination, source, length, max_length, pos=None): [ "with", "_l", - max_length, # CMC 20210917 shouldn't this just be length + max_length, # CMC 20210917 shouldn't this just be length ["pop", ["call", ["gas"], 4, 0, source, "_l", destination, "_l"]], ], typ=None, @@ -311,7 +311,9 @@ def _abi_helper(member_t, ofst, clamp=True): # offset is statically known. ofst_lll = _add_ofst(parent, unwrap_location(ofst_lll)) - x = LLLnode.from_list(["ofst"], typ=member_t, location=parent.location, annotation=f"{typ}->{member_t}") + x = LLLnode.from_list( + ["ofst"], typ=member_t, location=parent.location, annotation=f"{typ}->{member_t}" + ) if clamp and _needs_clamp(member_t): # special handling for args that need clamping From 43ee4a38c20cea89db74362aad2ec3d4bef6ef3a Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 17 Sep 2021 21:00:24 +0000 Subject: [PATCH 108/163] clarify variable ofst annotation a bit it's like a variable reference in C --- vyper/old_codegen/parser_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 3b79bade48..08944d490c 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -312,7 +312,7 @@ def _abi_helper(member_t, ofst, clamp=True): ofst_lll = _add_ofst(parent, unwrap_location(ofst_lll)) x = LLLnode.from_list( - ["ofst"], typ=member_t, location=parent.location, annotation=f"{typ}->{member_t}" + ["ofst"], typ=member_t, location=parent.location, annotation=f"&({typ}->{member_t})" ) if clamp and _needs_clamp(member_t): From 756d272af8abfaf54f80802841771bee940cfd25 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sat, 18 Sep 2021 01:19:55 +0000 Subject: [PATCH 109/163] wip fix the jumptable for external kwarg handlers --- .../function_definitions/external_function.py | 54 +++++++++---------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index cf735bf5a9..edca3c31fa 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -20,15 +20,9 @@ # register function args with the local calling context. # also allocate the ones that live in memory (i.e. kwargs) -# returns an LLLnode with copy operations for base args which -# need to be copied to memory (they don't play nicely with -# downstream code). -def _register_function_args(context: Context, sig: FunctionSignature) -> List[Any]: - - ret = ["seq"] - +def _register_function_args(context: Context, sig: FunctionSignature): if len(sig.args) == 0: - return ret + return base_args_t = TupleType([arg.typ for arg in sig.base_args]) @@ -54,8 +48,6 @@ def _register_function_args(context: Context, sig: FunctionSignature) -> List[An encoding=Encoding.ABI, ) - return ret - def _annotated_method_id(abi_sig): method_id = util.abi_method_id(abi_sig) @@ -68,6 +60,11 @@ def _generate_kwarg_handlers(context: Context, sig: FunctionSignature, pos: Any) # since they might come in thru calldata or be default, # allocate them in memory and then fill it in based on calldata or default, # depending on the signature + # a kwarg handler looks like + # (if (eq _method_id ) + # copy calldata args to memory + # write default args to memory + # goto external_function_common_lll def handler_for(calldata_kwargs, default_kwargs): calldata_args = sig.base_args + calldata_kwargs @@ -111,7 +108,7 @@ def handler_for(calldata_kwargs, default_kwargs): ret = ["if", ["eq", "_calldata_method_id", method_id], ret] return ret - ret = [] + ret = ["seq"] keyword_args = sig.default_args @@ -121,6 +118,8 @@ def handler_for(calldata_kwargs, default_kwargs): ret.append(handler_for(calldata_kwargs, default_kwargs)) + ret.append(handler_for(keyword_args, [])) + return ret @@ -139,23 +138,17 @@ def generate_lll_for_external_function(code, sig, context, check_nonpayable): func_type = code._metadata["type"] pos = getpos(code) - base_arg_handlers = _register_function_args(context, sig) + _register_function_args(context, sig) nonreentrant_pre, nonreentrant_post = get_nonreentrant_lock(func_type) - kwarg_handlers = _generate_kwarg_handlers(context, sig, pos) - entrance = [] - # once args have been handled - if len(kwarg_handlers) > 0: - entrance += [["label", sig.external_function_base_entry_label]] - else: - # otherwise, the label is redundant since there is only - # one control flow path into the external method - pass + kwarg_handlers = _generate_kwarg_handlers(context, sig, pos) - entrance += base_arg_handlers + # once optional args have been handled, + # generate the main body of the function + entrance += [["label", sig.external_function_base_entry_label]] if check_nonpayable and sig.mutability != "payable": # if the contract contains payable functions, but this is not one of them @@ -175,12 +168,17 @@ def generate_lll_for_external_function(code, sig, context, check_nonpayable): # ret_ofst and ret_len stack items passed by function body; consume using 'pass' exit += [["return", "pass", "pass"]] - ret = ["seq"] + kwarg_handlers + entrance + body + exit + # the lll which comprises the main body of the function, + # besides any kwarg handling + func_common_lll = ["seq"] + entrance + body + exit - if len(kwarg_handlers) == 0 and not sig.is_default_func and not sig.is_init_func: - assert len(sig.default_args) == 0 # sanity check - abi_sig = sig.base_signature - method_id = _annotated_method_id(abi_sig) - ret = ["if", ["eq", "_calldata_method_id", method_id], ret] + if sig.is_default_func or sig.is_init_func: + # default and init funcs have special entries generated by parser.py + ret = func_common_lll + else: + ret = kwarg_handlers + # sneak the base code into the kwarg handler + # TODO rethink this / make it clearer + ret[-1][-1].append(func_common_lll) return LLLnode.from_list(ret, pos=getpos(code)) From 3a508ea310790ab6e1f18db51600a491159e57e2 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 20 Sep 2021 18:15:54 +0000 Subject: [PATCH 110/163] add py-evm debug2 logging for tests --- tests/conftest.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3ca78c07b9..ec128bd6d6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,6 +3,7 @@ import pytest from eth_tester import EthereumTester +from eth_utils import setup_DEBUG2_logging from web3 import Web3 from web3.providers.eth_tester import EthereumTesterProvider @@ -25,12 +26,14 @@ def set_evm_verbose_logging(): - logger = logging.getLogger("evm") - logger.setLevel("TRACE") + logger = logging.getLogger("eth.vm.computation.Computation") + setup_DEBUG2_logging() + logger.setLevel("DEBUG2") # Useful options to comment out whilst working: # set_evm_verbose_logging() +# # from vdb import vdb # vdb.set_evm_opcode_debugger() From 23a498514b9ba611f5a4d50072fba0e1a0041ef6 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 20 Sep 2021 21:47:47 +0000 Subject: [PATCH 111/163] add structured goto to LLL seq_unchecked induces a POP when it is wrapped inside a seq (since it has nonzero valency). so instead of passing arguments on the stack via the pattern seq_unchecked x y z goto my_subroutine put it into a macro: goto my_subroutine z y x easier to reason about for LLL writers. --- vyper/evm/opcodes.py | 1 - vyper/lll/compile_lll.py | 8 ++++++-- vyper/old_codegen/lll_node.py | 10 ++++++++++ vyper/old_codegen/self_call.py | 15 ++++++++++----- 4 files changed, 26 insertions(+), 8 deletions(-) diff --git a/vyper/evm/opcodes.py b/vyper/evm/opcodes.py index 6cf908cb8f..e92871b6c7 100644 --- a/vyper/evm/opcodes.py +++ b/vyper/evm/opcodes.py @@ -209,7 +209,6 @@ "NE": (None, 2, 1, 6), "DEBUGGER": (None, 0, 0, 0), "LABEL": (None, 1, 0, 1), - "GOTO": (None, 1, 0, 8), } COMB_OPCODES: OpcodeMap = {**OPCODES, **PSEUDO_OPCODES} diff --git a/vyper/lll/compile_lll.py b/vyper/lll/compile_lll.py index e9746770b6..2a0eac71ba 100644 --- a/vyper/lll/compile_lll.py +++ b/vyper/lll/compile_lll.py @@ -493,9 +493,13 @@ def _compile_to_assembly(code, withargs=None, existing_labels=None, break_dest=N break_dest, height, ) - # # jump to a symbol + # # jump to a symbol, and push variable arguments onto stack elif code.value == "goto": - return ["_sym_" + str(code.args[0]), "JUMP"] + o = [] + for i, c in enumerate(reversed(code.args[1:])): + o.extend(_compile_to_assembly(c, withargs, existing_labels, break_dest, height + i)) + o.extend(["_sym_" + str(code.args[0]), "JUMP"]) + return o elif isinstance(code.value, str) and is_symbol(code.value): return [code.value] # set a symbol as a location. diff --git a/vyper/old_codegen/lll_node.py b/vyper/old_codegen/lll_node.py index 819fcf065d..c1480bb952 100644 --- a/vyper/old_codegen/lll_node.py +++ b/vyper/old_codegen/lll_node.py @@ -222,6 +222,16 @@ def __init__( elif self.value == "seq": self.valency = self.args[-1].valency if self.args else 0 self.gas = sum([arg.gas for arg in self.args]) + 30 + + # GOTO is a jump with args + # e.g. (goto my_label x y z) will push x y and z onto the stack, + # then JUMP to my_label. + elif self.value == "goto": + for arg in self.args: + if not arg.valency and arg.value != "pass": + raise CompilerPanic(f"zerovalent argument to goto {self}") + self.valency = 0 + self.gas = sum([arg.gas for arg in self.args]) # Multi statements: multi ... elif self.value == "multi": for arg in self.args: diff --git a/vyper/old_codegen/self_call.py b/vyper/old_codegen/self_call.py index 793c394a1d..2526f92ccf 100644 --- a/vyper/old_codegen/self_call.py +++ b/vyper/old_codegen/self_call.py @@ -14,7 +14,9 @@ def _generate_label(name: str) -> str: def lll_for_self_call(stmt_expr, context): - from vyper.old_codegen.expr import Expr # TODO rethink this circular import + from vyper.old_codegen.expr import ( + Expr, # TODO rethink this circular import + ) pos = getpos(stmt_expr) @@ -66,11 +68,14 @@ def lll_for_self_call(stmt_expr, context): copy_args = make_setter(args_dst, args_as_tuple, "memory", pos) call_sequence = [ - "seq_unchecked", + "seq", copy_args, - push_label_to_stack(return_label), # pass return label to subroutine - return_buffer, # pass return buffer to subroutine - ["goto", sig.internal_function_label], + [ + "goto", + sig.internal_function_label, + return_buffer, # pass return buffer to subroutine + push_label_to_stack(return_label), # pass return label to subroutine + ], ["label", return_label], return_buffer, # push return buffer location to stack ] From e84c427c9918de21f3f8fce55224ada5776c37eb Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 20 Sep 2021 21:50:01 +0000 Subject: [PATCH 112/163] fix some lint --- vyper/builtin_functions/functions.py | 2 +- vyper/old_codegen/external_call.py | 12 ++++++++---- vyper/old_codegen/return_.py | 1 - vyper/semantics/validation/local.py | 1 - 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/vyper/builtin_functions/functions.py b/vyper/builtin_functions/functions.py index 111a748d9a..84b1f9c97a 100644 --- a/vyper/builtin_functions/functions.py +++ b/vyper/builtin_functions/functions.py @@ -35,8 +35,8 @@ add_variable_offset, get_bytearray_length, getpos, - load_op, lll_tuple_from_args, + load_op, make_byte_array_copier, make_byte_slice_copier, unwrap_location, diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index 4de2084db7..468f1c3f80 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -8,9 +8,9 @@ from vyper.old_codegen.abi import abi_encode, abi_type_of from vyper.old_codegen.lll_node import Encoding, LLLnode from vyper.old_codegen.parser_utils import ( - getpos, - calculate_type_for_external_return, add_variable_offset, + calculate_type_for_external_return, + getpos, unwrap_location, ) from vyper.old_codegen.types import ( @@ -174,7 +174,9 @@ def _external_call_helper( # TODO push me up to expr.py def get_gas_and_value(stmt_expr, context): - from vyper.old_codegen.expr import Expr # TODO rethink this circular import + from vyper.old_codegen.expr import ( + Expr, # TODO rethink this circular import + ) value, gas = None, None for kw in stmt_expr.keywords: @@ -188,7 +190,9 @@ def get_gas_and_value(stmt_expr, context): def lll_for_external_call(stmt_expr, context): - from vyper.old_codegen.expr import Expr # TODO rethink this circular import + from vyper.old_codegen.expr import ( + Expr, # TODO rethink this circular import + ) pos = getpos(stmt_expr) value, gas = get_gas_and_value(stmt_expr, context) diff --git a/vyper/old_codegen/return_.py b/vyper/old_codegen/return_.py index 5077184eb7..a6c2edf1a8 100644 --- a/vyper/old_codegen/return_.py +++ b/vyper/old_codegen/return_.py @@ -11,7 +11,6 @@ from vyper.old_codegen.types import get_type_for_exact_size from vyper.old_codegen.types.check import check_assign - Stmt = Any # mypy kludge diff --git a/vyper/semantics/validation/local.py b/vyper/semantics/validation/local.py index b02c7ff585..861517add6 100644 --- a/vyper/semantics/validation/local.py +++ b/vyper/semantics/validation/local.py @@ -19,7 +19,6 @@ VariableDeclarationException, VyperException, ) - # TODO consolidate some of these imports from vyper.semantics.environment import ( CONSTANT_ENVIRONMENT_VARS, From 21560251ecefd2fbee324caaa0013daa501e1336 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 20 Sep 2021 22:09:23 +0000 Subject: [PATCH 113/163] fix unwrapping of tuples for real tuples not just return types which are wrapped in a tuple for the ABI --- vyper/old_codegen/external_call.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index 468f1c3f80..8f3ffe28b6 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -68,6 +68,10 @@ def _unpack_returndata(buf, contract_sig, context, pos): return ["pass"], 0, 0 return_t = calculate_type_for_external_return(return_t) + # if the abi signature has a different type than + # the vyper type, we need to wrap and unwrap the type + # so that the ABI decoding works correctly + should_unwrap_abi_tuple = return_t != contract_sig.return_type abi_return_t = abi_type_of(return_t) @@ -108,7 +112,9 @@ def _unpack_returndata(buf, contract_sig, context, pos): # in the special case where the return type has been wrapped # in a tuple AND its ABI type is dynamic, it expands to buf+32. buf = LLLnode(buf, typ=return_t, encoding=Encoding.ABI, location="memory") - buf = add_variable_offset(buf, 0, pos=None, array_bounds_check=False) + + if should_unwrap_abi_tuple: + buf = add_variable_offset(buf, 0, pos=None, array_bounds_check=False) ret += [buf] From ca7065cec794721238feb4147955c205b75107ef Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 20 Sep 2021 22:16:15 +0000 Subject: [PATCH 114/163] fix nested abi encode size --- tests/functional/codegen/test_abi_encode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/codegen/test_abi_encode.py b/tests/functional/codegen/test_abi_encode.py index 3f1507fdfd..1e29f7e7ba 100644 --- a/tests/functional/codegen/test_abi_encode.py +++ b/tests/functional/codegen/test_abi_encode.py @@ -31,7 +31,7 @@ def abi_encode( pet_metadata: bytes32, ensure_tuple: bool, include_method_id: bool -) -> Bytes[260]: +) -> Bytes[548]: human: Human = Human({ name: name, pet: Animal({ From 1827c4b3e8605977745c6111f695bb1fcff51889 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 21 Sep 2021 01:14:06 +0000 Subject: [PATCH 115/163] fix memory allocation for abi_encode builtin this was not a bug before because Bytes had an extra 32 bytes allocated to it. --- vyper/builtin_functions/functions.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vyper/builtin_functions/functions.py b/vyper/builtin_functions/functions.py index 84b1f9c97a..fa980bf803 100644 --- a/vyper/builtin_functions/functions.py +++ b/vyper/builtin_functions/functions.py @@ -1852,6 +1852,8 @@ def build_LLL(self, expr, context): input_abi_t = abi_type_of(encode_input.typ) maxlen = input_abi_t.size_bound() + if method_id is not None: + maxlen += 4 buf_t = ByteArrayType(maxlen=maxlen) buf = context.new_internal_variable(buf_t) @@ -1860,6 +1862,7 @@ def build_LLL(self, expr, context): ret = ["seq"] if method_id is not None: + # <32 bytes length> | <4 bytes method_id> | # write the unaligned method_id first, then we will # overwrite the 28 bytes of zeros with the bytestring length ret += [["mstore", buf + 4, method_id]] From 5926e88ff6086f287577436712713f2c1749e4ca Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 21 Sep 2021 15:55:43 +0000 Subject: [PATCH 116/163] add a note about dead code --- vyper/old_codegen/parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vyper/old_codegen/parser.py b/vyper/old_codegen/parser.py index d374ebbfc8..4e245777d2 100644 --- a/vyper/old_codegen/parser.py +++ b/vyper/old_codegen/parser.py @@ -231,6 +231,7 @@ def parse_tree_to_lll(global_ctx: GlobalContext) -> Tuple[LLLnode, LLLnode]: return LLLnode.from_list(o), LLLnode.from_list(runtime) +# TODO this function is dead code def parse_to_lll( source_code: str, runtime_only: bool = False, interface_codes: Optional[InterfaceImports] = None ) -> LLLnode: From de4d06c96033ab08247eee7988544202555a687f Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 21 Sep 2021 16:01:31 +0000 Subject: [PATCH 117/163] wip reorganize parser lll --- vyper/old_codegen/parser.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/vyper/old_codegen/parser.py b/vyper/old_codegen/parser.py index 4e245777d2..618104a0df 100644 --- a/vyper/old_codegen/parser.py +++ b/vyper/old_codegen/parser.py @@ -108,9 +108,9 @@ def parse_regular_functions( check_per_function = is_default_payable and has_nonpayable # generate LLL for regular functions - payable_func_sub = ["seq"] - external_func_sub = ["seq"] - internal_func_sub = ["seq"] + payable_funcs = [] + nonpayable_funcs = [] + internal_funcs = [] add_gas = func_init_lll().gas for func_node in regular_functions: @@ -120,15 +120,15 @@ def parse_regular_functions( ) if func_type.visibility == FunctionVisibility.INTERNAL: - internal_func_sub.append(func_lll) + internal_funcs.append(func_lll) elif func_type.mutability == StateMutability.PAYABLE: add_gas += 30 # CMC 20210910 why? - payable_func_sub.append(func_lll) + payable_funcs.append(func_lll) else: - external_func_sub.append(func_lll) add_gas += 30 # CMC 20210910 why? + nonpayable_funcs.append(func_lll) func_lll.total_gas += add_gas @@ -155,16 +155,17 @@ def parse_regular_functions( fallback_lll = LLLnode.from_list(["revert", 0, 0], typ=None, annotation="Default function") if check_per_function: - external_seq = ["seq", payable_func_sub, external_func_sub] + external_seq = ["seq"] + payable_funcs + nonpayable_funcs else: # payable functions are placed prior to nonpayable functions # and seperated by a nonpayable assertion external_seq = ["seq"] if has_payable: - external_seq.append(payable_func_sub) + external_seq += payable_funcs if has_nonpayable: - external_seq.extend([["assert", ["iszero", "callvalue"]], external_func_sub]) + external_seq.append(["assert", ["iszero", "callvalue"]]) + external_seq += nonpayable_funcs # bytecode is organized by: external functions, fallback fn, internal functions # this way we save gas and reduce bytecode by not jumping over internal functions @@ -173,8 +174,8 @@ def parse_regular_functions( func_init_lll(), ["with", "_calldata_method_id", ["mload", 0], external_seq], ["seq_unchecked", ["label", "fallback"], fallback_lll], - internal_func_sub, ] + runtime.extend(internal_funcs) # TODO CMC 20210911 why does the lll have a trailing 0 o.append(["return", 0, ["lll", runtime, 0]]) From d95ec07c7fc5715e8e379195a281cf944803f435 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 21 Sep 2021 16:26:28 +0000 Subject: [PATCH 118/163] remove "optimization" which doesn't do anything --- vyper/lll/optimizer.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/vyper/lll/optimizer.py b/vyper/lll/optimizer.py index 78403f0e6a..131680c75f 100644 --- a/vyper/lll/optimizer.py +++ b/vyper/lll/optimizer.py @@ -191,23 +191,6 @@ def apply_general_optimizations(node: LLLnode) -> LLLnode: annotation=node.annotation, # let from_list handle valency and gas_estimate ) - elif node.value == "seq": - xs: List[Any] = [] - for arg in argz: - if arg.value == "seq": - xs.extend(arg.args) - else: - xs.append(arg) - return LLLnode( - node.value, - xs, - node.typ, - node.location, - node.pos, - node.annotation, - add_gas_estimate=node.add_gas_estimate, - valency=node.valency, - ) elif node.total_gas is not None: o = LLLnode( node.value, From 8a416c885f386ed10ead02f03bd8910e73e029ca Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 21 Sep 2021 16:36:51 +0000 Subject: [PATCH 119/163] fix assert_reason for new bytestring length --- vyper/old_codegen/stmt.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/vyper/old_codegen/stmt.py b/vyper/old_codegen/stmt.py index 516cb29bd8..922eeddd0a 100644 --- a/vyper/old_codegen/stmt.py +++ b/vyper/old_codegen/stmt.py @@ -19,7 +19,7 @@ get_size_of_type, parse_type, ) -from vyper.utils import bytes_to_int, fourbytes_to_int, keccak256 +import vyper.utils as util class Stmt: @@ -70,7 +70,7 @@ def parse_AnnAssign(self): # If bytes[32] to bytes32 assignment rewrite sub as bytes32. if is_literal_bytes32_assign: sub = LLLnode( - bytes_to_int(self.stmt.value.s), typ=BaseType("bytes32"), pos=getpos(self.stmt), + util.bytes_to_int(self.stmt.value.s), typ=BaseType("bytes32"), pos=getpos(self.stmt), ) variable_loc = LLLnode.from_list(pos, typ=typ, location="memory", pos=getpos(self.stmt),) @@ -146,18 +146,21 @@ def _assert_reason(self, test_expr, msg): reason_str_type = ByteArrayType(len(msg.value.strip())) + # abi encode the reason string sig_placeholder = self.context.new_internal_variable(BaseType(32)) + # offset of bytes in (bytes,) arg_placeholder = self.context.new_internal_variable(BaseType(32)) placeholder_bytes = Expr(msg, self.context).lll_node - method_id = fourbytes_to_int(keccak256(b"Error(string)")[:4]) + method_id = util.abi_method_id("Error(string)") + # abi encode method_id + bytestring revert_seq = [ "seq", ["mstore", sig_placeholder, method_id], ["mstore", arg_placeholder, 32], placeholder_bytes, - ["revert", sig_placeholder + 28, int(4 + get_size_of_type(reason_str_type) * 32)], + ["revert", sig_placeholder + 28, int(32 + 4 + get_size_of_type(reason_str_type) * 32)], ] if test_expr: lll_node = ["if", ["iszero", test_expr], revert_seq] From 1c6c60567c798f0e8b4a8874dafd80a8149cfa3d Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 21 Sep 2021 17:36:57 +0000 Subject: [PATCH 120/163] fix sar emulation for pre-constantinople --- vyper/old_codegen/arg_clamps.py | 6 +++++- vyper/old_codegen/parser_utils.py | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/vyper/old_codegen/arg_clamps.py b/vyper/old_codegen/arg_clamps.py index 67e3caa7fd..e7c061b775 100644 --- a/vyper/old_codegen/arg_clamps.py +++ b/vyper/old_codegen/arg_clamps.py @@ -12,7 +12,11 @@ def _shr(x, bits): def _sar(x, bits): if version_check(begin="constantinople"): return ["sar", bits, x] - return ["sdiv", x, ["exp", 2, bits]] + + # emulate for older arches. keep in mind note from EIP 145: + # This is not equivalent to PUSH1 2 EXP SDIV, since it rounds + # differently. See SDIV(-1, 2) == 0, while SAR(-1, 1) == -1. + return ["sdiv", ["add", ["slt", x, 0], x], ["exp", 2, bits]] def address_clamp(lll_node): diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 08944d490c..1cbdf1283b 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -805,7 +805,11 @@ def _shr(x, bits): def _sar(x, bits): if version_check(begin="constantinople"): return ["sar", bits, x] - return ["sdiv", x, ["exp", 2, bits]] + + # emulate for older arches. keep in mind note from EIP 145: + # This is not equivalent to PUSH1 2 EXP SDIV, since it rounds + # differently. See SDIV(-1, 2) == 0, while SAR(-1, 1) == -1. + return ["sdiv", ["add", ["slt", x, 0], x], ["exp", 2, bits]] def _needs_clamp(t): From 868ad4cb871f5b5508045285d40492d3c7556791 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 21 Sep 2021 17:47:28 +0000 Subject: [PATCH 121/163] fix clamp for abi-encoded static arrays --- vyper/old_codegen/parser_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 1cbdf1283b..cf58eb7ead 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -609,7 +609,9 @@ def make_setter(left, right, location, pos): return LLLnode.from_list(["with", "_L", left, ["seq"] + subs], typ=None) # If the right side is a variable else: - right_token = LLLnode.from_list("_R", typ=right.typ, location=right.location) + right_token = LLLnode.from_list( + "_R", typ=right.typ, location=right.location, encoding=right.encoding + ) subs = [] for i in range(left.typ.count): lhs_setter = _make_array_index_setter(left, left_token, pos, left.location, i) From 2f83f088c367cfc9808127207c245ac6b2de128b Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 21 Sep 2021 17:48:03 +0000 Subject: [PATCH 122/163] fix lint --- vyper/lll/optimizer.py | 2 +- vyper/old_codegen/parser.py | 4 ++-- vyper/old_codegen/stmt.py | 6 ++++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/vyper/lll/optimizer.py b/vyper/lll/optimizer.py index 131680c75f..f1d18b2551 100644 --- a/vyper/lll/optimizer.py +++ b/vyper/lll/optimizer.py @@ -1,5 +1,5 @@ import operator -from typing import Any, List, Optional +from typing import List, Optional from vyper.old_codegen.parser_utils import LLLnode from vyper.utils import LOADED_LIMITS diff --git a/vyper/old_codegen/parser.py b/vyper/old_codegen/parser.py index 618104a0df..8c92395aab 100644 --- a/vyper/old_codegen/parser.py +++ b/vyper/old_codegen/parser.py @@ -108,9 +108,9 @@ def parse_regular_functions( check_per_function = is_default_payable and has_nonpayable # generate LLL for regular functions - payable_funcs = [] + payable_funcs = [] nonpayable_funcs = [] - internal_funcs = [] + internal_funcs = [] add_gas = func_init_lll().gas for func_node in regular_functions: diff --git a/vyper/old_codegen/stmt.py b/vyper/old_codegen/stmt.py index 922eeddd0a..398331c498 100644 --- a/vyper/old_codegen/stmt.py +++ b/vyper/old_codegen/stmt.py @@ -1,4 +1,5 @@ import vyper.old_codegen.events as events +import vyper.utils as util from vyper import ast as vy_ast from vyper.builtin_functions import STMT_DISPATCH_TABLE from vyper.exceptions import StructureException, TypeCheckFailure @@ -19,7 +20,6 @@ get_size_of_type, parse_type, ) -import vyper.utils as util class Stmt: @@ -70,7 +70,9 @@ def parse_AnnAssign(self): # If bytes[32] to bytes32 assignment rewrite sub as bytes32. if is_literal_bytes32_assign: sub = LLLnode( - util.bytes_to_int(self.stmt.value.s), typ=BaseType("bytes32"), pos=getpos(self.stmt), + util.bytes_to_int(self.stmt.value.s), + typ=BaseType("bytes32"), + pos=getpos(self.stmt), ) variable_loc = LLLnode.from_list(pos, typ=typ, location="memory", pos=getpos(self.stmt),) From f7e38b3eead970bde38018281c71c318ef99045c Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 21 Sep 2021 17:51:06 +0000 Subject: [PATCH 123/163] fix tests for new struct output format --- tests/functional/test_storage_slots.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/functional/test_storage_slots.py b/tests/functional/test_storage_slots.py index f2a4925fcd..38049073ab 100644 --- a/tests/functional/test_storage_slots.py +++ b/tests/functional/test_storage_slots.py @@ -55,18 +55,18 @@ def with_other_lock(): def test_storage_slots(get_contract): c = get_contract(code) - assert c.a() == ["ok", [4, 5, 6]] + assert c.a() == ("ok", [4, 5, 6]) assert [c.b(i) for i in range(2)] == [7, 8] assert c.c() == b"thisisthirtytwobytesokhowdoyoudo" assert [c.d(i) for i in range(4)] == [-1, -2, -3, -4] assert c.e() == "A realllllly long string but we wont use it all" assert c.f(0) == 33 - assert c.g(0) == [b"hello", [-66, 420], "another string"] - assert c.g(1) == [ + assert c.g(0) == (b"hello", [-66, 420], "another string") + assert c.g(1) == ( b"gbye", [1337, 888], "whatifthisstringtakesuptheentirelengthwouldthatbesobadidothinkso", - ] + ) assert [c.foo(0, i) for i in range(3)] == [987, 654, 321] assert [c.foo(1, i) for i in range(3)] == [123, 456, 789] assert c.h(0) == 123456789 @@ -80,18 +80,18 @@ def test_reentrancy_lock(get_contract): c.with_lock() c.with_other_lock() - assert c.a() == ["ok", [4, 5, 6]] + assert c.a() == ("ok", [4, 5, 6]) assert [c.b(i) for i in range(2)] == [7, 8] assert c.c() == b"thisisthirtytwobytesokhowdoyoudo" assert [c.d(i) for i in range(4)] == [-1, -2, -3, -4] assert c.e() == "A realllllly long string but we wont use it all" assert c.f(0) == 33 - assert c.g(0) == [b"hello", [-66, 420], "another string"] - assert c.g(1) == [ + assert c.g(0) == (b"hello", [-66, 420], "another string") + assert c.g(1) == ( b"gbye", [1337, 888], "whatifthisstringtakesuptheentirelengthwouldthatbesobadidothinkso", - ] + ) assert [c.foo(0, i) for i in range(3)] == [987, 654, 321] assert [c.foo(1, i) for i in range(3)] == [123, 456, 789] assert c.h(0) == 123456789 From 88e9e883b4477acc01b04647632c52416a334e16 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 21 Sep 2021 17:54:50 +0000 Subject: [PATCH 124/163] fix sha256 for non-memory args --- vyper/builtin_functions/functions.py | 30 +++++++++++++--------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/vyper/builtin_functions/functions.py b/vyper/builtin_functions/functions.py index fa980bf803..8886ad1493 100644 --- a/vyper/builtin_functions/functions.py +++ b/vyper/builtin_functions/functions.py @@ -613,13 +613,8 @@ def build_LLL(self, expr, args, kwargs, context): add_gas_estimate=SHA256_BASE_GAS + 1 * SHA256_PER_WORD_GAS, ) # bytearay-like input - if sub.location == "storage": - # Copy storage to memory - placeholder = context.new_internal_variable(sub.typ) - placeholder_node = LLLnode.from_list(placeholder, typ=sub.typ, location="memory") - copier = make_byte_array_copier( - placeholder_node, LLLnode.from_list("_sub", typ=sub.typ, location=sub.location), - ) + # special case if it's already in memory + if sub.location == "memory": return LLLnode.from_list( [ "with", @@ -627,10 +622,9 @@ def build_LLL(self, expr, args, kwargs, context): sub, [ "seq", - copier, _make_sha256_call( - inp_start=["add", placeholder, 32], - inp_len=["mload", placeholder], + inp_start=["add", "_sub", 32], + inp_len=["mload", "_sub"], out_start=MemoryPositions.FREE_VAR_SPACE, out_len=32, ), @@ -641,7 +635,13 @@ def build_LLL(self, expr, args, kwargs, context): pos=getpos(expr), add_gas_estimate=SHA256_BASE_GAS + sub.typ.maxlen * SHA256_PER_WORD_GAS, ) - elif sub.location == "memory": + else: + # otherwise, copy it to memory and then call the precompile + placeholder = context.new_internal_variable(sub.typ) + placeholder_node = LLLnode.from_list(placeholder, typ=sub.typ, location="memory") + copier = make_byte_array_copier( + placeholder_node, LLLnode.from_list("_sub", typ=sub.typ, location=sub.location), + ) return LLLnode.from_list( [ "with", @@ -649,9 +649,10 @@ def build_LLL(self, expr, args, kwargs, context): sub, [ "seq", + copier, _make_sha256_call( - inp_start=["add", "_sub", 32], - inp_len=["mload", "_sub"], + inp_start=["add", placeholder, 32], + inp_len=["mload", placeholder], out_start=MemoryPositions.FREE_VAR_SPACE, out_len=32, ), @@ -662,9 +663,6 @@ def build_LLL(self, expr, args, kwargs, context): pos=getpos(expr), add_gas_estimate=SHA256_BASE_GAS + sub.typ.maxlen * SHA256_PER_WORD_GAS, ) - else: - # This should never happen, but just left here for future compiler-writers. - raise Exception(f"Unsupported location: {sub.location}") # pragma: no test class MethodID: From d8fd8aacfab9148ceefd88c0224be7ba01a22ce8 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 21 Sep 2021 18:01:58 +0000 Subject: [PATCH 125/163] fix test for calldataload now we use calldatacopy to grab the method_id --- tests/parser/features/test_init.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/parser/features/test_init.py b/tests/parser/features/test_init.py index 3fe86ce834..6256ab321c 100644 --- a/tests/parser/features/test_init.py +++ b/tests/parser/features/test_init.py @@ -14,9 +14,10 @@ def __init__(a: uint256): assert c.val() == 123 - # Make sure the init signature has no unecessary CALLDATLOAD copy. + # Make sure the init code does not access calldata opcodes = vyper.compile_code(code, ["opcodes"])["opcodes"].split(" ") lll_return_idx = opcodes.index("JUMP") - assert "CALLDATALOAD" in opcodes + assert "CALLDATACOPY" in opcodes + assert "CALLDATACOPY" not in opcodes[:lll_return_idx] assert "CALLDATALOAD" not in opcodes[:lll_return_idx] From b0d92a226f1d33070a094a79bbc53b68a4003e4c Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 21 Sep 2021 18:25:19 +0000 Subject: [PATCH 126/163] fix calculation of kwargs needed --- vyper/old_codegen/context.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/vyper/old_codegen/context.py b/vyper/old_codegen/context.py index 7ca85cd747..c22a3b21c5 100644 --- a/vyper/old_codegen/context.py +++ b/vyper/old_codegen/context.py @@ -244,12 +244,11 @@ def _check(cond, s="Unreachable"): # more sanity check, that the types match # _check(all(l.typ == r.typ for (l, r) in zip(args_lll, sig.args)) - num_provided_args = len(args_lll) - total_args = len(sig.args) + num_provided_kwargs = len(args_lll) - len(sig.base_args) num_kwargs = len(sig.default_args) - args_needed = total_args - num_provided_args + kwargs_needed = num_kwargs - num_provided_kwargs - kw_vals = list(sig.default_values.values())[: num_kwargs - args_needed] + kw_vals = list(sig.default_values.values())[:kwargs_needed] return sig, kw_vals From 12931207ef4e25b3468b9bf5dccc29e3b2d221f5 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 21 Sep 2021 18:51:47 +0000 Subject: [PATCH 127/163] fix type bug in self call arg passing routine --- vyper/ast/signatures/function_signature.py | 5 +++++ vyper/old_codegen/self_call.py | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/vyper/ast/signatures/function_signature.py b/vyper/ast/signatures/function_signature.py index 68078d4190..65d3eb4713 100644 --- a/vyper/ast/signatures/function_signature.py +++ b/vyper/ast/signatures/function_signature.py @@ -41,6 +41,11 @@ def __init__( self.defined_at = defined_at # source code location variable record was defined. self.is_internal = is_internal + def __repr__(self): + ret = vars(self) + ret["allocated"] = self.size * 32 + return f"VariableRecord(f{ret})" + @property def size(self): if hasattr(self.typ, "size_in_bytes"): diff --git a/vyper/old_codegen/self_call.py b/vyper/old_codegen/self_call.py index 2526f92ccf..b7372a8eda 100644 --- a/vyper/old_codegen/self_call.py +++ b/vyper/old_codegen/self_call.py @@ -64,7 +64,9 @@ def lll_for_self_call(stmt_expr, context): ) return_buffer = LLLnode.from_list([return_buffer], annotation=f"{return_label}_return_buf") - args_dst = LLLnode(sig.frame_start, typ=args_tuple_t, location="memory") + # note: dst_tuple_t != args_tuple_t + dst_tuple_t = TupleType([arg.typ for arg in sig.args]) + args_dst = LLLnode(sig.frame_start, typ=dst_tuple_t, location="memory") copy_args = make_setter(args_dst, args_as_tuple, "memory", pos) call_sequence = [ From 53d9c18f3c095df12907c0ad8f368abb1e96bed9 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 21 Sep 2021 18:52:47 +0000 Subject: [PATCH 128/163] fix some output struct formats --- tests/parser/features/decorators/test_private.py | 2 +- .../external_contracts/test_external_contract_calls.py | 4 ++-- .../features/external_contracts/test_self_call_struct.py | 8 ++++---- vyper/old_codegen/function_definitions/common.py | 1 + 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/parser/features/decorators/test_private.py b/tests/parser/features/decorators/test_private.py index 06888dedde..7a9568b346 100644 --- a/tests/parser/features/decorators/test_private.py +++ b/tests/parser/features/decorators/test_private.py @@ -580,7 +580,7 @@ def foo() -> A: return self._foo([1, 2, 3, 4], 5) """, (), - [[1, 2, 3, 4], 5], + ([1, 2, 3, 4], 5), ), ( """ diff --git a/tests/parser/features/external_contracts/test_external_contract_calls.py b/tests/parser/features/external_contracts/test_external_contract_calls.py index 15e1842066..bd5ecce405 100644 --- a/tests/parser/features/external_contracts/test_external_contract_calls.py +++ b/tests/parser/features/external_contracts/test_external_contract_calls.py @@ -829,7 +829,7 @@ def test(addr: address) -> (int128, address): c1 = get_contract_with_gas_estimation(contract_1) c2 = get_contract_with_gas_estimation(contract_2) - assert c1.out_literals() == [1, "0x0000000000000000000000000000000000012345"] + assert c1.out_literals() == (1, "0x0000000000000000000000000000000000012345") assert c2.test(c1.address) == list(c1.out_literals()) @@ -862,7 +862,7 @@ def test(addr: address) -> (int128, String[{ln}], Bytes[{ln}]): c1 = get_contract_with_gas_estimation(contract_1) c2 = get_contract_with_gas_estimation(contract_2) - assert c1.get_struct_x() == [i, s, bytes(s, "utf-8")] + assert c1.get_struct_x() == (i, s, bytes(s, "utf-8")) assert c2.test(c1.address) == list(c1.get_struct_x()) diff --git a/tests/parser/features/external_contracts/test_self_call_struct.py b/tests/parser/features/external_contracts/test_self_call_struct.py index da14a80dba..ad3cd79079 100644 --- a/tests/parser/features/external_contracts/test_self_call_struct.py +++ b/tests/parser/features/external_contracts/test_self_call_struct.py @@ -24,11 +24,11 @@ def wrap_get_my_struct_BROKEN(_e1: decimal) -> MyStruct: return self.get_my_struct(_e1, block.timestamp) """ c = get_contract(code) - assert c.wrap_get_my_struct_WORKING(Decimal("0.1")) == [ + assert c.wrap_get_my_struct_WORKING(Decimal("0.1")) == ( Decimal("0.1"), w3.eth.getBlock(w3.eth.blockNumber)["timestamp"], - ] - assert c.wrap_get_my_struct_BROKEN(Decimal("0.1")) == [ + ) + assert c.wrap_get_my_struct_BROKEN(Decimal("0.1")) == ( Decimal("0.1"), w3.eth.getBlock(w3.eth.blockNumber)["timestamp"], - ] + ) diff --git a/vyper/old_codegen/function_definitions/common.py b/vyper/old_codegen/function_definitions/common.py index a62cd10a26..1f6c2d8158 100644 --- a/vyper/old_codegen/function_definitions/common.py +++ b/vyper/old_codegen/function_definitions/common.py @@ -33,6 +33,7 @@ def generate_lll_for_function( sigs: Dict[str, Dict[str, FunctionSignature]], global_ctx: GlobalContext, check_nonpayable: bool, + # CMC 20210921 TODO _vars can probably be removed _vars: Optional[Dict[str, VariableRecord]] = None, ) -> Tuple[LLLnode, int, int]: """ From 441167904789ecfe74bad47f3f3e040a779b20b3 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 21 Sep 2021 21:58:45 +0000 Subject: [PATCH 129/163] remove max bound check on returndatasize --- vyper/old_codegen/external_call.py | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index 8f3ffe28b6..df93be7ff0 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -77,30 +77,16 @@ def _unpack_returndata(buf, contract_sig, context, pos): min_return_size = abi_return_t.min_size() max_return_size = abi_return_t.size_bound() + assert 0 < min_return_size <= max_return_size ret_ofst = buf - ret_len = abi_return_t.size_bound() + ret_len = max_return_size # revert when returndatasize is not in bounds ret = [] - assert 0 < min_return_size <= max_return_size - if contract_sig.is_from_json: - # we don't have max_size information for json abis - # runtime: min_return_size <= returndatasize - ret += [["assert", ["gt", "returndatasize", min_return_size - 1]]] - else: - # runtime: min_return_size <= returndatasize <= max_return_size - # TODO move the +-1 optimization to LLL optimizer - ret += [ - [ - "assert", - [ - "and", - ["gt", "returndatasize", min_return_size - 1], - ["lt", "returndatasize", max_return_size + 1], - ], - ] - ] + # runtime: min_return_size <= returndatasize + # TODO move the -1 optimization to LLL optimizer + ret += [["assert", ["gt", "returndatasize", min_return_size - 1]]] # add as the last LLLnode a pointer to the return data structure From 817b9ea80822e3ed2cbc4651660ad46e1a0b792b Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 21 Sep 2021 23:22:04 +0000 Subject: [PATCH 130/163] fix method_id calculation for external call --- vyper/old_codegen/external_call.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index df93be7ff0..43ca71d1b4 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -18,6 +18,7 @@ canonicalize_type, get_type_for_exact_size, ) +from vyper.old_codegen.types.check import check_assign def _pack_arguments(contract_sig, args, context, pos): @@ -26,6 +27,11 @@ def _pack_arguments(contract_sig, args, context, pos): args_as_tuple = LLLnode.from_list(["multi"] + [x for x in args], typ=args_tuple_t) args_abi_t = abi_type_of(args_tuple_t) + # sanity typecheck - make sure the arguments can be assigned + dst_tuple_t = TupleType([arg.typ for arg in contract_sig.args][:len(args)]) + _tmp = LLLnode("fake node", location="memory", typ=dst_tuple_t) + check_assign(_tmp, args_as_tuple, pos) + if contract_sig.return_type is not None: return_abi_t = abi_type_of(calculate_type_for_external_return(contract_sig.return_type)) @@ -43,7 +49,7 @@ def _pack_arguments(contract_sig, args, context, pos): args_ofst = buf + 28 args_len = args_abi_t.size_bound() + 4 - abi_signature = contract_sig.name + canonicalize_type(args_tuple_t) + abi_signature = contract_sig.name + canonicalize_type(dst_tuple_t) # layout: # 32 bytes | args From fa2bc12469e116af728913516727a3ea9fca3c8a Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 21 Sep 2021 23:29:28 +0000 Subject: [PATCH 131/163] only allocate kwarg slots once --- .../function_definitions/external_function.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index edca3c31fa..42c29eac3a 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -91,13 +91,14 @@ def handler_for(calldata_kwargs, default_kwargs): for i, arg_meta in enumerate(calldata_kwargs): k = n_base_args + i - dst = context.new_variable(arg_meta.name, arg_meta.typ, is_mutable=False) + dst = context.lookup_var(arg_meta.name).pos + lhs = LLLnode(dst, location="memory", typ=arg_meta.typ) rhs = add_variable_offset(calldata_kwargs_ofst, k, pos=None, array_bounds_check=False) ret.append(make_setter(lhs, rhs, lhs_location, pos)) for x in default_kwargs: - dst = context.new_variable(x.name, x.typ, is_mutable=False) + dst = context.lookup_var(x.name).pos lhs = LLLnode(dst, location="memory", typ=x.typ) kw_ast_val = sig.default_values[x.name] # e.g. `3` in x: int = 3 rhs = Expr(kw_ast_val, context).lll_node @@ -112,6 +113,10 @@ def handler_for(calldata_kwargs, default_kwargs): keyword_args = sig.default_args + # allocate variable slots in memory + for arg in keyword_args: + context.new_variable(arg.name, arg.typ, is_mutable=False) + for i, _ in enumerate(keyword_args): calldata_kwargs = keyword_args[:i] default_kwargs = keyword_args[i:] From dba73c1e5eba673753b34675abb6e13122fbf186 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 21 Sep 2021 23:46:02 +0000 Subject: [PATCH 132/163] fix abi test for new format --- tests/parser/functions/test_abi.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/parser/functions/test_abi.py b/tests/parser/functions/test_abi.py index 6e177ae86e..9431a84e55 100644 --- a/tests/parser/functions/test_abi.py +++ b/tests/parser/functions/test_abi.py @@ -75,13 +75,19 @@ def foo(s: MyStruct) -> MyStruct: func_abi = abi[0] assert func_abi["name"] == "foo" - expected = { + + expected_output = [{ "type": "tuple", "name": "", "components": [{"type": "address", "name": "a"}, {"type": "uint256", "name": "b"}], - } + }] + + assert func_abi["outputs"] == expected_output - assert func_abi["outputs"] == expected["components"] + expected_input = { + "type": "tuple", + "name": "s", + "components": [{"type": "address", "name": "a"}, {"type": "uint256", "name": "b"}], + } - expected["name"] = "s" - assert func_abi["inputs"][0] == expected + assert func_abi["inputs"][0] == expected_input From 91e19297111762945e88f19124f7ac5d69ae9744 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 22 Sep 2021 00:34:36 +0000 Subject: [PATCH 133/163] fix returning out of a loop --- vyper/old_codegen/return_.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vyper/old_codegen/return_.py b/vyper/old_codegen/return_.py index a6c2edf1a8..7bf76b8187 100644 --- a/vyper/old_codegen/return_.py +++ b/vyper/old_codegen/return_.py @@ -38,8 +38,9 @@ def finalize(fill_return_buffer): fill_return_buffer = LLLnode.from_list( fill_return_buffer, annotation=f"fill return buffer {sig._lll_identifier}" ) + cleanup_loops = "exit_repeater" if context.forvars else "pass" return LLLnode.from_list( - ["seq_unchecked", fill_return_buffer, jump_to_exit], typ=None, pos=_pos, + ["seq_unchecked", cleanup_loops, fill_return_buffer, jump_to_exit], typ=None, pos=_pos, ) if context.return_type is None: From a8170f15b1687eb7f234fb787e0039c9aa97add3 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 22 Sep 2021 00:34:48 +0000 Subject: [PATCH 134/163] make return out of loop test a bit harder --- tests/parser/features/iteration/test_repeater.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/parser/features/iteration/test_repeater.py b/tests/parser/features/iteration/test_repeater.py index 192633af9c..eee57ef8d4 100644 --- a/tests/parser/features/iteration/test_repeater.py +++ b/tests/parser/features/iteration/test_repeater.py @@ -106,8 +106,10 @@ def test_return_inside_repeater(get_contract, typ): @internal def _final(a: {typ}) -> {typ}: for i in range(10): - if i > a: - return i + for j in range(10): + if j > 5: + if i > a: + return i return 31337 @internal From 7ebf01e6eca4e610432d6236a9b936c8b2f0aadd Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 22 Sep 2021 00:35:11 +0000 Subject: [PATCH 135/163] fix lint --- tests/parser/functions/test_abi.py | 12 +++++++----- vyper/old_codegen/external_call.py | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/parser/functions/test_abi.py b/tests/parser/functions/test_abi.py index 9431a84e55..4333bbbe9f 100644 --- a/tests/parser/functions/test_abi.py +++ b/tests/parser/functions/test_abi.py @@ -76,11 +76,13 @@ def foo(s: MyStruct) -> MyStruct: assert func_abi["name"] == "foo" - expected_output = [{ - "type": "tuple", - "name": "", - "components": [{"type": "address", "name": "a"}, {"type": "uint256", "name": "b"}], - }] + expected_output = [ + { + "type": "tuple", + "name": "", + "components": [{"type": "address", "name": "a"}, {"type": "uint256", "name": "b"}], + } + ] assert func_abi["outputs"] == expected_output diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index 43ca71d1b4..8823bbc153 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -28,7 +28,7 @@ def _pack_arguments(contract_sig, args, context, pos): args_abi_t = abi_type_of(args_tuple_t) # sanity typecheck - make sure the arguments can be assigned - dst_tuple_t = TupleType([arg.typ for arg in contract_sig.args][:len(args)]) + dst_tuple_t = TupleType([arg.typ for arg in contract_sig.args][: len(args)]) _tmp = LLLnode("fake node", location="memory", typ=dst_tuple_t) check_assign(_tmp, args_as_tuple, pos) From 86bb6bff0c27485911d023699a3e274c25d80f15 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 22 Sep 2021 00:40:46 +0000 Subject: [PATCH 136/163] add more cases to byte_array_to_num --- vyper/old_codegen/parser_utils.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index cf58eb7ead..93a1c61ff7 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -242,16 +242,19 @@ def make_byte_slice_copier(destination, source, length, max_length, pos=None): def byte_array_to_num( arg, expr, out_type, offset=32, ): - if arg.location == "memory": - lengetter = LLLnode.from_list(["mload", "_sub"], typ=BaseType("int256")) - first_el_getter = LLLnode.from_list(["mload", ["add", 32, "_sub"]], typ=BaseType("int256")) - elif arg.location == "storage": + if arg.location == "storage": lengetter = LLLnode.from_list(["sload", "_sub"], typ=BaseType("int256")) first_el_getter = LLLnode.from_list(["sload", ["add", 1, "_sub"]], typ=BaseType("int256")) + else: + op = load_op(arg.location) + lengetter = LLLnode.from_list([op, "_sub"], typ=BaseType("int256")) + first_el_getter = LLLnode.from_list([op, ["add", 32, "_sub"]], typ=BaseType("int256")) + if out_type == "int128": result = int128_clamp(["div", "_el1", ["exp", 256, ["sub", 32, "_len"]]]) elif out_type in ("int256", "uint256"): result = ["div", "_el1", ["exp", 256, ["sub", offset, "_len"]]] + # TODO decimal clamp? return LLLnode.from_list( [ "with", From 988628883f88ce5de59c055eeef68c8d230144f2 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 22 Sep 2021 00:49:30 +0000 Subject: [PATCH 137/163] add more location cases to extract32 --- vyper/builtin_functions/functions.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/vyper/builtin_functions/functions.py b/vyper/builtin_functions/functions.py index 8886ad1493..9f6d05f811 100644 --- a/vyper/builtin_functions/functions.py +++ b/vyper/builtin_functions/functions.py @@ -819,10 +819,13 @@ def build_LLL(self, expr, args, kwargs, context): return o -def _memory_element_getter(index): - return LLLnode.from_list( - ["mload", ["add", "_sub", ["add", 32, ["mul", 32, index]]]], typ=BaseType("int128"), - ) +def _generic_element_getter(op): + def f(index): + return LLLnode.from_list( + [op, ["add", "_sub", ["add", 32, ["mul", 32, index]]]], typ=BaseType("int128"), + ) + + return f def _storage_element_getter(index): @@ -854,14 +857,14 @@ def build_LLL(self, expr, args, kwargs, context): sub, index = args ret_type = kwargs["output_type"] # Get length and specific element - if sub.location == "memory": - lengetter = LLLnode.from_list(["mload", "_sub"], typ=BaseType("int128")) - elementgetter = _memory_element_getter - elif sub.location == "storage": + if sub.location == "storage": lengetter = LLLnode.from_list(["sload", "_sub"], typ=BaseType("int128")) elementgetter = _storage_element_getter - # TODO: unclosed if/elif clause. Undefined behavior if `sub.location` - # isn't one of `memory`/`storage` + + else: + op = load_op(sub.location) + lengetter = LLLnode.from_list([op, "_sub"], typ=BaseType("int128")) + elementgetter = _generic_element_getter(op) # Special case: index known to be a multiple of 32 if isinstance(index.value, int) and not index.value % 32: From 377fde587920bf8c87e6cb2f5d513fa8bfac2090 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 22 Sep 2021 01:27:49 +0000 Subject: [PATCH 138/163] add a JSON_ABI encoding to differentiate bytestrings which don't need clamping --- vyper/old_codegen/external_call.py | 10 ++++++++-- vyper/old_codegen/lll_node.py | 2 ++ vyper/old_codegen/parser_utils.py | 12 ++++++++---- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index 8823bbc153..42a4ebc160 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -68,6 +68,12 @@ def _pack_arguments(contract_sig, args, context, pos): return buf, mstore_method_id + [encode_args], args_ofst, args_len +def _returndata_encoding(contract_sig): + if contract_sig.is_from_json: + return Encoding.JSON_ABI + return Encoding.ABI + + def _unpack_returndata(buf, contract_sig, context, pos): return_t = contract_sig.return_type if return_t is None: @@ -103,7 +109,7 @@ def _unpack_returndata(buf, contract_sig, context, pos): # in most cases, this simply will evaluate to ret. # in the special case where the return type has been wrapped # in a tuple AND its ABI type is dynamic, it expands to buf+32. - buf = LLLnode(buf, typ=return_t, encoding=Encoding.ABI, location="memory") + buf = LLLnode(buf, typ=return_t, encoding=_returndata_encoding(contract_sig), location="memory") if should_unwrap_abi_tuple: buf = add_variable_offset(buf, 0, pos=None, array_bounds_check=False) @@ -163,7 +169,7 @@ def _external_call_helper( sub, typ=contract_sig.return_type, location="memory", - encoding=Encoding.ABI, + encoding=_returndata_encoding(contract_sig), pos=pos, ) diff --git a/vyper/old_codegen/lll_node.py b/vyper/old_codegen/lll_node.py index c1480bb952..e938d13d49 100644 --- a/vyper/old_codegen/lll_node.py +++ b/vyper/old_codegen/lll_node.py @@ -46,6 +46,8 @@ class Encoding(Enum): VYPER = auto() # abi encoded, default for args/return values from external funcs ABI = auto() + # abi encoded, same as ABI but no clamps for bytestrings + JSON_ABI = auto() # future: packed diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 93a1c61ff7..7c638f1b31 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -318,7 +318,7 @@ def _abi_helper(member_t, ofst, clamp=True): ["ofst"], typ=member_t, location=parent.location, annotation=f"&({typ}->{member_t})" ) - if clamp and _needs_clamp(member_t): + if clamp and _needs_clamp(member_t, parent.encoding): # special handling for args that need clamping ret = ["with", x, ofst_lll, ["seq", clamp_basetype(x), x]] else: @@ -352,7 +352,7 @@ def _abi_helper(member_t, ofst, clamp=True): assert parent.encoding != Encoding.ABI, "no abi-encoded literals" return parent.args[index] - if parent.encoding == Encoding.ABI: + if parent.encoding in (Encoding.ABI, Encoding.JSON_ABI): if parent.location == "storage": raise CompilerPanic("storage variables should not be abi encoded") @@ -410,7 +410,7 @@ def _abi_helper(member_t, ofst, clamp=True): # an array index, and the clamp will throw an error. sub = ["uclamplt", k, typ.count] - if parent.encoding == Encoding.ABI: + if parent.encoding in (Encoding.ABI, Encoding.JSON_ABI): if parent.location == "storage": raise CompilerPanic("storage variables should not be abi encoded") @@ -817,8 +817,12 @@ def _sar(x, bits): return ["sdiv", ["add", ["slt", x, 0], x], ["exp", 2, bits]] -def _needs_clamp(t): +def _needs_clamp(t, encoding): + assert encoding in (Encoding.ABI, Encoding.JSON_ABI) if isinstance(t, ByteArrayLike): + if encoding == Encoding.JSON_ABI: + # don't have bytestring size bound from json, don't clamp + return False return True if isinstance(t, BaseType) and t.typ not in ("int256", "uint256", "bytes32"): return True From 837edcce19650d703827bbee1af1be0f178e3816 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 22 Sep 2021 01:33:13 +0000 Subject: [PATCH 139/163] fix a broken assert --- vyper/builtin_functions/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vyper/builtin_functions/functions.py b/vyper/builtin_functions/functions.py index 9f6d05f811..ef776555ba 100644 --- a/vyper/builtin_functions/functions.py +++ b/vyper/builtin_functions/functions.py @@ -282,7 +282,7 @@ def build_LLL(self, expr, args, kwargs, context): # special handling for slice(msg.data) if sub.location == "calldata" and sub.value == 0: - assert expr.value.id == "msg" and expr.attr == "data" + assert expr.args[0].value.id == "msg" and expr.args[0].attr == "data" # if we are slicing msg.data, the length should # be a constant, since msg.data can be of dynamic length # we can't use its length as the maxlen From 2a9fbdb83d69f558083d06ad9610ad1192bdf8ed Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 22 Sep 2021 01:36:30 +0000 Subject: [PATCH 140/163] add more locations to convert builtin --- vyper/builtin_functions/convert.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/vyper/builtin_functions/convert.py b/vyper/builtin_functions/convert.py index 48f664e2e1..e606804c50 100644 --- a/vyper/builtin_functions/convert.py +++ b/vyper/builtin_functions/convert.py @@ -7,7 +7,12 @@ from vyper.evm.opcodes import version_check from vyper.exceptions import InvalidLiteral, StructureException, TypeMismatch from vyper.old_codegen.arg_clamps import address_clamp, int128_clamp -from vyper.old_codegen.parser_utils import LLLnode, byte_array_to_num, getpos +from vyper.old_codegen.parser_utils import ( + LLLnode, + byte_array_to_num, + getpos, + load_op, +) from vyper.old_codegen.types import ( BaseType, ByteArrayType, @@ -325,10 +330,11 @@ def to_bytes32(expr, args, kwargs, context): f"Unable to convert bytes[{_len}] to bytes32, max length is too " "large." ) - if in_arg.location == "memory": - return LLLnode.from_list(["mload", ["add", in_arg, 32]], typ=BaseType("bytes32")) - elif in_arg.location == "storage": + if in_arg.location == "storage": return LLLnode.from_list(["sload", ["add", in_arg, 1]], typ=BaseType("bytes32")) + else: + op = load_op(in_arg.location) + return LLLnode.from_list([op, ["add", in_arg, 32]], typ=BaseType("bytes32")) else: return LLLnode( From 464ca44c28e325bce6c9b87fd81330ccb7e90d31 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 22 Sep 2021 01:36:49 +0000 Subject: [PATCH 141/163] fix formatting on some test struct results --- tests/parser/functions/test_return_struct.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/parser/functions/test_return_struct.py b/tests/parser/functions/test_return_struct.py index 196f76bd68..98b7fa66d1 100644 --- a/tests/parser/functions/test_return_struct.py +++ b/tests/parser/functions/test_return_struct.py @@ -24,7 +24,7 @@ def test() -> Voter: c = get_contract_with_gas_estimation(code) - assert c.test() == [123, True] + assert c.test() == (123, True) def test_struct_return(get_contract_with_gas_estimation): @@ -74,13 +74,13 @@ def pub6() -> Foo: foo: Foo = Foo({x: 123, y: 456}) return self.return_arg(foo) """ - foo = [123, 456] + foo = (123, 456) c = get_contract_with_gas_estimation(code) - assert c.pub1() == [1, 2] - assert c.pub2() == [3, 4] - assert c.pub3() == [5, 6] - assert c.pub4() == [7, 8] + assert c.pub1() == (1, 2) + assert c.pub2() == (3, 4) + assert c.pub3() == (5, 6) + assert c.pub4() == (7, 8) assert c.pub5(foo) == foo assert c.pub6() == foo From 7f76cc03e32be20bdf7bd030a52571ddd4f419f7 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 22 Sep 2021 16:03:47 +0000 Subject: [PATCH 142/163] protect argument buffer in self call --- vyper/old_codegen/lll_node.py | 7 ++++++- vyper/old_codegen/self_call.py | 23 ++++++++++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/vyper/old_codegen/lll_node.py b/vyper/old_codegen/lll_node.py index e938d13d49..84d5ee2501 100644 --- a/vyper/old_codegen/lll_node.py +++ b/vyper/old_codegen/lll_node.py @@ -1,6 +1,7 @@ import re from enum import Enum, auto from typing import Any, List, Optional, Tuple, Union +from functools import cached_property from vyper.compiler.settings import VYPER_COLOR_OUTPUT from vyper.evm.opcodes import get_comb_opcodes @@ -269,13 +270,17 @@ def __init__( self.gas += self.add_gas_estimate - # may contain some side effects + # the LLL should be cached. @property def is_complex_lll(self): return isinstance(self.value, str) and ( self.value.lower() in VALID_LLL_MACROS or self.value.upper() in get_comb_opcodes() ) + @cached_property + def contains_self_call(self): + return getattr(self, "is_self_call", False) or any(x.contains_self_call for x in self.args) + def __getitem__(self, i): return self.to_list()[i] diff --git a/vyper/old_codegen/self_call.py b/vyper/old_codegen/self_call.py index b7372a8eda..16a9eeaaf7 100644 --- a/vyper/old_codegen/self_call.py +++ b/vyper/old_codegen/self_call.py @@ -67,7 +67,27 @@ def lll_for_self_call(stmt_expr, context): # note: dst_tuple_t != args_tuple_t dst_tuple_t = TupleType([arg.typ for arg in sig.args]) args_dst = LLLnode(sig.frame_start, typ=dst_tuple_t, location="memory") - copy_args = make_setter(args_dst, args_as_tuple, "memory", pos) + + # if one of the arguments is a self call, the argument + # buffer could get borked. to prevent against that, + # write args to a temporary buffer until all the arguments + # are fully evaluated. + if args_as_tuple.contains_self_call: + copy_args = ["seq"] + # TODO deallocate me + tmp_args_buf = LLLnode( + context.new_internal_variable(dst_tuple_t), + typ=dst_tuple_t, + location="memory", + ) + copy_args.append(make_setter(tmp_args_buf, args_as_tuple, "memory", pos)) + + # --> args evaluate here <-- + + copy_args.append(make_setter(args_dst, tmp_args_buf, "memory", pos)) + + else: + copy_args = make_setter(args_dst, args_as_tuple, "memory", pos) call_sequence = [ "seq", @@ -90,4 +110,5 @@ def lll_for_self_call(stmt_expr, context): annotation=stmt_expr.get("node_source_code"), add_gas_estimate=sig.gas, ) + o.is_self_call = True return o From 5e6c3873e71767f02ce213ac490792e1da7b87c7 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 22 Sep 2021 16:07:23 +0000 Subject: [PATCH 143/163] fix tests for get_size_of(bytes) --- tests/parser/types/test_node_types.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/parser/types/test_node_types.py b/tests/parser/types/test_node_types.py index afa7b1b707..01af7d3c23 100644 --- a/tests/parser/types/test_node_types.py +++ b/tests/parser/types/test_node_types.py @@ -60,8 +60,8 @@ def test_canonicalize_type(): def test_get_size_of_type(): assert get_size_of_type(BaseType("int128")) == 1 - assert get_size_of_type(ByteArrayType(12)) == 3 - assert get_size_of_type(ByteArrayType(33)) == 4 + assert get_size_of_type(ByteArrayType(12)) == 2 + assert get_size_of_type(ByteArrayType(33)) == 3 assert get_size_of_type(ListType(BaseType("int128"), 10)) == 10 _tuple = TupleType([BaseType("int128"), BaseType("decimal")]) From f2ebdf78fee7419a64296ddf97bd7c76e7c45fac Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 22 Sep 2021 16:12:16 +0000 Subject: [PATCH 144/163] handle more locations in Expr.parse_Compare --- vyper/old_codegen/expr.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/vyper/old_codegen/expr.py b/vyper/old_codegen/expr.py index d623c394a0..320c3f2c68 100644 --- a/vyper/old_codegen/expr.py +++ b/vyper/old_codegen/expr.py @@ -18,6 +18,7 @@ add_variable_offset, get_number_as_fraction, getpos, + load_op, make_setter, unwrap_location, ) @@ -836,10 +837,11 @@ def parse_Compare(self): else: def load_bytearray(side): - if side.location == "memory": - return ["mload", ["add", 32, side]] - elif side.location == "storage": + if side.location == "storage": return ["sload", ["add", 1, side]] + else: + load = load_op(side.location) + return [load, ["add", 32, side]] return LLLnode.from_list( [op, load_bytearray(left), load_bytearray(right)], From d8c9314d72642b7138cda1746f55ce463c6576b0 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 22 Sep 2021 16:18:37 +0000 Subject: [PATCH 145/163] fix constant folding test return buffer is somewhere else, assign it to a variable we know the location of --- tests/parser/types/numbers/test_constants.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/parser/types/numbers/test_constants.py b/tests/parser/types/numbers/test_constants.py index 23db2cb256..68beea504f 100644 --- a/tests/parser/types/numbers/test_constants.py +++ b/tests/parser/types/numbers/test_constants.py @@ -175,11 +175,12 @@ def test_constant_folds(search_for_sublist): def test() -> uint256: # calculate some constant which is really unlikely to be randomly # in bytecode - return 2**SOME_CONSTANT * SOME_PRIME + ret: uint256 = 2**SOME_CONSTANT * SOME_PRIME + return ret """ lll = compile_code(code, ["ir"])["ir"] - assert search_for_sublist(lll, ["mstore", [0], [2 ** 12 * some_prime]]) + assert search_for_sublist(lll, ["mstore", [320], [2 ** 12 * some_prime]]) def test_constant_lists(get_contract): From 5f4ac74cc78328791fd458bb714d59bf1e78809c Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 22 Sep 2021 16:26:13 +0000 Subject: [PATCH 146/163] fix mypy/lint --- vyper/ast/signatures/function_signature.py | 16 ---------------- .../function_definitions/external_function.py | 2 +- vyper/old_codegen/lll_node.py | 2 +- vyper/old_codegen/self_call.py | 4 +--- vyper/semantics/types/function.py | 2 +- vyper/semantics/types/indexable/sequence.py | 2 ++ 6 files changed, 6 insertions(+), 22 deletions(-) diff --git a/vyper/ast/signatures/function_signature.py b/vyper/ast/signatures/function_signature.py index 65d3eb4713..0b3ec53654 100644 --- a/vyper/ast/signatures/function_signature.py +++ b/vyper/ast/signatures/function_signature.py @@ -1,7 +1,6 @@ import math from dataclasses import dataclass from functools import cached_property -from typing import List from vyper import ast as vy_ast from vyper.exceptions import StructureException @@ -112,21 +111,6 @@ def abi_signature_for_kwargs(self, kwargs): args = self.base_args + kwargs return self.name + "(" + ",".join([canonicalize_type(arg.typ) for arg in args]) + ")" - @cached_property - def all_kwarg_sigs(self) -> List[str]: - assert not self.internal, "abi_signatures only make sense for external functions" - ret = [] - - kwargz = [] - - ret.append(self.abi_signature_for_kwargs(kwargz)) - - for kwarg in self.default_args: - kwargz.append(kwarg) - ret.append(self.abi_signature_for_args(kwargz)) - - return ret - @cached_property def base_signature(self): return self.abi_signature_for_kwargs([]) diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index 42c29eac3a..8c4ace8324 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -20,7 +20,7 @@ # register function args with the local calling context. # also allocate the ones that live in memory (i.e. kwargs) -def _register_function_args(context: Context, sig: FunctionSignature): +def _register_function_args(context: Context, sig: FunctionSignature) -> None: if len(sig.args) == 0: return diff --git a/vyper/old_codegen/lll_node.py b/vyper/old_codegen/lll_node.py index 84d5ee2501..3965beb6ff 100644 --- a/vyper/old_codegen/lll_node.py +++ b/vyper/old_codegen/lll_node.py @@ -1,7 +1,7 @@ import re from enum import Enum, auto -from typing import Any, List, Optional, Tuple, Union from functools import cached_property +from typing import Any, List, Optional, Tuple, Union from vyper.compiler.settings import VYPER_COLOR_OUTPUT from vyper.evm.opcodes import get_comb_opcodes diff --git a/vyper/old_codegen/self_call.py b/vyper/old_codegen/self_call.py index 16a9eeaaf7..63958f1d18 100644 --- a/vyper/old_codegen/self_call.py +++ b/vyper/old_codegen/self_call.py @@ -76,9 +76,7 @@ def lll_for_self_call(stmt_expr, context): copy_args = ["seq"] # TODO deallocate me tmp_args_buf = LLLnode( - context.new_internal_variable(dst_tuple_t), - typ=dst_tuple_t, - location="memory", + context.new_internal_variable(dst_tuple_t), typ=dst_tuple_t, location="memory", ) copy_args.append(make_setter(tmp_args_buf, args_as_tuple, "memory", pos)) diff --git a/vyper/semantics/types/function.py b/vyper/semantics/types/function.py index 6c9926cb38..fdeab12a5a 100644 --- a/vyper/semantics/types/function.py +++ b/vyper/semantics/types/function.py @@ -481,7 +481,7 @@ def to_abi_dict(self) -> List[Dict]: typ = self.return_type if typ is None: abi_dict["outputs"] = [] - elif isinstance(typ, TupleDefinition) and len(typ.value_type) > 1: + elif isinstance(typ, TupleDefinition) and len(typ.value_type) > 1: # type: ignore abi_dict["outputs"] = [_generate_abi_type(i) for i in typ.value_type] # type: ignore else: abi_dict["outputs"] = [_generate_abi_type(typ)] diff --git a/vyper/semantics/types/indexable/sequence.py b/vyper/semantics/types/indexable/sequence.py index 28981eb047..b27e1d24d1 100644 --- a/vyper/semantics/types/indexable/sequence.py +++ b/vyper/semantics/types/indexable/sequence.py @@ -116,7 +116,9 @@ def __init__(self, value_type: Tuple[BaseTypeDefinition, ...]) -> None: # always use the most restrictive location re: modification location = sorted((i.location for i in value_type), key=lambda k: k.value)[-1] is_immutable = next((True for i in value_type if getattr(i, "is_immutable", None)), False) + super().__init__( + # TODO fix the typing on value_type value_type, # type: ignore len(value_type), f"{value_type}", From 13fea45219638f5b79e702b141607ff7362a79f9 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 22 Sep 2021 16:36:58 +0000 Subject: [PATCH 147/163] clarify a comment --- vyper/old_codegen/self_call.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/vyper/old_codegen/self_call.py b/vyper/old_codegen/self_call.py index 63958f1d18..4bc1cdfe4d 100644 --- a/vyper/old_codegen/self_call.py +++ b/vyper/old_codegen/self_call.py @@ -78,9 +78,10 @@ def lll_for_self_call(stmt_expr, context): tmp_args_buf = LLLnode( context.new_internal_variable(dst_tuple_t), typ=dst_tuple_t, location="memory", ) - copy_args.append(make_setter(tmp_args_buf, args_as_tuple, "memory", pos)) - - # --> args evaluate here <-- + copy_args.append( + # --> args evaluate here <-- + make_setter(tmp_args_buf, args_as_tuple, "memory", pos) + ) copy_args.append(make_setter(args_dst, tmp_args_buf, "memory", pos)) From c9a93973e70cb350deac3855eff9700207d19a22 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 22 Sep 2021 16:58:56 +0000 Subject: [PATCH 148/163] backport cached_property import --- setup.py | 7 ++++++- vyper/ast/signatures/function_signature.py | 3 +-- vyper/old_codegen/lll_node.py | 3 +-- vyper/utils.py | 11 +++++++++++ 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 0378e17c99..dd6bec4488 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,12 @@ packages=find_packages(exclude=("tests", "docs")), python_requires=">=3.6", py_modules=["vyper"], - install_requires=["asttokens==2.0.4", "pycryptodome>=3.5.1,<4", "semantic-version==2.8.5"], + install_requires=[ + "asttokens==2.0.4", + "pycryptodome>=3.5.1,<4", + "semantic-version==2.8.5", + "cached-property==1.5.2", + ], setup_requires=["pytest-runner"], tests_require=extras_require["test"], extras_require=extras_require, diff --git a/vyper/ast/signatures/function_signature.py b/vyper/ast/signatures/function_signature.py index 0b3ec53654..957af12367 100644 --- a/vyper/ast/signatures/function_signature.py +++ b/vyper/ast/signatures/function_signature.py @@ -1,6 +1,5 @@ import math from dataclasses import dataclass -from functools import cached_property from vyper import ast as vy_ast from vyper.exceptions import StructureException @@ -11,7 +10,7 @@ get_size_of_type, parse_type, ) -from vyper.utils import mkalphanum +from vyper.utils import mkalphanum, cached_property # Function variable diff --git a/vyper/old_codegen/lll_node.py b/vyper/old_codegen/lll_node.py index 3965beb6ff..86a155db94 100644 --- a/vyper/old_codegen/lll_node.py +++ b/vyper/old_codegen/lll_node.py @@ -1,13 +1,12 @@ import re from enum import Enum, auto -from functools import cached_property from typing import Any, List, Optional, Tuple, Union from vyper.compiler.settings import VYPER_COLOR_OUTPUT from vyper.evm.opcodes import get_comb_opcodes from vyper.exceptions import CompilerPanic from vyper.old_codegen.types import BaseType, NodeType, ceil32 -from vyper.utils import VALID_LLL_MACROS +from vyper.utils import VALID_LLL_MACROS, cached_property # Set default string representation for ints in LLL output. AS_HEX_DEFAULT = False diff --git a/vyper/utils.py b/vyper/utils.py index d4ab3c9348..bfca38305b 100644 --- a/vyper/utils.py +++ b/vyper/utils.py @@ -15,6 +15,12 @@ keccak256 = lambda x: _sha3.sha3_256(x).digest() # noqa: E731 +try: + # available py3.8+ + from functools import cached_property +except ImportError: + from cached_property import cached_property # type: ignore + # Converts four bytes to an integer def fourbytes_to_int(inp): @@ -316,3 +322,8 @@ def annotate_source_code( cleanup_lines += [""] * (num_lines - len(cleanup_lines)) return "\n".join(cleanup_lines) + + +__all__ = [ + "cached_property", +] From e0db36530fbe15dc9ade10b212d0efd43f0a83ba Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 22 Sep 2021 17:29:18 +0000 Subject: [PATCH 149/163] drop python3.6 support Major operating systems (Ubuntu, MacOS) have already moved on, plus it doesn't have dataclasses. --- .github/workflows/test.yml | 23 ----------------------- pyproject.toml | 2 +- tox.ini | 3 +-- vyper/__init__.py | 4 ++-- 4 files changed, 4 insertions(+), 28 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e4311cde9c..ba41b3c1a1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -55,29 +55,6 @@ jobs: - name: Run Tox run: TOXENV=mypy tox -r - py36-core: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v1 - - - name: Set up Python 3.6 - uses: actions/setup-python@v1 - with: - python-version: 3.6 - - - name: Install Tox - run: pip install tox - - - name: Run Tox - run: TOXENV=py36-core tox -r - - - name: Upload Coverage - uses: codecov/codecov-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - file: ./coverage.xml - py37-core: runs-on: ubuntu-latest diff --git a/pyproject.toml b/pyproject.toml index 49a8195dc7..f646c2f664 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ [tool.black] line-length = 100 -target-version = ['py36', 'py37', 'py38'] +target-version = ['py37', 'py38', 'py39'] include = '\.pyi?$' exclude = ''' /( diff --git a/tox.ini b/tox.ini index 9459b7fbe2..f0a54f9826 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - py{36,37,38,39}-core + py{37,38,39}-core lint mypy docs @@ -10,7 +10,6 @@ usedevelop = True commands = core: pytest -m "not fuzzing" --showlocals {posargs:tests/} basepython = - py36: python3.6 py37: python3.7 py38: python3.8 py39: python3.9 diff --git a/vyper/__init__.py b/vyper/__init__.py index 8ef6d3bcd4..d62f546aa1 100644 --- a/vyper/__init__.py +++ b/vyper/__init__.py @@ -5,9 +5,9 @@ from vyper.compiler import compile_code, compile_codes # noqa: F401 -if (_sys.version_info.major, _sys.version_info.minor) < (3, 6): +if (_sys.version_info.major, _sys.version_info.minor) < (3, 7): # Can't be tested, as our test harness is using python3.6. - raise Exception("Requires python3.6+") # pragma: no cover + raise Exception("Requires python3.7+") # pragma: no cover _version_file = _Path(__file__).parent.joinpath("vyper_git_version.txt") From 4bd0eca5358eef2d7a6034dabdcf45787908b4cc Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 24 Sep 2021 18:14:21 +0000 Subject: [PATCH 150/163] clarify extcodesize check --- vyper/old_codegen/external_call.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index 42a4ebc160..050d8f712a 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -148,10 +148,12 @@ def _external_call_helper( sub += arg_packer if contract_sig.return_type is None: - # if we do not expect return data, check that a contract exists at the target address - # we can omit this when we _do_ expect return data because we later check `returndatasize` - # CMC 20210907 do we need to check this before the call, or can we defer until after? - # if we can defer, this code can be pushed down into unpack_returndata + # if we do not expect return data, check that a contract exists at the + # target address. we must perform this check BEFORE the call because + # the contract might selfdestruct. on the other hand we can omit this + # when we _do_ expect return data because we later check + # `returndatasize` (that check works even if the contract + # selfdestructs). sub.append(["assert", ["extcodesize", contract_address]]) if context.is_constant() or contract_sig.mutability in ("view", "pure"): From 82efdbd5ceb291be131a5e9413223daa2f47dae3 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 24 Sep 2021 13:54:37 -0700 Subject: [PATCH 151/163] import cached-property conditionally Co-authored-by: El De-dog-lo <3859395+fubuloubu@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index dd6bec4488..c96dc2bcde 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,7 @@ "asttokens==2.0.4", "pycryptodome>=3.5.1,<4", "semantic-version==2.8.5", - "cached-property==1.5.2", + "cached-property==1.5.2 ; python_version<'3.8'", ], setup_requires=["pytest-runner"], tests_require=extras_require["test"], From 100e48434ae2f60a290ab6fa164f0038bb3cdb61 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 24 Sep 2021 20:56:05 +0000 Subject: [PATCH 152/163] comment clarifying maxlen for abi_encode+method_id --- vyper/builtin_functions/functions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vyper/builtin_functions/functions.py b/vyper/builtin_functions/functions.py index e363f2b858..a1af3b43de 100644 --- a/vyper/builtin_functions/functions.py +++ b/vyper/builtin_functions/functions.py @@ -1841,6 +1841,7 @@ def fetch_call_return(self, node): maxlen = arg_abi_t.size_bound() if self._method_id(node) is not None: + # the output includes 4 bytes for the method_id. maxlen += 4 ret = BytesArrayDefinition() From 57aa48cd69548d83172ee29309e9df4d26e8f241 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 28 Sep 2021 20:20:48 +0000 Subject: [PATCH 153/163] add annotation to int_clamp --- vyper/old_codegen/parser_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 7c638f1b31..fe4e4205ca 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -874,4 +874,4 @@ def int_clamp(lll_node, bits, signed=False): else: ret = ["assert", ["iszero", _shr(lll_node, bits)]] - return ret + return LLLnode.from_list(ret, annotation=f"int_clamp {lll_node.typ}") From cee64b07db23aaab042d540c313b5e833ac5331a Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 24 Sep 2021 21:15:37 +0000 Subject: [PATCH 154/163] strictly decode calldata args that needs clamps otherwise they get repeatedly clamped. --- vyper/old_codegen/abi.py | 9 ++- .../function_definitions/external_function.py | 65 +++++++++++++------ vyper/old_codegen/parser_utils.py | 9 ++- 3 files changed, 58 insertions(+), 25 deletions(-) diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index e41cbde541..f4e2ecd340 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -1,10 +1,11 @@ import vyper.semantics.types as vy from vyper.exceptions import CompilerPanic -from vyper.old_codegen.lll_node import LLLnode +from vyper.old_codegen.lll_node import LLLnode, Encoding from vyper.old_codegen.parser_utils import ( add_variable_offset, clamp_basetype, make_setter, + _needs_clamp, unwrap_location, zero_pad, ) @@ -521,8 +522,10 @@ def abi_decode(lll_node, src, clamp=True, pos=None): else: - if clamp: - lll_ret.append(clamp_basetype(unwrap_location(src_loc))) + if clamp and _needs_clamp(o.typ, Encoding.ABI): + src_loc = LLLnode.from_list(["with", "src_loc", src_loc, ["seq", clamp_basetype(src_loc), src_loc]], typ=src_loc.typ, location=src_loc.location) + else: + pass lll_ret.append(make_setter(o, src_loc, location=o.location, pos=pos)) diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index 8c4ace8324..365370d4fe 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -15,15 +15,32 @@ make_setter, ) from vyper.old_codegen.stmt import parse_body -from vyper.old_codegen.types.types import TupleType +from vyper.old_codegen.types.types import TupleType, BaseType, ByteArrayLike, ListType, TupleLike + + +def _should_decode(typ): + # either a basetype which needs to be clamped + # or a complex type which contains something that + # needs to be clamped. + if isinstance(typ, BaseType): + return typ.typ not in ("int256", "uint256", "bytes32") + if isinstance(typ, ByteArrayLike): + return True + if isinstance(typ, ListType): + return _should_decode(typ.subtype) + if isinstance(typ, TupleLike): + return any(_should_decode(t) for t in typ.tuple_members()) + raise CompilerPanic(f"_should_decode({typ})") # register function args with the local calling context. # also allocate the ones that live in memory (i.e. kwargs) def _register_function_args(context: Context, sig: FunctionSignature) -> None: - if len(sig.args) == 0: - return + pos = None + + ret = [] + # the type of the calldata base_args_t = TupleType([arg.typ for arg in sig.base_args]) # tuple with the abi_encoded args @@ -35,18 +52,26 @@ def _register_function_args(context: Context, sig: FunctionSignature) -> None: base_args_ofst = LLLnode(4, location="calldata", typ=base_args_t, encoding=Encoding.ABI) for i, arg in enumerate(sig.base_args): - arg_lll = add_variable_offset(base_args_ofst, i, pos=None, array_bounds_check=False) - assert arg.typ == arg_lll.typ, (arg.typ, arg_lll.typ) - - # register the record in the local namespace, no copy needed - context.vars[arg.name] = VariableRecord( - name=arg.name, - pos=arg_lll, - typ=arg.typ, - mutable=False, - location=arg_lll.location, - encoding=Encoding.ABI, - ) + + arg_lll = add_variable_offset(base_args_ofst, i, pos=pos) + + if _should_decode(arg.typ): + # allocate a memory slot for it and copy + p = context.new_variable(arg.name, arg.typ, is_mutable=False) + dst = LLLnode(p, typ=arg.typ, location="memory") + ret.append(make_setter(dst, arg_lll, "memory", pos=pos)) + else: + # leave it in place + context.vars[arg.name] = VariableRecord( + name=arg.name, + pos=arg_lll, + typ=arg.typ, + mutable=False, + location=arg_lll.location, + encoding=Encoding.ABI, + ) + + return ret def _annotated_method_id(abi_sig): @@ -143,17 +168,19 @@ def generate_lll_for_external_function(code, sig, context, check_nonpayable): func_type = code._metadata["type"] pos = getpos(code) - _register_function_args(context, sig) - nonreentrant_pre, nonreentrant_post = get_nonreentrant_lock(func_type) - entrance = [] + # generate handlers for base args and register the variable records + handle_base_args = _register_function_args(context, sig) + # generate handlers for kwargs and register the variable records kwarg_handlers = _generate_kwarg_handlers(context, sig, pos) # once optional args have been handled, # generate the main body of the function - entrance += [["label", sig.external_function_base_entry_label]] + entrance = [["label", sig.external_function_base_entry_label]] + + entrance += handle_base_args if check_nonpayable and sig.mutability != "payable": # if the contract contains payable functions, but this is not one of them diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index fe4e4205ca..85747b048b 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -85,7 +85,7 @@ def _codecopy_gas_bound(num_bytes): def make_byte_array_copier(destination, source, pos=None): if not isinstance(source.typ, ByteArrayLike): btype = "byte array" if isinstance(destination.typ, ByteArrayType) else "string" - raise TypeMismatch(f"Can only set a {btype} to another {btype}", pos) + raise TypeMismatch(f"Cannot cast from {source.typ} to {destination.typ}", pos) if isinstance(source.typ, ByteArrayLike) and source.typ.maxlen > destination.typ.maxlen: raise TypeMismatch( f"Cannot cast from greater max-length {source.typ.maxlen} to shorter " @@ -319,7 +319,10 @@ def _abi_helper(member_t, ofst, clamp=True): ) if clamp and _needs_clamp(member_t, parent.encoding): - # special handling for args that need clamping + # special handling for unsanitized external data that need clamping + # TODO optimize me. this results in a double dereference because + # it returns a pointer and not a value. probably the best thing + # is to move the clamp to make_setter ret = ["with", x, ofst_lll, ["seq", clamp_basetype(x), x]] else: ret = ofst_lll @@ -478,7 +481,7 @@ def load_op(location): return "calldataload" if location == "code": return "codeload" - raise CompilerPanic("unreachable", location) # pragma: no test + raise CompilerPanic(f"unreachable {location}") # pragma: no test # Unwrap location From 43a3bc7acb8fb02dc787427b7c63cbb0e25ef67a Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 29 Sep 2021 14:32:45 +0000 Subject: [PATCH 155/163] fix copy op for bytestrings located in code --- vyper/old_codegen/parser_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 85747b048b..6baab17228 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -110,7 +110,7 @@ def make_byte_array_copier(destination, source, pos=None): copy_op = ["calldatacopy", destination, "src", "sz"] gas_bound = _calldatacopy_gas_bound(source.typ.maxlen) elif source.location == "code": - copy_op = ["code", destination, "src", "sz"] + copy_op = ["codecopy", destination, "src", "sz"] gas_bound = _codecopy_gas_bound(source.typ.maxlen) _sz_lll = ["add", 32, [load_op(source.location), "src"]] o = LLLnode.from_list( From eb0088d5557bae3ef5b7655e9cb732c698c1e5d9 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 29 Sep 2021 18:02:09 +0000 Subject: [PATCH 156/163] fix lint and mypy --- vyper/ast/signatures/function_signature.py | 2 +- vyper/old_codegen/abi.py | 10 +++++++--- .../function_definitions/external_function.py | 11 +++++++++-- vyper/old_codegen/parser_utils.py | 2 -- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/vyper/ast/signatures/function_signature.py b/vyper/ast/signatures/function_signature.py index 957af12367..f55e74b02e 100644 --- a/vyper/ast/signatures/function_signature.py +++ b/vyper/ast/signatures/function_signature.py @@ -10,7 +10,7 @@ get_size_of_type, parse_type, ) -from vyper.utils import mkalphanum, cached_property +from vyper.utils import cached_property, mkalphanum # Function variable diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index f4e2ecd340..b20c968331 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -1,11 +1,11 @@ import vyper.semantics.types as vy from vyper.exceptions import CompilerPanic -from vyper.old_codegen.lll_node import LLLnode, Encoding +from vyper.old_codegen.lll_node import Encoding, LLLnode from vyper.old_codegen.parser_utils import ( + _needs_clamp, add_variable_offset, clamp_basetype, make_setter, - _needs_clamp, unwrap_location, zero_pad, ) @@ -523,7 +523,11 @@ def abi_decode(lll_node, src, clamp=True, pos=None): else: if clamp and _needs_clamp(o.typ, Encoding.ABI): - src_loc = LLLnode.from_list(["with", "src_loc", src_loc, ["seq", clamp_basetype(src_loc), src_loc]], typ=src_loc.typ, location=src_loc.location) + src_loc = LLLnode.from_list( + ["with", "src_loc", src_loc, ["seq", clamp_basetype(src_loc), src_loc]], + typ=src_loc.typ, + location=src_loc.location, + ) else: pass diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index 365370d4fe..a9a4f535f5 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -5,6 +5,7 @@ FunctionSignature, VariableRecord, ) +from vyper.exceptions import CompilerPanic from vyper.old_codegen.context import Context from vyper.old_codegen.expr import Expr from vyper.old_codegen.function_definitions.utils import get_nonreentrant_lock @@ -15,7 +16,13 @@ make_setter, ) from vyper.old_codegen.stmt import parse_body -from vyper.old_codegen.types.types import TupleType, BaseType, ByteArrayLike, ListType, TupleLike +from vyper.old_codegen.types.types import ( + BaseType, + ByteArrayLike, + ListType, + TupleLike, + TupleType, +) def _should_decode(typ): @@ -35,7 +42,7 @@ def _should_decode(typ): # register function args with the local calling context. # also allocate the ones that live in memory (i.e. kwargs) -def _register_function_args(context: Context, sig: FunctionSignature) -> None: +def _register_function_args(context: Context, sig: FunctionSignature) -> List[LLLnode]: pos = None ret = [] diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 6baab17228..fca233ee1b 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -14,7 +14,6 @@ from vyper.old_codegen.types import ( BaseType, ByteArrayLike, - ByteArrayType, ListType, MappingType, StructType, @@ -84,7 +83,6 @@ def _codecopy_gas_bound(num_bytes): # Copy byte array word-for-word (including layout) def make_byte_array_copier(destination, source, pos=None): if not isinstance(source.typ, ByteArrayLike): - btype = "byte array" if isinstance(destination.typ, ByteArrayType) else "string" raise TypeMismatch(f"Cannot cast from {source.typ} to {destination.typ}", pos) if isinstance(source.typ, ByteArrayLike) and source.typ.maxlen > destination.typ.maxlen: raise TypeMismatch( From f2715e00d9bd41499a7f09205e59087b9ca05161 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 1 Oct 2021 20:33:01 +0000 Subject: [PATCH 157/163] Prune inefficient jumps The new function codegen generates a lot of code like ``` goto function_common label function_common ``` This prunes that code. It also prunes labels that are never used, e.g. there is a join point at the end of all functions which is like ``` label function__cleanup ``` If there is no return statement in the function the label is unnecessary. --- vyper/lll/compile_lll.py | 41 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/vyper/lll/compile_lll.py b/vyper/lll/compile_lll.py index 2a0eac71ba..d0601eaf29 100644 --- a/vyper/lll/compile_lll.py +++ b/vyper/lll/compile_lll.py @@ -561,6 +561,17 @@ def _prune_unreachable_code(assembly): i += 1 +def _prune_inefficient_jumps(assembly): + # prune sequences `_sym_x JUMP _sym_x JUMPDEST` to `_sym_x JUMPDEST` + i = 0 + while i < len(assembly) - 4: + if is_symbol(assembly[i]) and assembly[i+1] == "JUMP" and assembly[i] == assembly[i+2] and assembly[i+3]=="JUMPDEST": + # delete _sym_x JUMP + del assembly[i:i+2] + else: + i += 1 + + def _merge_jumpdests(assembly): # When a nested subroutine finishes and is the final action within it's # parent subroutine, we end up with multiple simultaneous JUMPDEST @@ -596,14 +607,36 @@ def _merge_iszero(assembly): else: i += 1 +def _prune_unused_jumpdests(assembly): + used_jumpdests = set() -# Assembles assembly into EVM -def assembly_to_evm(assembly, start_pos=0): - _prune_unreachable_code(assembly) + # find all used jumpdests + for i in range(len(assembly) - 1): + if is_symbol(assembly[i]) and assembly[i+1]!="JUMPDEST": + used_jumpdests.add(assembly[i]) - _merge_iszero(assembly) + # delete jumpdests that aren't used + i = 0 + while i < len(assembly) - 2: + if is_symbol(assembly[i]) and assembly[i] not in used_jumpdests: + del assembly[i:i+2] + else: + i+=1 + + +# optimize assembly, in place +def _optimize_assembly(assembly): + _prune_unreachable_code(assembly) + _merge_iszero(assembly) _merge_jumpdests(assembly) + _prune_inefficient_jumps(assembly) + _prune_unused_jumpdests(assembly) + + +# Assembles assembly into EVM +def assembly_to_evm(assembly, start_pos=0): + _optimize_assembly(assembly) line_number_map = { "breakpoints": set(), From e680e443c7855a2649e560b443e8b58dcde8f63d Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 1 Oct 2021 21:06:29 +0000 Subject: [PATCH 158/163] fix a breaking test --- tests/compiler/test_source_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/compiler/test_source_map.py b/tests/compiler/test_source_map.py index 88d773792f..af708fc860 100644 --- a/tests/compiler/test_source_map.py +++ b/tests/compiler/test_source_map.py @@ -32,7 +32,7 @@ def test_jump_map(): pos_map = source_map["pc_pos_map"] jump_map = source_map["pc_jump_map"] - assert len([v for v in jump_map.values() if v == "o"]) == 3 + assert len([v for v in jump_map.values() if v == "o"]) == 1 assert len([v for v in jump_map.values() if v == "i"]) == 2 code_lines = [i + "\n" for i in TEST_CODE.split("\n")] From d2808d060a0b078d9d7c5aa81205b75b963c1b80 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 1 Oct 2021 21:22:23 +0000 Subject: [PATCH 159/163] elide copies of zero-element tuples Sounds weird but zero-element tuples are generated all the time in codegen (e.g. zero function args). The code is cleaner to handle it in make_setter than at all the call sites. --- vyper/old_codegen/parser_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index fca233ee1b..3997e1b81b 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -650,6 +650,9 @@ def make_setter(left, right, location, pos): left_token = LLLnode.from_list("_L", typ=left.typ, location=left.location) keyz = left.typ.tuple_keys() + if len(keyz) == 0: + return LLLnode.from_list(["pass"]) + # If the left side is complex if left.value == "multi": locations = [arg.location for arg in left.args] From 5b8af9037a38c9461428feec31f1af70c3cf3934 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 1 Oct 2021 21:30:33 +0000 Subject: [PATCH 160/163] rename add_variable_offset to get_element_ptr reuse terminology from LLVM so that if somebody is familiar with LLVM they can understand what the function does from the name. --- vyper/builtin_functions/functions.py | 18 +++++++++--------- vyper/old_codegen/abi.py | 4 ++-- vyper/old_codegen/expr.py | 6 +++--- vyper/old_codegen/external_call.py | 4 ++-- .../function_definitions/external_function.py | 8 ++++---- vyper/old_codegen/parser_utils.py | 17 +++++++++-------- 6 files changed, 29 insertions(+), 28 deletions(-) diff --git a/vyper/builtin_functions/functions.py b/vyper/builtin_functions/functions.py index a1af3b43de..69aae8d495 100644 --- a/vyper/builtin_functions/functions.py +++ b/vyper/builtin_functions/functions.py @@ -32,7 +32,7 @@ from vyper.old_codegen.keccak256_helper import keccak256_helper from vyper.old_codegen.parser_utils import ( LLLnode, - add_variable_offset, + get_element_ptr, get_bytearray_length, getpos, lll_tuple_from_args, @@ -751,8 +751,8 @@ def build_LLL(self, expr, args, kwargs, context): ) -def avo(arg, ind, pos): - return unwrap_location(add_variable_offset(arg, LLLnode.from_list(ind, "int128"), pos=pos)) +def _getelem(arg, ind, pos): + return unwrap_location(get_element_ptr(arg, LLLnode.from_list(ind, "int128"), pos=pos)) class ECAdd(_SimpleBuiltinFunction): @@ -775,10 +775,10 @@ def build_LLL(self, expr, args, kwargs, context): o = LLLnode.from_list( [ "seq", - ["mstore", placeholder_node, avo(args[0], 0, pos)], - ["mstore", ["add", placeholder_node, 32], avo(args[0], 1, pos)], - ["mstore", ["add", placeholder_node, 64], avo(args[1], 0, pos)], - ["mstore", ["add", placeholder_node, 96], avo(args[1], 1, pos)], + ["mstore", placeholder_node, _getelem(args[0], 0, pos)], + ["mstore", ["add", placeholder_node, 32], _getelem(args[0], 1, pos)], + ["mstore", ["add", placeholder_node, 64], _getelem(args[1], 0, pos)], + ["mstore", ["add", placeholder_node, 96], _getelem(args[1], 1, pos)], ["assert", ["staticcall", ["gas"], 6, placeholder_node, 128, placeholder_node, 64]], placeholder_node, ], @@ -806,8 +806,8 @@ def build_LLL(self, expr, args, kwargs, context): o = LLLnode.from_list( [ "seq", - ["mstore", placeholder_node, avo(args[0], 0, pos)], - ["mstore", ["add", placeholder_node, 32], avo(args[0], 1, pos)], + ["mstore", placeholder_node, _getelem(args[0], 0, pos)], + ["mstore", ["add", placeholder_node, 32], _getelem(args[0], 1, pos)], ["mstore", ["add", placeholder_node, 64], args[1]], ["assert", ["staticcall", ["gas"], 7, placeholder_node, 96, placeholder_node, 64]], placeholder_node, diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index b20c968331..0b7a8c5f20 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -3,7 +3,7 @@ from vyper.old_codegen.lll_node import Encoding, LLLnode from vyper.old_codegen.parser_utils import ( _needs_clamp, - add_variable_offset, + get_element_ptr, clamp_basetype, make_setter, unwrap_location, @@ -364,7 +364,7 @@ def o_list(lll_node, pos=None): else [LLLnode.from_list(i, "uint256") for i in range(lll_t.count)] ) - ret = [add_variable_offset(lll_node, k, pos, array_bounds_check=False) for k in ks] + ret = [get_element_ptr(lll_node, k, pos, array_bounds_check=False) for k in ks] return ret else: return [lll_node] diff --git a/vyper/old_codegen/expr.py b/vyper/old_codegen/expr.py index 320c3f2c68..ff012f7af5 100644 --- a/vyper/old_codegen/expr.py +++ b/vyper/old_codegen/expr.py @@ -15,7 +15,7 @@ from vyper.old_codegen.keccak256_helper import keccak256_helper from vyper.old_codegen.lll_node import LLLnode from vyper.old_codegen.parser_utils import ( - add_variable_offset, + get_element_ptr, get_number_as_fraction, getpos, load_op, @@ -412,7 +412,7 @@ def parse_Attribute(self): if isinstance(sub.typ, InterfaceType): return sub if isinstance(sub.typ, StructType) and self.expr.attr in sub.typ.members: - return add_variable_offset(sub, self.expr.attr, pos=getpos(self.expr)) + return get_element_ptr(sub, self.expr.attr, pos=getpos(self.expr)) def parse_Subscript(self): sub = Expr.parse_variable_location(self.expr.value, self.context) @@ -434,7 +434,7 @@ def parse_Subscript(self): return else: return - lll_node = add_variable_offset(sub, index, pos=getpos(self.expr)) + lll_node = get_element_ptr(sub, index, pos=getpos(self.expr)) lll_node.mutable = sub.mutable return lll_node diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index 050d8f712a..34df5e4119 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -8,7 +8,7 @@ from vyper.old_codegen.abi import abi_encode, abi_type_of from vyper.old_codegen.lll_node import Encoding, LLLnode from vyper.old_codegen.parser_utils import ( - add_variable_offset, + get_element_ptr, calculate_type_for_external_return, getpos, unwrap_location, @@ -112,7 +112,7 @@ def _unpack_returndata(buf, contract_sig, context, pos): buf = LLLnode(buf, typ=return_t, encoding=_returndata_encoding(contract_sig), location="memory") if should_unwrap_abi_tuple: - buf = add_variable_offset(buf, 0, pos=None, array_bounds_check=False) + buf = get_element_ptr(buf, 0, pos=None, array_bounds_check=False) ret += [buf] diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index a9a4f535f5..319672d19b 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -11,7 +11,7 @@ from vyper.old_codegen.function_definitions.utils import get_nonreentrant_lock from vyper.old_codegen.lll_node import Encoding, LLLnode from vyper.old_codegen.parser_utils import ( - add_variable_offset, + get_element_ptr, getpos, make_setter, ) @@ -60,7 +60,7 @@ def _register_function_args(context: Context, sig: FunctionSignature) -> List[LL for i, arg in enumerate(sig.base_args): - arg_lll = add_variable_offset(base_args_ofst, i, pos=pos) + arg_lll = get_element_ptr(base_args_ofst, i, pos=pos) if _should_decode(arg.typ): # allocate a memory slot for it and copy @@ -100,7 +100,7 @@ def _generate_kwarg_handlers(context: Context, sig: FunctionSignature, pos: Any) def handler_for(calldata_kwargs, default_kwargs): calldata_args = sig.base_args + calldata_kwargs - # create a fake type so that add_variable_offset works + # create a fake type so that get_element_ptr works calldata_args_t = TupleType(list(arg.typ for arg in calldata_args)) abi_sig = sig.abi_signature_for_kwargs(calldata_kwargs) @@ -126,7 +126,7 @@ def handler_for(calldata_kwargs, default_kwargs): dst = context.lookup_var(arg_meta.name).pos lhs = LLLnode(dst, location="memory", typ=arg_meta.typ) - rhs = add_variable_offset(calldata_kwargs_ofst, k, pos=None, array_bounds_check=False) + rhs = get_element_ptr(calldata_kwargs_ofst, k, pos=None, array_bounds_check=False) ret.append(make_setter(lhs, rhs, lhs_location, pos)) for x in default_kwargs: diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 3997e1b81b..31936ec974 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -294,9 +294,10 @@ def _add_ofst(loc, ofst): # Take a value representing a memory or storage location, and descend down to # an element or member variable +# This is analogous (but not necessarily equivalent to) getelementptr in LLVM. # TODO refactor / streamline this code, especially the ABI decoding @type_check_wrapper -def add_variable_offset(parent, key, pos, array_bounds_check=True): +def get_element_ptr(parent, key, pos, array_bounds_check=True): # TODO rethink this circular import from vyper.old_codegen.abi import abi_type_of @@ -499,7 +500,7 @@ def _make_array_index_setter(target, target_token, pos, location, offset): offset = target.value + 32 * get_size_of_type(target.typ.subtype) * offset return LLLnode.from_list([offset], typ=target.typ.subtype, location=location, pos=pos) else: - return add_variable_offset( + return get_element_ptr( target_token, LLLnode.from_list(offset, typ="int256"), pos=pos, @@ -599,7 +600,7 @@ def make_setter(left, right, location, pos): for i in range(left.typ.count): subs.append( make_setter( - add_variable_offset( + get_element_ptr( left_token, LLLnode.from_list(i, typ="int256"), pos=pos, @@ -673,7 +674,7 @@ def make_setter(left, right, location, pos): for (key, loc) in zip(keyz, locations): subs.append( make_setter( - add_variable_offset(left_token, key, pos=pos), + get_element_ptr(left_token, key, pos=pos), right_args[key], loc, pos=pos, @@ -692,7 +693,7 @@ def make_setter(left, right, location, pos): for key, loc in zip(keyz, locations): subs.append( make_setter( - add_variable_offset(left_token, key, pos=pos), + get_element_ptr(left_token, key, pos=pos), LLLnode.from_list(None, typ=right.typ.members[key]), loc, pos=pos, @@ -716,7 +717,7 @@ def make_setter(left, right, location, pos): for left_arg, key, loc in zip(left.args, keyz, locations): subs.append( make_setter( - left_arg, add_variable_offset(right_token, key, pos=pos), loc, pos=pos + left_arg, get_element_ptr(right_token, key, pos=pos), loc, pos=pos ) ) @@ -730,8 +731,8 @@ def make_setter(left, right, location, pos): for typ, loc in zip(keyz, locations): subs.append( make_setter( - add_variable_offset(left_token, typ, pos=pos), - add_variable_offset(right_token, typ, pos=pos), + get_element_ptr(left_token, typ, pos=pos), + get_element_ptr(right_token, typ, pos=pos), loc, pos=pos, ) From 2ab392d18209ab7e68dc2770ceb931225a0d1787 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sat, 2 Oct 2021 18:06:29 +0000 Subject: [PATCH 161/163] fix lint --- vyper/builtin_functions/functions.py | 2 +- vyper/lll/compile_lll.py | 19 ++++++++++++------- vyper/old_codegen/abi.py | 2 +- vyper/old_codegen/external_call.py | 2 +- .../function_definitions/external_function.py | 6 +----- vyper/old_codegen/parser_utils.py | 9 ++------- 6 files changed, 18 insertions(+), 22 deletions(-) diff --git a/vyper/builtin_functions/functions.py b/vyper/builtin_functions/functions.py index 69aae8d495..45b1a2dab4 100644 --- a/vyper/builtin_functions/functions.py +++ b/vyper/builtin_functions/functions.py @@ -32,8 +32,8 @@ from vyper.old_codegen.keccak256_helper import keccak256_helper from vyper.old_codegen.parser_utils import ( LLLnode, - get_element_ptr, get_bytearray_length, + get_element_ptr, getpos, lll_tuple_from_args, load_op, diff --git a/vyper/lll/compile_lll.py b/vyper/lll/compile_lll.py index d0601eaf29..a7c3407de5 100644 --- a/vyper/lll/compile_lll.py +++ b/vyper/lll/compile_lll.py @@ -565,9 +565,14 @@ def _prune_inefficient_jumps(assembly): # prune sequences `_sym_x JUMP _sym_x JUMPDEST` to `_sym_x JUMPDEST` i = 0 while i < len(assembly) - 4: - if is_symbol(assembly[i]) and assembly[i+1] == "JUMP" and assembly[i] == assembly[i+2] and assembly[i+3]=="JUMPDEST": - # delete _sym_x JUMP - del assembly[i:i+2] + if ( + is_symbol(assembly[i]) + and assembly[i + 1] == "JUMP" + and assembly[i] == assembly[i + 2] + and assembly[i + 3] == "JUMPDEST" + ): + # delete _sym_x JUMP + del assembly[i : i + 2] # noqa: E203 else: i += 1 @@ -607,22 +612,22 @@ def _merge_iszero(assembly): else: i += 1 + def _prune_unused_jumpdests(assembly): used_jumpdests = set() # find all used jumpdests for i in range(len(assembly) - 1): - if is_symbol(assembly[i]) and assembly[i+1]!="JUMPDEST": + if is_symbol(assembly[i]) and assembly[i + 1] != "JUMPDEST": used_jumpdests.add(assembly[i]) - # delete jumpdests that aren't used i = 0 while i < len(assembly) - 2: if is_symbol(assembly[i]) and assembly[i] not in used_jumpdests: - del assembly[i:i+2] + del assembly[i : i + 2] # noqa: E203 else: - i+=1 + i += 1 # optimize assembly, in place diff --git a/vyper/old_codegen/abi.py b/vyper/old_codegen/abi.py index 0b7a8c5f20..8e8a403580 100644 --- a/vyper/old_codegen/abi.py +++ b/vyper/old_codegen/abi.py @@ -3,8 +3,8 @@ from vyper.old_codegen.lll_node import Encoding, LLLnode from vyper.old_codegen.parser_utils import ( _needs_clamp, - get_element_ptr, clamp_basetype, + get_element_ptr, make_setter, unwrap_location, zero_pad, diff --git a/vyper/old_codegen/external_call.py b/vyper/old_codegen/external_call.py index 34df5e4119..5308a94d3c 100644 --- a/vyper/old_codegen/external_call.py +++ b/vyper/old_codegen/external_call.py @@ -8,8 +8,8 @@ from vyper.old_codegen.abi import abi_encode, abi_type_of from vyper.old_codegen.lll_node import Encoding, LLLnode from vyper.old_codegen.parser_utils import ( - get_element_ptr, calculate_type_for_external_return, + get_element_ptr, getpos, unwrap_location, ) diff --git a/vyper/old_codegen/function_definitions/external_function.py b/vyper/old_codegen/function_definitions/external_function.py index 319672d19b..a5e9c697f6 100644 --- a/vyper/old_codegen/function_definitions/external_function.py +++ b/vyper/old_codegen/function_definitions/external_function.py @@ -10,11 +10,7 @@ from vyper.old_codegen.expr import Expr from vyper.old_codegen.function_definitions.utils import get_nonreentrant_lock from vyper.old_codegen.lll_node import Encoding, LLLnode -from vyper.old_codegen.parser_utils import ( - get_element_ptr, - getpos, - make_setter, -) +from vyper.old_codegen.parser_utils import get_element_ptr, getpos, make_setter from vyper.old_codegen.stmt import parse_body from vyper.old_codegen.types.types import ( BaseType, diff --git a/vyper/old_codegen/parser_utils.py b/vyper/old_codegen/parser_utils.py index 31936ec974..31924d0a91 100644 --- a/vyper/old_codegen/parser_utils.py +++ b/vyper/old_codegen/parser_utils.py @@ -674,10 +674,7 @@ def make_setter(left, right, location, pos): for (key, loc) in zip(keyz, locations): subs.append( make_setter( - get_element_ptr(left_token, key, pos=pos), - right_args[key], - loc, - pos=pos, + get_element_ptr(left_token, key, pos=pos), right_args[key], loc, pos=pos, ) ) return LLLnode.from_list(["with", "_L", left, ["seq"] + subs], typ=None) @@ -716,9 +713,7 @@ def make_setter(left, right, location, pos): ) for left_arg, key, loc in zip(left.args, keyz, locations): subs.append( - make_setter( - left_arg, get_element_ptr(right_token, key, pos=pos), loc, pos=pos - ) + make_setter(left_arg, get_element_ptr(right_token, key, pos=pos), loc, pos=pos) ) return LLLnode.from_list(["with", "_R", right, ["seq"] + subs], typ=None) From a022c958b34db8343efc2c4bd3a9c3ebe038ffdd Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sat, 2 Oct 2021 14:23:35 -0700 Subject: [PATCH 162/163] fix nonreentrancy codegen for private functions we should probably deprecate nonreentrant keys on private functions but in the meantime keep them working --- vyper/old_codegen/function_definitions/internal_function.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vyper/old_codegen/function_definitions/internal_function.py b/vyper/old_codegen/function_definitions/internal_function.py index a3cb9031ed..98712316db 100644 --- a/vyper/old_codegen/function_definitions/internal_function.py +++ b/vyper/old_codegen/function_definitions/internal_function.py @@ -55,7 +55,7 @@ def generate_lll_for_internal_function( # jump to the label which was passed in via stack stop_func = LLLnode.from_list(["jump", "pass"], annotation="jump to return address") - enter = [["label", function_entry_label], nonreentrant_pre] + enter = [["label", function_entry_label]] + nonreentrant_pre body = [parse_body(c, context) for c in code.body] From 3e036aaf49dc2dc7b051f30f3ea2b88ed61e29b2 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sat, 2 Oct 2021 14:24:28 -0700 Subject: [PATCH 163/163] improve annotation for `in` expressions --- vyper/old_codegen/expr.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vyper/old_codegen/expr.py b/vyper/old_codegen/expr.py index ff012f7af5..d9083da5f2 100644 --- a/vyper/old_codegen/expr.py +++ b/vyper/old_codegen/expr.py @@ -768,7 +768,8 @@ def build_in_comparator(self): # for `not in`, invert the result compare_sequence = ["iszero", compare_sequence] - return LLLnode.from_list(compare_sequence, typ="bool", annotation="in comparator") + annotation = self.expr.get("node_source_code") + return LLLnode.from_list(compare_sequence, typ="bool", annotation=annotation) @staticmethod def _signed_to_unsigned_comparision_op(op):