From 658f0c4bd8ebd80a6e98fb01121ad5801bdd6661 Mon Sep 17 00:00:00 2001 From: HodanPlodky <36966616+HodanPlodky@users.noreply.github.com> Date: Fri, 25 Oct 2024 17:26:10 +0000 Subject: [PATCH 01/16] refactor[venom]: refactor sccp pass to use dfg (#4329) use `DFGAnalysis` in `SCCP` instead of duplicating the logic to compute uses in the `SCCP` itself. also use `OrderedSet` for var uses, this ensures we don't add the same instruction multiple times (as in `add %2 %2`) to a var's use set, and also enables a cheaper `remove_use()` implementation. --- vyper/venom/analysis/dfg.py | 18 ++++++++++-------- vyper/venom/passes/sccp/sccp.py | 28 +++++++--------------------- 2 files changed, 17 insertions(+), 29 deletions(-) diff --git a/vyper/venom/analysis/dfg.py b/vyper/venom/analysis/dfg.py index 328ed47c72..f49b2ac6ac 100644 --- a/vyper/venom/analysis/dfg.py +++ b/vyper/venom/analysis/dfg.py @@ -1,5 +1,6 @@ from typing import Optional +from vyper.utils import OrderedSet from vyper.venom.analysis.analysis import IRAnalysesCache, IRAnalysis from vyper.venom.analysis.liveness import LivenessAnalysis from vyper.venom.basicblock import IRInstruction, IRVariable @@ -7,7 +8,7 @@ class DFGAnalysis(IRAnalysis): - _dfg_inputs: dict[IRVariable, list[IRInstruction]] + _dfg_inputs: dict[IRVariable, OrderedSet[IRInstruction]] _dfg_outputs: dict[IRVariable, IRInstruction] def __init__(self, analyses_cache: IRAnalysesCache, function: IRFunction): @@ -16,19 +17,19 @@ def __init__(self, analyses_cache: IRAnalysesCache, function: IRFunction): self._dfg_outputs = dict() # return uses of a given variable - def get_uses(self, op: IRVariable) -> list[IRInstruction]: - return self._dfg_inputs.get(op, []) + def get_uses(self, op: IRVariable) -> OrderedSet[IRInstruction]: + return self._dfg_inputs.get(op, OrderedSet()) # the instruction which produces this variable. def get_producing_instruction(self, op: IRVariable) -> Optional[IRInstruction]: return self._dfg_outputs.get(op) def add_use(self, op: IRVariable, inst: IRInstruction): - uses = self._dfg_inputs.setdefault(op, []) - uses.append(inst) + uses = self._dfg_inputs.setdefault(op, OrderedSet()) + uses.add(inst) def remove_use(self, op: IRVariable, inst: IRInstruction): - uses = self._dfg_inputs.get(op, []) + uses: OrderedSet = self._dfg_inputs.get(op, OrderedSet()) uses.remove(inst) @property @@ -48,10 +49,11 @@ def analyze(self): res = inst.get_outputs() for op in operands: - inputs = self._dfg_inputs.setdefault(op, []) - inputs.append(inst) + inputs = self._dfg_inputs.setdefault(op, OrderedSet()) + inputs.add(inst) for op in res: # type: ignore + assert isinstance(op, IRVariable) self._dfg_outputs[op] = inst def as_graph(self) -> str: diff --git a/vyper/venom/passes/sccp/sccp.py b/vyper/venom/passes/sccp/sccp.py index 19d373f81a..d85e09c9b4 100644 --- a/vyper/venom/passes/sccp/sccp.py +++ b/vyper/venom/passes/sccp/sccp.py @@ -5,7 +5,7 @@ from vyper.exceptions import CompilerPanic, StaticAssertionException from vyper.utils import OrderedSet -from vyper.venom.analysis import CFGAnalysis, DominatorTreeAnalysis, IRAnalysesCache +from vyper.venom.analysis import CFGAnalysis, DFGAnalysis, DominatorTreeAnalysis, IRAnalysesCache from vyper.venom.basicblock import ( IRBasicBlock, IRInstruction, @@ -51,7 +51,7 @@ class SCCP(IRPass): fn: IRFunction dom: DominatorTreeAnalysis - uses: dict[IRVariable, OrderedSet[IRInstruction]] + dfg: DFGAnalysis lattice: Lattice work_list: list[WorkListItem] cfg_in_exec: dict[IRBasicBlock, OrderedSet[IRBasicBlock]] @@ -67,7 +67,7 @@ def __init__(self, analyses_cache: IRAnalysesCache, function: IRFunction): def run_pass(self): self.fn = self.function self.dom = self.analyses_cache.request_analysis(DominatorTreeAnalysis) - self._compute_uses() + self.dfg = self.analyses_cache.request_analysis(DFGAnalysis) self._calculate_sccp(self.fn.entry) self._propagate_constants() @@ -75,6 +75,8 @@ def run_pass(self): self.analyses_cache.force_analysis(CFGAnalysis) self._fix_phi_nodes() + self.analyses_cache.invalidate_analysis(DFGAnalysis) + def _calculate_sccp(self, entry: IRBasicBlock): """ This method is the main entry point for the SCCP algorithm. It @@ -92,7 +94,7 @@ def _calculate_sccp(self, entry: IRBasicBlock): self.work_list.append(FlowWorkItem(dummy, entry)) # Initialize the lattice with TOP values for all variables - for v in self.uses.keys(): + for v in self.dfg._dfg_outputs: self.lattice[v] = LatticeEnum.TOP # Iterate over the work list until it is empty @@ -258,25 +260,9 @@ def _eval(self, inst) -> LatticeItem: return ret # type: ignore def _add_ssa_work_items(self, inst: IRInstruction): - for target_inst in self._get_uses(inst.output): # type: ignore + for target_inst in self.dfg.get_uses(inst.output): # type: ignore self.work_list.append(SSAWorkListItem(target_inst)) - def _compute_uses(self): - """ - This method computes the uses for each variable in the IR. - It iterates over the dominator tree and collects all the - instructions that use each variable. - """ - self.uses = {} - for bb in self.dom.dfs_walk: - for var, insts in bb.get_uses().items(): - self._get_uses(var).update(insts) - - def _get_uses(self, var: IRVariable): - if var not in self.uses: - self.uses[var] = OrderedSet() - return self.uses[var] - def _propagate_constants(self): """ This method iterates over the IR and replaces constant values From 063f1900a1b2b32bf8d0f091027dd0fb16e99de8 Mon Sep 17 00:00:00 2001 From: sandbubbles <160503471+sandbubbles@users.noreply.github.com> Date: Sat, 26 Oct 2024 01:57:30 +0200 Subject: [PATCH 02/16] docs[venom]: expand venom docs (#4314) expand venom docs. document venom instructions and structure of a venom program. --------- Co-authored-by: Charles Cooper --- vyper/venom/README.md | 313 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 313 insertions(+) diff --git a/vyper/venom/README.md b/vyper/venom/README.md index 5d98b22dd6..4e4f5ca3d1 100644 --- a/vyper/venom/README.md +++ b/vyper/venom/README.md @@ -160,3 +160,316 @@ A number of passes that are planned to be implemented, or are implemented for im ### Function inlining ### Load-store elimination + +--- + +## Structure of a venom program + +### IRContext +An `IRContext` consists of multiple `IRFunctions`, with one designated as the main entry point of the program. +Additionally, the `IRContext` maintains its own representation of the data segment. + +### IRFunction +An `IRFunction` is composed of a name and multiple `IRBasicBlocks`, with one marked as the entry point to the function. + +### IRBasicBlock +An `IRBasicBlock` contains a label and a sequence of `IRInstructions`. +Each `IRBasicBlock` has a single entry point and exit point. +The exit point must be one of the following terminator instructions: +- `jmp` +- `djmp` +- `jnz` +- `ret` +- `return` +- `stop` +- `exit` + +Normalized basic blocks cannot have multiple predecessors and successors. It has either one (or zero) predecessors and potentially multiple successors or vice versa. + +### IRInstruction +An `IRInstruction` consists of an opcode, a list of operands, and an optional return value. +An operand can be a label, a variable, or a literal. + +By convention, variables have a `%-` prefix, e.g. `%1` is a valid variable. However, the prefix is not required. + +## Instructions +To enable Venom IR in Vyper, use the `--experimental-codegen` flag. To view the Venom IR output, use `-f bb_runtime` for the runtime code, or `-f bb` to see the deploy code. To get a dot file (for use e.g. with `xdot -`), use `-f cfg` or `-f cfg_runtime`. + +Assembly can be inspected with `-f asm`, whereas an opcode view of the final bytecode can be seen with `-f opcodes` or `-f opcodes_runtime`, respectively. + +### Special instructions + +- `invoke` + - ``` + invoke offset, label + ``` + - Causes control flow to jump to a function denoted by the `label`. + - Return values are passed in the return buffer at the `offset` address. + - Used for internal functions. + - Effectively translates to `JUMP`, and marks the call site as a valid return destination (for callee to jump back to) by `JUMPDEST`. +- `alloca` + - ``` + out = alloca size, offset + ``` + - Allocates memory of a given `size` at a given `offset` in memory. + - The output is the offset value itself. + - Because the SSA form does not allow changing values of registers, handling mutable variables can be tricky. The `alloca` instruction is meant to simplify that. + +- `palloca` + - ``` + out = palloca size, offset + ``` + - Like the `alloca` instruction but only used for parameters of internal functions which are passed by memory. +- `iload` + - ``` + out = iload offset + ``` + - Loads value at an immutable section of memory denoted by `offset` into `out` variable. + - The operand can be either a literal, which is a statically computed offset, or a variable. + - Essentially translates to `MLOAD` on an immutable section of memory. So, for example + ``` + %op = 12 + %out = iload %op + ``` + could compile into `PUSH1 12 _mem_deploy_end ADD MLOAD`. + - When `offset` is a literal the location is computed statically during compilation from assembly to bytecode. +- `istore` + - ``` + istore offset value + ``` + - Represents a store into immutable section of memory. + - Like in `iload`, the offset operand can be a literal. + - Essentially translates to `MSTORE` on an immutable section of memory. For example, + ``` + %op = 12 + istore 24 %op + ``` + could compile to + `PUSH1 12 PUSH1 24 _mem_deploy_end ADD MSTORE`. +- `phi` + - ``` + out = phi %var_a, label_a, %var_b, label_b + ``` + - Because in SSA form each variable is assigned just once, it is tricky to handle that variables may be assigned to something different based on which program path was taken. + - Therefore, we use `phi` instructions. They are are magic instructions, used in basic blocks where the control flow path merges. + - In this example, essentially the `out` variable is set to `%var_a` if the program entered the current block from `label_a` or to `%var_b` when it went through `label_b`. +- `offset` + - ``` + ret = offset label, op + ``` + - Statically compute offset before compiling into bytecode. Useful for `mstore`, `mload` and such. + - Basically `label` + `op`. + - The `asm` output could show something like `_OFST _sym_ label`. +- `param` + - ``` + out = param + ``` + - The `param` instruction is used to represent function arguments passed by the stack. + - We assume the argument is on the stack and the `param` instruction is used to ensure we represent the argument by the `out` variable. +- `store` + - ``` + out = op + ``` + - Store variable value or literal into `out` variable. +- `dbname` + - ``` + dbname label + ``` + - Mark memory with a `label` in the data segment so it can be referenced. +- `db` + - ``` + db data + ``` + - Store `data` into data segment. +- `dloadbytes` + - Alias for `codecopy` for legacy reasons. May be removed in future versions. + - Translates to `CODECOPY`. +- `ret` + - ``` + ret op + ``` + - Represents return from an internal call. + - Jumps to a location given by `op`. + - If `op` is a label it can effectively translate into `op JUMP`. +- `exit` + - ``` + exit + ``` + - Similar to `stop`, but used for constructor exit. The assembler is expected to jump to a special initcode sequence which returns the runtime code. + - Might translate to something like `_sym__ctor_exit JUMP`. +- `sha3_64` + - ``` + out = sha3_64 x y + ``` + - Shortcut to access the `SHA3` EVM opcode where `out` is the result. + - Essentially translates to + ``` + PUSH y PUSH FREE_VAR_SPACE MSTORE + PUSH x PUSH FREE_VAR_SPACE2 MSTORE + PUSH 64 PUSH FREE_VAR_SPACE SHA3 + ``` + where `FREE_VAR_SPACE` and `FREE_VAR_SPACE2` are locations reserved by the compiler, set to 0 and 32 respectively. + +- `assert` + - ``` + assert op + ``` + - Assert that `op` is zero. If it is not, revert. + - Calls that terminate this way receive a gas refund. + - For example + ``` + %op = 13 + assert %op + ``` + could compile to + `PUSH1 13 ISZERO _sym___revert JUMPI`. +- `assert_unreachable` + - ``` + assert_unreachable op + ``` + - Check that `op` is zero. If it is not, terminate with `0xFE` ("INVALID" opcode). + - Calls that end this way do not receive a gas refund. + - Could translate to `op reachable JUMPI INVALID reachable JUMPDEST`. + - For example + ``` + %op = 13 + assert_unreachable %op + ``` + could compile to + ``` + PUSH1 13 _sym_reachable1 JUMPI + INVALID + _sym_reachable1 JUMPDEST + ``` +- `log` + - ``` + log offset, size, [topic] * topic_count , topic_count + ``` + - Corresponds to the `LOGX` instruction in EVM. + - Depending on the `topic_count` value (which can be only from 0 to 4) translates to `LOG0` ... `LOG4`. + - The rest of the operands correspond to the `LOGX` instructions. + - For example + ``` + log %53, 32, 64, %56, 2 + ``` + could translate to: + ``` + %56, 64, 32, %53 LOG2 + ``` +- `nop` + - ``` + nop + ``` + - No operation, does nothing. +- `offset` + - ``` + %2 = offset %1 label1 + - Similar to `add`, but takes a label as the second argument. If the first argument is a literal, the addition will get optimized at assembly time. + +### Jump instructions + +- `jmp` + - ``` + jmp label + ``` + - Unconditional jump to code denoted by given `label`. + - Translates to `label JUMP`. +- `jnz` + - ``` + jnz label1, label2, op + ``` + - A conditional jump depending on the value of `op`. + - Jumps to `label2` when `op` is not zero, otherwise jumps to `label1`. + - For example + ``` + %op = 15 + jnz label1, label2, %op + ``` + could translate to: `PUSH1 15 label2 JUMPI label1 JUMP`. +- `djmp` + - ``` + djmp %var, label1, label2, label3, ... + ``` + - Dynamic jump to an address specified by the variable operand, constrained to the provided labels. + - Accepts a variable number of labels. + - The target is not a fixed label but rather a value stored in a variable, making the jump dynamic. + - The jump target can be any of the provided labels. + - Translates to `JUMP`. + +### EVM instructions + +The following instructions map one-to-one with [EVM instructions](https://www.evm.codes/). +Operands correspond to stack inputs in the same order. Stack outputs are the instruction's output. +Instructions have the same effects. +- `return` +- `revert` +- `coinbase` +- `calldatasize` +- `calldatacopy` +- `mcopy` +- `calldataload` +- `gas` +- `gasprice` +- `gaslimit` +- `chainid` +- `address` +- `origin` +- `number` +- `extcodesize` +- `extcodehash` +- `extcodecopy` +- `returndatasize` +- `returndatacopy` +- `callvalue` +- `selfbalance` +- `sload` +- `sstore` +- `mload` +- `mstore` +- `tload` +- `tstore` +- `timestamp` +- `caller` +- `blockhash` +- `selfdestruct` +- `signextend` +- `stop` +- `shr` +- `shl` +- `sar` +- `and` +- `xor` +- `or` +- `add` +- `sub` +- `mul` +- `div` +- `smul` +- `sdiv` +- `mod` +- `smod` +- `exp` +- `addmod` +- `mulmod` +- `eq` +- `iszero` +- `not` +- `lt` +- `gt` +- `slt` +- `sgt` +- `create` +- `create2` +- `msize` +- `balance` +- `call` +- `staticcall` +- `delegatecall` +- `codesize` +- `basefee` +- `blobhash` +- `blobbasefee` +- `prevrandao` +- `difficulty` +- `invalid` +- `sha3` From 3fba8c735e1ab124e928609296f70606261e7113 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 29 Oct 2024 09:32:31 -0400 Subject: [PATCH 03/16] refactor[venom]: update translator for `deploy` instruction (#4318) this commit updates `ir_node_to_venom.py` for the `deploy` instruction to properly emit the `exit` instruction. it also removes places in the translator where the `exit` instruction was improperly emitted. misc: - update style to continue early in the loop rather than nest. --- vyper/venom/function.py | 22 ++++++++++++---------- vyper/venom/ir_node_to_venom.py | 6 ++---- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/vyper/venom/function.py b/vyper/venom/function.py index fb0dabc99a..2f2d477460 100644 --- a/vyper/venom/function.py +++ b/vyper/venom/function.py @@ -195,17 +195,19 @@ def chain_basic_blocks(self) -> None: """ bbs = list(self.get_basic_blocks()) for i, bb in enumerate(bbs): - if not bb.is_terminated: - if i < len(bbs) - 1: - # TODO: revisit this. When contructor calls internal functions they - # are linked to the last ctor block. Should separate them before this - # so we don't have to handle this here - if bbs[i + 1].label.value.startswith("internal"): - bb.append_instruction("stop") - else: - bb.append_instruction("jmp", bbs[i + 1].label) + if bb.is_terminated: + continue + + if i < len(bbs) - 1: + # TODO: revisit this. When contructor calls internal functions + # they are linked to the last ctor block. Should separate them + # before this so we don't have to handle this here + if bbs[i + 1].label.value.startswith("internal"): + bb.append_instruction("stop") else: - bb.append_instruction("exit") + bb.append_instruction("jmp", bbs[i + 1].label) + else: + bb.append_instruction("stop") def copy(self): new = IRFunction(self.name) diff --git a/vyper/venom/ir_node_to_venom.py b/vyper/venom/ir_node_to_venom.py index e30f27f480..02a9f4d1f7 100644 --- a/vyper/venom/ir_node_to_venom.py +++ b/vyper/venom/ir_node_to_venom.py @@ -255,6 +255,7 @@ def _convert_ir_bb(fn, ir, symbols): elif ir.value == "deploy": ctx.ctor_mem_size = ir.args[0].value ctx.immutables_len = ir.args[2].value + fn.get_basic_block().append_instruction("exit") return None elif ir.value == "seq": if len(ir.args) == 0: @@ -398,10 +399,7 @@ def _convert_ir_bb(fn, ir, symbols): bb = IRBasicBlock(label, fn) fn.append_basic_block(bb) code = ir.args[2] - if code.value == "pass": - bb.append_instruction("exit") - else: - _convert_ir_bb(fn, code, symbols) + _convert_ir_bb(fn, code, symbols) elif ir.value == "exit_to": args = _convert_ir_bb_list(fn, ir.args[1:], symbols) var_list = args From fcddb70b6a796757709fa68a36f86ce729d18f77 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 29 Oct 2024 09:33:13 -0400 Subject: [PATCH 04/16] feat[venom]: reduce legacy opts when venom is enabled (#4336) reduce legacy IR optimization when venom is enabled. the use of `IRnode._optimized` was there to decide if it was safe to inline an IRnode expression or it required a `with` statement; with venom, we don't need to inline the expressions, since the venom optimizer is more powerful. this leads to a 50% improvement in AST -> IRnode generation, which is a ~15% performance improvement in overall end-to-end compile time. --- vyper/codegen/ir_node.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vyper/codegen/ir_node.py b/vyper/codegen/ir_node.py index 6f9eb0359b..ff721fafcb 100644 --- a/vyper/codegen/ir_node.py +++ b/vyper/codegen/ir_node.py @@ -6,7 +6,7 @@ from typing import Any, List, Optional, Union import vyper.ast as vy_ast -from vyper.compiler.settings import VYPER_COLOR_OUTPUT +from vyper.compiler.settings import VYPER_COLOR_OUTPUT, get_global_settings from vyper.evm.address_space import AddrSpace from vyper.evm.opcodes import get_ir_opcodes from vyper.exceptions import CodegenPanic, CompilerPanic @@ -426,6 +426,10 @@ def is_pointer(self) -> bool: @property # probably could be cached_property but be paranoid def _optimized(self): + if get_global_settings().experimental_codegen: + # in venom pipeline, we don't need to inline constants. + return self + # TODO figure out how to fix this circular import from vyper.ir.optimizer import optimize From 471a556f617e550a7882b138a0fdd75dabf34279 Mon Sep 17 00:00:00 2001 From: sandbubbles <160503471+sandbubbles@users.noreply.github.com> Date: Tue, 5 Nov 2024 14:14:07 +0100 Subject: [PATCH 05/16] refactor[ux]: add `venom` as `experimental-codegen` alias (#4337) Make "venom" an alternative alias to the "experimental_codegen" flag. Add the alias to cli, pre-parser, and json parser. --- tests/unit/ast/test_pre_parser.py | 28 +++++++++++++++ .../unit/cli/vyper_json/test_compile_json.py | 36 +++++++++++++++++++ vyper/ast/pre_parser.py | 4 +-- vyper/cli/vyper_compile.py | 1 + vyper/cli/vyper_json.py | 6 ++++ vyper/compiler/settings.py | 2 +- 6 files changed, 74 insertions(+), 3 deletions(-) diff --git a/tests/unit/ast/test_pre_parser.py b/tests/unit/ast/test_pre_parser.py index 4190725f7e..5d3f30481c 100644 --- a/tests/unit/ast/test_pre_parser.py +++ b/tests/unit/ast/test_pre_parser.py @@ -191,6 +191,25 @@ def test_parse_pragmas(code, pre_parse_settings, compiler_data_settings, mock_ve assert compiler_data.settings == compiler_data_settings +pragma_venom = [ + """ + #pragma venom + """, + """ + #pragma experimental-codegen + """, +] + + +@pytest.mark.parametrize("code", pragma_venom) +def test_parse_venom_pragma(code): + pre_parse_result = pre_parse(code) + assert pre_parse_result.settings.experimental_codegen is True + + compiler_data = CompilerData(code) + assert compiler_data.settings.experimental_codegen is True + + invalid_pragmas = [ # evm-versionnn """ @@ -218,6 +237,15 @@ def test_parse_pragmas(code, pre_parse_settings, compiler_data_settings, mock_ve # pragma evm-version cancun # pragma evm-version shanghai """, + # duplicate setting of venom + """ + #pragma venom + #pragma experimental-codegen + """, + """ + #pragma venom + #pragma venom + """, ] diff --git a/tests/unit/cli/vyper_json/test_compile_json.py b/tests/unit/cli/vyper_json/test_compile_json.py index 5da98cf20f..7802ee7955 100644 --- a/tests/unit/cli/vyper_json/test_compile_json.py +++ b/tests/unit/cli/vyper_json/test_compile_json.py @@ -9,6 +9,7 @@ compile_json, exc_handler_to_dict, get_inputs, + get_settings, ) from vyper.compiler import OUTPUT_FORMATS, compile_code, compile_from_file_input from vyper.compiler.input_bundle import JSONInputBundle @@ -319,3 +320,38 @@ def test_compile_json_with_abi_top(make_input_bundle): """ input_bundle = make_input_bundle({"stream.json": stream, "code.vy": code}) vyper.compiler.compile_code(code, input_bundle=input_bundle) + + +def test_compile_json_with_experimental_codegen(): + code = { + "language": "Vyper", + "sources": {"foo.vy": {"content": "@external\ndef foo() -> bool:\n return True"}}, + "settings": { + "evmVersion": "cancun", + "optimize": "gas", + "venom": True, + "search_paths": [], + "outputSelection": {"*": ["ast"]}, + }, + } + + settings = get_settings(code) + assert settings.experimental_codegen is True + + +def test_compile_json_with_both_venom_aliases(): + code = { + "language": "Vyper", + "sources": {"foo.vy": {"content": ""}}, + "settings": { + "evmVersion": "cancun", + "optimize": "gas", + "experimentalCodegen": False, + "venom": False, + "search_paths": [], + "outputSelection": {"*": ["ast"]}, + }, + } + with pytest.raises(JSONError) as e: + get_settings(code) + assert e.value.args[0] == "both experimentalCodegen and venom cannot be set" diff --git a/vyper/ast/pre_parser.py b/vyper/ast/pre_parser.py index 07ba1d2d0d..5d2abcf645 100644 --- a/vyper/ast/pre_parser.py +++ b/vyper/ast/pre_parser.py @@ -265,10 +265,10 @@ def pre_parse(code: str) -> PreParseResult: if evm_version not in EVM_VERSIONS: raise StructureException(f"Invalid evm version: `{evm_version}`", start) settings.evm_version = evm_version - elif pragma.startswith("experimental-codegen"): + elif pragma.startswith("experimental-codegen") or pragma.startswith("venom"): if settings.experimental_codegen is not None: raise StructureException( - "pragma experimental-codegen specified twice!", start + "pragma experimental-codegen/venom specified twice!", start ) settings.experimental_codegen = True elif pragma.startswith("enable-decimals"): diff --git a/vyper/cli/vyper_compile.py b/vyper/cli/vyper_compile.py index 5999aed178..fde35f781e 100755 --- a/vyper/cli/vyper_compile.py +++ b/vyper/cli/vyper_compile.py @@ -176,6 +176,7 @@ def _parse_args(argv): parser.add_argument("-o", help="Set the output path", dest="output_path") parser.add_argument( "--experimental-codegen", + "--venom", help="The compiler use the new IR codegen. This is an experimental feature.", action="store_true", dest="experimental_codegen", diff --git a/vyper/cli/vyper_json.py b/vyper/cli/vyper_json.py index 7d0e6064af..9fcdf27baf 100755 --- a/vyper/cli/vyper_json.py +++ b/vyper/cli/vyper_json.py @@ -253,7 +253,13 @@ def get_settings(input_dict: dict) -> Settings: evm_version = get_evm_version(input_dict) optimize = input_dict["settings"].get("optimize") + experimental_codegen = input_dict["settings"].get("experimentalCodegen") + if experimental_codegen is None: + experimental_codegen = input_dict["settings"].get("venom") + elif input_dict["settings"].get("venom") is not None: + raise JSONError("both experimentalCodegen and venom cannot be set") + if isinstance(optimize, bool): # bool optimization level for backwards compatibility warnings.warn( diff --git a/vyper/compiler/settings.py b/vyper/compiler/settings.py index 7c20e03906..a8e28c1ed1 100644 --- a/vyper/compiler/settings.py +++ b/vyper/compiler/settings.py @@ -77,7 +77,7 @@ def as_cli(self): if self.optimize is not None: ret.append(" --optimize " + str(self.optimize)) if self.experimental_codegen is True: - ret.append(" --experimental-codegen") + ret.append(" --venom") if self.evm_version is not None: ret.append(" --evm-version " + self.evm_version) if self.debug is True: From 53f623b83d718fad78453142462736d99cdc6515 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 5 Nov 2024 09:17:02 -0500 Subject: [PATCH 06/16] fix[ci]: fix README encoding in `setup.py` (#4348) the call to `setup()` would fail on windows when there are unicode characters in `README.md`, because files are apparently opened with encoding `cp1252` by default on windows. this commit ensures the file is opened with `utf-8` encoding. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6e48129cba..5d6bd1db3a 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ extras_require["dev"] = extras_require["dev"] + extras_require["test"] + extras_require["lint"] -with open("README.md", "r") as f: +with open("README.md", "r", encoding="utf-8") as f: long_description = f.read() From 0abcf452b29f5348cb14233fd9a8444224392184 Mon Sep 17 00:00:00 2001 From: Rafael Abuawad Date: Tue, 5 Nov 2024 10:34:52 -0400 Subject: [PATCH 07/16] feat[docs]: add Telegram badge to README.md (#4342) --------- Co-authored-by: Charles Cooper --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index bcaa50b570..84c2948ceb 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ [![Build Status](https://github.com/vyperlang/vyper/workflows/Test/badge.svg)](https://github.com/vyperlang/vyper/actions/workflows/test.yml) [![Documentation Status](https://readthedocs.org/projects/vyper/badge/?version=latest)](http://docs.vyperlang.org/en/latest/?badge=latest "ReadTheDocs") [![Discord](https://img.shields.io/discord/969926564286459934.svg?label=%23vyper)](https://discord.gg/6tw7PTM7C2) +[![Telegram](https://img.shields.io/badge/Vyperholics🐍-Telegram-blue)](https://t.me/vyperlang) [![PyPI](https://badge.fury.io/py/vyper.svg)](https://pypi.org/project/vyper "PyPI") [![Docker](https://img.shields.io/docker/cloud/build/vyperlang/vyper)](https://hub.docker.com/r/vyperlang/vyper "DockerHub") From fee16e657abf9ab944c10ead75d5ce112fbf3a67 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 12 Nov 2024 10:31:56 +0400 Subject: [PATCH 08/16] feat[venom]: make cfg scheduler "stack aware" (#4356) this is a step towards making the cfg traversal order more "stack aware". previously we would blindly try to remove `iszero` instructions before a `jnz`. now, the heuristic is based on which basic block has a smaller sized dependency on this basic block. only if the two cfg_out blocks have the same sized dependency, then we try to remove the iszero. this creates a slight improvement to codesize due to fewer stack operations at `jnz` boundaries. --- .../compiler/venom/test_branch_optimizer.py | 6 ++-- vyper/venom/passes/branch_optimization.py | 30 ++++++++++++++----- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/tests/unit/compiler/venom/test_branch_optimizer.py b/tests/unit/compiler/venom/test_branch_optimizer.py index 82dff4777d..4c46127e1d 100644 --- a/tests/unit/compiler/venom/test_branch_optimizer.py +++ b/tests/unit/compiler/venom/test_branch_optimizer.py @@ -22,9 +22,9 @@ def test_simple_jump_case(): jnz_input = bb.append_instruction("iszero", op3) bb.append_instruction("jnz", jnz_input, br1.label, br2.label) - br1.append_instruction("add", op3, 10) + br1.append_instruction("add", op3, p1) br1.append_instruction("stop") - br2.append_instruction("add", op3, p1) + br2.append_instruction("add", op3, 10) br2.append_instruction("stop") term_inst = bb.instructions[-1] @@ -47,6 +47,6 @@ def test_simple_jump_case(): # Test that the dfg is updated correctly dfg = ac.request_analysis(DFGAnalysis) - assert dfg is old_dfg, "DFG should not be invalidated by BranchOptimizationPass" + assert dfg is not old_dfg, "DFG should be invalidated by BranchOptimizationPass" assert term_inst in dfg.get_uses(op3), "jnz not using the new condition" assert term_inst not in dfg.get_uses(jnz_input), "jnz still using the old condition" diff --git a/vyper/venom/passes/branch_optimization.py b/vyper/venom/passes/branch_optimization.py index d5b0ed9809..920dc5e431 100644 --- a/vyper/venom/passes/branch_optimization.py +++ b/vyper/venom/passes/branch_optimization.py @@ -1,4 +1,5 @@ -from vyper.venom.analysis import DFGAnalysis +from vyper.venom.analysis import CFGAnalysis, DFGAnalysis, LivenessAnalysis +from vyper.venom.basicblock import IRInstruction from vyper.venom.passes.base_pass import IRPass @@ -14,17 +15,30 @@ def _optimize_branches(self) -> None: if term_inst.opcode != "jnz": continue - prev_inst = self.dfg.get_producing_instruction(term_inst.operands[0]) - if prev_inst.opcode == "iszero": + fst, snd = bb.cfg_out + + fst_liveness = fst.instructions[0].liveness + snd_liveness = snd.instructions[0].liveness + + cost_a, cost_b = len(fst_liveness), len(snd_liveness) + + cond = term_inst.operands[0] + prev_inst = self.dfg.get_producing_instruction(cond) + if cost_a >= cost_b and prev_inst.opcode == "iszero": new_cond = prev_inst.operands[0] term_inst.operands = [new_cond, term_inst.operands[2], term_inst.operands[1]] - - # Since the DFG update is simple we do in place to avoid invalidating the DFG - # and having to recompute it (which is expensive(er)) - self.dfg.remove_use(prev_inst.output, term_inst) - self.dfg.add_use(new_cond, term_inst) + elif cost_a > cost_b: + new_cond = fn.get_next_variable() + inst = IRInstruction("iszero", [term_inst.operands[0]], output=new_cond) + bb.insert_instruction(inst, index=-1) + term_inst.operands = [new_cond, term_inst.operands[2], term_inst.operands[1]] def run_pass(self): + self.liveness = self.analyses_cache.request_analysis(LivenessAnalysis) + self.cfg = self.analyses_cache.request_analysis(CFGAnalysis) self.dfg = self.analyses_cache.request_analysis(DFGAnalysis) self._optimize_branches() + + self.analyses_cache.invalidate_analysis(LivenessAnalysis) + self.analyses_cache.invalidate_analysis(CFGAnalysis) From 48cb39bed2c3fd5403f64e4c89854f6c645231c1 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 12 Nov 2024 11:27:39 +0400 Subject: [PATCH 09/16] feat[venom]: improve liveness computation (#4330) traversing in reverse topsort order improves stack scheduling slightly this commit also adds a topsort method to CFGAnalysis, and speeds it up by only checking the terminator instruction instead of iterating over all the instructions in every basic block. additional refactors: - move dfs order calculation from domtree to cfg analysis. - remove unnecessary calculation of domtree in sccp - remove redundant IRFunction.compute_reachability - change cfg_out order - refactor shared phi fixup code - remove useless `__eq__()` and `__hash__()` for IRBasicBlock --- .../codegen/features/test_constructor.py | 4 -- .../codegen/types/test_dynamic_array.py | 2 - .../compiler/venom/test_branch_optimizer.py | 3 +- tests/unit/compiler/venom/test_sccp.py | 9 +-- vyper/venom/analysis/cfg.py | 47 ++++++++++----- vyper/venom/analysis/dominators.py | 29 +++------- vyper/venom/analysis/liveness.py | 14 ++--- vyper/venom/basicblock.py | 37 ++++++++---- vyper/venom/function.py | 58 +++++-------------- vyper/venom/passes/sccp/sccp.py | 35 +---------- vyper/venom/passes/simplify_cfg.py | 4 +- vyper/venom/venom_to_assembly.py | 10 +++- 12 files changed, 110 insertions(+), 142 deletions(-) diff --git a/tests/functional/codegen/features/test_constructor.py b/tests/functional/codegen/features/test_constructor.py index 3b86fe3460..6cc7007bb2 100644 --- a/tests/functional/codegen/features/test_constructor.py +++ b/tests/functional/codegen/features/test_constructor.py @@ -1,7 +1,4 @@ -import pytest - from tests.evm_backends.base_env import _compile -from vyper.exceptions import StackTooDeep from vyper.utils import method_id @@ -169,7 +166,6 @@ def get_foo() -> uint256: assert c.get_foo() == 39 -@pytest.mark.venom_xfail(raises=StackTooDeep, reason="stack scheduler regression") def test_nested_dynamic_array_constructor_arg_2(env, get_contract): code = """ foo: int128 diff --git a/tests/functional/codegen/types/test_dynamic_array.py b/tests/functional/codegen/types/test_dynamic_array.py index 2a0f4e77e5..2f647ac38c 100644 --- a/tests/functional/codegen/types/test_dynamic_array.py +++ b/tests/functional/codegen/types/test_dynamic_array.py @@ -11,7 +11,6 @@ CompilerPanic, ImmutableViolation, OverflowException, - StackTooDeep, StateAccessViolation, TypeMismatch, ) @@ -737,7 +736,6 @@ def test_array_decimal_return3() -> DynArray[DynArray[decimal, 2], 2]: ] -@pytest.mark.venom_xfail(raises=StackTooDeep, reason="stack scheduler regression") def test_mult_list(get_contract): code = """ nest3: DynArray[DynArray[DynArray[uint256, 2], 2], 2] diff --git a/tests/unit/compiler/venom/test_branch_optimizer.py b/tests/unit/compiler/venom/test_branch_optimizer.py index 4c46127e1d..a96ed0709c 100644 --- a/tests/unit/compiler/venom/test_branch_optimizer.py +++ b/tests/unit/compiler/venom/test_branch_optimizer.py @@ -16,6 +16,7 @@ def test_simple_jump_case(): fn.append_basic_block(br2) p1 = bb.append_instruction("param") + p2 = bb.append_instruction("param") op1 = bb.append_instruction("store", p1) op2 = bb.append_instruction("store", 64) op3 = bb.append_instruction("add", op1, op2) @@ -24,7 +25,7 @@ def test_simple_jump_case(): br1.append_instruction("add", op3, p1) br1.append_instruction("stop") - br2.append_instruction("add", op3, 10) + br2.append_instruction("add", op3, p2) br2.append_instruction("stop") term_inst = bb.instructions[-1] diff --git a/tests/unit/compiler/venom/test_sccp.py b/tests/unit/compiler/venom/test_sccp.py index 375dfd5dac..0d46b61acd 100644 --- a/tests/unit/compiler/venom/test_sccp.py +++ b/tests/unit/compiler/venom/test_sccp.py @@ -167,8 +167,8 @@ def test_cont_phi_case(): assert sccp.lattice[IRVariable("%2")].value == 32 assert sccp.lattice[IRVariable("%3")].value == 64 assert sccp.lattice[IRVariable("%4")].value == 96 - assert sccp.lattice[IRVariable("%5", version=1)].value == 106 - assert sccp.lattice[IRVariable("%5", version=2)] == LatticeEnum.BOTTOM + assert sccp.lattice[IRVariable("%5", version=2)].value == 106 + assert sccp.lattice[IRVariable("%5", version=1)] == LatticeEnum.BOTTOM assert sccp.lattice[IRVariable("%5")].value == 2 @@ -207,8 +207,9 @@ def test_cont_phi_const_case(): assert sccp.lattice[IRVariable("%2")].value == 32 assert sccp.lattice[IRVariable("%3")].value == 64 assert sccp.lattice[IRVariable("%4")].value == 96 - assert sccp.lattice[IRVariable("%5", version=1)].value == 106 - assert sccp.lattice[IRVariable("%5", version=2)].value == 97 + # dependent on cfg traversal order + assert sccp.lattice[IRVariable("%5", version=2)].value == 106 + assert sccp.lattice[IRVariable("%5", version=1)].value == 97 assert sccp.lattice[IRVariable("%5")].value == 2 diff --git a/vyper/venom/analysis/cfg.py b/vyper/venom/analysis/cfg.py index 90b18b353c..700fd73f26 100644 --- a/vyper/venom/analysis/cfg.py +++ b/vyper/venom/analysis/cfg.py @@ -1,6 +1,8 @@ +from typing import Iterator + from vyper.utils import OrderedSet from vyper.venom.analysis import IRAnalysis -from vyper.venom.basicblock import CFG_ALTERING_INSTRUCTIONS +from vyper.venom.basicblock import CFG_ALTERING_INSTRUCTIONS, IRBasicBlock class CFGAnalysis(IRAnalysis): @@ -8,28 +10,45 @@ class CFGAnalysis(IRAnalysis): Compute control flow graph information for each basic block in the function. """ + _dfs: OrderedSet[IRBasicBlock] + def analyze(self) -> None: fn = self.function + self._dfs = OrderedSet() + for bb in fn.get_basic_blocks(): bb.cfg_in = OrderedSet() bb.cfg_out = OrderedSet() bb.out_vars = OrderedSet() + bb.is_reachable = False for bb in fn.get_basic_blocks(): - assert len(bb.instructions) > 0, "Basic block should not be empty" - last_inst = bb.instructions[-1] - assert last_inst.is_bb_terminator, f"Last instruction should be a terminator {bb}" + assert bb.is_terminated - for inst in bb.instructions: - if inst.opcode in CFG_ALTERING_INSTRUCTIONS: - ops = inst.get_label_operands() - for op in ops: - fn.get_basic_block(op.value).add_cfg_in(bb) + term = bb.instructions[-1] + if term.opcode in CFG_ALTERING_INSTRUCTIONS: + ops = term.get_label_operands() + # order of cfg_out matters to performance! + for op in reversed(list(ops)): + next_bb = fn.get_basic_block(op.value) + bb.add_cfg_out(next_bb) + next_bb.add_cfg_in(bb) - # Fill in the "out" set for each basic block - for bb in fn.get_basic_blocks(): - for in_bb in bb.cfg_in: - in_bb.add_cfg_out(bb) + self._compute_dfs_r(self.function.entry) + + def _compute_dfs_r(self, bb): + if bb.is_reachable: + return + bb.is_reachable = True + + for out_bb in bb.cfg_out: + self._compute_dfs_r(out_bb) + + self._dfs.add(bb) + + @property + def dfs_walk(self) -> Iterator[IRBasicBlock]: + return iter(self._dfs) def invalidate(self): from vyper.venom.analysis import DFGAnalysis, DominatorTreeAnalysis, LivenessAnalysis @@ -37,5 +56,7 @@ def invalidate(self): self.analyses_cache.invalidate_analysis(DominatorTreeAnalysis) self.analyses_cache.invalidate_analysis(LivenessAnalysis) + self._dfs = None + # be conservative - assume cfg invalidation invalidates dfg self.analyses_cache.invalidate_analysis(DFGAnalysis) diff --git a/vyper/venom/analysis/dominators.py b/vyper/venom/analysis/dominators.py index e360df36b9..b60f9bdab9 100644 --- a/vyper/venom/analysis/dominators.py +++ b/vyper/venom/analysis/dominators.py @@ -1,3 +1,5 @@ +from functools import cached_property + from vyper.exceptions import CompilerPanic from vyper.utils import OrderedSet from vyper.venom.analysis import CFGAnalysis, IRAnalysis @@ -14,8 +16,6 @@ class DominatorTreeAnalysis(IRAnalysis): fn: IRFunction entry_block: IRBasicBlock - dfs_order: dict[IRBasicBlock, int] - dfs_walk: list[IRBasicBlock] dominators: dict[IRBasicBlock, OrderedSet[IRBasicBlock]] immediate_dominators: dict[IRBasicBlock, IRBasicBlock] dominated: dict[IRBasicBlock, OrderedSet[IRBasicBlock]] @@ -27,16 +27,13 @@ def analyze(self): """ self.fn = self.function self.entry_block = self.fn.entry - self.dfs_order = {} - self.dfs_walk = [] self.dominators = {} self.immediate_dominators = {} self.dominated = {} self.dominator_frontiers = {} - self.analyses_cache.request_analysis(CFGAnalysis) + self.cfg = self.analyses_cache.request_analysis(CFGAnalysis) - self._compute_dfs(self.entry_block, OrderedSet()) self._compute_dominators() self._compute_idoms() self._compute_df() @@ -131,21 +128,13 @@ def _intersect(self, bb1, bb2): bb2 = self.immediate_dominators[bb2] return bb1 - def _compute_dfs(self, entry: IRBasicBlock, visited): - """ - Depth-first search to compute the DFS order of the basic blocks. This - is used to compute the dominator tree. The sequence of basic blocks in - the DFS order is stored in `self.dfs_walk`. The DFS order of each basic - block is stored in `self.dfs_order`. - """ - visited.add(entry) - - for bb in entry.cfg_out: - if bb not in visited: - self._compute_dfs(bb, visited) + @cached_property + def dfs_walk(self) -> list[IRBasicBlock]: + return list(self.cfg.dfs_walk) - self.dfs_walk.append(entry) - self.dfs_order[entry] = len(self.dfs_walk) + @cached_property + def dfs_order(self) -> dict[IRBasicBlock, int]: + return {bb: idx for idx, bb in enumerate(self.dfs_walk)} def as_graph(self) -> str: """ diff --git a/vyper/venom/analysis/liveness.py b/vyper/venom/analysis/liveness.py index b5d65961b7..2ee28b9530 100644 --- a/vyper/venom/analysis/liveness.py +++ b/vyper/venom/analysis/liveness.py @@ -12,21 +12,21 @@ class LivenessAnalysis(IRAnalysis): """ def analyze(self): - self.analyses_cache.request_analysis(CFGAnalysis) + cfg = self.analyses_cache.request_analysis(CFGAnalysis) self._reset_liveness() - self._worklist = deque() - self._worklist.extend(self.function.get_basic_blocks()) + worklist = deque(cfg.dfs_walk) - while len(self._worklist) > 0: + while len(worklist) > 0: changed = False - bb = self._worklist.popleft() + + bb = worklist.popleft() changed |= self._calculate_out_vars(bb) changed |= self._calculate_liveness(bb) # recompute liveness for basic blocks pointing into # this basic block if changed: - self._worklist.extend(bb.cfg_in) + worklist.extend(bb.cfg_in) def _reset_liveness(self) -> None: for bb in self.function.get_basic_blocks(): @@ -64,7 +64,7 @@ def _calculate_out_vars(self, bb: IRBasicBlock) -> bool: bb.out_vars = OrderedSet() for out_bb in bb.cfg_out: target_vars = self.input_vars_from(bb, out_bb) - bb.out_vars = bb.out_vars.union(target_vars) + bb.out_vars.update(target_vars) return out_vars != bb.out_vars # calculate the input variables into self from source diff --git a/vyper/venom/basicblock.py b/vyper/venom/basicblock.py index eb4f7e67ba..f73f847a62 100644 --- a/vyper/venom/basicblock.py +++ b/vyper/venom/basicblock.py @@ -185,15 +185,6 @@ def __init__(self, value: str, is_symbol: bool = False) -> None: self.value = value self.is_symbol = is_symbol - def __eq__(self, other): - # no need for is_symbol to participate in equality - return super().__eq__(other) - - def __hash__(self): - # __hash__ is required when __eq__ is overridden -- - # https://docs.python.org/3/reference/datamodel.html#object.__hash__ - return super().__hash__() - class IRInstruction: """ @@ -393,7 +384,6 @@ class IRBasicBlock: # stack items which this basic block produces out_vars: OrderedSet[IRVariable] - reachable: OrderedSet["IRBasicBlock"] is_reachable: bool = False def __init__(self, label: IRLabel, parent: "IRFunction") -> None: @@ -404,7 +394,6 @@ def __init__(self, label: IRLabel, parent: "IRFunction") -> None: self.cfg_in = OrderedSet() self.cfg_out = OrderedSet() self.out_vars = OrderedSet() - self.reachable = OrderedSet() self.is_reachable = False def add_cfg_in(self, bb: "IRBasicBlock") -> None: @@ -495,6 +484,32 @@ def replace_operands(self, replacements: dict) -> None: for instruction in self.instructions: instruction.replace_operands(replacements) + def fix_phi_instructions(self): + cfg_in_labels = tuple(bb.label for bb in self.cfg_in) + + needs_sort = False + for inst in self.instructions: + if inst.opcode != "phi": + continue + + labels = inst.get_label_operands() + for label in labels: + if label not in cfg_in_labels: + needs_sort = True + inst.remove_phi_operand(label) + + op_len = len(inst.operands) + if op_len == 2: + inst.opcode = "store" + inst.operands = [inst.operands[1]] + elif op_len == 0: + inst.opcode = "nop" + inst.output = None + inst.operands = [] + + if needs_sort: + self.instructions.sort(key=lambda inst: inst.opcode != "phi") + def get_assignments(self): """ Get all assignments in basic block. diff --git a/vyper/venom/function.py b/vyper/venom/function.py index 2f2d477460..0c48c9740e 100644 --- a/vyper/venom/function.py +++ b/vyper/venom/function.py @@ -1,8 +1,7 @@ from typing import Iterator, Optional from vyper.codegen.ir_node import IRnode -from vyper.utils import OrderedSet -from vyper.venom.basicblock import CFG_ALTERING_INSTRUCTIONS, IRBasicBlock, IRLabel, IRVariable +from vyper.venom.basicblock import IRBasicBlock, IRLabel, IRVariable class IRFunction: @@ -89,60 +88,31 @@ def get_last_variable(self) -> str: return f"%{self.last_variable}" def remove_unreachable_blocks(self) -> int: - self._compute_reachability() + # Remove unreachable basic blocks + # pre: requires CFG analysis! + # NOTE: should this be a pass? - removed = [] + removed = set() - # Remove unreachable basic blocks for bb in self.get_basic_blocks(): if not bb.is_reachable: - removed.append(bb) + removed.add(bb) for bb in removed: self.remove_basic_block(bb) # Remove phi instructions that reference removed basic blocks - for bb in removed: - for out_bb in bb.cfg_out: - out_bb.remove_cfg_in(bb) - for inst in out_bb.instructions: - if inst.opcode != "phi": - continue - in_labels = inst.get_label_operands() - if bb.label in in_labels: - inst.remove_phi_operand(bb.label) - op_len = len(inst.operands) - if op_len == 2: - inst.opcode = "store" - inst.operands = [inst.operands[1]] - elif op_len == 0: - out_bb.remove_instruction(inst) - - return len(removed) - - def _compute_reachability(self) -> None: - """ - Compute reachability of basic blocks. - """ for bb in self.get_basic_blocks(): - bb.reachable = OrderedSet() - bb.is_reachable = False + for in_bb in list(bb.cfg_in): + if in_bb not in removed: + continue - self._compute_reachability_from(self.entry) + bb.remove_cfg_in(in_bb) - def _compute_reachability_from(self, bb: IRBasicBlock) -> None: - """ - Compute reachability of basic blocks from bb. - """ - if bb.is_reachable: - return - bb.is_reachable = True - for inst in bb.instructions: - if inst.opcode in CFG_ALTERING_INSTRUCTIONS: - for op in inst.get_label_operands(): - out_bb = self.get_basic_block(op.value) - bb.reachable.add(out_bb) - self._compute_reachability_from(out_bb) + # TODO: only run this if cfg_in changed + bb.fix_phi_instructions() + + return len(removed) @property def normalized(self) -> bool: diff --git a/vyper/venom/passes/sccp/sccp.py b/vyper/venom/passes/sccp/sccp.py index d85e09c9b4..2bdd0ace44 100644 --- a/vyper/venom/passes/sccp/sccp.py +++ b/vyper/venom/passes/sccp/sccp.py @@ -5,7 +5,7 @@ from vyper.exceptions import CompilerPanic, StaticAssertionException from vyper.utils import OrderedSet -from vyper.venom.analysis import CFGAnalysis, DFGAnalysis, DominatorTreeAnalysis, IRAnalysesCache +from vyper.venom.analysis import CFGAnalysis, DFGAnalysis, IRAnalysesCache from vyper.venom.basicblock import ( IRBasicBlock, IRInstruction, @@ -50,7 +50,6 @@ class SCCP(IRPass): """ fn: IRFunction - dom: DominatorTreeAnalysis dfg: DFGAnalysis lattice: Lattice work_list: list[WorkListItem] @@ -66,14 +65,13 @@ def __init__(self, analyses_cache: IRAnalysesCache, function: IRFunction): def run_pass(self): self.fn = self.function - self.dom = self.analyses_cache.request_analysis(DominatorTreeAnalysis) self.dfg = self.analyses_cache.request_analysis(DFGAnalysis) self._calculate_sccp(self.fn.entry) self._propagate_constants() if self.cfg_dirty: self.analyses_cache.force_analysis(CFGAnalysis) - self._fix_phi_nodes() + self.fn.remove_unreachable_blocks() self.analyses_cache.invalidate_analysis(DFGAnalysis) @@ -269,7 +267,7 @@ def _propagate_constants(self): with their actual values. It also replaces conditional jumps with unconditional jumps if the condition is a constant value. """ - for bb in self.dom.dfs_walk: + for bb in self.function.get_basic_blocks(): for inst in bb.instructions: self._replace_constants(inst) @@ -315,33 +313,6 @@ def _replace_constants(self, inst: IRInstruction): if isinstance(lat, IRLiteral): inst.operands[i] = lat - def _fix_phi_nodes(self): - # fix basic blocks whose cfg in was changed - # maybe this should really be done in _visit_phi - for bb in self.fn.get_basic_blocks(): - cfg_in_labels = OrderedSet(in_bb.label for in_bb in bb.cfg_in) - - needs_sort = False - for inst in bb.instructions: - if inst.opcode != "phi": - break - needs_sort |= self._fix_phi_inst(inst, cfg_in_labels) - - # move phi instructions to the top of the block - if needs_sort: - bb.instructions.sort(key=lambda inst: inst.opcode != "phi") - - def _fix_phi_inst(self, inst: IRInstruction, cfg_in_labels: OrderedSet): - operands = [op for label, op in inst.phi_operands if label in cfg_in_labels] - - if len(operands) != 1: - return False - - assert inst.output is not None - inst.opcode = "store" - inst.operands = operands - return True - def _meet(x: LatticeItem, y: LatticeItem) -> LatticeItem: if x == LatticeEnum.TOP: diff --git a/vyper/venom/passes/simplify_cfg.py b/vyper/venom/passes/simplify_cfg.py index acf37376e0..10535c2144 100644 --- a/vyper/venom/passes/simplify_cfg.py +++ b/vyper/venom/passes/simplify_cfg.py @@ -122,16 +122,16 @@ def run_pass(self): for _ in range(fn.num_basic_blocks): changes = self._optimize_empty_basicblocks() + self.analyses_cache.force_analysis(CFGAnalysis) changes += fn.remove_unreachable_blocks() if changes == 0: break else: raise CompilerPanic("Too many iterations removing empty basic blocks") - self.analyses_cache.force_analysis(CFGAnalysis) - for _ in range(fn.num_basic_blocks): # essentially `while True` self._collapse_chained_blocks(entry) + self.analyses_cache.force_analysis(CFGAnalysis) if fn.remove_unreachable_blocks() == 0: break else: diff --git a/vyper/venom/venom_to_assembly.py b/vyper/venom/venom_to_assembly.py index 264ec35eee..68c8fc3fd7 100644 --- a/vyper/venom/venom_to_assembly.py +++ b/vyper/venom/venom_to_assembly.py @@ -10,7 +10,12 @@ optimize_assembly, ) from vyper.utils import MemoryPositions, OrderedSet -from vyper.venom.analysis import IRAnalysesCache, LivenessAnalysis, VarEquivalenceAnalysis +from vyper.venom.analysis import ( + CFGAnalysis, + IRAnalysesCache, + LivenessAnalysis, + VarEquivalenceAnalysis, +) from vyper.venom.basicblock import ( IRBasicBlock, IRInstruction, @@ -152,6 +157,7 @@ def generate_evm(self, no_optimize: bool = False) -> list[str]: NormalizationPass(ac, fn).run_pass() self.liveness_analysis = ac.request_analysis(LivenessAnalysis) self.equivalence = ac.request_analysis(VarEquivalenceAnalysis) + ac.request_analysis(CFGAnalysis) assert fn.normalized, "Non-normalized CFG!" @@ -308,7 +314,7 @@ def _generate_evm_for_basicblock_r( ref.extend(asm) - for bb in basicblock.reachable: + for bb in basicblock.cfg_out: self._generate_evm_for_basicblock_r(ref, bb, stack.copy()) # pop values from stack at entry to bb From c32b9b4c6f0d8b8cdb103d3017ff540faf56a305 Mon Sep 17 00:00:00 2001 From: Harry Kalogirou Date: Tue, 12 Nov 2024 09:44:34 +0200 Subject: [PATCH 10/16] feat[venom]: new `DFTPass` algorithm (#4255) this commit upgrades the DFT algorithm to allow for more instruction movement and performs "multidimensional" fencing, which allows instructions to be reordered across volatile instructions if there is no effect barrier. since barriers do not truly live in the data dependency graph, it introduces a heuristic which chooses which barrier to recurse into first. it also removes the use of order ids and sorting, which improves performance. --------- Co-authored-by: Charles Cooper Co-authored-by: HodanPlodky <36966616+HodanPlodky@users.noreply.github.com> --- vyper/ir/compile_ir.py | 3 + vyper/venom/analysis/analysis.py | 2 +- vyper/venom/analysis/dfg.py | 8 +- vyper/venom/analysis/liveness.py | 2 +- vyper/venom/basicblock.py | 65 ++++++++++-- vyper/venom/effects.py | 6 ++ vyper/venom/passes/dft.py | 167 +++++++++++++++++++++---------- vyper/venom/venom_to_assembly.py | 8 +- 8 files changed, 195 insertions(+), 66 deletions(-) diff --git a/vyper/ir/compile_ir.py b/vyper/ir/compile_ir.py index 2cc951b188..e87cf1b310 100644 --- a/vyper/ir/compile_ir.py +++ b/vyper/ir/compile_ir.py @@ -1033,6 +1033,9 @@ def _stack_peephole_opts(assembly): if assembly[i] == "SWAP1" and assembly[i + 1].lower() in COMMUTATIVE_OPS: changed = True del assembly[i] + if assembly[i] == "DUP1" and assembly[i + 1] == "SWAP1": + changed = True + del assembly[i + 1] i += 1 return changed diff --git a/vyper/venom/analysis/analysis.py b/vyper/venom/analysis/analysis.py index f154993925..7bff6ba555 100644 --- a/vyper/venom/analysis/analysis.py +++ b/vyper/venom/analysis/analysis.py @@ -50,9 +50,9 @@ def request_analysis(self, analysis_cls: Type[IRAnalysis], *args, **kwargs): if analysis_cls in self.analyses_cache: return self.analyses_cache[analysis_cls] analysis = analysis_cls(self, self.function) + self.analyses_cache[analysis_cls] = analysis analysis.analyze(*args, **kwargs) - self.analyses_cache[analysis_cls] = analysis return analysis def invalidate_analysis(self, analysis_cls: Type[IRAnalysis]): diff --git a/vyper/venom/analysis/dfg.py b/vyper/venom/analysis/dfg.py index f49b2ac6ac..a2e050094d 100644 --- a/vyper/venom/analysis/dfg.py +++ b/vyper/venom/analysis/dfg.py @@ -3,7 +3,7 @@ from vyper.utils import OrderedSet from vyper.venom.analysis.analysis import IRAnalysesCache, IRAnalysis from vyper.venom.analysis.liveness import LivenessAnalysis -from vyper.venom.basicblock import IRInstruction, IRVariable +from vyper.venom.basicblock import IRBasicBlock, IRInstruction, IRVariable from vyper.venom.function import IRFunction @@ -20,6 +20,12 @@ def __init__(self, analyses_cache: IRAnalysesCache, function: IRFunction): def get_uses(self, op: IRVariable) -> OrderedSet[IRInstruction]: return self._dfg_inputs.get(op, OrderedSet()) + def get_uses_in_bb(self, op: IRVariable, bb: IRBasicBlock): + """ + Get uses of a given variable in a specific basic block. + """ + return [inst for inst in self.get_uses(op) if inst.parent == bb] + # the instruction which produces this variable. def get_producing_instruction(self, op: IRVariable) -> Optional[IRInstruction]: return self._dfg_outputs.get(op) diff --git a/vyper/venom/analysis/liveness.py b/vyper/venom/analysis/liveness.py index 2ee28b9530..0ccda3de2c 100644 --- a/vyper/venom/analysis/liveness.py +++ b/vyper/venom/analysis/liveness.py @@ -60,7 +60,7 @@ def _calculate_out_vars(self, bb: IRBasicBlock) -> bool: Compute out_vars of basic block. Returns True if out_vars changed """ - out_vars = bb.out_vars + out_vars = bb.out_vars.copy() bb.out_vars = OrderedSet() for out_bb in bb.cfg_out: target_vars = self.input_vars_from(bb, out_bb) diff --git a/vyper/venom/basicblock.py b/vyper/venom/basicblock.py index f73f847a62..c0abcefcb0 100644 --- a/vyper/venom/basicblock.py +++ b/vyper/venom/basicblock.py @@ -202,7 +202,6 @@ class IRInstruction: # set of live variables at this instruction liveness: OrderedSet[IRVariable] parent: "IRBasicBlock" - fence_id: int annotation: Optional[str] ast_source: Optional[IRnode] error_msg: Optional[str] @@ -219,7 +218,6 @@ def __init__( self.operands = list(operands) # in case we get an iterator self.output = output self.liveness = OrderedSet() - self.fence_id = -1 self.annotation = None self.ast_source = None self.error_msg = None @@ -236,6 +234,22 @@ def is_commutative(self) -> bool: def is_bb_terminator(self) -> bool: return self.opcode in BB_TERMINATORS + @property + def is_phi(self) -> bool: + return self.opcode == "phi" + + @property + def is_param(self) -> bool: + return self.opcode == "param" + + @property + def is_pseudo(self) -> bool: + """ + Check if instruction is pseudo, i.e. not an actual instruction but + a construct for intermediate representation like phi and param. + """ + return self.is_phi or self.is_param + def get_read_effects(self): return effects.reads.get(self.opcode, effects.EMPTY) @@ -321,6 +335,20 @@ def get_ast_source(self) -> Optional[IRnode]: return inst.ast_source return self.parent.parent.ast_source + def str_short(self) -> str: + s = "" + if self.output: + s += f"{self.output} = " + opcode = f"{self.opcode} " if self.opcode != "store" else "" + s += opcode + operands = self.operands + if opcode not in ["jmp", "jnz", "invoke"]: + operands = list(reversed(operands)) + s += ", ".join( + [(f"label %{op}" if isinstance(op, IRLabel) else str(op)) for op in operands] + ) + return s + def __repr__(self) -> str: s = "" if self.output: @@ -337,10 +365,7 @@ def __repr__(self) -> str: if self.annotation: s += f" <{self.annotation}>" - if self.liveness: - return f"{s: <30} # {self.liveness}" - - return s + return f"{s: <30}" def _ir_operand_from_value(val: Any) -> IROperand: @@ -477,6 +502,34 @@ def remove_instruction(self, instruction: IRInstruction) -> None: def clear_instructions(self) -> None: self.instructions = [] + @property + def phi_instructions(self) -> Iterator[IRInstruction]: + for inst in self.instructions: + if inst.opcode == "phi": + yield inst + else: + return + + @property + def non_phi_instructions(self) -> Iterator[IRInstruction]: + return (inst for inst in self.instructions if inst.opcode != "phi") + + @property + def param_instructions(self) -> Iterator[IRInstruction]: + for inst in self.instructions: + if inst.opcode == "param": + yield inst + else: + return + + @property + def pseudo_instructions(self) -> Iterator[IRInstruction]: + return (inst for inst in self.instructions if inst.is_pseudo) + + @property + def body_instructions(self) -> Iterator[IRInstruction]: + return (inst for inst in self.instructions[:-1] if not inst.is_pseudo) + def replace_operands(self, replacements: dict) -> None: """ Update operands with replacements. diff --git a/vyper/venom/effects.py b/vyper/venom/effects.py index 20cc0e4b02..a668ff5439 100644 --- a/vyper/venom/effects.py +++ b/vyper/venom/effects.py @@ -12,6 +12,11 @@ class Effects(Flag): BALANCE = auto() EXTCODE = auto() + def __iter__(self): + # python3.10 doesn't have an iter implementation. we can + # remove this once we drop python3.10 support. + return (m for m in self.__class__.__members__.values() if m in self) + EMPTY = Effects(0) ALL = ~EMPTY @@ -68,6 +73,7 @@ class Effects(Flag): "revert": MEMORY, "return": MEMORY, "sha3": MEMORY, + "sha3_64": MEMORY, "msize": MSIZE, } diff --git a/vyper/venom/passes/dft.py b/vyper/venom/passes/dft.py index 85f27867a7..2bf82810b6 100644 --- a/vyper/venom/passes/dft.py +++ b/vyper/venom/passes/dft.py @@ -1,81 +1,138 @@ +from collections import defaultdict + +import vyper.venom.effects as effects from vyper.utils import OrderedSet -from vyper.venom.analysis import DFGAnalysis -from vyper.venom.basicblock import IRBasicBlock, IRInstruction, IRVariable +from vyper.venom.analysis import DFGAnalysis, IRAnalysesCache, LivenessAnalysis +from vyper.venom.basicblock import IRBasicBlock, IRInstruction from vyper.venom.function import IRFunction from vyper.venom.passes.base_pass import IRPass class DFTPass(IRPass): function: IRFunction - inst_order: dict[IRInstruction, int] - inst_order_num: int + inst_offspring: dict[IRInstruction, OrderedSet[IRInstruction]] + visited_instructions: OrderedSet[IRInstruction] + ida: dict[IRInstruction, OrderedSet[IRInstruction]] + + def __init__(self, analyses_cache: IRAnalysesCache, function: IRFunction): + super().__init__(analyses_cache, function) + self.inst_offspring = {} + + def run_pass(self) -> None: + self.inst_offspring = {} + self.visited_instructions: OrderedSet[IRInstruction] = OrderedSet() + + self.dfg = self.analyses_cache.request_analysis(DFGAnalysis) + basic_blocks = list(self.function.get_basic_blocks()) + + self.function.clear_basic_blocks() + for bb in basic_blocks: + self._process_basic_block(bb) + + self.analyses_cache.invalidate_analysis(LivenessAnalysis) + + def _process_basic_block(self, bb: IRBasicBlock) -> None: + self.function.append_basic_block(bb) + + self._calculate_dependency_graphs(bb) + self.instructions = list(bb.pseudo_instructions) + non_phi_instructions = list(bb.non_phi_instructions) + + self.visited_instructions = OrderedSet() + for inst in non_phi_instructions: + self._calculate_instruction_offspring(inst) + + # Compute entry points in the graph of instruction dependencies + entry_instructions: OrderedSet[IRInstruction] = OrderedSet(non_phi_instructions) + for inst in non_phi_instructions: + to_remove = self.ida.get(inst, OrderedSet()) + if len(to_remove) > 0: + entry_instructions.dropmany(to_remove) + + entry_instructions_list = list(entry_instructions) - def _process_instruction_r(self, bb: IRBasicBlock, inst: IRInstruction, offset: int = 0): - for op in inst.get_outputs(): - assert isinstance(op, IRVariable), f"expected variable, got {op}" - uses = self.dfg.get_uses(op) + # Move the terminator instruction to the end of the list + self._move_terminator_to_end(entry_instructions_list) - for uses_this in uses: - if uses_this.parent != inst.parent or uses_this.fence_id != inst.fence_id: - # don't reorder across basic block or fence boundaries - continue + self.visited_instructions = OrderedSet() + for inst in entry_instructions_list: + self._process_instruction_r(self.instructions, inst) - # if the instruction is a terminator, we need to place - # it at the end of the basic block - # along with all the instructions that "lead" to it - self._process_instruction_r(bb, uses_this, offset) + bb.instructions = self.instructions + assert bb.is_terminated, f"Basic block should be terminated {bb}" + def _move_terminator_to_end(self, instructions: list[IRInstruction]) -> None: + terminator = next((inst for inst in instructions if inst.is_bb_terminator), None) + if terminator is None: + raise ValueError(f"Basic block should have a terminator instruction {self.function}") + instructions.remove(terminator) + instructions.append(terminator) + + def _process_instruction_r(self, instructions: list[IRInstruction], inst: IRInstruction): if inst in self.visited_instructions: return self.visited_instructions.add(inst) - self.inst_order_num += 1 - - if inst.is_bb_terminator: - offset = len(bb.instructions) - if inst.opcode == "phi": - # phi instructions stay at the beginning of the basic block - # and no input processing is needed - # bb.instructions.append(inst) - self.inst_order[inst] = 0 + if inst.is_pseudo: return - for op in inst.get_input_variables(): - target = self.dfg.get_producing_instruction(op) - assert target is not None, f"no producing instruction for {op}" - if target.parent != inst.parent or target.fence_id != inst.fence_id: - # don't reorder across basic block or fence boundaries - continue - self._process_instruction_r(bb, target, offset) + children = list(self.ida[inst]) - self.inst_order[inst] = self.inst_order_num + offset + def key(x): + cost = inst.operands.index(x.output) if x.output in inst.operands else 0 + return cost - len(self.inst_offspring[x]) * 0.5 - def _process_basic_block(self, bb: IRBasicBlock) -> None: - self.function.append_basic_block(bb) + # heuristic: sort by size of child dependency graph + children.sort(key=key) - for inst in bb.instructions: - inst.fence_id = self.fence_id - if inst.is_volatile: - self.fence_id += 1 + for dep_inst in children: + self._process_instruction_r(instructions, dep_inst) - # We go throught the instructions and calculate the order in which they should be executed - # based on the data flow graph. This order is stored in the inst_order dictionary. - # We then sort the instructions based on this order. - self.inst_order = {} - self.inst_order_num = 0 - for inst in bb.instructions: - self._process_instruction_r(bb, inst) + instructions.append(inst) - bb.instructions.sort(key=lambda x: self.inst_order[x]) + def _calculate_dependency_graphs(self, bb: IRBasicBlock) -> None: + # ida: instruction dependency analysis + self.ida = defaultdict(OrderedSet) - def run_pass(self) -> None: - self.dfg = self.analyses_cache.request_analysis(DFGAnalysis) + non_phis = list(bb.non_phi_instructions) - self.fence_id = 0 - self.visited_instructions: OrderedSet[IRInstruction] = OrderedSet() + # + # Compute dependency graph + # + last_write_effects: dict[effects.Effects, IRInstruction] = {} + last_read_effects: dict[effects.Effects, IRInstruction] = {} - basic_blocks = list(self.function.get_basic_blocks()) + for inst in non_phis: + for op in inst.operands: + dep = self.dfg.get_producing_instruction(op) + if dep is not None and dep.parent == bb: + self.ida[inst].add(dep) - self.function.clear_basic_blocks() - for bb in basic_blocks: - self._process_basic_block(bb) + write_effects = inst.get_write_effects() + read_effects = inst.get_read_effects() + + for write_effect in write_effects: + if write_effect in last_read_effects: + self.ida[inst].add(last_read_effects[write_effect]) + last_write_effects[write_effect] = inst + + for read_effect in read_effects: + if read_effect in last_write_effects and last_write_effects[read_effect] != inst: + self.ida[inst].add(last_write_effects[read_effect]) + last_read_effects[read_effect] = inst + + def _calculate_instruction_offspring(self, inst: IRInstruction): + if inst in self.inst_offspring: + return self.inst_offspring[inst] + + self.inst_offspring[inst] = self.ida[inst].copy() + + deps = self.ida[inst] + for dep_inst in deps: + assert inst.parent == dep_inst.parent + if dep_inst.opcode == "store": + continue + res = self._calculate_instruction_offspring(dep_inst) + self.inst_offspring[inst] |= res + + return self.inst_offspring[inst] diff --git a/vyper/venom/venom_to_assembly.py b/vyper/venom/venom_to_assembly.py index 68c8fc3fd7..9b52b842ba 100644 --- a/vyper/venom/venom_to_assembly.py +++ b/vyper/venom/venom_to_assembly.py @@ -600,10 +600,14 @@ def dup(self, assembly, stack, depth): assembly.append(_evm_dup_for(depth)) def swap_op(self, assembly, stack, op): - return self.swap(assembly, stack, stack.get_depth(op)) + depth = stack.get_depth(op) + assert depth is not StackModel.NOT_IN_STACK, f"Cannot swap non-existent operand {op}" + return self.swap(assembly, stack, depth) def dup_op(self, assembly, stack, op): - self.dup(assembly, stack, stack.get_depth(op)) + depth = stack.get_depth(op) + assert depth is not StackModel.NOT_IN_STACK, f"Cannot dup non-existent operand {op}" + self.dup(assembly, stack, depth) def _evm_swap_for(depth: int) -> str: From dbf9fa08e11d8b07df99bbca26973d5665249315 Mon Sep 17 00:00:00 2001 From: sudo rm -rf --no-preserve-root / Date: Tue, 19 Nov 2024 18:41:31 +0100 Subject: [PATCH 11/16] chore[docs]: add `method_id` to `abi_encode` signature (#4355) this commit adds the missing `method_id` parameter to the function signature of `abi_encode` in the documentation --- docs/built-in-functions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/built-in-functions.rst b/docs/built-in-functions.rst index a0e424adb4..7a79379d08 100644 --- a/docs/built-in-functions.rst +++ b/docs/built-in-functions.rst @@ -1023,7 +1023,7 @@ Utilities >>> ExampleContract.foo() 0xa9059cbb -.. py:function:: abi_encode(*args, ensure_tuple: bool = True) -> Bytes[] +.. py:function:: abi_encode(*args, ensure_tuple: bool = True, method_id: Bytes[4] = None) -> Bytes[] Takes a variable number of args as input, and returns the ABIv2-encoded bytestring. Used for packing arguments to raw_call, EIP712 and other cases where a consistent and efficient serialization method is needed. Once this function has seen more use we provisionally plan to put it into the ``ethereum.abi`` namespace. From 7d54f326076f4bbaffab56d8582d2e991b5a4f11 Mon Sep 17 00:00:00 2001 From: sandbubbles <160503471+sandbubbles@users.noreply.github.com> Date: Tue, 19 Nov 2024 18:06:50 +0000 Subject: [PATCH 12/16] fix[ux]: fix empty hints in error messages (#4351) remove empty hint message from exceptions. this regression was introduced in 03095ce3921636. the root cause of the issue, however, was that `hint=""` could be constructed in the first place. this commit fixes the `get_levenshtein_error_suggestions` helper so that it returns `None` on failure to find a suggestion rather than the empty string `""`. --------- Co-authored-by: Charles Cooper --- .../exceptions/test_undeclared_definition.py | 3 ++- .../syntax/exceptions/test_unknown_type.py | 15 +++++++++++++++ vyper/semantics/analysis/levenshtein_utils.py | 8 ++++---- 3 files changed, 21 insertions(+), 5 deletions(-) create mode 100644 tests/functional/syntax/exceptions/test_unknown_type.py diff --git a/tests/functional/syntax/exceptions/test_undeclared_definition.py b/tests/functional/syntax/exceptions/test_undeclared_definition.py index f90aa4137b..5786b37b1f 100644 --- a/tests/functional/syntax/exceptions/test_undeclared_definition.py +++ b/tests/functional/syntax/exceptions/test_undeclared_definition.py @@ -66,5 +66,6 @@ def foo(): @pytest.mark.parametrize("bad_code", fail_list) def test_undeclared_def_exception(bad_code): - with pytest.raises(UndeclaredDefinition): + with pytest.raises(UndeclaredDefinition) as e: compiler.compile_code(bad_code) + assert "(hint: )" not in str(e.value) diff --git a/tests/functional/syntax/exceptions/test_unknown_type.py b/tests/functional/syntax/exceptions/test_unknown_type.py new file mode 100644 index 0000000000..cd8866d5cb --- /dev/null +++ b/tests/functional/syntax/exceptions/test_unknown_type.py @@ -0,0 +1,15 @@ +import pytest + +from vyper import compiler +from vyper.exceptions import UnknownType + + +def test_unknown_type_exception(): + code = """ +@internal +def foobar(token: IERC20): + pass + """ + with pytest.raises(UnknownType) as e: + compiler.compile_code(code) + assert "(hint: )" not in str(e.value) diff --git a/vyper/semantics/analysis/levenshtein_utils.py b/vyper/semantics/analysis/levenshtein_utils.py index fc6e497d43..ac4fe4fab3 100644 --- a/vyper/semantics/analysis/levenshtein_utils.py +++ b/vyper/semantics/analysis/levenshtein_utils.py @@ -1,4 +1,4 @@ -from typing import Any, Callable +from typing import Any, Callable, Optional def levenshtein_norm(source: str, target: str) -> float: @@ -79,7 +79,7 @@ def get_levenshtein_error_suggestions(*args, **kwargs) -> Callable: def _get_levenshtein_error_suggestions( key: str, namespace: dict[str, Any], threshold: float -) -> str: +) -> Optional[str]: """ Generate an error message snippet for the suggested closest values in the provided namespace with the shortest normalized Levenshtein distance from the given key if that distance @@ -100,11 +100,11 @@ def _get_levenshtein_error_suggestions( """ if key is None or key == "": - return "" + return None distances = sorted([(i, levenshtein_norm(key, i)) for i in namespace], key=lambda k: k[1]) if len(distances) > 0 and distances[0][1] <= threshold: if len(distances) > 1 and distances[1][1] <= threshold: return f"Did you mean '{distances[0][0]}', or maybe '{distances[1][0]}'?" return f"Did you mean '{distances[0][0]}'?" - return "" + return None From 9697bae7dd95078bbcc59cb10fc1a85ca486b93f Mon Sep 17 00:00:00 2001 From: tserg <8017125+tserg@users.noreply.github.com> Date: Wed, 20 Nov 2024 02:21:15 +0800 Subject: [PATCH 13/16] refactor[ux]: refactor preparser (#4293) this PR refactors the pre-parsing routine to use a new `PreParser` object. this will make it easier in the future to keep track of state during pre-parsing. --------- Co-authored-by: Charles Cooper --- tests/functional/grammar/test_grammar.py | 7 +- .../ast/test_annotate_and_optimize_ast.py | 11 +- tests/unit/ast/test_pre_parser.py | 14 ++- vyper/ast/parse.py | 39 +++--- vyper/ast/pre_parser.py | 112 +++++++----------- 5 files changed, 84 insertions(+), 99 deletions(-) diff --git a/tests/functional/grammar/test_grammar.py b/tests/functional/grammar/test_grammar.py index 0ff8c23477..871ba4547f 100644 --- a/tests/functional/grammar/test_grammar.py +++ b/tests/functional/grammar/test_grammar.py @@ -9,7 +9,7 @@ from vyper.ast import Module, parse_to_ast from vyper.ast.grammar import parse_vyper_source, vyper_grammar -from vyper.ast.pre_parser import pre_parse +from vyper.ast.pre_parser import PreParser def test_basic_grammar(): @@ -102,6 +102,7 @@ def has_no_docstrings(c): max_examples=500, suppress_health_check=[HealthCheck.too_slow, HealthCheck.filter_too_much] ) def test_grammar_bruteforce(code): - pre_parse_result = pre_parse(code + "\n") - tree = parse_to_ast(pre_parse_result.reformatted_code) + pre_parser = PreParser() + pre_parser.parse(code + "\n") + tree = parse_to_ast(pre_parser.reformatted_code) assert isinstance(tree, Module) diff --git a/tests/unit/ast/test_annotate_and_optimize_ast.py b/tests/unit/ast/test_annotate_and_optimize_ast.py index 39ea899bd9..afba043113 100644 --- a/tests/unit/ast/test_annotate_and_optimize_ast.py +++ b/tests/unit/ast/test_annotate_and_optimize_ast.py @@ -1,6 +1,6 @@ import ast as python_ast -from vyper.ast.parse import annotate_python_ast, pre_parse +from vyper.ast.parse import PreParser, annotate_python_ast class AssertionVisitor(python_ast.NodeVisitor): @@ -28,12 +28,13 @@ def foo() -> int128: def get_contract_info(source_code): - pre_parse_result = pre_parse(source_code) - py_ast = python_ast.parse(pre_parse_result.reformatted_code) + pre_parser = PreParser() + pre_parser.parse(source_code) + py_ast = python_ast.parse(pre_parser.reformatted_code) - annotate_python_ast(py_ast, pre_parse_result.reformatted_code, pre_parse_result) + annotate_python_ast(py_ast, pre_parser.reformatted_code, pre_parser) - return py_ast, pre_parse_result.reformatted_code + return py_ast, pre_parser.reformatted_code def test_it_annotates_ast_with_source_code(): diff --git a/tests/unit/ast/test_pre_parser.py b/tests/unit/ast/test_pre_parser.py index 5d3f30481c..73712aadb8 100644 --- a/tests/unit/ast/test_pre_parser.py +++ b/tests/unit/ast/test_pre_parser.py @@ -1,7 +1,7 @@ import pytest from vyper import compile_code -from vyper.ast.pre_parser import pre_parse, validate_version_pragma +from vyper.ast.pre_parser import PreParser, validate_version_pragma from vyper.compiler.phases import CompilerData from vyper.compiler.settings import OptimizationLevel, Settings from vyper.exceptions import StructureException, VersionException @@ -174,9 +174,10 @@ def test_prerelease_invalid_version_pragma(file_version, mock_version): @pytest.mark.parametrize("code, pre_parse_settings, compiler_data_settings", pragma_examples) def test_parse_pragmas(code, pre_parse_settings, compiler_data_settings, mock_version): mock_version("0.3.10") - pre_parse_result = pre_parse(code) + pre_parser = PreParser() + pre_parser.parse(code) - assert pre_parse_result.settings == pre_parse_settings + assert pre_parser.settings == pre_parse_settings compiler_data = CompilerData(code) @@ -203,8 +204,9 @@ def test_parse_pragmas(code, pre_parse_settings, compiler_data_settings, mock_ve @pytest.mark.parametrize("code", pragma_venom) def test_parse_venom_pragma(code): - pre_parse_result = pre_parse(code) - assert pre_parse_result.settings.experimental_codegen is True + pre_parser = PreParser() + pre_parser.parse(code) + assert pre_parser.settings.experimental_codegen is True compiler_data = CompilerData(code) assert compiler_data.settings.experimental_codegen is True @@ -252,7 +254,7 @@ def test_parse_venom_pragma(code): @pytest.mark.parametrize("code", invalid_pragmas) def test_invalid_pragma(code): with pytest.raises(StructureException): - pre_parse(code) + PreParser().parse(code) def test_version_exception_in_import(make_input_bundle): diff --git a/vyper/ast/parse.py b/vyper/ast/parse.py index 1e88241186..5d62072b9e 100644 --- a/vyper/ast/parse.py +++ b/vyper/ast/parse.py @@ -6,7 +6,7 @@ import asttokens from vyper.ast import nodes as vy_ast -from vyper.ast.pre_parser import PreParseResult, pre_parse +from vyper.ast.pre_parser import PreParser from vyper.compiler.settings import Settings from vyper.exceptions import CompilerPanic, ParserException, SyntaxException from vyper.utils import sha256sum, vyper_warn @@ -54,9 +54,10 @@ def parse_to_ast_with_settings( """ if "\x00" in vyper_source: raise ParserException("No null bytes (\\x00) allowed in the source code.") - pre_parse_result = pre_parse(vyper_source) + pre_parser = PreParser() + pre_parser.parse(vyper_source) try: - py_ast = python_ast.parse(pre_parse_result.reformatted_code) + py_ast = python_ast.parse(pre_parser.reformatted_code) except SyntaxError as e: # TODO: Ensure 1-to-1 match of source_code:reformatted_code SyntaxErrors raise SyntaxException(str(e), vyper_source, e.lineno, e.offset) from None @@ -72,20 +73,20 @@ def parse_to_ast_with_settings( annotate_python_ast( py_ast, vyper_source, - pre_parse_result, + pre_parser, source_id=source_id, module_path=module_path, resolved_path=resolved_path, ) # postcondition: consumed all the for loop annotations - assert len(pre_parse_result.for_loop_annotations) == 0 + assert len(pre_parser.for_loop_annotations) == 0 # Convert to Vyper AST. module = vy_ast.get_node(py_ast) assert isinstance(module, vy_ast.Module) # mypy hint - return pre_parse_result.settings, module + return pre_parser.settings, module def ast_to_dict(ast_struct: Union[vy_ast.VyperNode, List]) -> Union[Dict, List]: @@ -116,7 +117,7 @@ def dict_to_ast(ast_struct: Union[Dict, List]) -> Union[vy_ast.VyperNode, List]: def annotate_python_ast( parsed_ast: python_ast.AST, vyper_source: str, - pre_parse_result: PreParseResult, + pre_parser: PreParser, source_id: int = 0, module_path: Optional[str] = None, resolved_path: Optional[str] = None, @@ -130,8 +131,8 @@ def annotate_python_ast( The AST to be annotated and optimized. vyper_source: str The original vyper source code - pre_parse_result: PreParseResult - Outputs from pre-parsing. + pre_parser: PreParser + PreParser object. Returns ------- @@ -142,7 +143,7 @@ def annotate_python_ast( tokens.mark_tokens(parsed_ast) visitor = AnnotatingVisitor( vyper_source, - pre_parse_result, + pre_parser, tokens, source_id, module_path=module_path, @@ -155,12 +156,12 @@ def annotate_python_ast( class AnnotatingVisitor(python_ast.NodeTransformer): _source_code: str - _pre_parse_result: PreParseResult + _pre_parser: PreParser def __init__( self, source_code: str, - pre_parse_result: PreParseResult, + pre_parser: PreParser, tokens: asttokens.ASTTokens, source_id: int, module_path: Optional[str] = None, @@ -171,7 +172,7 @@ def __init__( self._module_path = module_path self._resolved_path = resolved_path self._source_code = source_code - self._pre_parse_result = pre_parse_result + self._pre_parser = pre_parser self.counter: int = 0 @@ -265,7 +266,7 @@ def visit_ClassDef(self, node): """ self.generic_visit(node) - node.ast_type = self._pre_parse_result.modification_offsets[(node.lineno, node.col_offset)] + node.ast_type = self._pre_parser.modification_offsets[(node.lineno, node.col_offset)] return node def visit_For(self, node): @@ -274,7 +275,7 @@ def visit_For(self, node): the pre-parser """ key = (node.lineno, node.col_offset) - annotation_tokens = self._pre_parse_result.for_loop_annotations.pop(key) + annotation_tokens = self._pre_parser.for_loop_annotations.pop(key) if not annotation_tokens: # a common case for people migrating to 0.4.0, provide a more @@ -342,14 +343,14 @@ def visit_Expr(self, node): # CMC 2024-03-03 consider unremoving this from the enclosing Expr node = node.value key = (node.lineno, node.col_offset) - node.ast_type = self._pre_parse_result.modification_offsets[key] + node.ast_type = self._pre_parser.modification_offsets[key] return node def visit_Await(self, node): start_pos = node.lineno, node.col_offset # grab these before generic_visit modifies them self.generic_visit(node) - node.ast_type = self._pre_parse_result.modification_offsets[start_pos] + node.ast_type = self._pre_parser.modification_offsets[start_pos] return node def visit_Call(self, node): @@ -394,10 +395,10 @@ def visit_Constant(self, node): node.ast_type = "NameConstant" elif isinstance(node.value, str): key = (node.lineno, node.col_offset) - if key in self._pre_parse_result.native_hex_literal_locations: + if key in self._pre_parser.hex_string_locations: if len(node.value) % 2 != 0: raise SyntaxException( - "Native hex string must have an even number of characters", + "Hex string must have an even number of characters", self._source_code, node.lineno, node.col_offset, diff --git a/vyper/ast/pre_parser.py b/vyper/ast/pre_parser.py index 5d2abcf645..dbeb6181f9 100644 --- a/vyper/ast/pre_parser.py +++ b/vyper/ast/pre_parser.py @@ -158,67 +158,52 @@ def consume(self, token, result): CUSTOM_EXPRESSION_TYPES = {"extcall": "ExtCall", "staticcall": "StaticCall"} -class PreParseResult: +class PreParser: # Compilation settings based on the directives in the source code settings: Settings # A mapping of class names to their original class types. modification_offsets: dict[tuple[int, int], str] # A mapping of line/column offsets of `For` nodes to the annotation of the for loop target for_loop_annotations: dict[tuple[int, int], list[TokenInfo]] - # A list of line/column offsets of native hex literals - native_hex_literal_locations: list[tuple[int, int]] + # A list of line/column offsets of hex string literals + hex_string_locations: list[tuple[int, int]] # Reformatted python source string. reformatted_code: str - def __init__( - self, - settings, - modification_offsets, - for_loop_annotations, - native_hex_literal_locations, - reformatted_code, - ): - self.settings = settings - self.modification_offsets = modification_offsets - self.for_loop_annotations = for_loop_annotations - self.native_hex_literal_locations = native_hex_literal_locations - self.reformatted_code = reformatted_code - - -def pre_parse(code: str) -> PreParseResult: - """ - Re-formats a vyper source string into a python source string and performs - some validation. More specifically, - - * Translates "interface", "struct", "flag", and "event" keywords into python "class" keyword - * Validates "@version" pragma against current compiler version - * Prevents direct use of python "class" keyword - * Prevents use of python semi-colon statement separator - * Extracts type annotation of for loop iterators into a separate dictionary - - Also returns a mapping of detected interface and struct names to their - respective vyper class types ("interface" or "struct"), and a mapping of line numbers - of for loops to the type annotation of their iterators. - - Parameters - ---------- - code : str - The vyper source code to be re-formatted. - - Returns - ------- - PreParseResult - Outputs for transforming the python AST to vyper AST - """ - result: list[TokenInfo] = [] - modification_offsets: dict[tuple[int, int], str] = {} - settings = Settings() - for_parser = ForParser(code) - native_hex_parser = HexStringParser() + def parse(self, code: str): + """ + Re-formats a vyper source string into a python source string and performs + some validation. More specifically, + + * Translates "interface", "struct", "flag", and "event" keywords into python "class" keyword + * Validates "@version" pragma against current compiler version + * Prevents direct use of python "class" keyword + * Prevents use of python semi-colon statement separator + * Extracts type annotation of for loop iterators into a separate dictionary + + Stores a mapping of detected interface and struct names to their + respective vyper class types ("interface" or "struct"), and a mapping of line numbers + of for loops to the type annotation of their iterators. + + Parameters + ---------- + code : str + The vyper source code to be re-formatted. + """ + try: + self._parse(code) + except TokenError as e: + raise SyntaxException(e.args[0], code, e.args[1][0], e.args[1][1]) from e + + def _parse(self, code: str): + result: list[TokenInfo] = [] + modification_offsets: dict[tuple[int, int], str] = {} + settings = Settings() + for_parser = ForParser(code) + hex_string_parser = HexStringParser() + + _col_adjustments: dict[int, int] = defaultdict(lambda: 0) - _col_adjustments: dict[int, int] = defaultdict(lambda: 0) - - try: code_bytes = code.encode("utf-8") token_list = list(tokenize(io.BytesIO(code_bytes).readline)) @@ -301,7 +286,7 @@ def pre_parse(code: str) -> PreParseResult: # a bit cursed technique to get untokenize to put # the new tokens in the right place so that modification_offsets # will work correctly. - # (recommend comparing the result of pre_parse with the + # (recommend comparing the result of parse with the # source code side by side to visualize the whitespace) new_keyword = "await" vyper_type = CUSTOM_EXPRESSION_TYPES[string] @@ -322,20 +307,15 @@ def pre_parse(code: str) -> PreParseResult: if (typ, string) == (OP, ";"): raise SyntaxException("Semi-colon statements not allowed", code, start[0], start[1]) - if not for_parser.consume(token) and not native_hex_parser.consume(token, result): + if not for_parser.consume(token) and not hex_string_parser.consume(token, result): result.extend(toks) - except TokenError as e: - raise SyntaxException(e.args[0], code, e.args[1][0], e.args[1][1]) from e - - for_loop_annotations = {} - for k, v in for_parser.annotations.items(): - for_loop_annotations[k] = v.copy() + for_loop_annotations = {} + for k, v in for_parser.annotations.items(): + for_loop_annotations[k] = v.copy() - return PreParseResult( - settings, - modification_offsets, - for_loop_annotations, - native_hex_parser.locations, - untokenize(result).decode("utf-8"), - ) + self.settings = settings + self.modification_offsets = modification_offsets + self.for_loop_annotations = for_loop_annotations + self.hex_string_locations = hex_string_parser.locations + self.reformatted_code = untokenize(result).decode("utf-8") From 0c23b214018ac3a09bdc2df31843a29467ae3695 Mon Sep 17 00:00:00 2001 From: Harry Kalogirou Date: Wed, 20 Nov 2024 16:27:34 +0200 Subject: [PATCH 14/16] refactor[venom]: optimize lattice evaluation (#4368) this commit improves the `_eval()` method in the SCCP pass by trying to return early when possible and avoid unnecessary set operations. it also performs some minor code clean up and dead code removal. --- vyper/venom/passes/sccp/sccp.py | 48 +++++++++++++++++---------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/vyper/venom/passes/sccp/sccp.py b/vyper/venom/passes/sccp/sccp.py index 2bdd0ace44..2be84ce502 100644 --- a/vyper/venom/passes/sccp/sccp.py +++ b/vyper/venom/passes/sccp/sccp.py @@ -227,35 +227,37 @@ def _eval(self, inst) -> LatticeItem: instruction to the SSA work list if the knowledge about the variable changed. """ - opcode = inst.opcode - ops = [] + def finalize(ret): + # Update the lattice if the value changed + old_val = self.lattice.get(inst.output, LatticeEnum.TOP) + if old_val != ret: + self.lattice[inst.output] = ret + self._add_ssa_work_items(inst) + return ret + + opcode = inst.opcode + ops: list[IROperand] = [] for op in inst.operands: - if isinstance(op, IRVariable): - ops.append(self.lattice[op]) - elif isinstance(op, IRLabel): - return LatticeEnum.BOTTOM + # Evaluate the operand according to the lattice + if isinstance(op, IRLabel): + return finalize(LatticeEnum.BOTTOM) + elif isinstance(op, IRVariable): + eval_result = self.lattice[op] else: - ops.append(op) + eval_result = op - ret = None - if LatticeEnum.BOTTOM in ops: - ret = LatticeEnum.BOTTOM - else: - if opcode in ARITHMETIC_OPS: - fn = ARITHMETIC_OPS[opcode] - ret = IRLiteral(fn(ops)) # type: ignore - elif len(ops) > 0: - ret = ops[0] # type: ignore - else: - raise CompilerPanic("Bad constant evaluation") + # If any operand is BOTTOM, the whole operation is BOTTOM + # and we can stop the evaluation early + if eval_result is LatticeEnum.BOTTOM: + return finalize(LatticeEnum.BOTTOM) - old_val = self.lattice.get(inst.output, LatticeEnum.TOP) - if old_val != ret: - self.lattice[inst.output] = ret # type: ignore - self._add_ssa_work_items(inst) + assert isinstance(eval_result, IROperand) + ops.append(eval_result) - return ret # type: ignore + # If we haven't found BOTTOM yet, evaluate the operation + fn = ARITHMETIC_OPS[opcode] + return finalize(IRLiteral(fn(ops))) def _add_ssa_work_items(self, inst: IRInstruction): for target_inst in self.dfg.get_uses(inst.output): # type: ignore From 215de1da6c53fcc97d00f38621d13b622d4a224f Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 20 Nov 2024 22:29:18 +0700 Subject: [PATCH 15/16] feat[venom]: make dft pass commutative aware (#4358) this commit performs some code cleanup for `DFTPass`. it separates the effects graph from the data dependency graph, and applies the heuristic for barriers to commutative and comparator instructions as well. --- vyper/venom/basicblock.py | 25 ++++++++++++ vyper/venom/passes/dft.py | 86 ++++++++++++++++++--------------------- 2 files changed, 64 insertions(+), 47 deletions(-) diff --git a/vyper/venom/basicblock.py b/vyper/venom/basicblock.py index c0abcefcb0..968ce42bdf 100644 --- a/vyper/venom/basicblock.py +++ b/vyper/venom/basicblock.py @@ -84,6 +84,8 @@ COMMUTATIVE_INSTRUCTIONS = frozenset(["add", "mul", "smul", "or", "xor", "and", "eq"]) +COMPARATOR_INSTRUCTIONS = ("gt", "lt", "sgt", "slt") + if TYPE_CHECKING: from vyper.venom.function import IRFunction @@ -230,6 +232,14 @@ def is_volatile(self) -> bool: def is_commutative(self) -> bool: return self.opcode in COMMUTATIVE_INSTRUCTIONS + @property + def is_comparator(self) -> bool: + return self.opcode in COMPARATOR_INSTRUCTIONS + + @property + def flippable(self) -> bool: + return self.is_commutative or self.is_comparator + @property def is_bb_terminator(self) -> bool: return self.opcode in BB_TERMINATORS @@ -282,6 +292,21 @@ def get_outputs(self) -> list[IROperand]: """ return [self.output] if self.output else [] + def flip(self): + """ + Flip operands for commutative or comparator opcodes + """ + assert self.flippable + self.operands.reverse() + + if self.is_commutative: + return + + if self.opcode in ("gt", "sgt"): + self.opcode = self.opcode.replace("g", "l") + else: + self.opcode = self.opcode.replace("l", "g") + def replace_operands(self, replacements: dict) -> None: """ Update operands with replacements. diff --git a/vyper/venom/passes/dft.py b/vyper/venom/passes/dft.py index 2bf82810b6..a8d68ad676 100644 --- a/vyper/venom/passes/dft.py +++ b/vyper/venom/passes/dft.py @@ -2,7 +2,7 @@ import vyper.venom.effects as effects from vyper.utils import OrderedSet -from vyper.venom.analysis import DFGAnalysis, IRAnalysesCache, LivenessAnalysis +from vyper.venom.analysis import DFGAnalysis, LivenessAnalysis from vyper.venom.basicblock import IRBasicBlock, IRInstruction from vyper.venom.function import IRFunction from vyper.venom.passes.base_pass import IRPass @@ -10,50 +10,41 @@ class DFTPass(IRPass): function: IRFunction - inst_offspring: dict[IRInstruction, OrderedSet[IRInstruction]] + data_offspring: dict[IRInstruction, OrderedSet[IRInstruction]] visited_instructions: OrderedSet[IRInstruction] - ida: dict[IRInstruction, OrderedSet[IRInstruction]] - - def __init__(self, analyses_cache: IRAnalysesCache, function: IRFunction): - super().__init__(analyses_cache, function) - self.inst_offspring = {} + # "data dependency analysis" + dda: dict[IRInstruction, OrderedSet[IRInstruction]] + # "effect dependency analysis" + eda: dict[IRInstruction, OrderedSet[IRInstruction]] def run_pass(self) -> None: - self.inst_offspring = {} + self.data_offspring = {} self.visited_instructions: OrderedSet[IRInstruction] = OrderedSet() self.dfg = self.analyses_cache.request_analysis(DFGAnalysis) - basic_blocks = list(self.function.get_basic_blocks()) - self.function.clear_basic_blocks() - for bb in basic_blocks: + for bb in self.function.get_basic_blocks(): self._process_basic_block(bb) self.analyses_cache.invalidate_analysis(LivenessAnalysis) def _process_basic_block(self, bb: IRBasicBlock) -> None: - self.function.append_basic_block(bb) - self._calculate_dependency_graphs(bb) self.instructions = list(bb.pseudo_instructions) non_phi_instructions = list(bb.non_phi_instructions) self.visited_instructions = OrderedSet() - for inst in non_phi_instructions: - self._calculate_instruction_offspring(inst) + for inst in bb.instructions: + self._calculate_data_offspring(inst) # Compute entry points in the graph of instruction dependencies entry_instructions: OrderedSet[IRInstruction] = OrderedSet(non_phi_instructions) for inst in non_phi_instructions: - to_remove = self.ida.get(inst, OrderedSet()) - if len(to_remove) > 0: - entry_instructions.dropmany(to_remove) + to_remove = self.dda.get(inst, OrderedSet()) | self.eda.get(inst, OrderedSet()) + entry_instructions.dropmany(to_remove) entry_instructions_list = list(entry_instructions) - # Move the terminator instruction to the end of the list - self._move_terminator_to_end(entry_instructions_list) - self.visited_instructions = OrderedSet() for inst in entry_instructions_list: self._process_instruction_r(self.instructions, inst) @@ -61,13 +52,6 @@ def _process_basic_block(self, bb: IRBasicBlock) -> None: bb.instructions = self.instructions assert bb.is_terminated, f"Basic block should be terminated {bb}" - def _move_terminator_to_end(self, instructions: list[IRInstruction]) -> None: - terminator = next((inst for inst in instructions if inst.is_bb_terminator), None) - if terminator is None: - raise ValueError(f"Basic block should have a terminator instruction {self.function}") - instructions.remove(terminator) - instructions.append(terminator) - def _process_instruction_r(self, instructions: list[IRInstruction], inst: IRInstruction): if inst in self.visited_instructions: return @@ -76,14 +60,23 @@ def _process_instruction_r(self, instructions: list[IRInstruction], inst: IRInst if inst.is_pseudo: return - children = list(self.ida[inst]) + children = list(self.dda[inst] | self.eda[inst]) - def key(x): - cost = inst.operands.index(x.output) if x.output in inst.operands else 0 - return cost - len(self.inst_offspring[x]) * 0.5 + def cost(x: IRInstruction) -> int | float: + if x in self.eda[inst] or inst.flippable: + ret = -1 * int(len(self.data_offspring[x]) > 0) + else: + assert x in self.dda[inst] # sanity check + assert x.output is not None # help mypy + ret = inst.operands.index(x.output) + return ret # heuristic: sort by size of child dependency graph - children.sort(key=key) + orig_children = children.copy() + children.sort(key=cost) + + if inst.flippable and (orig_children != children): + inst.flip() for dep_inst in children: self._process_instruction_r(instructions, dep_inst) @@ -92,7 +85,8 @@ def key(x): def _calculate_dependency_graphs(self, bb: IRBasicBlock) -> None: # ida: instruction dependency analysis - self.ida = defaultdict(OrderedSet) + self.dda = defaultdict(OrderedSet) + self.eda = defaultdict(OrderedSet) non_phis = list(bb.non_phi_instructions) @@ -106,33 +100,31 @@ def _calculate_dependency_graphs(self, bb: IRBasicBlock) -> None: for op in inst.operands: dep = self.dfg.get_producing_instruction(op) if dep is not None and dep.parent == bb: - self.ida[inst].add(dep) + self.dda[inst].add(dep) write_effects = inst.get_write_effects() read_effects = inst.get_read_effects() for write_effect in write_effects: if write_effect in last_read_effects: - self.ida[inst].add(last_read_effects[write_effect]) + self.eda[inst].add(last_read_effects[write_effect]) last_write_effects[write_effect] = inst for read_effect in read_effects: if read_effect in last_write_effects and last_write_effects[read_effect] != inst: - self.ida[inst].add(last_write_effects[read_effect]) + self.eda[inst].add(last_write_effects[read_effect]) last_read_effects[read_effect] = inst - def _calculate_instruction_offspring(self, inst: IRInstruction): - if inst in self.inst_offspring: - return self.inst_offspring[inst] + def _calculate_data_offspring(self, inst: IRInstruction): + if inst in self.data_offspring: + return self.data_offspring[inst] - self.inst_offspring[inst] = self.ida[inst].copy() + self.data_offspring[inst] = self.dda[inst].copy() - deps = self.ida[inst] + deps = self.dda[inst] for dep_inst in deps: assert inst.parent == dep_inst.parent - if dep_inst.opcode == "store": - continue - res = self._calculate_instruction_offspring(dep_inst) - self.inst_offspring[inst] |= res + res = self._calculate_data_offspring(dep_inst) + self.data_offspring[inst] |= res - return self.inst_offspring[inst] + return self.data_offspring[inst] From e0fc53a10e69abd55ad563f98477bcf9568d76b0 Mon Sep 17 00:00:00 2001 From: sandbubbles <160503471+sandbubbles@users.noreply.github.com> Date: Wed, 20 Nov 2024 20:42:34 +0000 Subject: [PATCH 16/16] chore[docs]: mention the `--venom` flag in venom docs (#4353) add some notes to the relevant docs that `venom` is an alias for `experimental-codegen` --- docs/compiling-a-contract.rst | 7 ++++++- docs/structure-of-a-contract.rst | 10 ++++++++++ vyper/cli/vyper_compile.py | 6 +++++- vyper/venom/README.md | 2 +- 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/docs/compiling-a-contract.rst b/docs/compiling-a-contract.rst index c2cd3ed22c..7132cff58d 100644 --- a/docs/compiling-a-contract.rst +++ b/docs/compiling-a-contract.rst @@ -31,7 +31,7 @@ Include the ``-f`` flag to specify which output formats to return. Use ``vyper - .. code:: shell - $ vyper -f abi,abi_python,bytecode,bytecode_runtime,blueprint_bytecode,interface,external_interface,ast,annotated_ast,integrity,ir,ir_json,ir_runtime,asm,opcodes,opcodes_runtime,source_map,source_map_runtime,archive,solc_json,method_identifiers,userdoc,devdoc,metadata,combined_json,layout yourFileName.vy + $ vyper -f abi,abi_python,bb,bb_runtime,bytecode,bytecode_runtime,blueprint_bytecode,cfg,cfg_runtime,interface,external_interface,ast,annotated_ast,integrity,ir,ir_json,ir_runtime,asm,opcodes,opcodes_runtime,source_map,source_map_runtime,archive,solc_json,method_identifiers,userdoc,devdoc,metadata,combined_json,layout yourFileName.vy .. note:: The ``opcodes`` and ``opcodes_runtime`` output of the compiler has been returning incorrect opcodes since ``0.2.0`` due to a lack of 0 padding (patched via `PR 3735 `_). If you rely on these functions for debugging, please use the latest patched versions. @@ -134,6 +134,11 @@ In codesize optimized mode, the compiler will try hard to minimize codesize by * out-lining code, and * using more loops for data copies. +Enabling Experimental Code Generation +=========================== + +When compiling, you can use the CLI flag ``--experimental-codegen`` or its alias ``--venom`` to activate the new `Venom IR `_. +Venom IR is inspired by LLVM IR and enables new advanced analysis and optimizations. .. _evm-version: diff --git a/docs/structure-of-a-contract.rst b/docs/structure-of-a-contract.rst index fc817cf4b6..7e599d677b 100644 --- a/docs/structure-of-a-contract.rst +++ b/docs/structure-of-a-contract.rst @@ -54,6 +54,16 @@ EVM Version The EVM version can be set with the ``evm-version`` pragma, which is documented in :ref:`evm-version`. +Experimental Code Generation +----------------- +The new experimental code generation feature can be activated using the following directive: + +.. code-block:: vyper + + #pragma experimental-codegen + +Alternatively, you can use the alias ``"venom"`` instead of ``"experimental-codegen"`` to enable this feature. + Imports ======= diff --git a/vyper/cli/vyper_compile.py b/vyper/cli/vyper_compile.py index fde35f781e..046cac2c0b 100755 --- a/vyper/cli/vyper_compile.py +++ b/vyper/cli/vyper_compile.py @@ -34,6 +34,8 @@ layout - Storage layout of a Vyper contract ast - AST (not yet annotated) in JSON format annotated_ast - Annotated AST in JSON format +cfg - Control flow graph of deployable bytecode +cfg_runtime - Control flow graph of runtime bytecode interface - Vyper interface of a contract external_interface - External interface of a contract, used for outside contract calls opcodes - List of opcodes as a string @@ -41,6 +43,8 @@ ir - Intermediate representation in list format ir_json - Intermediate representation in JSON format ir_runtime - Intermediate representation of runtime bytecode in list format +bb - Basic blocks of Venom IR for deployable bytecode +bb_runtime - Basic blocks of Venom IR for runtime bytecode asm - Output the EVM assembly of the deployable bytecode integrity - Output the integrity hash of the source code archive - Output the build as an archive file @@ -177,7 +181,7 @@ def _parse_args(argv): parser.add_argument( "--experimental-codegen", "--venom", - help="The compiler use the new IR codegen. This is an experimental feature.", + help="The compiler uses the new IR codegen. This is an experimental feature.", action="store_true", dest="experimental_codegen", ) diff --git a/vyper/venom/README.md b/vyper/venom/README.md index 4e4f5ca3d1..6f3b318c9b 100644 --- a/vyper/venom/README.md +++ b/vyper/venom/README.md @@ -193,7 +193,7 @@ An operand can be a label, a variable, or a literal. By convention, variables have a `%-` prefix, e.g. `%1` is a valid variable. However, the prefix is not required. ## Instructions -To enable Venom IR in Vyper, use the `--experimental-codegen` flag. To view the Venom IR output, use `-f bb_runtime` for the runtime code, or `-f bb` to see the deploy code. To get a dot file (for use e.g. with `xdot -`), use `-f cfg` or `-f cfg_runtime`. +To enable Venom IR in Vyper, use the `--experimental-codegen` CLI flag or its alias `--venom`, or the corresponding pragma statements (e.g. `#pragma experimental-codegen`). To view the Venom IR output, use `-f bb_runtime` for the runtime code, or `-f bb` to see the deploy code. To get a dot file (for use e.g. with `xdot -`), use `-f cfg` or `-f cfg_runtime`. Assembly can be inspected with `-f asm`, whereas an opcode view of the final bytecode can be seen with `-f opcodes` or `-f opcodes_runtime`, respectively.