Tests for venom codesize and better chosing of optimization

tests/unit/compiler/venom/test_literals_codesize.py

@@ -6,9 +6,13 @@
 from vyper.venom.context import IRContext
 from vyper.venom.passes import ReduceLiteralsCodesize
 
+should_invert = [2**256 - 1] + [((1 << i) - 1) << (256 - i) for i in range(121, 256 + 1)]
 
-@pytest.mark.parametrize("orig_value", [0xFFFF << 240, 2**256 - 1])
-def test_ff_inversion(orig_value):
+
+@pytest.mark.parametrize("orig_value", should_invert)
+def test_literal_codesize_ff_inversion(orig_value):
+    print(hex(orig_value))
+    print(hex(orig_value % 2**256))
     ctx = IRContext()
     fn = ctx.create_function("_global")
     bb = fn.get_basic_block()
@@ -22,11 +26,13 @@ def test_ff_inversion(orig_value):
     assert evm_not(bb.instructions[0].operands[0].value) == orig_value
 
 
-should_not_invert = [1, 0xFE << 248 | (2**248 - 1)]  # 0xfeff...ff
+should_not_invert = [1, 0xFE << 248 | (2**248 - 1)] + [
+    ((2**255 - 1) >> i) << i for i in range(0, 3 * 8)
+]
 
 
 @pytest.mark.parametrize("orig_value", should_not_invert)
-def test_no_inversion(orig_value):
+def test_literal_codesize_no_inversion(orig_value):
     ctx = IRContext()
     fn = ctx.create_function("_global")
     bb = fn.get_basic_block()
@@ -40,11 +46,15 @@ def test_no_inversion(orig_value):
     assert bb.instructions[0].operands[0].value == orig_value
 
 
-should_shl = [0x01_000000]  # saves 3 bytes
+should_shl = (
+    [1 << i for i in range(3 * 8, 255)]
+    + [((1 << i) - 1) << (256 - i) for i in range(1, 121)]
+    + [((2**255 - 1) >> i) << i for i in range(3 * 8, 254)]
+)
 
 
 @pytest.mark.parametrize("orig_value", should_shl)
-def test_shl(orig_value):
+def test_literal_codesize_shl(orig_value):
     ctx = IRContext()
     fn = ctx.create_function("_global")
     bb = fn.get_basic_block()
@@ -59,11 +69,11 @@ def test_shl(orig_value):
     assert op0.value << op1.value == orig_value
 
 
-should_not_shl = [0x01_00]  # only saves 2 bytes
+should_not_shl = [0x0, 2 ** (256 - 2) - 1 << (2 * 8) ^ 2**255] + [1 << i for i in range(0, 3 * 8)]
 
 
 @pytest.mark.parametrize("orig_value", should_not_shl)
-def test_no_shl(orig_value):
+def test_literal_codesize_no_shl(orig_value):
     ctx = IRContext()
     fn = ctx.create_function("_global")
     bb = fn.get_basic_block()

vyper/venom/passes/literals_codesize.py

@@ -30,20 +30,28 @@ def _process_bb(self, bb):
 
             val = op.value % (2**256)
 
-            # transform things like 0xffff...01 to (not 0xfe)
-            if len(hex(val)) // 2 - len(hex(evm_not(val))) // 2 > NOT_THRESHOLD:
-                inst.opcode = "not"
-                op.value = evm_not(val)
-                continue
+            # calculate amount of bits saved by not optimization
+            not_benefit = ((len(hex(val)) // 2 - len(hex(evm_not(val))) // 2) - NOT_THRESHOLD) * 8
 
-            # transform things like 0x123400....000 to 0x1234 << ...
+            # calculate amount of bits saved by shl optimization
             binz = bin(val)[2:]
-            if (ix := len(binz) - binz.rfind("1")) > SHL_THRESHOLD * 8:
-                ix -= 1
-                # sanity check
-                assert (val >> ix) << ix == val, val
-                assert (val >> ix) & 1 == 1, val
-
-                inst.opcode = "shl"
-                inst.operands = [IRLiteral(val >> ix), IRLiteral(ix)]
-                continue
+            ix = len(binz) - binz.rfind("1")
+            shl_benefit = ix - SHL_THRESHOLD * 8
+
+            if not_benefit >= shl_benefit:
+                # transform things like 0xffff...01 to (not 0xfe)
+                if not_benefit > 0:
+                    inst.opcode = "not"
+                    op.value = evm_not(val)
+                    continue
+            else:
+                # transform things like 0x123400....000 to 0x1234 << ...
+                if shl_benefit > 0:
+                    ix -= 1
+                    # sanity check
+                    assert (val >> ix) << ix == val, val
+                    assert (val >> ix) & 1 == 1, val
+
+                    inst.opcode = "shl"
+                    inst.operands = [IRLiteral(val >> ix), IRLiteral(ix)]
+                    continue