From 205902bc3e1a3a59039a04fa80bbbadddc37a082 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Wed, 25 Jan 2023 17:34:39 -0800 Subject: [PATCH] Move `run` stuff to submodule. --- clvm/EvalError.py | 5 +- clvm/SExp.py | 4 +- clvm/__init__.py | 4 +- clvm/casts.py | 25 +---- clvm/operators.py | 205 +--------------------------------- clvm/run/EvalError.py | 4 + clvm/run/__init__.py | 0 clvm/run/casts.py | 24 ++++ clvm/{ => run}/core_ops.py | 3 +- clvm/{ => run}/costs.py | 0 clvm/{ => run}/more_ops.py | 2 +- clvm/{ => run}/op_utils.py | 0 clvm/run/operators.py | 204 +++++++++++++++++++++++++++++++++ clvm/{ => run}/run_program.py | 4 +- setup.py | 2 +- tests/as_python_test.py | 2 +- tests/operatordict_test.py | 2 +- tests/operators_test.py | 6 +- tests/run_program_test.py | 2 +- 19 files changed, 251 insertions(+), 247 deletions(-) create mode 100644 clvm/run/EvalError.py create mode 100644 clvm/run/__init__.py create mode 100644 clvm/run/casts.py rename clvm/{ => run}/core_ops.py (98%) rename clvm/{ => run}/costs.py (100%) rename clvm/{ => run}/more_ops.py (99%) rename clvm/{ => run}/op_utils.py (100%) create mode 100644 clvm/run/operators.py rename clvm/{ => run}/run_program.py (98%) diff --git a/clvm/EvalError.py b/clvm/EvalError.py index f71f912a..d9e61612 100644 --- a/clvm/EvalError.py +++ b/clvm/EvalError.py @@ -1,4 +1 @@ -class EvalError(Exception): - def __init__(self, message: str, sexp): - super().__init__(message) - self._sexp = sexp +from .run.EvalError import EvalError diff --git a/clvm/SExp.py b/clvm/SExp.py index 398fe108..08c4534c 100644 --- a/clvm/SExp.py +++ b/clvm/SExp.py @@ -6,9 +6,9 @@ from .as_python import as_python from .CLVMObject import CLVMObject -from .EvalError import EvalError +from .run.EvalError import EvalError -from .casts import ( +from .run.casts import ( int_from_bytes, int_to_bytes, ) diff --git a/clvm/__init__.py b/clvm/__init__.py index a7062e1e..5747866e 100644 --- a/clvm/__init__.py +++ b/clvm/__init__.py @@ -1,10 +1,10 @@ from .SExp import SExp -from .operators import ( # noqa +from .run.operators import ( # noqa QUOTE_ATOM, KEYWORD_TO_ATOM, KEYWORD_FROM_ATOM, ) -from .run_program import run_program # noqa +from .run.run_program import run_program # noqa from .version import __version__ # noqa to_sexp_f = SExp.to # noqa diff --git a/clvm/casts.py b/clvm/casts.py index d142c3a4..6a12240f 100644 --- a/clvm/casts.py +++ b/clvm/casts.py @@ -1,24 +1 @@ -def int_from_bytes(blob): - size = len(blob) - if size == 0: - return 0 - return int.from_bytes(blob, "big", signed=True) - - -def int_to_bytes(v): - byte_count = (v.bit_length() + 8) >> 3 - if v == 0: - return b"" - r = v.to_bytes(byte_count, "big", signed=True) - # make sure the string returned is minimal - # ie. no leading 00 or ff bytes that are unnecessary - while len(r) > 1 and r[0] == (0xFF if r[1] & 0x80 else 0): - r = r[1:] - return r - - -def limbs_for_int(v): - """ - Return the number of bytes required to represent this integer. - """ - return (v.bit_length() + 7) >> 3 +from .run.casts import int_from_bytes, int_to_bytes diff --git a/clvm/operators.py b/clvm/operators.py index a63e6a88..c279a81a 100644 --- a/clvm/operators.py +++ b/clvm/operators.py @@ -1,204 +1 @@ -from typing import Dict, Tuple - -from . import core_ops, more_ops - -from .CLVMObject import CLVMObject -from .SExp import SExp -from .EvalError import EvalError - -from .casts import int_to_bytes -from .op_utils import operators_for_module - -from .costs import ( - ARITH_BASE_COST, - ARITH_COST_PER_BYTE, - ARITH_COST_PER_ARG, - MUL_BASE_COST, - MUL_COST_PER_OP, - MUL_LINEAR_COST_PER_BYTE, - MUL_SQUARE_COST_PER_BYTE_DIVIDER, - CONCAT_BASE_COST, - CONCAT_COST_PER_ARG, - CONCAT_COST_PER_BYTE, -) - -KEYWORDS = ( - # core opcodes 0x01-x08 - ". q a i c f r l x " - - # opcodes on atoms as strings 0x09-0x0f - "= >s sha256 substr strlen concat . " - - # opcodes on atoms as ints 0x10-0x17 - "+ - * / divmod > ash lsh " - - # opcodes on atoms as vectors of bools 0x18-0x1c - "logand logior logxor lognot . " - - # opcodes for bls 1381 0x1d-0x1f - "point_add pubkey_for_exp . " - - # bool opcodes 0x20-0x23 - "not any all . " - - # misc 0x24 - "softfork " -).split() - -KEYWORD_FROM_ATOM = {int_to_bytes(k): v for k, v in enumerate(KEYWORDS)} -KEYWORD_TO_ATOM = {v: k for k, v in KEYWORD_FROM_ATOM.items()} - -OP_REWRITE = { - "+": "add", - "-": "subtract", - "*": "multiply", - "/": "div", - "i": "if", - "c": "cons", - "f": "first", - "r": "rest", - "l": "listp", - "x": "raise", - "=": "eq", - ">": "gr", - ">s": "gr_bytes", -} - - -def args_len(op_name, args): - for arg in args.as_iter(): - if arg.pair: - raise EvalError("%s requires int args" % op_name, arg) - yield len(arg.as_atom()) - - -# unknown ops are reserved if they start with 0xffff -# otherwise, unknown ops are no-ops, but they have costs. The cost is computed -# like this: - -# byte index (reverse): -# | 4 | 3 | 2 | 1 | 0 | -# +---+---+---+---+------------+ -# | multiplier |XX | XXXXXX | -# +---+---+---+---+---+--------+ -# ^ ^ ^ -# | | + 6 bits ignored when computing cost -# cost_multiplier | -# + 2 bits -# cost_function - -# 1 is always added to the multiplier before using it to multiply the cost, this -# is since cost may not be 0. - -# cost_function is 2 bits and defines how cost is computed based on arguments: -# 0: constant, cost is 1 * (multiplier + 1) -# 1: computed like operator add, multiplied by (multiplier + 1) -# 2: computed like operator mul, multiplied by (multiplier + 1) -# 3: computed like operator concat, multiplied by (multiplier + 1) - -# this means that unknown ops where cost_function is 1, 2, or 3, may still be -# fatal errors if the arguments passed are not atoms. - -def default_unknown_op(op: bytes, args: CLVMObject) -> Tuple[int, CLVMObject]: - # any opcode starting with ffff is reserved (i.e. fatal error) - # opcodes are not allowed to be empty - if len(op) == 0 or op[:2] == b"\xff\xff": - raise EvalError("reserved operator", args.to(op)) - - # all other unknown opcodes are no-ops - # the cost of the no-ops is determined by the opcode number, except the - # 6 least significant bits. - - cost_function = (op[-1] & 0b11000000) >> 6 - # the multiplier cannot be 0. it starts at 1 - - if len(op) > 5: - raise EvalError("invalid operator", args.to(op)) - - cost_multiplier = int.from_bytes(op[:-1], "big", signed=False) + 1 - - # 0 = constant - # 1 = like op_add/op_sub - # 2 = like op_multiply - # 3 = like op_concat - if cost_function == 0: - cost = 1 - elif cost_function == 1: - # like op_add - cost = ARITH_BASE_COST - arg_size = 0 - for length in args_len("unknown op", args): - arg_size += length - cost += ARITH_COST_PER_ARG - cost += arg_size * ARITH_COST_PER_BYTE - elif cost_function == 2: - # like op_multiply - cost = MUL_BASE_COST - operands = args_len("unknown op", args) - try: - vs = next(operands) - for rs in operands: - cost += MUL_COST_PER_OP - cost += (rs + vs) * MUL_LINEAR_COST_PER_BYTE - cost += (rs * vs) // MUL_SQUARE_COST_PER_BYTE_DIVIDER - # this is an estimate, since we don't want to actually multiply the - # values - vs += rs - except StopIteration: - pass - - elif cost_function == 3: - # like concat - cost = CONCAT_BASE_COST - length = 0 - for arg in args.as_iter(): - if arg.pair: - raise EvalError("unknown op on list", arg) - cost += CONCAT_COST_PER_ARG - length += len(arg.atom) - cost += length * CONCAT_COST_PER_BYTE - - cost *= cost_multiplier - if cost >= 2**32: - raise EvalError("invalid operator", args.to(op)) - - return (cost, SExp.null()) - - -class OperatorDict(dict): - """ - This is a nice hack that adds `__call__` to a dictionary, so - operators can be added dynamically. - """ - - def __new__(class_, d: Dict, *args, **kwargs): - """ - `quote_atom` and `apply_atom` must be set - `unknown_op_handler` has a default implementation - We do not check if quote and apply are distinct - We do not check if the opcode values for quote and apply exist in the passed-in dict - """ - self = super(OperatorDict, class_).__new__(class_, d) - self.quote_atom = kwargs["quote"] if "quote" in kwargs else d.quote_atom - self.apply_atom = kwargs["apply"] if "apply" in kwargs else d.apply_atom - if "unknown_op_handler" in kwargs: - self.unknown_op_handler = kwargs["unknown_op_handler"] - else: - self.unknown_op_handler = default_unknown_op - return self - - def __call__(self, op: bytes, arguments: CLVMObject) -> Tuple[int, CLVMObject]: - f = self.get(op) - if f is None: - return self.unknown_op_handler(op, arguments) - else: - return f(arguments) - - -QUOTE_ATOM = KEYWORD_TO_ATOM["q"] -APPLY_ATOM = KEYWORD_TO_ATOM["a"] - -OPERATOR_LOOKUP = OperatorDict( - operators_for_module(KEYWORD_TO_ATOM, core_ops, OP_REWRITE), quote=QUOTE_ATOM, apply=APPLY_ATOM -) -OPERATOR_LOOKUP.update(operators_for_module(KEYWORD_TO_ATOM, more_ops, OP_REWRITE)) +from .run.operators import OP_REWRITE, OperatorDict, OPERATOR_LOOKUP diff --git a/clvm/run/EvalError.py b/clvm/run/EvalError.py new file mode 100644 index 00000000..f71f912a --- /dev/null +++ b/clvm/run/EvalError.py @@ -0,0 +1,4 @@ +class EvalError(Exception): + def __init__(self, message: str, sexp): + super().__init__(message) + self._sexp = sexp diff --git a/clvm/run/__init__.py b/clvm/run/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/clvm/run/casts.py b/clvm/run/casts.py new file mode 100644 index 00000000..d142c3a4 --- /dev/null +++ b/clvm/run/casts.py @@ -0,0 +1,24 @@ +def int_from_bytes(blob): + size = len(blob) + if size == 0: + return 0 + return int.from_bytes(blob, "big", signed=True) + + +def int_to_bytes(v): + byte_count = (v.bit_length() + 8) >> 3 + if v == 0: + return b"" + r = v.to_bytes(byte_count, "big", signed=True) + # make sure the string returned is minimal + # ie. no leading 00 or ff bytes that are unnecessary + while len(r) > 1 and r[0] == (0xFF if r[1] & 0x80 else 0): + r = r[1:] + return r + + +def limbs_for_int(v): + """ + Return the number of bytes required to represent this integer. + """ + return (v.bit_length() + 7) >> 3 diff --git a/clvm/core_ops.py b/clvm/run/core_ops.py similarity index 98% rename from clvm/core_ops.py rename to clvm/run/core_ops.py index 850e5ce7..9820548f 100644 --- a/clvm/core_ops.py +++ b/clvm/run/core_ops.py @@ -1,5 +1,6 @@ +from ..SExp import SExp + from .EvalError import EvalError -from .SExp import SExp from .costs import ( IF_COST, diff --git a/clvm/costs.py b/clvm/run/costs.py similarity index 100% rename from clvm/costs.py rename to clvm/run/costs.py diff --git a/clvm/more_ops.py b/clvm/run/more_ops.py similarity index 99% rename from clvm/more_ops.py rename to clvm/run/more_ops.py index 43df6cb5..1b14718c 100644 --- a/clvm/more_ops.py +++ b/clvm/run/more_ops.py @@ -5,7 +5,7 @@ from .EvalError import EvalError from .casts import limbs_for_int -from .SExp import SExp +from ..SExp import SExp from .costs import ( ARITH_BASE_COST, diff --git a/clvm/op_utils.py b/clvm/run/op_utils.py similarity index 100% rename from clvm/op_utils.py rename to clvm/run/op_utils.py diff --git a/clvm/run/operators.py b/clvm/run/operators.py new file mode 100644 index 00000000..289f8b45 --- /dev/null +++ b/clvm/run/operators.py @@ -0,0 +1,204 @@ +from typing import Dict, Tuple + +from . import core_ops, more_ops + +from ..CLVMObject import CLVMObject +from ..SExp import SExp +from .EvalError import EvalError + +from .casts import int_to_bytes +from .op_utils import operators_for_module + +from .costs import ( + ARITH_BASE_COST, + ARITH_COST_PER_BYTE, + ARITH_COST_PER_ARG, + MUL_BASE_COST, + MUL_COST_PER_OP, + MUL_LINEAR_COST_PER_BYTE, + MUL_SQUARE_COST_PER_BYTE_DIVIDER, + CONCAT_BASE_COST, + CONCAT_COST_PER_ARG, + CONCAT_COST_PER_BYTE, +) + +KEYWORDS = ( + # core opcodes 0x01-x08 + ". q a i c f r l x " + + # opcodes on atoms as strings 0x09-0x0f + "= >s sha256 substr strlen concat . " + + # opcodes on atoms as ints 0x10-0x17 + "+ - * / divmod > ash lsh " + + # opcodes on atoms as vectors of bools 0x18-0x1c + "logand logior logxor lognot . " + + # opcodes for bls 1381 0x1d-0x1f + "point_add pubkey_for_exp . " + + # bool opcodes 0x20-0x23 + "not any all . " + + # misc 0x24 + "softfork " +).split() + +KEYWORD_FROM_ATOM = {int_to_bytes(k): v for k, v in enumerate(KEYWORDS)} +KEYWORD_TO_ATOM = {v: k for k, v in KEYWORD_FROM_ATOM.items()} + +OP_REWRITE = { + "+": "add", + "-": "subtract", + "*": "multiply", + "/": "div", + "i": "if", + "c": "cons", + "f": "first", + "r": "rest", + "l": "listp", + "x": "raise", + "=": "eq", + ">": "gr", + ">s": "gr_bytes", +} + + +def args_len(op_name, args): + for arg in args.as_iter(): + if arg.pair: + raise EvalError("%s requires int args" % op_name, arg) + yield len(arg.as_atom()) + + +# unknown ops are reserved if they start with 0xffff +# otherwise, unknown ops are no-ops, but they have costs. The cost is computed +# like this: + +# byte index (reverse): +# | 4 | 3 | 2 | 1 | 0 | +# +---+---+---+---+------------+ +# | multiplier |XX | XXXXXX | +# +---+---+---+---+---+--------+ +# ^ ^ ^ +# | | + 6 bits ignored when computing cost +# cost_multiplier | +# + 2 bits +# cost_function + +# 1 is always added to the multiplier before using it to multiply the cost, this +# is since cost may not be 0. + +# cost_function is 2 bits and defines how cost is computed based on arguments: +# 0: constant, cost is 1 * (multiplier + 1) +# 1: computed like operator add, multiplied by (multiplier + 1) +# 2: computed like operator mul, multiplied by (multiplier + 1) +# 3: computed like operator concat, multiplied by (multiplier + 1) + +# this means that unknown ops where cost_function is 1, 2, or 3, may still be +# fatal errors if the arguments passed are not atoms. + +def default_unknown_op(op: bytes, args: CLVMObject) -> Tuple[int, CLVMObject]: + # any opcode starting with ffff is reserved (i.e. fatal error) + # opcodes are not allowed to be empty + if len(op) == 0 or op[:2] == b"\xff\xff": + raise EvalError("reserved operator", args.to(op)) + + # all other unknown opcodes are no-ops + # the cost of the no-ops is determined by the opcode number, except the + # 6 least significant bits. + + cost_function = (op[-1] & 0b11000000) >> 6 + # the multiplier cannot be 0. it starts at 1 + + if len(op) > 5: + raise EvalError("invalid operator", args.to(op)) + + cost_multiplier = int.from_bytes(op[:-1], "big", signed=False) + 1 + + # 0 = constant + # 1 = like op_add/op_sub + # 2 = like op_multiply + # 3 = like op_concat + if cost_function == 0: + cost = 1 + elif cost_function == 1: + # like op_add + cost = ARITH_BASE_COST + arg_size = 0 + for length in args_len("unknown op", args): + arg_size += length + cost += ARITH_COST_PER_ARG + cost += arg_size * ARITH_COST_PER_BYTE + elif cost_function == 2: + # like op_multiply + cost = MUL_BASE_COST + operands = args_len("unknown op", args) + try: + vs = next(operands) + for rs in operands: + cost += MUL_COST_PER_OP + cost += (rs + vs) * MUL_LINEAR_COST_PER_BYTE + cost += (rs * vs) // MUL_SQUARE_COST_PER_BYTE_DIVIDER + # this is an estimate, since we don't want to actually multiply the + # values + vs += rs + except StopIteration: + pass + + elif cost_function == 3: + # like concat + cost = CONCAT_BASE_COST + length = 0 + for arg in args.as_iter(): + if arg.pair: + raise EvalError("unknown op on list", arg) + cost += CONCAT_COST_PER_ARG + length += len(arg.atom) + cost += length * CONCAT_COST_PER_BYTE + + cost *= cost_multiplier + if cost >= 2**32: + raise EvalError("invalid operator", args.to(op)) + + return (cost, SExp.null()) + + +class OperatorDict(dict): + """ + This is a nice hack that adds `__call__` to a dictionary, so + operators can be added dynamically. + """ + + def __new__(class_, d: Dict, *args, **kwargs): + """ + `quote_atom` and `apply_atom` must be set + `unknown_op_handler` has a default implementation + We do not check if quote and apply are distinct + We do not check if the opcode values for quote and apply exist in the passed-in dict + """ + self = super(OperatorDict, class_).__new__(class_, d) + self.quote_atom = kwargs["quote"] if "quote" in kwargs else d.quote_atom + self.apply_atom = kwargs["apply"] if "apply" in kwargs else d.apply_atom + if "unknown_op_handler" in kwargs: + self.unknown_op_handler = kwargs["unknown_op_handler"] + else: + self.unknown_op_handler = default_unknown_op + return self + + def __call__(self, op: bytes, arguments: CLVMObject) -> Tuple[int, CLVMObject]: + f = self.get(op) + if f is None: + return self.unknown_op_handler(op, arguments) + else: + return f(arguments) + + +QUOTE_ATOM = KEYWORD_TO_ATOM["q"] +APPLY_ATOM = KEYWORD_TO_ATOM["a"] + +OPERATOR_LOOKUP = OperatorDict( + operators_for_module(KEYWORD_TO_ATOM, core_ops, OP_REWRITE), quote=QUOTE_ATOM, apply=APPLY_ATOM +) +OPERATOR_LOOKUP.update(operators_for_module(KEYWORD_TO_ATOM, more_ops, OP_REWRITE)) diff --git a/clvm/run_program.py b/clvm/run/run_program.py similarity index 98% rename from clvm/run_program.py rename to clvm/run/run_program.py index 20f4b75c..7e1383f6 100644 --- a/clvm/run_program.py +++ b/clvm/run/run_program.py @@ -1,8 +1,8 @@ from typing import Any, Callable, List, Tuple -from .CLVMObject import CLVMObject +from ..CLVMObject import CLVMObject +from ..SExp import SExp from .EvalError import EvalError -from .SExp import SExp from .costs import ( APPLY_COST, diff --git a/setup.py b/setup.py index c3428e42..9b9314c8 100755 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ setup( name="clvm", - packages=["clvm",], + packages=["clvm","clvm.run"], author="Chia Network, Inc.", author_email="hello@chia.net", url="https://github.com/Chia-Network/clvm", diff --git a/tests/as_python_test.py b/tests/as_python_test.py index 389c58f1..cbd0d9e3 100644 --- a/tests/as_python_test.py +++ b/tests/as_python_test.py @@ -3,7 +3,7 @@ from clvm import SExp from clvm.CLVMObject import CLVMObject from blspy import G1Element -from clvm.EvalError import EvalError +from clvm.run.EvalError import EvalError class dummy_class: diff --git a/tests/operatordict_test.py b/tests/operatordict_test.py index 897f6ffe..d67c3147 100644 --- a/tests/operatordict_test.py +++ b/tests/operatordict_test.py @@ -1,6 +1,6 @@ import unittest -from clvm.operators import OperatorDict +from clvm.run.operators import OperatorDict class OperatorDictTest(unittest.TestCase): diff --git a/tests/operators_test.py b/tests/operators_test.py index 9c84d719..177b5dbe 100644 --- a/tests/operators_test.py +++ b/tests/operators_test.py @@ -1,9 +1,9 @@ import unittest -from clvm.operators import (OPERATOR_LOOKUP, KEYWORD_TO_ATOM, default_unknown_op, OperatorDict) -from clvm.EvalError import EvalError from clvm import SExp -from clvm.costs import CONCAT_BASE_COST +from clvm.run.operators import (OPERATOR_LOOKUP, KEYWORD_TO_ATOM, default_unknown_op, OperatorDict) +from clvm.run.EvalError import EvalError +from clvm.run.costs import CONCAT_BASE_COST class OperatorsTest(unittest.TestCase): diff --git a/tests/run_program_test.py b/tests/run_program_test.py index d64462ca..e8882a5b 100644 --- a/tests/run_program_test.py +++ b/tests/run_program_test.py @@ -1,6 +1,6 @@ import unittest -from clvm.run_program import msb_mask +from clvm.run.run_program import msb_mask class BitTest(unittest.TestCase):