From 8d76d260333485912e51710b797964c0a8c47d98 Mon Sep 17 00:00:00 2001 From: Bruce Merry Date: Mon, 18 Mar 2024 11:42:50 +0200 Subject: [PATCH 1/7] Initial prototyping with katcp-codec --- .pre-commit-config.yaml | 1 + setup.cfg | 1 + src/aiokatcp/core.py | 50 +++++++++++++---------------------------- 3 files changed, 17 insertions(+), 35 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 865b504..bfc298e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,6 +36,7 @@ repos: args: [] additional_dependencies: [ 'async-timeout==4.0.3', + 'katcp-codec==0.1.0b2', 'pytest==7.4.2', 'types-decorator==5.1.1', 'typing-extensions==4.7.1' diff --git a/setup.cfg b/setup.cfg index 7d171b7..1cbdb6c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,6 +27,7 @@ packages = find: install_requires = async-timeout decorator>=4.1 + katcp-codec typing-extensions python_requires = >=3.8 zip_safe = False # For py.typed diff --git a/src/aiokatcp/core.py b/src/aiokatcp/core.py index 1847919..503789b 100644 --- a/src/aiokatcp/core.py +++ b/src/aiokatcp/core.py @@ -47,6 +47,9 @@ Union, ) +import katcp_codec +from typing_extensions import TypeAlias + _T = TypeVar("_T") _E = TypeVar("_E", bound=enum.Enum) _F = TypeVar("_F", bound=numbers.Real) @@ -465,12 +468,7 @@ def __init__(self, message: str, raw: Optional[bytes] = None) -> None: class Message: __slots__ = ["mtype", "name", "arguments", "mid"] - class Type(enum.Enum): - """Message type""" - - REQUEST = 1 - REPLY = 2 - INFORM = 3 + Type: TypeAlias = katcp_codec.MessageType _TYPE_SYMBOLS = { Type.REQUEST: b"?", @@ -581,39 +579,21 @@ def parse(cls, raw) -> "Message": If `raw` is not validly encoded. """ try: - if not raw or raw[:1] not in b"?#!": - raise KatcpSyntaxError("message does not start with message type") if raw[-1:] not in (b"\r", b"\n"): - raise KatcpSyntaxError("message does not end with newline") - clean = raw[:-1].replace(b"\t", b" ") - match = cls._SPECIAL_RE.search(clean) - if match: - raise KatcpSyntaxError(f"unescaped special {match.group()!r}") - # NB: don't use split() without an argument, as it will also split - # on whitespace other than space or tab (e.g. form feed). - parts = [part for part in clean.split(b" ") if part] - match = cls._HEADER_RE.match(parts[0]) - if not match: - raise KatcpSyntaxError("could not parse name and message ID") - name = match.group(1).decode("ascii") - mid_raw = match.group(2) - if mid_raw is not None: - mid = int(mid_raw) - else: - mid = None - mtype = cls._REVERSE_TYPE_SYMBOLS[clean[:1]] + raise ValueError("message does not end with newline") + parser = katcp_codec.Parser(len(raw)) + msgs = parser.append(raw) + if not msgs: + raise ValueError("no message") + if len(msgs) > 1 or parser.buffer_size > 0: + raise ValueError("internal newline") + if isinstance(msgs[0], Exception): + raise msgs[0] # Create the message first without arguments, to avoid the argument # encoding and let us store raw bytes. - msg = cls(mtype, name, mid=mid) - # Performance: copy functions to local variables to avoid doing - # attribute lookup for every element of parts. - sub = cls._UNESCAPE_RE.sub - unescape_match = cls._unescape_match - msg.arguments = [sub(unescape_match, arg) for arg in parts[1:]] + msg = cls(msgs[0].mtype, msgs[0].name.decode("ascii"), mid=msgs[0].mid) + msg.arguments = msgs[0].arguments return msg - except KatcpSyntaxError as error: - error.raw = raw - raise error except ValueError as error: raise KatcpSyntaxError(str(error), raw) from error From 4b9adec96950fa2a61dd85bcc5cfcd3a5c3ff16d Mon Sep 17 00:00:00 2001 From: Bruce Merry Date: Tue, 16 Apr 2024 10:56:34 +0200 Subject: [PATCH 2/7] Use katcp_codec for Message.__bytes__ too --- src/aiokatcp/core.py | 26 +++----------------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/src/aiokatcp/core.py b/src/aiokatcp/core.py index 503789b..ebacdbd 100644 --- a/src/aiokatcp/core.py +++ b/src/aiokatcp/core.py @@ -27,7 +27,6 @@ import enum import functools -import io import ipaddress import logging import numbers @@ -470,21 +469,10 @@ class Message: Type: TypeAlias = katcp_codec.MessageType - _TYPE_SYMBOLS = { - Type.REQUEST: b"?", - Type.REPLY: b"!", - Type.INFORM: b"#", - } - _REVERSE_TYPE_SYMBOLS = {value: key for (key, value) in _TYPE_SYMBOLS.items()} - _NAME_RE = re.compile("^[A-Za-z][A-Za-z0-9-]*$", re.ASCII) - _HEADER_RE = re.compile(rb"^[!#?]([A-Za-z][A-Za-z0-9-]*)(?:\[([1-9][0-9]*)\])?$") #: Characters that must be escaped in an argument _ESCAPE_RE = re.compile(rb"[\\ \0\n\r\x1b\t]") _UNESCAPE_RE = re.compile(rb"\\(.)?") # ? so that it also matches trailing backslash - #: Characters not allowed to appear in an argument - # (space, tab are omitted because they are split on already) - _SPECIAL_RE = re.compile(rb"[\0\r\n\x1b]") _ESCAPE_LOOKUP = { b"\\": b"\\", @@ -599,17 +587,9 @@ def parse(cls, raw) -> "Message": def __bytes__(self) -> bytes: """Return Message as serialised for transmission""" - - output = io.BytesIO() - output.write(self._TYPE_SYMBOLS[self.mtype]) - output.write(self.name.encode("ascii")) - if self.mid is not None: - output.write(b"[" + str(self.mid).encode("ascii") + b"]") - for arg in self.arguments: - output.write(b" ") - output.write(self.escape_argument(arg)) - output.write(b"\n") - return output.getvalue() + return bytes( + katcp_codec.Message(self.mtype, self.name.encode("ascii"), self.mid, self.arguments) + ) def __repr__(self) -> str: return ( From f1fbba28c1a142980b6dbf679cc75cbb319251cf Mon Sep 17 00:00:00 2001 From: Bruce Merry Date: Tue, 16 Apr 2024 11:08:25 +0200 Subject: [PATCH 3/7] Create a changelog for 1.10.0b1 --- doc/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/changelog.rst b/doc/changelog.rst index 7269c3f..6fd9821 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -1,6 +1,13 @@ Changelog ========= +.. rubric:: Version 1.10.0b1 + +- Use `katcp-codec`_ to provide the low-level encoding and decoding of + katcp messages, yielding a significant speedup. + +.. _katcp-codec: https://katcp-codec.readthedocs.io/en/latest/ + .. rubric:: Version 1.9.0 - Drop support for end-of-life Python 3.7. From f6509b15fb9791b24491823314cde3f9ae06a81c Mon Sep 17 00:00:00 2001 From: Bruce Merry Date: Tue, 16 Apr 2024 11:12:47 +0200 Subject: [PATCH 4/7] Remove claim that Python 3.5+ is needed The minimum version is already listed in the README and setup.cfg, so I deleted this incorrect copy rather than trying to maintain it in yet another place. --- doc/intro.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/intro.rst b/doc/intro.rst index 78ea56b..501164a 100644 --- a/doc/intro.rst +++ b/doc/intro.rst @@ -2,9 +2,8 @@ Introduction to aiokatcp ======================== aiokatcp is an implementation of the `katcp`_ protocol based around the Python -asyncio system module. It requires Python 3.5 or later, as it makes extensive -uses of coroutines and type annotations. It is loosely inspired by the `Python -2 bindings`_, but has a much narrower scope. +asyncio system module. It is loosely inspired by the `Python 2 bindings`_, but +has a much narrower scope. .. _katcp: https://katcp-python.readthedocs.io/en/latest/_downloads/361189acb383a294be20d6c10c257cb4/NRF-KAT7-6.0-IFCE-002-Rev5-1.pdf From d69184d2a677efa594080204d1d2b52b7f8bc8c3 Mon Sep 17 00:00:00 2001 From: Bruce Merry Date: Tue, 16 Apr 2024 13:29:27 +0200 Subject: [PATCH 5/7] Switch from katversion to setuptools-scm for versioning katversion doesn't support the full range of version specifiers including beta versions. This also allows setup.py to finally be removed. --- .gitignore | 1 + mypy.ini | 3 --- pyproject.toml | 5 ++++- setup.py | 30 ------------------------------ src/aiokatcp/__init__.py | 15 ++------------- 5 files changed, 7 insertions(+), 47 deletions(-) delete mode 100644 setup.py diff --git a/.gitignore b/.gitignore index d7c5f31..cacc34a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ /.coverage /.eggs /.mypy_cache +/src/aiokatcp/_version.py src/aiokatcp.egg-info *.pyc __pycache__ diff --git a/mypy.ini b/mypy.ini index d3a2047..c0ead5e 100644 --- a/mypy.ini +++ b/mypy.ini @@ -4,6 +4,3 @@ files = src/aiokatcp, examples, tests [mypy-async_solipsism.*] ignore_missing_imports = True - -[mypy-katversion.*] -ignore_missing_imports = True diff --git a/pyproject.toml b/pyproject.toml index 34d130d..cf529a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,8 @@ [build-system] -requires = ["setuptools", "katversion", "setuptools_scm"] +requires = ["setuptools", "setuptools_scm"] + +[tool.setuptools_scm] +version_file = "src/aiokatcp/_version.py" [tool.isort] profile = "black" diff --git a/setup.py b/setup.py deleted file mode 100644 index 476caea..0000000 --- a/setup.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2017, 2022 National Research Foundation (SARAO) -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its contributors -# may be used to endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from setuptools import setup - -setup(use_katversion=True) diff --git a/src/aiokatcp/__init__.py b/src/aiokatcp/__init__.py index 7d6c085..fed86b7 100644 --- a/src/aiokatcp/__init__.py +++ b/src/aiokatcp/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2017, 2022, 2023 National Research Foundation (SARAO) +# Copyright 2017, 2022-2024 National Research Foundation (SARAO) # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -25,18 +25,7 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# BEGIN VERSION CHECK -# Get package version when locally imported from repo or via -e develop install -try: - import katversion as _katversion -except ImportError: - import time as _time - - __version__ = "0.0+unknown.{}".format(_time.strftime("%Y%m%d%H%M")) -else: - __version__ = _katversion.get_version(__path__[0]) -# END VERSION CHECK - +from ._version import __version__ from .client import ( # noqa: F401 AbstractSensorWatcher, Client, From e4bfe7a06ccebff7cce5ab81e11abf30ac7b68d4 Mon Sep 17 00:00:00 2001 From: Bruce Merry Date: Tue, 16 Apr 2024 13:38:18 +0200 Subject: [PATCH 6/7] Add katcp-codec to requirements.in / requirements.txt --- requirements.in | 1 + requirements.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/requirements.in b/requirements.in index 2ffcc5b..234d89b 100644 --- a/requirements.in +++ b/requirements.in @@ -2,6 +2,7 @@ async-solipsism async-timeout coveralls decorator +katcp-codec pre-commit pytest pytest-asyncio diff --git a/requirements.txt b/requirements.txt index 160f934..09a7be3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,6 +36,8 @@ idna==3.7 # via requests iniconfig==2.0.0 # via pytest +katcp-codec==0.1.0b2 + # via -r requirements.in nodeenv==1.8.0 # via pre-commit packaging==23.1 From 456953c1b8d41b87d88ceda9cad8a1e9fa392ef4 Mon Sep 17 00:00:00 2001 From: Bruce Merry Date: Tue, 16 Apr 2024 17:06:07 +0200 Subject: [PATCH 7/7] Introduce an intermediate variable --- src/aiokatcp/core.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/aiokatcp/core.py b/src/aiokatcp/core.py index ebacdbd..386ec23 100644 --- a/src/aiokatcp/core.py +++ b/src/aiokatcp/core.py @@ -575,13 +575,14 @@ def parse(cls, raw) -> "Message": raise ValueError("no message") if len(msgs) > 1 or parser.buffer_size > 0: raise ValueError("internal newline") - if isinstance(msgs[0], Exception): - raise msgs[0] + msg = msgs[0] + if isinstance(msg, Exception): + raise msg # Create the message first without arguments, to avoid the argument # encoding and let us store raw bytes. - msg = cls(msgs[0].mtype, msgs[0].name.decode("ascii"), mid=msgs[0].mid) - msg.arguments = msgs[0].arguments - return msg + ret = cls(msg.mtype, msg.name.decode("ascii"), mid=msg.mid) + ret.arguments = msg.arguments + return ret except ValueError as error: raise KatcpSyntaxError(str(error), raw) from error