diff --git a/.github/workflows/python-publish-ezmsg-sigproc.yml b/.github/workflows/python-publish-ezmsg-sigproc.yml deleted file mode 100644 index 7dcbdd81..00000000 --- a/.github/workflows/python-publish-ezmsg-sigproc.yml +++ /dev/null @@ -1,41 +0,0 @@ -# This workflow will upload a Python Package using Twine when a release is created -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries - -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -name: Upload Python Package - ezmsg-sigproc - -on: - release: - types: [published] - workflow_dispatch: - -permissions: - contents: read - -jobs: - deploy: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.8" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install build - - - name: Build ezmsg-sigproc - run: python -m build extensions/ezmsg-sigproc - - name: Publish ezmsg-sigproc - uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 - with: - password: ${{ secrets.PYPI_API_TOKEN_SIGPROC }} - packages_dir: extensions/ezmsg-sigproc/dist diff --git a/.github/workflows/python-publish-ezmsg-websocket.yml b/.github/workflows/python-publish-ezmsg-websocket.yml deleted file mode 100644 index 34c6e1f3..00000000 --- a/.github/workflows/python-publish-ezmsg-websocket.yml +++ /dev/null @@ -1,41 +0,0 @@ -# This workflow will upload a Python Package using Twine when a release is created -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries - -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -name: Upload Python Package - ezmsg-websocket - -on: - release: - types: [published] - workflow_dispatch: - -permissions: - contents: read - -jobs: - deploy: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.8" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install build - - - name: Build ezmsg-websocket - run: python -m build extensions/ezmsg-websocket - - name: Publish ezmsg-websocket - uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 - with: - password: ${{ secrets.PYPI_API_TOKEN_WEBSOCKET }} - packages_dir: extensions/ezmsg-websocket/dist diff --git a/.github/workflows/python-publish-ezmsg-zmq.yml b/.github/workflows/python-publish-ezmsg-zmq.yml deleted file mode 100644 index 85c6f446..00000000 --- a/.github/workflows/python-publish-ezmsg-zmq.yml +++ /dev/null @@ -1,41 +0,0 @@ -# This workflow will upload a Python Package using Twine when a release is created -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries - -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -name: Upload Python Package - ezmsg-zmq - -on: - release: - types: [published] - workflow_dispatch: - -permissions: - contents: read - -jobs: - deploy: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.8" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install build - - - name: Build ezmsg-zmq - run: python -m build extensions/ezmsg-zmq - - name: Publish ezmsg-zmq - uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 - with: - password: ${{ secrets.PYPI_API_TOKEN_ZMQ }} - packages_dir: extensions/ezmsg-zmq/dist diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 4d42136b..3407af48 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -37,7 +37,7 @@ jobs: - name: Install dependencies with Poetry run: | - poetry install -E zmq -E websocket -E sigproc --with test + poetry install --with test - name: Lint with flake8 run: | @@ -49,7 +49,3 @@ jobs: - name: Test ezmsg run: | poetry run python -m pytest -v tests - - - name: Test ezmsg-sigproc - run: | - poetry run python -m pytest -v extensions/ezmsg-sigproc/tests diff --git a/README.md b/README.md index 52366697..29ed95b6 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ $ source env/bin/activate (env) $ python -m pytest tests # Optionally, Perform tests ``` -Note that it is generally recommended to install poetry into it's own standalone venv via the `pipx` cli tool. +Note that it is generally recommended to install poetry into its own standalone venv via the `pipx` cli tool. ## Documentation @@ -50,7 +50,7 @@ pip install "ezmsg[all_ext]" ``` This will install all the available public extension packages for `ezmsg` that are listed in `pyproject.toml`. -If you prefer to install the extension packages individually, you can use the following command: +If you prefer to install a subset of extension packages, you can use the following command: ```bash pip install "ezmsg[zmq,sigproc,...]" @@ -58,17 +58,14 @@ pip install "ezmsg[zmq,sigproc,...]" Please note that the `ezmsg` package itself can still be installed without any additional extensions using `pip install ezmsg`. -See the extension directory for more details - -- `ezmsg-sigproc` -- Timeseries signal processing modules -- `ezmsg-websocket` -- Websocket server and client nodes for `ezmsg` graphs -- `ezmsg-zmq` -- ZeroMQ pub and sub nodes for `ezmsg` graphs -- ... More to come! - -Additionally, the following extensions are contained in external repositories: +Extensions can be managed manually as well. Here are some of the extensions we manage or are aware of: +- [ezmsg-sigproc](https://github.com/ezmsg-org/ezmsg-sigproc) -- Timeseries signal processing modules +- [ezmsg-websocket](https://github.com/ezmsg-org/ezmsg-websocket) -- Websocket server and client nodes for `ezmsg` graphs +- [ezmsg-zmq](https://github.com/ezmsg-org/ezmsg-zmq) -- ZeroMQ pub and sub nodes for `ezmsg` graphs - [ezmsg-panel](https://github.com/griffinmilsap/ezmsg-panel) -- Plotting tools for `ezmsg` that use [panel](https://github.com/holoviz/panel) - [ezmsg-blackrock](https://github.com/griffinmilsap/ezmsg-blackrock) -- Interface for Blackrock Cerebus ecosystem (incl. Neuroport) using `pycbsdk` +- [ezmsg-lsl](https://github.com/ezmsg-org/ezmsg-lsl) -- Source unit for LSL Inlet and sink unit for LSL Outlet - [ezmsg-unicorn](https://github.com/griffinmilsap/ezmsg-unicorn) -- g.tec Unicorn Hybrid Black integration for `ezmsg` - [ezmsg-gadget](https://github.com/griffinmilsap/ezmsg-gadget) -- USB-gadget with HID control integration for Raspberry Pi (Zero/W/2W, 4, CM4) - [ezmsg-openbci](https://github.com/griffinmilsap/ezmsg-openbci) -- OpenBCI Cyton serial interface for `ezmsg` diff --git a/docs/source/api.rst b/docs/source/api.rst index 8e4a8fd0..3dde159f 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -1,240 +1,84 @@ API === -An ``ezmsg`` pipeline is created from a few basic components. ``ezmsg`` provides a framework for you to define your own graphs using its building blocks. Inherit from its base components to define a pipeline that works for your project. +An ``ezmsg`` pipeline is created from a few basic components. +``ezmsg`` provides a framework for you to define your own graphs using its building blocks. +Inherit from its base components to define a pipeline that works for your project. +.. automodule:: ezmsg.core -Collection ----------- - -Connects ``Units`` together by defining a graph which connects ``OutputStreams`` to ``InputStreams``. - -Lifecycle Hooks -^^^^^^^^^^^^^^^ - -The following lifecycle hooks in the ``Collection`` class can be overridden: - -.. py:method:: configure( self ) - - Runs when the ``Collection`` is instantiated. This is the best place to call ``Unit.apply_settings()`` on each member ``Unit`` of the ``Collection``. - -Overridable Methods -^^^^^^^^^^^^^^^^^^^^ - -.. py:method:: network( self ) -> NetworkDefinition - - In this method, define and return a ``NetworkDefinition`` which defines how ``InputStreams`` and ``OutputStreams`` from member ``Units`` will be connected. - -.. py:method:: process_components( self ) -> Tuple[Unit|Collection, ...] - - In this method, define and return a tuple which contains ``Units`` and ``Collections`` which should run in their own processes. - -Component ---------- - -Metaclass which ``Units`` and ``Collections`` inherit from. - -Complete --------- - -.. py:class:: Complete - - A type of ``Exception`` which signals to ``ezmsg`` that the function can be shut down gracefully. If all functions in all ``Units`` raise ``Complete``, the entire pipeline will terminate execution. - - -NetworkDefinition ------------------- - -.. py:class:: NetworkDefinition - - Wrapper on ``Iterator[Tuple[OutputStream, InputStream]]``. - - -NormalTermination ------------------ - -.. py:class:: NormalTermination - - A type of ``Exception`` which signals to ``ezmsg`` that the pipeline can be shut down gracefully. - -run ---- - -.. py:method:: run(components: {str: Component} = None, root_name: str = None, connections: NetworkDefinition = None, process_components: [Component] = None, backend_process: BackendProcess = DefaultBackendProcess, graph_address: (str, int) = None, force_single_process: bool = False, **components_kwargs: Component) -> None - - `The old method` run_system() `has been deprecated and uses` run() `instead.` - - Begin execution of a set of ``Components``. - - `components` represents the nodes in the directed acyclic graph. It is a dictionary which contains the ``Components`` to be run mapped to string names. On initialization, ``ezmsg`` will call ``initialize()`` for each ``Unit`` and ``configure()`` for each ``Collection``, if defined. - - `connections` represents the edges is a ``NetworkDefinition`` which connects ``OutputStreams`` to ``InputStreams``. On initialization, ``ezmsg`` will create a directed acyclic graph using the contents of this parameter. - - `process_components` is a list of ``Components`` which should live in their own process. - - `backend_process` is currently under development. - - `graph_address` is a tuple which contains the hostname and port of the graph server which ``ezmsg`` should connect to. If not defined, ``ezmsg`` will start a new graph server at 127.0.0.1:25978. - - `force_single_process` will run all ``Components`` in one process. This is necessary when running ``ezmsg`` in a notebook. - -Settings --------- - -To pass parameters into a ``Component``, inherit from ``Settings``. +Most ``ezmsg`` classes intended for use in building pipelines are available in ``ezmsg.core``. +It is convention to ``import ezmsg.core as ez`` and then use this shorthand in your code. e.g., .. code-block:: python - class YourSettings(Settings): - setting1: int - setting2: float + class MyUnit(ez.Unit): + ... -To use, declare the ``Settings`` object for a ``Component`` as a member variable called (all-caps!) ``SETTINGS``. ``ezmsg`` will monitor the variable called ``SETTINGS`` in the background, so it is important to name it correctly. +Components +---------- -.. code-block:: python +.. autoclass:: Component - class YourUnit(Unit): +.. autoclass:: Collection + :show-inheritance: + :members: - SETTINGS: YourSettings +.. autoclass:: NetworkDefinition -A ``Unit`` can accept a ``Settings`` object as a parameter on instantiation. +.. autoclass:: Unit + :show-inheritance: + :members: + :inherited-members: -.. code-block:: python - class YourCollection(Collection): +Unit Function Decorators +^^^^^^^^^^^^^^^^^^^^^^^^ - YOUR_UNIT = YourUnit( - YourSettings( - setting1: int, - setting2: float - ) - ) +These function decorators can be added to member functions. -.. note:: - ``Settings`` uses type hints to define member variables, but does not enforce type checking. +.. automethod:: ezmsg.core.subscriber -State ------ +.. automethod:: ezmsg.core.publisher -To track a mutable state for a ``Component``, inherit from ``State``. +.. automethod:: ezmsg.core.main -.. code-block:: python +.. automethod:: unit.thread - class YourState(State): - state1: int - state2: float +.. automethod:: ezmsg.core.task -To use, declare the ``State`` object for a ``Component`` as a member variable called (all-caps!) ``STATE``. ``ezmsg`` will monitor the variable called ``STATE`` in the background, so it is important to name it correctly. +.. automethod:: ezmsg.core.process -Member functions can then access and mutate ``STATE`` as needed during function execution. It is recommended to initialize state values inside the ``initialize()`` or ``configure()`` lifecycle hooks if defaults are not defined. +.. automethod:: ezmsg.core.timeit -.. code-block:: python - class YourUnit(Unit): +Component Interaction +--------------------- - STATE: YourState +.. autoclass:: Settings - def initialize(self): - this.STATE.state1 = 0 - this.STATE.state2 = 0.0 +.. autoclass:: State -.. note:: - ``State`` uses type hints to define member variables, but does not enforce type checking. Stream ------ -Facilitates a flow of ``Messages`` into or out of a ``Component``. - -.. class:: InputStream(Message) - - Can be added to any ``Component`` as a member variable. Methods may subscribe to it. - - -.. class:: OutputStream(Message) - - Can be added to any ``Component`` as a member variable. Methods may publish to it. - +Facilitates a flow of ``Messages`` into or out of a ``Component``. -Unit ----- - -Represents a single step in the graph. To create a ``Unit``, inherit from the ``Unit`` class. - -Lifecycle Hooks -^^^^^^^^^^^^^^^ - -The following lifecycle hooks in the ``Unit`` class can be overridden. Both can be run as ``async`` functions by simply adding the ``async`` keyword when overriding. - -.. py:method:: initialize( self ) - - Runs when the ``Unit`` is instantiated. - -.. py:method:: shutdown( self ) - - Runs when the ``Unit`` terminates. - -Function Decorators -^^^^^^^^^^^^^^^^^^^ - -These function decorators can be added to member functions. +.. autoclass:: InputStream -.. py:method:: @subscriber(InputStream) +.. autoclass:: OutputStream - An async function will run once per message received from the ``InputStream`` it subscribes to. Example: - .. code-block:: python - - INPUT = ez.InputStream(Message) - - @subscriber(INPUT) - async def print_message(self, message: Message) -> None: - print(message) - - A function can have both ``@subscriber`` and ``@publisher`` decorators. - -.. py:method:: @publisher(OutputStream) - - An async function will yield messages on the designated ``OutputStream``. - - .. code-block:: python - - from typing import AsyncGenerator - - OUTPUT = OutputStream(ez.Message) - - @publisher(OUTPUT) - async def send_message(self) -> AsyncGenerator: - message = Message() - yield(OUTPUT, message) - - A function can have both ``@subscriber`` and ``@publisher`` decorators. - -.. py:method:: @main - - Designates this function to run as the main thread for this ``Unit``. A ``Unit`` may only have one of these. - -.. py:method:: @thread - - Designates this function to run as a background thread for this ``Unit``. - -.. py:method:: @task - - Designates this function to run as a task in the task/messaging thread. - -.. py:method:: @process - - Designates this function to run in its own process. - -.. py:method:: @timeit +Custom Exceptions +----------------- - ``ezmsg`` will log the amount of time this function takes to execute. +.. autoclass:: Complete -Public Methods -^^^^^^^^^^^^^^ +.. autoclass:: NormalTermination -A class which inherits from ``Unit`` also inherits one public method: -.. function:: Unit.apply_settings( self, settings: Settings ) +Entry Point +----------- - Update a ``Unit`` 's ``Settings`` object. +.. automethod:: ezmsg.core.run diff --git a/docs/source/conf.py b/docs/source/conf.py index ea7d5f88..4f4d9b51 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -6,8 +6,8 @@ copyright = "2022, JHU/APL" author = "JHU/APL" -release = "3.0" -version = "3.0.0" +release = "3.3.4" +version = "3.3.4" # -- General configuration @@ -17,11 +17,15 @@ "sphinx.ext.autodoc", "sphinx.ext.autosummary", "sphinx.ext.intersphinx", + "sphinx.ext.linkcode", + "sphinx.ext.napoleon" ] intersphinx_mapping = { "python": ("https://docs.python.org/3/", None), "sphinx": ("https://www.sphinx-doc.org/en/master/", None), + "numpy": ("https://docs.scipy.org/doc/numpy", None), + "scipy": ("https://docs.scipy.org/doc/scipy/reference", None), } intersphinx_disabled_domains = ["std"] @@ -33,3 +37,24 @@ # -- Options for EPUB output epub_show_urls = "footnote" + +add_module_names = False + +branch = "dev" +code_url = f"https://github.com/iscoe/ezmsg/blob/{branch}/" +sigproc_code_url = f"https://github.com/ezmsg-org/ezmsg-sigproc/blob/{branch}/" + + +def linkcode_resolve(domain, info): + if domain != 'py': + return None + if not info['module']: + return None + filename = info['module'].replace('.', '/') + if "sigproc" in filename: + return f"{sigproc_code_url}src/{filename}.py" + elif "core" in filename: + return f"{code_url}src/ezmsg/core/__init__.py" + else: + return f"{code_url}src/{filename}.py" + diff --git a/docs/source/developer.rst b/docs/source/developer.rst new file mode 100644 index 00000000..bbe675c4 --- /dev/null +++ b/docs/source/developer.rst @@ -0,0 +1,12 @@ +Developer Info +============== + +ezmsg developers will want to clone this repo locally then create an env with dependencies using `poetry install`. + +Documentation +------------- + +Documentation is built using Sphinx. + +`poetry install --with docs` +`poetry run docs/make html` diff --git a/docs/source/extensions.rst b/docs/source/extensions.rst index 3348634b..124e682e 100644 --- a/docs/source/extensions.rst +++ b/docs/source/extensions.rst @@ -1,2 +1,7 @@ Extensions -========== \ No newline at end of file +========== + +.. toctree:: + :maxdepth: 1 + + extensions/sigproc diff --git a/docs/source/extensions/sigproc.rst b/docs/source/extensions/sigproc.rst new file mode 100644 index 00000000..1d3853c0 --- /dev/null +++ b/docs/source/extensions/sigproc.rst @@ -0,0 +1,114 @@ +ezmsg-sigproc +============= + +ezmsg.sigproc.affinetransform +----------------------------- + +.. automodule:: ezmsg.sigproc.affinetransform + :members: + + +ezmsg.sigproc.aggregate +----------------------- + +.. automodule:: ezmsg.sigproc.aggregate + :members: + :undoc-members: + + +ezmsg.sigproc.bandpower +----------------------- + +.. automodule:: ezmsg.sigproc.bandpower + :members: + + +ezmsg.sigproc.filter +-------------------- + +.. automodule:: ezmsg.sigproc.filter + :members: + +ezmsg.sigproc.butterworthfilter +------------------------------- + +.. automodule:: ezmsg.sigproc.butterworthfilter + :members: + + +ezmsg.sigproc.decimate +---------------------- + +.. automodule:: ezmsg.sigproc.decimate + :members: + + +ezmsg.sigproc.downsample +------------------------ + +.. automodule:: ezmsg.sigproc.downsample + :members: + + +ezmsg.sigproc.ewmfilter +----------------------- + +.. automodule:: ezmsg.sigproc.ewmfilter + :members: + + +ezmsg.sigproc.sampler +--------------------- + +.. automodule:: ezmsg.sigproc.sampler + :members: + + +ezmsg.sigproc.scaler +-------------------- + +.. automodule:: ezmsg.sigproc.scaler + :members: + + +ezmsg.sigproc.signalinjector +---------------------------- + +.. automodule:: ezmsg.sigproc.signalinjector + :members: + + +ezmsg.sigproc.slicer +--------------------- + +.. automodule:: ezmsg.sigproc.slicer + :members: + + +ezmsg.sigproc.spectrum +---------------------- + +.. automodule:: ezmsg.sigproc.spectrum + :members: + :undoc-members: + + +ezmsg.sigproc.spectrogram +------------------------- + +.. automodule:: ezmsg.sigproc.spectrogram + :members: + + +ezmsg.sigproc.synth +------------------- + +.. automodule:: ezmsg.sigproc.synth + :members: + + +ezmsg.sigproc.window +-------------------- + +.. automodule:: ezmsg.sigproc.window + :members: diff --git a/docs/source/index.rst b/docs/source/index.rst index ea0456fa..76be523b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -27,3 +27,5 @@ Contents api utils other + extensions + developer diff --git a/docs/source/utils.rst b/docs/source/utils.rst index d8a1d9c8..b1ba15a6 100644 --- a/docs/source/utils.rst +++ b/docs/source/utils.rst @@ -3,310 +3,56 @@ Utility Classes Classes which implement functionality which is commonly useful for ``ezmsg`` pipelines. -DebugLog --------- - -Inherits from ``Units``. Logs messages that pass through. - -Input/OutputStreams -^^^^^^^^^^^^^^^^^^^ - -.. object:: INPUT - - `Type:` ``InputStream(Any)`` - - Send messages to log here. - -.. object:: OUTPUT - - `Type:` ``OutputStream(Any)`` - - Send messages back out to continue through the graph. - -Settings -^^^^^^^^ - -.. py:class:: DebugLogSettings(name: str = "DEBUG", max_length: Optional[int] = 400) - - Inherits from ``Settings``. Define to customize. - - `name` is included in the logstring so that if multiple DebugLogs are used in one pipeline, their messages can be differentiated. - - `max_length` sets a maximum number of chars which will be printed from the message. If the message is longer, the log message will be truncated. - -Message Collector ------------------ +AxisArray +--------- -Inherits from ``Unit``. Collects ``Messages`` into a local list. +.. automodule:: ezmsg.util.messages.axisarray + :show-inheritance: + :members: -Input/OutputStreams -^^^^^^^^^^^^^^^^^^^ - -.. object:: INPUT_MESSAGE - - Send messages here to be collected. - -.. object:: OUTPUT_MESSAGE +DebugLog +-------- - Messages will pass straight through after being recorded and be published here. +.. automodule:: ezmsg.util.debuglog + :show-inheritance: + :members: -Methods -^^^^^^^ -.. py:method:: messages() -> [Any] +MessageReplay +------------- - Returns a list of messages which have been collected. +.. automodule:: ezmsg.util.messagereplay + :show-inheritance: + :members: MessageGate ----------- -Inherits from ``Unit``. Blocks ``Messages`` from continuing through the system. Can be set as open, closed, open after n messages, or closed after n messages. - -Messages -^^^^^^^^ - -.. py:class:: GateMessage(open: bool) - - Send this message to ``INPUT_GATE`` to open or close the gate. - -Input/OutputStreams -^^^^^^^^^^^^^^^^^^^ - -.. object:: INPUT_GATE - - `Type:` ``InputStream(GateMessage)`` - - Stop or start message flow. If ``GateMessage.open == True``, messages will flow through. If ``GateMessage.open == False``, messages will be discarded. - -.. object:: INPUT - - `Type:` ``InputStream(Any)`` - - Messages which will flow through or be discarded, depending on gate status. +.. automodule:: ezmsg.util.messagegate + :show-inheritance: + :members: -.. object:: OUTPUT - - `Type`: ``OutputStream(Any)`` - - Publishes messages which flow through. - -Settings -^^^^^^^^ - -.. py:class:: MessageGateSettings(start_open: bool = False, default_open: bool = False, default_after: Optional[int] = None) - - Inherits from ``Settings``. Define to customize ``MessageGate`` behavior. - - `start_open` sets the gate's initial state to allow messages to flow through or be discarded. ``True`` will allow messages to flow through initially, ``False`` will discard messages initially. - - `default_open` sets the gate's behavior after the `default_after` number of messages have flowed through. ``True`` will allow messages to flow through, ``False`` will discard messages. - - `default_after` sets the number of messages after which the `default_open` state will be applied. MessageLogger ------------- -Inherits from ``Unit``. Logs all messages it receives to a file. File path can be set in ``SETTINGS`` or set dynamically by passing a `pathlib.Path `_ to ``INPUT_START``. - -Input/OutputStreams -^^^^^^^^^^^^^^^^^^^ - -.. object:: INPUT_START - - `Type:` ``InputStream(pathlib.Path)`` - - Pass a `pathlib.Path `_ to begin logging messages to that path. If the file path already exists, the existing file will be truncated to 0 length. If the file is already open, nothing will happen. - -.. object:: INPUT_STOP - - `Type:` ``InputStream(pathlib.Path)`` - - Pass a `pathlib.Path `_ to stop logging messages to that path. - -.. object:: INPUT_MESSAGE +.. automodule:: ezmsg.util.messagelogger + :show-inheritance: + :members: - `Type:` ``InputStream(Any)`` - - Pass a piece of data to log it to every open file which the ``MessageLogger`` is using. - -.. object:: OUTPUT_MESSAGE - - `Type`: ``OutputStream(Any)`` - - Messages which are sent to ``INPUT_MESSAGE`` will pass through and be published on ``OUTPUT_MESSAGE``. - -.. object:: OUTPUT_START - - `Type:` ``OutputStream(pathlib.Path)`` - - If a file passed to ``INPUT_START`` is successfully opened, its path will be published to ``OUTPUT_START``, otherwise ``None``. - -.. object:: OUTPUT_STOP - - `Type:` ``OutputStream(pathlib.Path)`` - - If a file passed to ``INPUT_STOP`` is successfully closed, its path will be published to ``OUTPUT_STOP``, otherwise ``None``. - - -Settings -^^^^^^^^ - -.. py:class:: MessageLoggerSettings(output: Optional[Path] = None) - - Pass a `pathlib.Path `_ for a file where the messages will be logged. If the file path already exists, the existing file will be truncated to 0 length. MessageQueue ------------ -Inherits from ``Unit``. Place between two other ``Units`` to induce backpressure. - -Input/OutputStreams -^^^^^^^^^^^^^^^^^^^ - -.. object:: INPUT - - `Type:` ``InputStream(Any)`` - - Send messages to queue here. - -.. object:: OUTPUT - - `Type:` ``OutputStream(Any)`` - - Subscribe to pull messages out of the queue. - -Settings -^^^^^^^^ - -.. py:class:: MessageQueueSettings(maxsize: int = 0, leaky: bool = False) - -`maxsize` indicates the maximum number of items which the queue will hold. - -`leaky` indicates whether the queue will drop new messages when it reaches its maxsize, or whether it will wait for space to open for them. - -MessageReplay -------------- - -Inherits from ``Unit``. Stream messages from files created by ``MessageLogger``. Stores a queue of files to stream and streams from them in order. - -Messages -^^^^^^^^ -.. py:class:: ReplayStatusMessage(filename: Path, idx: int, total: int, done: bool = False) - - Message which gives the status of a file replay. - - `filename` is the file currently being replayed. - - `idx` is the line number of the message that was just published. - - `total` is the total number of messages in the file. - - `done` denotes whether the file has finished replaying. - -.. py:class:: FileReplayMessage(filename: Optional[Path] = None, rate: Optional[float] = None) - - Add a file to the queue. - - `filename` is the path of the file to replay. - - `rate` in Hertz at which the messages will be published. If not specified, messages will publish as fast as possible. - -Input/OutputStreams -^^^^^^^^^^^^^^^^^^^ - -.. object:: INPUT_FILE - - `Type:` ``InputStream(FileReplayMessage)`` - - Add a new file to the queue. - -.. object:: INPUT_PAUSED - - `Type:` ``InputStream(bool)`` - - Send ``True`` to pause the stream, ``False`` to restart the stream. - -.. object:: INPUT_STOP - - `Type:` ``InputStream(bool)`` - - Stop the stream. Send ``True`` to also clear the queue. Send ``False`` to reset to the beginning of the current file. - -.. object:: OUTPUT_MESSAGE - - `Type:` ``OutputStream(Any)`` - - The output on which the messages from the files will be streamed. - -.. object:: OUTPUT_TOTAL - - `Type:` ``OutputStream(int)`` - - Publishes an integer total of messages which have been published on OUTPUT_MESSAGE from a single file. Resets when a file completes. - -.. object:: OUTPUT_REPLAY_STATUS - - `Type:` ``OutputStream(ReplayStatusMessage)`` - - Publishes status messages. - -Settings -^^^^^^^^ - -.. py:class:: MessageReplaySettings(filename: Optional[Path] = None, rate: Optional[float] = None, progress: bool = False): - - `filename` is the path of the file to replay. - - `rate` in Hertz at which the messages will be published. If not specified, messages will publish as fast as possible. - - `progress` will use tqdm to indicate progress through the file. Tqdm must be installed. - -TerminateOnTimeout ------------------- - -End a pipeline execution when a certain amount of time has passed without receiving a message. - -Input/OutputStreams -^^^^^^^^^^^^^^^^^^^ - -.. object:: INPUT - - Send messages here. - -Settings -^^^^^^^^ - -.. py:class:: TerminateOnTimeoutSettings(time: float = 2.0, poll_rate: float = 4.0) - - `time` in seconds after which the pipeline will be terminated if no messages have been received - - `poll_rate` - - -TerminateOnTotal ----------------- - -End a pipeline execution once a certain number of messages have been received. - -Input/OutputStreams -^^^^^^^^^^^^^^^^^^^ - -.. object:: INPUT_MESSAGE - - `Type:` ``InputStream(Any)`` - - Send messages here. - -.. object:: INPUT_TOTAL - - `Type:` ``InputStream(int)`` - - Change the total number of messages to terminate after. If this number has already been reached, termination will occur immediately. +.. automodule:: ezmsg.util.messagequeue + :show-inheritance: + :members: -Settings -^^^^^^^^ -.. py:class:: TerminateOnTotalSettings(total: int = None) +Terminate +--------- - `total` represents the total number of messages to terminate after \ No newline at end of file +.. automodule:: ezmsg.util.terminate + :show-inheritance: + :members: diff --git a/extensions/ezmsg-sigproc/LICENSE.txt b/extensions/ezmsg-sigproc/LICENSE.txt deleted file mode 100644 index a21312ac..00000000 --- a/extensions/ezmsg-sigproc/LICENSE.txt +++ /dev/null @@ -1,9 +0,0 @@ -MIT License - -Copyright (c) 2022 Johns Hopkins University Applied Physics Lab - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/extensions/ezmsg-sigproc/README.md b/extensions/ezmsg-sigproc/README.md deleted file mode 100644 index 12251336..00000000 --- a/extensions/ezmsg-sigproc/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# ezmsg.sigproc - -Timeseries signal processing implementations for ezmsg - -## Installation -`pip install ezmsg-sigproc` - -## Dependencies -* `ezmsg` -* `numpy` -* `scipy` - -## Setup (Development) -1. Install `ezmsg` either using `pip install ezmsg` or set up the repo for development as described in the `ezmsg` readme. -2. `cd` to this directory (`ezmsg-sigproc`) and run `pip install -e .` -3. Signal processing modules are available under `import ezmsg.sigproc` - diff --git a/extensions/ezmsg-sigproc/poetry.lock b/extensions/ezmsg-sigproc/poetry.lock deleted file mode 100644 index d543c14a..00000000 --- a/extensions/ezmsg-sigproc/poetry.lock +++ /dev/null @@ -1,290 +0,0 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. - -[[package]] -name = "colorama" -version = "0.4.6" -description = "Cross-platform colored terminal text." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] - -[[package]] -name = "coverage" -version = "7.3.4" -description = "Code coverage measurement for Python" -optional = false -python-versions = ">=3.8" -files = [ - {file = "coverage-7.3.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:aff2bd3d585969cc4486bfc69655e862028b689404563e6b549e6a8244f226df"}, - {file = "coverage-7.3.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4353923f38d752ecfbd3f1f20bf7a3546993ae5ecd7c07fd2f25d40b4e54571"}, - {file = "coverage-7.3.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea473c37872f0159294f7073f3fa72f68b03a129799f3533b2bb44d5e9fa4f82"}, - {file = "coverage-7.3.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5214362abf26e254d749fc0c18af4c57b532a4bfde1a057565616dd3b8d7cc94"}, - {file = "coverage-7.3.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f99b7d3f7a7adfa3d11e3a48d1a91bb65739555dd6a0d3fa68aa5852d962e5b1"}, - {file = "coverage-7.3.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:74397a1263275bea9d736572d4cf338efaade2de9ff759f9c26bcdceb383bb49"}, - {file = "coverage-7.3.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:f154bd866318185ef5865ace5be3ac047b6d1cc0aeecf53bf83fe846f4384d5d"}, - {file = "coverage-7.3.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e0d84099ea7cba9ff467f9c6f747e3fc3906e2aadac1ce7b41add72e8d0a3712"}, - {file = "coverage-7.3.4-cp310-cp310-win32.whl", hash = "sha256:3f477fb8a56e0c603587b8278d9dbd32e54bcc2922d62405f65574bd76eba78a"}, - {file = "coverage-7.3.4-cp310-cp310-win_amd64.whl", hash = "sha256:c75738ce13d257efbb6633a049fb2ed8e87e2e6c2e906c52d1093a4d08d67c6b"}, - {file = "coverage-7.3.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:997aa14b3e014339d8101b9886063c5d06238848905d9ad6c6eabe533440a9a7"}, - {file = "coverage-7.3.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8a9c5bc5db3eb4cd55ecb8397d8e9b70247904f8eca718cc53c12dcc98e59fc8"}, - {file = "coverage-7.3.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27ee94f088397d1feea3cb524e4313ff0410ead7d968029ecc4bc5a7e1d34fbf"}, - {file = "coverage-7.3.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ce03e25e18dd9bf44723e83bc202114817f3367789052dc9e5b5c79f40cf59d"}, - {file = "coverage-7.3.4-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85072e99474d894e5df582faec04abe137b28972d5e466999bc64fc37f564a03"}, - {file = "coverage-7.3.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a877810ef918d0d345b783fc569608804f3ed2507bf32f14f652e4eaf5d8f8d0"}, - {file = "coverage-7.3.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:9ac17b94ab4ca66cf803f2b22d47e392f0977f9da838bf71d1f0db6c32893cb9"}, - {file = "coverage-7.3.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:36d75ef2acab74dc948d0b537ef021306796da551e8ac8b467810911000af66a"}, - {file = "coverage-7.3.4-cp311-cp311-win32.whl", hash = "sha256:47ee56c2cd445ea35a8cc3ad5c8134cb9bece3a5cb50bb8265514208d0a65928"}, - {file = "coverage-7.3.4-cp311-cp311-win_amd64.whl", hash = "sha256:11ab62d0ce5d9324915726f611f511a761efcca970bd49d876cf831b4de65be5"}, - {file = "coverage-7.3.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:33e63c578f4acce1b6cd292a66bc30164495010f1091d4b7529d014845cd9bee"}, - {file = "coverage-7.3.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:782693b817218169bfeb9b9ba7f4a9f242764e180ac9589b45112571f32a0ba6"}, - {file = "coverage-7.3.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c4277ddaad9293454da19121c59f2d850f16bcb27f71f89a5c4836906eb35ef"}, - {file = "coverage-7.3.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3d892a19ae24b9801771a5a989fb3e850bd1ad2e2b6e83e949c65e8f37bc67a1"}, - {file = "coverage-7.3.4-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3024ec1b3a221bd10b5d87337d0373c2bcaf7afd86d42081afe39b3e1820323b"}, - {file = "coverage-7.3.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a1c3e9d2bbd6f3f79cfecd6f20854f4dc0c6e0ec317df2b265266d0dc06535f1"}, - {file = "coverage-7.3.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:e91029d7f151d8bf5ab7d8bfe2c3dbefd239759d642b211a677bc0709c9fdb96"}, - {file = "coverage-7.3.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:6879fe41c60080aa4bb59703a526c54e0412b77e649a0d06a61782ecf0853ee1"}, - {file = "coverage-7.3.4-cp312-cp312-win32.whl", hash = "sha256:fd2f8a641f8f193968afdc8fd1697e602e199931012b574194052d132a79be13"}, - {file = "coverage-7.3.4-cp312-cp312-win_amd64.whl", hash = "sha256:d1d0ce6c6947a3a4aa5479bebceff2c807b9f3b529b637e2b33dea4468d75fc7"}, - {file = "coverage-7.3.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:36797b3625d1da885b369bdaaa3b0d9fb8865caed3c2b8230afaa6005434aa2f"}, - {file = "coverage-7.3.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bfed0ec4b419fbc807dec417c401499ea869436910e1ca524cfb4f81cf3f60e7"}, - {file = "coverage-7.3.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f97ff5a9fc2ca47f3383482858dd2cb8ddbf7514427eecf5aa5f7992d0571429"}, - {file = "coverage-7.3.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:607b6c6b35aa49defaebf4526729bd5238bc36fe3ef1a417d9839e1d96ee1e4c"}, - {file = "coverage-7.3.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8e258dcc335055ab59fe79f1dec217d9fb0cdace103d6b5c6df6b75915e7959"}, - {file = "coverage-7.3.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a02ac7c51819702b384fea5ee033a7c202f732a2a2f1fe6c41e3d4019828c8d3"}, - {file = "coverage-7.3.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:b710869a15b8caf02e31d16487a931dbe78335462a122c8603bb9bd401ff6fb2"}, - {file = "coverage-7.3.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:c6a23ae9348a7a92e7f750f9b7e828448e428e99c24616dec93a0720342f241d"}, - {file = "coverage-7.3.4-cp38-cp38-win32.whl", hash = "sha256:758ebaf74578b73f727acc4e8ab4b16ab6f22a5ffd7dd254e5946aba42a4ce76"}, - {file = "coverage-7.3.4-cp38-cp38-win_amd64.whl", hash = "sha256:309ed6a559bc942b7cc721f2976326efbfe81fc2b8f601c722bff927328507dc"}, - {file = "coverage-7.3.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:aefbb29dc56317a4fcb2f3857d5bce9b881038ed7e5aa5d3bcab25bd23f57328"}, - {file = "coverage-7.3.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:183c16173a70caf92e2dfcfe7c7a576de6fa9edc4119b8e13f91db7ca33a7923"}, - {file = "coverage-7.3.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a4184dcbe4f98d86470273e758f1d24191ca095412e4335ff27b417291f5964"}, - {file = "coverage-7.3.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:93698ac0995516ccdca55342599a1463ed2e2d8942316da31686d4d614597ef9"}, - {file = "coverage-7.3.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb220b3596358a86361139edce40d97da7458412d412e1e10c8e1970ee8c09ab"}, - {file = "coverage-7.3.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d5b14abde6f8d969e6b9dd8c7a013d9a2b52af1235fe7bebef25ad5c8f47fa18"}, - {file = "coverage-7.3.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:610afaf929dc0e09a5eef6981edb6a57a46b7eceff151947b836d869d6d567c1"}, - {file = "coverage-7.3.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d6ed790728fb71e6b8247bd28e77e99d0c276dff952389b5388169b8ca7b1c28"}, - {file = "coverage-7.3.4-cp39-cp39-win32.whl", hash = "sha256:c15fdfb141fcf6a900e68bfa35689e1256a670db32b96e7a931cab4a0e1600e5"}, - {file = "coverage-7.3.4-cp39-cp39-win_amd64.whl", hash = "sha256:38d0b307c4d99a7aca4e00cad4311b7c51b7ac38fb7dea2abe0d182dd4008e05"}, - {file = "coverage-7.3.4-pp38.pp39.pp310-none-any.whl", hash = "sha256:b1e0f25ae99cf247abfb3f0fac7ae25739e4cd96bf1afa3537827c576b4847e5"}, - {file = "coverage-7.3.4.tar.gz", hash = "sha256:020d56d2da5bc22a0e00a5b0d54597ee91ad72446fa4cf1b97c35022f6b6dbf0"}, -] - -[package.dependencies] -tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} - -[package.extras] -toml = ["tomli"] - -[[package]] -name = "exceptiongroup" -version = "1.2.0" -description = "Backport of PEP 654 (exception groups)" -optional = false -python-versions = ">=3.7" -files = [ - {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, - {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, -] - -[package.extras] -test = ["pytest (>=6)"] - -[[package]] -name = "ezmsg" -version = "3.3.3" -description = "A simple DAG-based computation model" -optional = false -python-versions = ">=3.8" -files = [ - {file = "ezmsg-3.3.3-py3-none-any.whl", hash = "sha256:62920470d8a692fcd986e980a80e27d0ec3c0a36677d2068ee75b8cf301a0cde"}, - {file = "ezmsg-3.3.3.tar.gz", hash = "sha256:411dd4e027e37bb322bfbcef75264d47134ea64efa2b428522c257d959ca439f"}, -] - -[package.dependencies] -typing-extensions = "*" - -[package.extras] -all-ext = ["ezmsg-sigproc", "ezmsg-websocket", "ezmsg-zmq"] -test = ["numpy", "pytest", "pytest-asyncio", "pytest-cov"] - -[[package]] -name = "iniconfig" -version = "2.0.0" -description = "brain-dead simple config-ini parsing" -optional = false -python-versions = ">=3.7" -files = [ - {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, - {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, -] - -[[package]] -name = "numpy" -version = "1.24.4" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.8" -files = [ - {file = "numpy-1.24.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0bfb52d2169d58c1cdb8cc1f16989101639b34c7d3ce60ed70b19c63eba0b64"}, - {file = "numpy-1.24.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ed094d4f0c177b1b8e7aa9cba7d6ceed51c0e569a5318ac0ca9a090680a6a1b1"}, - {file = "numpy-1.24.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79fc682a374c4a8ed08b331bef9c5f582585d1048fa6d80bc6c35bc384eee9b4"}, - {file = "numpy-1.24.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ffe43c74893dbf38c2b0a1f5428760a1a9c98285553c89e12d70a96a7f3a4d6"}, - {file = "numpy-1.24.4-cp310-cp310-win32.whl", hash = "sha256:4c21decb6ea94057331e111a5bed9a79d335658c27ce2adb580fb4d54f2ad9bc"}, - {file = "numpy-1.24.4-cp310-cp310-win_amd64.whl", hash = "sha256:b4bea75e47d9586d31e892a7401f76e909712a0fd510f58f5337bea9572c571e"}, - {file = "numpy-1.24.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f136bab9c2cfd8da131132c2cf6cc27331dd6fae65f95f69dcd4ae3c3639c810"}, - {file = "numpy-1.24.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e2926dac25b313635e4d6cf4dc4e51c8c0ebfed60b801c799ffc4c32bf3d1254"}, - {file = "numpy-1.24.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:222e40d0e2548690405b0b3c7b21d1169117391c2e82c378467ef9ab4c8f0da7"}, - {file = "numpy-1.24.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7215847ce88a85ce39baf9e89070cb860c98fdddacbaa6c0da3ffb31b3350bd5"}, - {file = "numpy-1.24.4-cp311-cp311-win32.whl", hash = "sha256:4979217d7de511a8d57f4b4b5b2b965f707768440c17cb70fbf254c4b225238d"}, - {file = "numpy-1.24.4-cp311-cp311-win_amd64.whl", hash = "sha256:b7b1fc9864d7d39e28f41d089bfd6353cb5f27ecd9905348c24187a768c79694"}, - {file = "numpy-1.24.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1452241c290f3e2a312c137a9999cdbf63f78864d63c79039bda65ee86943f61"}, - {file = "numpy-1.24.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:04640dab83f7c6c85abf9cd729c5b65f1ebd0ccf9de90b270cd61935eef0197f"}, - {file = "numpy-1.24.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5425b114831d1e77e4b5d812b69d11d962e104095a5b9c3b641a218abcc050e"}, - {file = "numpy-1.24.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd80e219fd4c71fc3699fc1dadac5dcf4fd882bfc6f7ec53d30fa197b8ee22dc"}, - {file = "numpy-1.24.4-cp38-cp38-win32.whl", hash = "sha256:4602244f345453db537be5314d3983dbf5834a9701b7723ec28923e2889e0bb2"}, - {file = "numpy-1.24.4-cp38-cp38-win_amd64.whl", hash = "sha256:692f2e0f55794943c5bfff12b3f56f99af76f902fc47487bdfe97856de51a706"}, - {file = "numpy-1.24.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2541312fbf09977f3b3ad449c4e5f4bb55d0dbf79226d7724211acc905049400"}, - {file = "numpy-1.24.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9667575fb6d13c95f1b36aca12c5ee3356bf001b714fc354eb5465ce1609e62f"}, - {file = "numpy-1.24.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3a86ed21e4f87050382c7bc96571755193c4c1392490744ac73d660e8f564a9"}, - {file = "numpy-1.24.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d11efb4dbecbdf22508d55e48d9c8384db795e1b7b51ea735289ff96613ff74d"}, - {file = "numpy-1.24.4-cp39-cp39-win32.whl", hash = "sha256:6620c0acd41dbcb368610bb2f4d83145674040025e5536954782467100aa8835"}, - {file = "numpy-1.24.4-cp39-cp39-win_amd64.whl", hash = "sha256:befe2bf740fd8373cf56149a5c23a0f601e82869598d41f8e188a0e9869926f8"}, - {file = "numpy-1.24.4-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:31f13e25b4e304632a4619d0e0777662c2ffea99fcae2029556b17d8ff958aef"}, - {file = "numpy-1.24.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95f7ac6540e95bc440ad77f56e520da5bf877f87dca58bd095288dce8940532a"}, - {file = "numpy-1.24.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:e98f220aa76ca2a977fe435f5b04d7b3470c0a2e6312907b37ba6068f26787f2"}, - {file = "numpy-1.24.4.tar.gz", hash = "sha256:80f5e3a4e498641401868df4208b74581206afbee7cf7b8329daae82676d9463"}, -] - -[[package]] -name = "packaging" -version = "23.2" -description = "Core utilities for Python packages" -optional = false -python-versions = ">=3.7" -files = [ - {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, - {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, -] - -[[package]] -name = "pluggy" -version = "1.3.0" -description = "plugin and hook calling mechanisms for python" -optional = false -python-versions = ">=3.8" -files = [ - {file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"}, - {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"}, -] - -[package.extras] -dev = ["pre-commit", "tox"] -testing = ["pytest", "pytest-benchmark"] - -[[package]] -name = "pytest" -version = "7.4.3" -description = "pytest: simple powerful testing with Python" -optional = false -python-versions = ">=3.7" -files = [ - {file = "pytest-7.4.3-py3-none-any.whl", hash = "sha256:0d009c083ea859a71b76adf7c1d502e4bc170b80a8ef002da5806527b9591fac"}, - {file = "pytest-7.4.3.tar.gz", hash = "sha256:d989d136982de4e3b29dabcc838ad581c64e8ed52c11fbe86ddebd9da0818cd5"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "sys_platform == \"win32\""} -exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} -iniconfig = "*" -packaging = "*" -pluggy = ">=0.12,<2.0" -tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} - -[package.extras] -testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] - -[[package]] -name = "pytest-cov" -version = "4.1.0" -description = "Pytest plugin for measuring coverage." -optional = false -python-versions = ">=3.7" -files = [ - {file = "pytest-cov-4.1.0.tar.gz", hash = "sha256:3904b13dfbfec47f003b8e77fd5b589cd11904a21ddf1ab38a64f204d6a10ef6"}, - {file = "pytest_cov-4.1.0-py3-none-any.whl", hash = "sha256:6ba70b9e97e69fcc3fb45bfeab2d0a138fb65c4d0d6a41ef33983ad114be8c3a"}, -] - -[package.dependencies] -coverage = {version = ">=5.2.1", extras = ["toml"]} -pytest = ">=4.6" - -[package.extras] -testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"] - -[[package]] -name = "scipy" -version = "1.9.3" -description = "Fundamental algorithms for scientific computing in Python" -optional = false -python-versions = ">=3.8" -files = [ - {file = "scipy-1.9.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1884b66a54887e21addf9c16fb588720a8309a57b2e258ae1c7986d4444d3bc0"}, - {file = "scipy-1.9.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:83b89e9586c62e787f5012e8475fbb12185bafb996a03257e9675cd73d3736dd"}, - {file = "scipy-1.9.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a72d885fa44247f92743fc20732ae55564ff2a519e8302fb7e18717c5355a8b"}, - {file = "scipy-1.9.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d01e1dd7b15bd2449c8bfc6b7cc67d630700ed655654f0dfcf121600bad205c9"}, - {file = "scipy-1.9.3-cp310-cp310-win_amd64.whl", hash = "sha256:68239b6aa6f9c593da8be1509a05cb7f9efe98b80f43a5861cd24c7557e98523"}, - {file = "scipy-1.9.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b41bc822679ad1c9a5f023bc93f6d0543129ca0f37c1ce294dd9d386f0a21096"}, - {file = "scipy-1.9.3-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:90453d2b93ea82a9f434e4e1cba043e779ff67b92f7a0e85d05d286a3625df3c"}, - {file = "scipy-1.9.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83c06e62a390a9167da60bedd4575a14c1f58ca9dfde59830fc42e5197283dab"}, - {file = "scipy-1.9.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:abaf921531b5aeaafced90157db505e10345e45038c39e5d9b6c7922d68085cb"}, - {file = "scipy-1.9.3-cp311-cp311-win_amd64.whl", hash = "sha256:06d2e1b4c491dc7d8eacea139a1b0b295f74e1a1a0f704c375028f8320d16e31"}, - {file = "scipy-1.9.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5a04cd7d0d3eff6ea4719371cbc44df31411862b9646db617c99718ff68d4840"}, - {file = "scipy-1.9.3-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:545c83ffb518094d8c9d83cce216c0c32f8c04aaf28b92cc8283eda0685162d5"}, - {file = "scipy-1.9.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d54222d7a3ba6022fdf5773931b5d7c56efe41ede7f7128c7b1637700409108"}, - {file = "scipy-1.9.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cff3a5295234037e39500d35316a4c5794739433528310e117b8a9a0c76d20fc"}, - {file = "scipy-1.9.3-cp38-cp38-win_amd64.whl", hash = "sha256:2318bef588acc7a574f5bfdff9c172d0b1bf2c8143d9582e05f878e580a3781e"}, - {file = "scipy-1.9.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d644a64e174c16cb4b2e41dfea6af722053e83d066da7343f333a54dae9bc31c"}, - {file = "scipy-1.9.3-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:da8245491d73ed0a994ed9c2e380fd058ce2fa8a18da204681f2fe1f57f98f95"}, - {file = "scipy-1.9.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4db5b30849606a95dcf519763dd3ab6fe9bd91df49eba517359e450a7d80ce2e"}, - {file = "scipy-1.9.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c68db6b290cbd4049012990d7fe71a2abd9ffbe82c0056ebe0f01df8be5436b0"}, - {file = "scipy-1.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:5b88e6d91ad9d59478fafe92a7c757d00c59e3bdc3331be8ada76a4f8d683f58"}, - {file = "scipy-1.9.3.tar.gz", hash = "sha256:fbc5c05c85c1a02be77b1ff591087c83bc44579c6d2bd9fb798bb64ea5e1a027"}, -] - -[package.dependencies] -numpy = ">=1.18.5,<1.26.0" - -[package.extras] -dev = ["flake8", "mypy", "pycodestyle", "typing_extensions"] -doc = ["matplotlib (>2)", "numpydoc", "pydata-sphinx-theme (==0.9.0)", "sphinx (!=4.1.0)", "sphinx-panels (>=0.5.2)", "sphinx-tabs"] -test = ["asv", "gmpy2", "mpmath", "pytest", "pytest-cov", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] - -[[package]] -name = "tomli" -version = "2.0.1" -description = "A lil' TOML parser" -optional = false -python-versions = ">=3.7" -files = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, -] - -[[package]] -name = "typing-extensions" -version = "4.9.0" -description = "Backported and Experimental Type Hints for Python 3.8+" -optional = false -python-versions = ">=3.8" -files = [ - {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, - {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, -] - -[metadata] -lock-version = "2.0" -python-versions = "^3.8" -content-hash = "460365db2418dc58ef305ed0c89e5581187cd17112b77997b53b5636d7b839b3" diff --git a/extensions/ezmsg-sigproc/pyproject.toml b/extensions/ezmsg-sigproc/pyproject.toml deleted file mode 100644 index 060cbeb0..00000000 --- a/extensions/ezmsg-sigproc/pyproject.toml +++ /dev/null @@ -1,30 +0,0 @@ -[tool.poetry] -name = "ezmsg-sigproc" -version = "1.2.3" -description = "Timeseries signal processing implementations in ezmsg" -authors = [ - "Milsap, Griffin ", - "Peranich, Preston ", -] -license = "MIT" -readme = "README.md" -packages = [{ include = "ezmsg", from = "src" }] - -[tool.poetry.dependencies] -python = "^3.8" -ezmsg = "^3.3.0" -numpy = "^1.19.5" -scipy = "^1.6.3" - -[tool.poetry.group.test.dependencies] -pytest = "^7.0.0" -pytest-cov = "*" - -[tool.pytest.ini_options] -addopts = ["--import-mode=importlib"] -pythonpath = ["src", "tests"] -testpaths = "tests" - -[build-system] -requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" diff --git a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/__init__.py b/extensions/ezmsg-sigproc/src/ezmsg/sigproc/__init__.py deleted file mode 100644 index 684cdda9..00000000 --- a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -import importlib.metadata - - -__version__ = importlib.metadata.version("ezmsg-sigproc") diff --git a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/affinetransform.py b/extensions/ezmsg-sigproc/src/ezmsg/sigproc/affinetransform.py deleted file mode 100644 index 23f4007a..00000000 --- a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/affinetransform.py +++ /dev/null @@ -1,124 +0,0 @@ -from dataclasses import replace -import os -from pathlib import Path -from typing import Generator, Optional, Union - -import numpy as np -import ezmsg.core as ez -from ezmsg.util.messages.axisarray import AxisArray -from ezmsg.util.generator import consumer, GenAxisArray - - -@consumer -def affine_transform( - weights: Union[np.ndarray, str, Path], - axis: Optional[str] = None, - right_multiply: bool = True, -) -> Generator[AxisArray, AxisArray, None]: - axis_arr_in = AxisArray(np.array([]), dims=[""]) - axis_arr_out = AxisArray(np.array([]), dims=[""]) - - if isinstance(weights, str): - weights = Path(os.path.abspath(os.path.expanduser(weights))) - if isinstance(weights, Path): - weights = np.loadtxt(weights, delimiter=",") - if not right_multiply: - weights = weights.T - weights = np.ascontiguousarray(weights) - - while True: - axis_arr_in = yield axis_arr_out - - if axis is None: - axis = axis_arr_in.dims[-1] - axis_idx = -1 - else: - axis_idx = axis_arr_in.get_axis_idx(axis) - - data = axis_arr_in.data - - if data.shape[axis_idx] == (weights.shape[0] - 1): - # The weights are stacked A|B where A is the transform and B is a single row - # in the equation y = Ax + B. This supports NeuroKey's weights matrices. - sample_shape = data.shape[:axis_idx] + (1,) + data.shape[axis_idx+1:] - data = np.concatenate((data, np.ones(sample_shape).astype(data.dtype)), axis=axis_idx) - - if axis_idx in [-1, len(axis_arr_in.dims) - 1]: - data = np.matmul(data, weights) - else: - data = np.moveaxis(data, axis_idx, -1) - data = np.matmul(data, weights) - data = np.moveaxis(data, -1, axis_idx) - axis_arr_out = replace(axis_arr_in, data=data) - - -class AffineTransformSettings(ez.Settings): - weights: Union[np.ndarray, str, Path] - axis: Optional[str] = None - right_multiply: bool = True - - -class AffineTransform(GenAxisArray): - SETTINGS: AffineTransformSettings - - def construct_generator(self): - self.STATE.gen = affine_transform( - weights=self.SETTINGS.weights, - axis=self.SETTINGS.axis, - right_multiply=self.SETTINGS.right_multiply, - ) - - -@consumer -def common_rereference( - mode: str = "mean", axis: Optional[str] = None, include_current: bool = True -) -> Generator[AxisArray, AxisArray, None]: - axis_arr_in = AxisArray(np.array([]), dims=[""]) - axis_arr_out = AxisArray(np.array([]), dims=[""]) - - func = {"mean": np.mean, "median": np.median}[mode] - - while True: - axis_arr_in = yield axis_arr_out - - if axis is None: - axis = axis_arr_in.dims[-1] - axis_idx = -1 - else: - axis_idx = axis_arr_in.get_axis_idx(axis) - - ref_data = func(axis_arr_in.data, axis=axis_idx, keepdims=True) - - if not include_current: - # Typical `CAR = x[0]/N + x[1]/N + ... x[i-1]/N + x[i]/N + x[i+1]/N + ... + x[N-1]/N` - # and is the same for all i, so it is calculated only once in `ref_data`. - # However, if we had excluded the current channel, - # then we would have omitted the contribution of the current channel: - # `CAR[i] = x[0]/(N-1) + x[1]/(N-1) + ... x[i-1]/(N-1) + x[i+1]/(N-1) + ... + x[N-1]/(N-1)` - # The majority of the calculation is the same as when the current channel is included; - # we need only rescale CAR so the divisor is `N-1` instead of `N`, then subtract the contribution - # from the current channel (i.e., `x[i] / (N-1)`) - # i.e., `CAR[i] = (N / (N-1)) * common_CAR - x[i]/(N-1)` - # We can use broadcasting subtraction instead of looping over channels. - N = axis_arr_in.data.shape[axis_idx] - ref_data = (N / (N - 1)) * ref_data - axis_arr_in.data / (N - 1) - # Side note: I profiled using affine_transform and it's about 30x slower than this implementation. - - axis_arr_out = replace(axis_arr_in, data=axis_arr_in.data - ref_data) - - -class CommonRereferenceSettings(ez.Settings): - mode: str = "mean" - axis: Optional[str] = None - include_current: bool = True - - -class CommonRereference(GenAxisArray): - SETTINGS: CommonRereferenceSettings - - def construct_generator(self): - self.STATE.gen = common_rereference( - mode=self.SETTINGS.mode, - axis=self.SETTINGS.axis, - include_current=self.SETTINGS.include_current, - ) diff --git a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/aggregate.py b/extensions/ezmsg-sigproc/src/ezmsg/sigproc/aggregate.py deleted file mode 100644 index 8bded83c..00000000 --- a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/aggregate.py +++ /dev/null @@ -1,103 +0,0 @@ -from dataclasses import replace -import typing - -import numpy as np -import ezmsg.core as ez -from ezmsg.util.generator import consumer, GenAxisArray -from ezmsg.util.messages.axisarray import AxisArray, slice_along_axis -from ezmsg.sigproc.spectral import OptionsEnum - - -class AggregationFunction(OptionsEnum): - NONE = "None (all)" - MAX = "max" - MIN = "min" - MEAN = "mean" - MEDIAN = "median" - STD = "std" - NANMAX = "nanmax" - NANMIN = "nanmin" - NANMEAN = "nanmean" - NANMEDIAN = "nanmedian" - NANSTD = "nanstd" - - -AGGREGATORS = { - AggregationFunction.NONE: np.all, - AggregationFunction.MAX: np.max, - AggregationFunction.MIN: np.min, - AggregationFunction.MEAN: np.mean, - AggregationFunction.MEDIAN: np.median, - AggregationFunction.STD: np.std, - AggregationFunction.NANMAX: np.nanmax, - AggregationFunction.NANMIN: np.nanmin, - AggregationFunction.NANMEAN: np.nanmean, - AggregationFunction.NANMEDIAN: np.nanmedian, - AggregationFunction.NANSTD: np.nanstd -} - - -@consumer -def ranged_aggregate( - axis: typing.Optional[str] = None, - bands: typing.Optional[typing.List[typing.Tuple[float, float]]] = None, - operation: AggregationFunction = AggregationFunction.MEAN -): - axis_arr_in = AxisArray(np.array([]), dims=[""]) - axis_arr_out = AxisArray(np.array([]), dims=[""]) - - target_axis: typing.Optional[AxisArray.Axis] = None - out_axis = AxisArray.Axis() - slices: typing.Optional[typing.List[typing.Tuple[typing.Any, ...]]] = None - axis_name = "" - - while True: - axis_arr_in = yield axis_arr_out - if bands is None: - axis_arr_out = axis_arr_in - else: - if slices is None or target_axis != axis_arr_in.get_axis(axis_name): - # Calculate the slices. If we are operating on time axis then - axis_name = axis or axis_arr_in.dims[0] - ax_idx = axis_arr_in.get_axis_idx(axis_name) - target_axis = axis_arr_in.axes[axis_name] - - ax_vec = target_axis.offset + np.arange(axis_arr_in.data.shape[ax_idx]) * target_axis.gain - slices = [] - mids = [] - for (start, stop) in bands: - inds = np.where(np.logical_and(ax_vec >= start, ax_vec <= stop))[0] - mids.append(np.mean(inds) * target_axis.gain + target_axis.offset) - slices.append(np.s_[inds[0]:inds[-1] + 1]) - out_axis = AxisArray.Axis( - unit=target_axis.unit, offset=mids[0], gain=(mids[1] - mids[0]) if len(mids) > 1 else 1.0 - ) - - agg_func = AGGREGATORS[operation] - out_data = [ - agg_func(slice_along_axis(axis_arr_in.data, sl, axis=ax_idx), axis=ax_idx) - for sl in slices - ] - new_axes = {**axis_arr_in.axes, axis_name: out_axis} - axis_arr_out = replace( - axis_arr_in, - data=np.stack(out_data, axis=ax_idx), - axes=new_axes - ) - - -class RangedAggregateSettings(ez.Settings): - axis: typing.Optional[str] = None - bands: typing.Optional[typing.List[typing.Tuple[float, float]]] = None - operation: AggregationFunction = AggregationFunction.MEAN - - -class RangedAggregate(GenAxisArray): - SETTINGS: RangedAggregateSettings - - def construct_generator(self): - self.STATE.gen = ranged_aggregate( - axis=self.SETTINGS.axis, - bands=self.SETTINGS.bands, - operation=self.SETTINGS.operation - ) diff --git a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/bandpower.py b/extensions/ezmsg-sigproc/src/ezmsg/sigproc/bandpower.py deleted file mode 100644 index 5fd47305..00000000 --- a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/bandpower.py +++ /dev/null @@ -1,53 +0,0 @@ -from dataclasses import field -import typing - -import numpy as np -import ezmsg.core as ez -from ezmsg.util.messages.axisarray import AxisArray -from ezmsg.util.generator import consumer, compose, GenAxisArray - -from .spectrogram import spectrogram, SpectrogramSettings -from .aggregate import ranged_aggregate, AggregationFunction - - -@consumer -def bandpower( - spectrogram_settings: SpectrogramSettings, - bands: typing.Optional[typing.List[typing.Tuple[float, float]]] = [(17, 30), (70, 170)] -) -> typing.Generator[AxisArray, AxisArray, None]: - axis_arr_in = AxisArray(np.array([]), dims=[""]) - axis_arr_out = AxisArray(np.array([]), dims=[""]) - - f_spec = spectrogram( - window_dur=spectrogram_settings.window_dur, - window_shift=spectrogram_settings.window_shift, - window=spectrogram_settings.window, - transform=spectrogram_settings.transform, - output=spectrogram_settings.output - ) - f_agg = ranged_aggregate( - axis="freq", - bands=bands, - operation=AggregationFunction.MEAN - ) - pipeline = compose(f_spec, f_agg) - - while True: - axis_arr_in = yield axis_arr_out - axis_arr_out = pipeline(axis_arr_in) - - -class BandPowerSettings(ez.Settings): - spectrogram_settings: SpectrogramSettings = field(default_factory=SpectrogramSettings) - bands: typing.Optional[typing.List[typing.Tuple[float, float]]] = ( - field(default_factory=lambda: [(17, 30), (70, 170)])) - - -class BandPower(GenAxisArray): - SETTINGS: BandPowerSettings - - def construct_generator(self): - self.STATE.gen = bandpower( - spectrogram_settings=self.SETTINGS.spectrogram_settings, - bands=self.SETTINGS.bands - ) diff --git a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/butterworthfilter.py b/extensions/ezmsg-sigproc/src/ezmsg/sigproc/butterworthfilter.py deleted file mode 100644 index b16d2c32..00000000 --- a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/butterworthfilter.py +++ /dev/null @@ -1,101 +0,0 @@ -import typing - -import ezmsg.core as ez -import scipy.signal -import numpy as np - -from .filter import filtergen, Filter, FilterState, FilterSettingsBase - -from ezmsg.util.messages.axisarray import AxisArray -from ezmsg.util.generator import consumer - - -class ButterworthFilterSettings(FilterSettingsBase): - order: int = 0 - cuton: typing.Optional[float] = None # Hz - cutoff: typing.Optional[float] = None # Hz - - def filter_specs(self) -> typing.Optional[typing.Tuple[str, typing.Union[float, typing.Tuple[float, float]]]]: - if self.cuton is None and self.cutoff is None: - return None - elif self.cuton is None and self.cutoff is not None: - return "lowpass", self.cutoff - elif self.cuton is not None and self.cutoff is None: - return "highpass", self.cuton - elif self.cuton is not None and self.cutoff is not None: - if self.cuton <= self.cutoff: - return "bandpass", (self.cuton, self.cutoff) - else: - return "bandstop", (self.cutoff, self.cuton) - - -@consumer -def butter( - axis: typing.Optional[str], - order: int = 0, - cuton: typing.Optional[float] = None, - cutoff: typing.Optional[float] = None, - coef_type: str = "ba", -) -> typing.Generator[AxisArray, AxisArray, None]: - # IO - axis_arr_in = AxisArray(np.array([]), dims=[""]) - axis_arr_out = AxisArray(np.array([]), dims=[""]) - - btype, cutoffs = ButterworthFilterSettings( - order=order, cuton=cuton, cutoff=cutoff - ).filter_specs() - - # We cannot calculate coefs yet because we do not know input sample rate - coefs = None - filter_gen = filtergen(axis, coefs, coef_type) # Passthrough. - - while True: - axis_arr_in = yield axis_arr_out - if coefs is None and order > 0: - fs = 1 / axis_arr_in.axes[axis or axis_arr_in.dims[0]].gain - coefs = scipy.signal.butter( - order, Wn=cutoffs, btype=btype, fs=fs, output=coef_type - ) - filter_gen = filtergen(axis, coefs, coef_type) - - axis_arr_out = filter_gen.send(axis_arr_in) - - -class ButterworthFilterState(FilterState): - design: ButterworthFilterSettings - - -class ButterworthFilter(Filter): - SETTINGS: ButterworthFilterSettings - STATE: ButterworthFilterState - - INPUT_FILTER = ez.InputStream(ButterworthFilterSettings) - - def initialize(self) -> None: - self.STATE.design = self.SETTINGS - self.STATE.filt_designed = True - super().initialize() - - def design_filter(self) -> typing.Optional[typing.Tuple[np.ndarray, np.ndarray]]: - specs = self.STATE.design.filter_specs() - if self.STATE.design.order > 0 and specs is not None: - btype, cut = specs - return scipy.signal.butter( - self.STATE.design.order, - Wn=cut, - btype=btype, - fs=self.STATE.fs, - output="ba", - ) - else: - return None - - @ez.subscriber(INPUT_FILTER) - async def redesign(self, message: ButterworthFilterSettings) -> None: - if type(message) is not ButterworthFilterSettings: - return - - if self.STATE.design.order != message.order: - self.STATE.zi = None - self.STATE.design = message - self.update_filter() diff --git a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/decimate.py b/extensions/ezmsg-sigproc/src/ezmsg/sigproc/decimate.py deleted file mode 100644 index b975b021..00000000 --- a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/decimate.py +++ /dev/null @@ -1,40 +0,0 @@ -import ezmsg.core as ez - -import scipy.signal - -from ezmsg.util.messages.axisarray import AxisArray - -from .downsample import Downsample, DownsampleSettings -from .filter import Filter, FilterCoefficients, FilterSettings - - -class Decimate(ez.Collection): - SETTINGS: DownsampleSettings - - INPUT_SIGNAL = ez.InputStream(AxisArray) - OUTPUT_SIGNAL = ez.OutputStream(AxisArray) - - FILTER = Filter() - DOWNSAMPLE = Downsample() - - def configure(self) -> None: - self.DOWNSAMPLE.apply_settings(self.SETTINGS) - - if self.SETTINGS.factor < 1: - raise ValueError("Decimation factor must be >= 1 (no decimation") - elif self.SETTINGS.factor == 1: - filt = FilterCoefficients() - else: - # See scipy.signal.decimate for IIR Filter Condition - b, a = scipy.signal.cheby1(8, 0.05, 0.8 / self.SETTINGS.factor) - system = scipy.signal.dlti(b, a) - filt = FilterCoefficients(b=system.num, a=system.den) # type: ignore - - self.FILTER.apply_settings(FilterSettings(filt=filt)) - - def network(self) -> ez.NetworkDefinition: - return ( - (self.INPUT_SIGNAL, self.FILTER.INPUT_SIGNAL), - (self.FILTER.OUTPUT_SIGNAL, self.DOWNSAMPLE.INPUT_SIGNAL), - (self.DOWNSAMPLE.OUTPUT_SIGNAL, self.OUTPUT_SIGNAL), - ) diff --git a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/downsample.py b/extensions/ezmsg-sigproc/src/ezmsg/sigproc/downsample.py deleted file mode 100644 index bb37011f..00000000 --- a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/downsample.py +++ /dev/null @@ -1,89 +0,0 @@ -from dataclasses import replace -import traceback -from typing import AsyncGenerator, Optional, Generator - -import numpy as np - -from ezmsg.util.messages.axisarray import AxisArray -from ezmsg.util.generator import consumer -import ezmsg.core as ez - - -@consumer -def downsample( - axis: Optional[str] = None, factor: int = 1 -) -> Generator[AxisArray, AxisArray, None]: - axis_arr_in = AxisArray(np.array([]), dims=[""]) - axis_arr_out = AxisArray(np.array([]), dims=[""]) - - # state variables - s_idx = 0 - - while True: - axis_arr_in = yield axis_arr_out - - if axis is None: - axis = axis_arr_in.dims[0] - axis_info = axis_arr_in.get_axis(axis) - axis_idx = axis_arr_in.get_axis_idx(axis) - - samples = np.arange(axis_arr_in.data.shape[axis_idx]) + s_idx - samples = samples % factor - s_idx = samples[-1] + 1 - - pub_samples = np.where(samples == 0)[0] - if len(pub_samples) > 0: - new_axes = {ax_name: axis_arr_in.get_axis(ax_name) for ax_name in axis_arr_in.dims} - new_offset = axis_info.offset + (axis_info.gain * pub_samples[0].item()) - new_gain = axis_info.gain * factor - new_axes[axis] = replace(axis_info, gain=new_gain, offset=new_offset) - down_data = np.take(axis_arr_in.data, pub_samples, axis=axis_idx) - axis_arr_out = replace(axis_arr_in, data=down_data, dims=axis_arr_in.dims, axes=new_axes) - else: - axis_arr_out = None - - -class DownsampleSettings(ez.Settings): - axis: Optional[str] = None - factor: int = 1 - - -class DownsampleState(ez.State): - cur_settings: DownsampleSettings - gen: Generator - - -class Downsample(ez.Unit): - SETTINGS: DownsampleSettings - STATE: DownsampleState - - INPUT_SETTINGS = ez.InputStream(DownsampleSettings) - INPUT_SIGNAL = ez.InputStream(AxisArray) - OUTPUT_SIGNAL = ez.OutputStream(AxisArray) - - def construct_generator(self): - self.STATE.gen = downsample(axis=self.STATE.cur_settings.axis, factor=self.STATE.cur_settings.factor) - - def initialize(self) -> None: - self.STATE.cur_settings = self.SETTINGS - self.construct_generator() - - @ez.subscriber(INPUT_SETTINGS) - async def on_settings(self, msg: DownsampleSettings) -> None: - self.STATE.cur_settings = msg - self.construct_generator() - - @ez.subscriber(INPUT_SIGNAL, zero_copy=True) - @ez.publisher(OUTPUT_SIGNAL) - async def on_signal(self, msg: AxisArray) -> AsyncGenerator: - if self.STATE.cur_settings.factor < 1: - raise ValueError("Downsample factor must be at least 1 (no downsampling)") - - try: - out_msg = self.STATE.gen.send(msg) - if out_msg is not None: - yield self.OUTPUT_SIGNAL, out_msg - except (StopIteration, GeneratorExit): - ez.logger.debug(f"Downsample closed in {self.address}") - except Exception: - ez.logger.info(traceback.format_exc()) diff --git a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/ewmfilter.py b/extensions/ezmsg-sigproc/src/ezmsg/sigproc/ewmfilter.py deleted file mode 100644 index 7de82ca5..00000000 --- a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/ewmfilter.py +++ /dev/null @@ -1,135 +0,0 @@ -import asyncio -from dataclasses import replace - -import ezmsg.core as ez -from ezmsg.util.messages.axisarray import AxisArray - -import numpy as np - -from .window import Window, WindowSettings - -from typing import AsyncGenerator, Optional - - -class EWMSettings(ez.Settings): - axis: Optional[str] = None - zero_offset: bool = True # If true, we assume zero DC offset - - -class EWMState(ez.State): - buffer_queue: "asyncio.Queue[AxisArray]" - signal_queue: "asyncio.Queue[AxisArray]" - - -class EWM(ez.Unit): - """ - Exponentially Weighted Moving Average Standardization - - References https://stackoverflow.com/a/42926270 - """ - - SETTINGS: EWMSettings - STATE: EWMState - - INPUT_SIGNAL = ez.InputStream(AxisArray) - INPUT_BUFFER = ez.InputStream(AxisArray) - OUTPUT_SIGNAL = ez.OutputStream(AxisArray) - - def initialize(self) -> None: - self.STATE.signal_queue = asyncio.Queue() - self.STATE.buffer_queue = asyncio.Queue() - - @ez.subscriber(INPUT_SIGNAL) - async def on_signal(self, message: AxisArray) -> None: - self.STATE.signal_queue.put_nowait(message) - - @ez.subscriber(INPUT_BUFFER) - async def on_buffer(self, message: AxisArray) -> None: - self.STATE.buffer_queue.put_nowait(message) - - @ez.publisher(OUTPUT_SIGNAL) - async def sync_output(self) -> AsyncGenerator: - while True: - signal = await self.STATE.signal_queue.get() - buffer = await self.STATE.buffer_queue.get() # includes signal - - axis_name = self.SETTINGS.axis - if axis_name is None: - axis_name = signal.dims[0] - - axis_idx = signal.get_axis_idx(axis_name) - - buffer_len = buffer.shape[axis_idx] - block_len = signal.shape[axis_idx] - window = buffer_len - block_len - - alpha = 2 / (window + 1.0) - alpha_rev = 1 - alpha - - pows = alpha_rev ** (np.arange(buffer_len + 1)) - scale_arr = 1 / pows[:-1] - pw0 = alpha * alpha_rev ** (buffer_len - 1) - - buffer_data = buffer.data - buffer_data = np.moveaxis(buffer_data, axis_idx, 0) - - while scale_arr.ndim < buffer_data.ndim: - scale_arr = scale_arr[..., None] - - def ewma(data: np.ndarray) -> np.ndarray: - mult = scale_arr * data * pw0 - out = scale_arr[::-1] * mult.cumsum(axis=0) - - if not self.SETTINGS.zero_offset: - out = (data[0, :, np.newaxis] * pows[1:]).T + out - - return out - - mean = ewma(buffer_data) - std = ewma((buffer_data - mean) ** 2.0) - - standardized = (buffer_data - mean) / np.sqrt(std).clip(1e-4) - standardized = standardized[-signal.shape[axis_idx] :, ...] - standardized = np.moveaxis(standardized, axis_idx, 0) - - yield self.OUTPUT_SIGNAL, replace(signal, data=standardized) - - -class EWMFilterSettings(ez.Settings): - history_dur: float # previous data to accumulate for standardization - axis: Optional[str] = None - zero_offset: bool = True # If true, we assume zero DC offset for input data - - -class EWMFilter(ez.Collection): - SETTINGS: EWMFilterSettings - - INPUT_SIGNAL = ez.InputStream(AxisArray) - OUTPUT_SIGNAL = ez.OutputStream(AxisArray) - - WINDOW = Window() - EWM = EWM() - - def configure(self) -> None: - self.EWM.apply_settings( - EWMSettings( - axis=self.SETTINGS.axis, - zero_offset=self.SETTINGS.zero_offset, - ) - ) - - self.WINDOW.apply_settings( - WindowSettings( - axis=self.SETTINGS.axis, - window_dur=self.SETTINGS.history_dur, - window_shift=None, # 1:1 mode - ) - ) - - def network(self) -> ez.NetworkDefinition: - return ( - (self.INPUT_SIGNAL, self.WINDOW.INPUT_SIGNAL), - (self.WINDOW.OUTPUT_SIGNAL, self.EWM.INPUT_BUFFER), - (self.INPUT_SIGNAL, self.EWM.INPUT_SIGNAL), - (self.EWM.OUTPUT_SIGNAL, self.OUTPUT_SIGNAL), - ) diff --git a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/filter.py b/extensions/ezmsg-sigproc/src/ezmsg/sigproc/filter.py deleted file mode 100644 index b9acfe96..00000000 --- a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/filter.py +++ /dev/null @@ -1,208 +0,0 @@ -import asyncio -import typing - -from dataclasses import dataclass, replace, field - -import ezmsg.core as ez -import scipy.signal - -import numpy as np -import numpy.typing as npt - -from ezmsg.util.messages.axisarray import AxisArray -from ezmsg.util.generator import consumer - -@dataclass -class FilterCoefficients: - b: np.ndarray = field(default_factory=lambda: np.array([1.0, 0.0])) - a: np.ndarray = field(default_factory=lambda: np.array([1.0, 0.0])) - -def _normalize_coefs( - coefs: typing.Union[FilterCoefficients, typing.Tuple[npt.NDArray, npt.NDArray],npt.NDArray] -) -> typing.Tuple[str, typing.Tuple[npt.NDArray,...]]: - coef_type = "ba" - if coefs is not None: - # scipy.signal functions called with first arg `*coefs`. - # Make sure we have a tuple of coefficients. - if isinstance(coefs, npt.NDArray): - coef_type = "sos" - coefs = (coefs,) # sos funcs just want a single ndarray. - elif isinstance(coefs, FilterCoefficients): - coefs = (FilterCoefficients.b, FilterCoefficients.a) - return coef_type, coefs - -@consumer -def filtergen( - axis: str, coefs: typing.Optional[typing.Tuple[np.ndarray]], coef_type: str -) -> typing.Generator[AxisArray, AxisArray, None]: - # Massage inputs - if coefs is not None and not isinstance(coefs, tuple): - # scipy.signal functions called with first arg `*coefs`, but sos coefs are a single ndarray. - coefs = (coefs,) - - # Init IO - axis_arr_in = AxisArray(np.array([]), dims=[""]) - axis_arr_out = AxisArray(np.array([]), dims=[""]) - - filt_func = {"ba": scipy.signal.lfilter, "sos": scipy.signal.sosfilt}[coef_type] - zi_func = {"ba": scipy.signal.lfilter_zi, "sos": scipy.signal.sosfilt_zi}[coef_type] - - # State variables - axis_idx = None - zi = None - expected_shape = None - - while True: - axis_arr_in = yield axis_arr_out - - if coefs is None: - # passthrough if we do not have a filter design. - axis_arr_out = axis_arr_in - continue - - if axis_idx is None: - axis_name = axis_arr_in.dims[0] if axis is None else axis - axis_idx = axis_arr_in.get_axis_idx(axis_name) - - dat_in = axis_arr_in.data - - # Re-calculate/reset zi if necessary - samp_shape = dat_in.shape[:axis_idx] + dat_in.shape[axis_idx + 1 :] - if zi is None or samp_shape != expected_shape: - expected_shape = samp_shape - n_tail = dat_in.ndim - axis_idx - 1 - zi = zi_func(*coefs) - zi_expand = (None,) * axis_idx + (slice(None),) + (None,) * n_tail - n_tile = dat_in.shape[:axis_idx] + (1,) + dat_in.shape[axis_idx + 1 :] - if coef_type == "sos": - # sos zi must keep its leading dimension (`order / 2` for low|high; `order` for bpass|bstop) - zi_expand = (slice(None),) + zi_expand - n_tile = (1,) + n_tile - zi = np.tile(zi[zi_expand], n_tile) - - dat_out, zi = filt_func(*coefs, dat_in, axis=axis_idx, zi=zi) - axis_arr_out = replace(axis_arr_in, data=dat_out) - - -class FilterSettingsBase(ez.Settings): - axis: typing.Optional[str] = None - fs: typing.Optional[float] = None - - -class FilterSettings(FilterSettingsBase): - # If you'd like to statically design a filter, define it in settings - filt: typing.Optional[FilterCoefficients] = None - - -class FilterState(ez.State): - axis: typing.Optional[str] = None - zi: typing.Optional[np.ndarray] = None - filt_designed: bool = False - filt: typing.Optional[FilterCoefficients] = None - filt_set: asyncio.Event = field(default_factory=asyncio.Event) - samp_shape: typing.Optional[typing.Tuple[int, ...]] = None - fs: typing.Optional[float] = None # Hz - - -class Filter(ez.Unit): - SETTINGS: FilterSettingsBase - STATE: FilterState - - INPUT_FILTER = ez.InputStream(FilterCoefficients) - INPUT_SIGNAL = ez.InputStream(AxisArray) - OUTPUT_SIGNAL = ez.OutputStream(AxisArray) - - def design_filter(self) -> typing.Optional[typing.Tuple[np.ndarray, np.ndarray]]: - raise NotImplementedError("Must implement 'design_filter' in Unit subclass!") - - # Set up filter with static initialization if specified - def initialize(self) -> None: - if self.SETTINGS.axis is not None: - self.STATE.axis = self.SETTINGS.axis - - if isinstance(self.SETTINGS, FilterSettings): - if self.SETTINGS.filt is not None: - self.STATE.filt = self.SETTINGS.filt - self.STATE.filt_set.set() - else: - self.STATE.filt_set.clear() - - if self.SETTINGS.fs is not None: - try: - self.update_filter() - except NotImplementedError: - ez.logger.debug("Using filter coefficients.") - - @ez.subscriber(INPUT_FILTER) - async def redesign(self, message: FilterCoefficients): - self.STATE.filt = message - - def update_filter(self): - try: - coefs = self.design_filter() - self.STATE.filt = ( - FilterCoefficients() if coefs is None else FilterCoefficients(*coefs) - ) - self.STATE.filt_set.set() - self.STATE.filt_designed = True - except NotImplementedError as e: - raise e - except Exception as e: - ez.logger.warning(f"Error when designing filter: {e}") - - @ez.subscriber(INPUT_SIGNAL) - @ez.publisher(OUTPUT_SIGNAL) - async def apply_filter(self, msg: AxisArray) -> typing.AsyncGenerator: - axis_name = msg.dims[0] if self.STATE.axis is None else self.STATE.axis - axis_idx = msg.get_axis_idx(axis_name) - axis = msg.get_axis(axis_name) - fs = 1.0 / axis.gain - - if self.STATE.fs != fs and self.STATE.filt_designed is True: - self.STATE.fs = fs - self.update_filter() - - # Ensure filter is defined - # TODO: Maybe have me be a passthrough filter until coefficients are received - if self.STATE.filt is None: - self.STATE.filt_set.clear() - ez.logger.info("Awaiting filter coefficients...") - await self.STATE.filt_set.wait() - ez.logger.info("Filter coefficients received.") - - assert self.STATE.filt is not None - - arr_in = msg.data - - # If the array is one dimensional, add a temporary second dimension so that the math works out - one_dimensional = False - if arr_in.ndim == 1: - arr_in = np.expand_dims(arr_in, axis=1) - one_dimensional = True - - # We will perform filter with time dimension as last axis - arr_in = np.moveaxis(arr_in, axis_idx, -1) - samp_shape = arr_in[..., 0].shape - - # Re-calculate/reset zi if necessary - if self.STATE.zi is None or samp_shape != self.STATE.samp_shape: - zi: np.ndarray = scipy.signal.lfilter_zi( - self.STATE.filt.b, self.STATE.filt.a - ) - self.STATE.samp_shape = samp_shape - self.STATE.zi = np.array([zi] * np.prod(self.STATE.samp_shape)) - self.STATE.zi = self.STATE.zi.reshape( - tuple(list(self.STATE.samp_shape) + [zi.shape[0]]) - ) - - arr_out, self.STATE.zi = scipy.signal.lfilter( - self.STATE.filt.b, self.STATE.filt.a, arr_in, zi=self.STATE.zi - ) - - arr_out = np.moveaxis(arr_out, -1, axis_idx) - - # Remove temporary first dimension if necessary - if one_dimensional: - arr_out = np.squeeze(arr_out, axis=1) - - yield self.OUTPUT_SIGNAL, replace(msg, data=arr_out), diff --git a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/messages.py b/extensions/ezmsg-sigproc/src/ezmsg/sigproc/messages.py deleted file mode 100644 index 7ffbc34c..00000000 --- a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/messages.py +++ /dev/null @@ -1,31 +0,0 @@ -import warnings -import time - -import numpy.typing as npt - -from ezmsg.util.messages.axisarray import AxisArray - -from typing import Optional - -# UPCOMING: TSMessage Deprecation -# TSMessage is deprecated because it doesn't handle multiple time axes well. -# AxisArray has an incompatible API but supports a superset of functionality. -warnings.warn( - "TimeSeriesMessage/TSMessage is deprecated. Please use ezmsg.utils.AxisArray", - DeprecationWarning, - stacklevel=2, -) - - -def TSMessage( - data: npt.NDArray, - fs: float = 1.0, - time_dim: int = 0, - timestamp: Optional[float] = None, -) -> AxisArray: - dims = [f"dim_{i}" for i in range(data.ndim)] - dims[time_dim] = "time" - offset = time.time() if timestamp is None else timestamp - offset_adj = data.shape[time_dim] / fs # offset corresponds to idx[0] on time_dim - axis = AxisArray.Axis.TimeAxis(fs, offset=offset - offset_adj) - return AxisArray(data, dims=dims, axes=dict(time=axis)) diff --git a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/sampler.py b/extensions/ezmsg-sigproc/src/ezmsg/sigproc/sampler.py deleted file mode 100644 index 0afbac5d..00000000 --- a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/sampler.py +++ /dev/null @@ -1,260 +0,0 @@ -from collections import deque -from dataclasses import dataclass, replace, field -import time -from typing import Optional, Any, Tuple, List, Union, AsyncGenerator, Generator - -import ezmsg.core as ez -import numpy as np - -from ezmsg.util.messages.axisarray import AxisArray, slice_along_axis -from ezmsg.util.generator import consumer - -# Dev/test apparatus -import asyncio - - -@dataclass(unsafe_hash=True) -class SampleTriggerMessage: - timestamp: float = field(default_factory=time.time) - period: Optional[Tuple[float, float]] = None - value: Any = None - - -@dataclass -class SampleMessage: - trigger: SampleTriggerMessage - sample: AxisArray - - -@consumer -def sampler( - buffer_dur: float, - axis: Optional[str] = None, - period: Optional[Tuple[float, float]] = None, - value: Any = None, - estimate_alignment: bool = True -) -> Generator[Union[AxisArray, SampleTriggerMessage], List[SampleMessage], None]: - """ - A generator function that samples data into a buffer, accepts triggers, and returns slices of sampled - data around the trigger time. - - Parameters: - - buffer_dur (float): The duration of the buffer in seconds. The buffer must be long enough to store the oldest - sample to be included in a window. e.g., a trigger lagged by 0.5 seconds with a period of (-1.0, +1.5) will - need a buffer of 0.5 + (1.5 - -1.0) = 3.0 seconds. It is best to at least double your estimate if memory allows. - - axis (Optional[str]): The axis along which to sample the data. - None (default) will choose the first axis in the first input. - - period (Optional[Tuple[float, float]]): The period in seconds during which to sample the data. - Defaults to None. Only used if not None and the trigger message does not define its own period. - - value (Any): The value to sample. Defaults to None. - - estimate_alignment (bool): Whether to estimate the sample alignment. Defaults to True. - If True, the trigger timestamp field is used to slice the buffer. - If False, the trigger timestamp is ignored and the next signal's .offset is used. - NOTE: For faster-than-realtime playback -- Signals and triggers must share the same (fast) clock for - estimate_alignment to operate correctly. - - Sends: - - AxisArray containing streaming data messages - - SampleTriggerMessage containing a trigger - Yields: - - list[SampleMessage]: The list of sample messages. - """ - msg_in = None - msg_out: Optional[list[SampleMessage]] = None - - # State variables (most shared between trigger- and data-processing. - triggers: deque[SampleTriggerMessage] = deque() - last_msg_stats = None - buffer = None - - while True: - msg_in = yield msg_out - msg_out = [] - if isinstance(msg_in, SampleTriggerMessage): - if last_msg_stats is None or buffer is None: - # We've yet to see any data; drop the trigger. - continue - fs = last_msg_stats["fs"] - axis_idx = last_msg_stats["axis_idx"] - - _period = msg_in.period if msg_in.period is not None else period - _value = msg_in.value if msg_in.value is not None else value - - if _period is None: - ez.logger.warning("Sampling failed: period not specified") - continue - - # Check that period is valid - if _period[0] >= _period[1]: - ez.logger.warning(f"Sampling failed: invalid period requested ({_period})") - continue - - # Check that period is compatible with buffer duration. - max_buf_len = int(buffer_dur * fs) - req_buf_len = int((_period[1] - _period[0]) * fs) - if req_buf_len >= max_buf_len: - ez.logger.warning( - f"Sampling failed: {period=} >= {buffer_dur=}" - ) - continue - - trigger_ts: float = msg_in.timestamp - if not estimate_alignment: - # Override the trigger timestamp with the next sample's likely timestamp. - trigger_ts = last_msg_stats["offset"] + (last_msg_stats["n_samples"] + 1) / fs - - new_trig_msg = replace(msg_in, timestamp=trigger_ts, period=_period, value=_value) - triggers.append(new_trig_msg) - - elif isinstance(msg_in, AxisArray): - if axis is None: - axis = msg_in.dims[0] - axis_idx = msg_in.get_axis_idx(axis) - axis_info = msg_in.get_axis(axis) - fs = 1.0 / axis_info.gain - sample_shape = msg_in.data.shape[:axis_idx] + msg_in.data.shape[axis_idx + 1:] - - # If the signal properties have changed in a breaking way then reset buffer and triggers. - if last_msg_stats is None or fs != last_msg_stats["fs"] or sample_shape != last_msg_stats["sample_shape"]: - last_msg_stats = { - "fs": fs, - "sample_shape": sample_shape, - "axis_idx": axis_idx, - "n_samples": msg_in.data.shape[axis_idx] - } - buffer = None - if len(triggers) > 0: - ez.logger.warning("Data stream changed: Discarding all triggers") - triggers.clear() - last_msg_stats["offset"] = axis_info.offset # Should be updated on every message. - - # Update buffer - buffer = msg_in.data if buffer is None else np.concatenate((buffer, msg_in.data), axis=axis_idx) - - # Calculate timestamps associated with buffer. - buffer_offset = np.arange(buffer.shape[axis_idx], dtype=float) - buffer_offset -= buffer_offset[-msg_in.data.shape[axis_idx]] - buffer_offset *= axis_info.gain - buffer_offset += axis_info.offset - - # ... for each trigger, collect the message (if possible) and append to msg_out - for trig in list(triggers): - if trig.period is None: - # This trigger was malformed; drop it. - triggers.remove(trig) - - # If the previous iteration had insufficient data for the trigger timestamp + period, - # and buffer-management removed data required for the trigger, then we will never be able - # to accommodate this trigger. Discard it. An increase in buffer_dur is recommended. - if (trig.timestamp + trig.period[0]) < buffer_offset[0]: - ez.logger.warning( - f"Sampling failed: Buffer span {buffer_offset[0]} is beyond the " - f"requested sample period start: {trig.timestamp + trig.period[0]}" - ) - triggers.remove(trig) - - # TODO: Speed up with searchsorted? - t_start = trig.timestamp + trig.period[0] - if t_start >= buffer_offset[0]: - start = np.searchsorted(buffer_offset, t_start) - stop = start + int(fs * (trig.period[1] - trig.period[0])) - if buffer.shape[axis_idx] > stop: - # Trigger period fully enclosed in buffer. - msg_out.append( - SampleMessage( - trigger=trig, - sample=replace( - msg_in, - data=slice_along_axis(buffer, slice(start, stop), axis_idx), - axes={**msg_in.axes, axis: replace(axis_info, offset=buffer_offset[start])} - ) - ) - ) - triggers.remove(trig) - - buf_len = int(buffer_dur * fs) - buffer = slice_along_axis(buffer, np.s_[-buf_len:], axis_idx) - - -class SamplerSettings(ez.Settings): - buffer_dur: float - axis: Optional[str] = None - period: Optional[ - Tuple[float, float] - ] = None # Optional default period if unspecified in SampleTriggerMessage - value: Any = None # Optional default value if unspecified in SampleTriggerMessage - - estimate_alignment: bool = True - # If true, use message timestamp fields and reported sampling rate to estimate - # sample-accurate alignment for samples. - # If false, sampling will be limited to incoming message rate -- "Block timing" - # NOTE: For faster-than-realtime playback -- Incoming timestamps must reflect - # "realtime" operation for estimate_alignment to operate correctly. - - -class SamplerState(ez.State): - cur_settings: SamplerSettings - gen: Generator[Union[AxisArray, SampleTriggerMessage], List[SampleMessage], None] - - -class Sampler(ez.Unit): - SETTINGS: SamplerSettings - STATE: SamplerState - - INPUT_TRIGGER = ez.InputStream(SampleTriggerMessage) - INPUT_SETTINGS = ez.InputStream(SamplerSettings) - INPUT_SIGNAL = ez.InputStream(AxisArray) - OUTPUT_SAMPLE = ez.OutputStream(SampleMessage) - - def construct_generator(self): - self.STATE.gen = sampler( - buffer_dur=self.STATE.cur_settings.buffer_dur, - axis=self.STATE.cur_settings.axis, - period=self.STATE.cur_settings.period, - value=self.STATE.cur_settings.value, - estimate_alignment=self.STATE.cur_settings.estimate_alignment - ) - - def initialize(self) -> None: - self.STATE.cur_settings = self.SETTINGS - self.construct_generator() - - @ez.subscriber(INPUT_SETTINGS) - async def on_settings(self, msg: SamplerSettings) -> None: - self.STATE.cur_settings = msg - self.construct_generator() - - @ez.subscriber(INPUT_TRIGGER) - async def on_trigger(self, msg: SampleTriggerMessage) -> None: - _ = self.STATE.gen.send(msg) - - @ez.subscriber(INPUT_SIGNAL) - @ez.publisher(OUTPUT_SAMPLE) - async def on_signal(self, msg: AxisArray) -> AsyncGenerator: - pub_samples = self.STATE.gen.send(msg) - for sample in pub_samples: - yield self.OUTPUT_SAMPLE, sample - - -class TriggerGeneratorSettings(ez.Settings): - period: Tuple[float, float] # sec - prewait: float = 0.5 # sec - publish_period: float = 5.0 # sec - - -class TriggerGenerator(ez.Unit): - SETTINGS: TriggerGeneratorSettings - - OUTPUT_TRIGGER = ez.OutputStream(SampleTriggerMessage) - - @ez.publisher(OUTPUT_TRIGGER) - async def generate(self) -> AsyncGenerator: - await asyncio.sleep(self.SETTINGS.prewait) - - output = 0 - while True: - out_msg = SampleTriggerMessage(period=self.SETTINGS.period, value=output) - yield self.OUTPUT_TRIGGER, out_msg - - await asyncio.sleep(self.SETTINGS.publish_period) - output += 1 diff --git a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/scaler.py b/extensions/ezmsg-sigproc/src/ezmsg/sigproc/scaler.py deleted file mode 100644 index aa348a8c..00000000 --- a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/scaler.py +++ /dev/null @@ -1,127 +0,0 @@ -from dataclasses import replace -from typing import Generator, Optional - -import numpy as np - -import ezmsg.core as ez -from ezmsg.util.messages.axisarray import AxisArray -from ezmsg.util.generator import consumer, GenAxisArray - - -def _tau_from_alpha(alpha: float, dt: float) -> float: - """ - Inverse of _alpha_from_tau. See that function for explanation. - """ - return -dt / np.log(1 - alpha) - - -def _alpha_from_tau(tau: float, dt: float) -> float: - """ - # https://en.wikipedia.org/wiki/Exponential_smoothing#Time_constant - :param tau: The amount of time for the smoothed response of a unit step function to reach - 1 - 1/e approx-eq 63.2%. - :param dt: sampling period, or 1 / sampling_rate. - :return: alpha, the "fading factor" in exponential smoothing. - """ - return 1 - np.exp(-dt / tau) - - -@consumer -def scaler(time_constant: float = 1.0, axis: Optional[str] = None) -> Generator[AxisArray, AxisArray, None]: - from river import preprocessing - axis_arr_in = AxisArray(np.array([]), dims=[""]) - axis_arr_out = AxisArray(np.array([]), dims=[""]) - _scaler = None - while True: - axis_arr_in = yield axis_arr_out - data = axis_arr_in.data - if axis is None: - axis = axis_arr_in.dims[0] - axis_idx = 0 - else: - axis_idx = axis_arr_in.get_axis_idx(axis) - if axis_idx != 0: - data = np.moveaxis(data, axis_idx, 0) - - if _scaler is None: - alpha = _alpha_from_tau(time_constant, axis_arr_in.axes[axis].gain) - _scaler = preprocessing.AdaptiveStandardScaler(fading_factor=alpha) - - result = [] - for sample in data: - x = {k: v for k, v in enumerate(sample.flatten().tolist())} - _scaler.learn_one(x) - y = _scaler.transform_one(x) - k = sorted(y.keys()) - result.append(np.array([y[_] for _ in k]).reshape(sample.shape)) - - result = np.stack(result) - result = np.moveaxis(result, 0, axis_idx) - axis_arr_out = replace(axis_arr_in, data=result) - - -@consumer -def scaler_np(time_constant: float = 1.0, axis: Optional[str] = None) -> Generator[AxisArray, AxisArray, None]: - # The only dependency is numpy. - # This is faster for multi-channel data but slower for single-channel data. - axis_arr_in = AxisArray(np.array([]), dims=[""]) - axis_arr_out = AxisArray(np.array([]), dims=[""]) - means = vars_means = vars_sq_means = None - alpha = None - - def _ew_update(arr, prev, _alpha): - if np.all(prev == 0): - return arr - # return _alpha * arr + (1 - _alpha) * prev - # Micro-optimization: sub, mult, add (below) is faster than sub, mult, mult, add (above) - return prev + _alpha * (arr - prev) - - while True: - axis_arr_in = yield axis_arr_out - - data = axis_arr_in.data - if axis is None: - axis = axis_arr_in.dims[0] - axis_idx = 0 - else: - axis_idx = axis_arr_in.get_axis_idx(axis) - data = np.moveaxis(data, axis_idx, 0) - - if alpha is None: - alpha = _alpha_from_tau(time_constant, axis_arr_in.axes[axis].gain) - - if means is None or means.shape != data[0].shape: - vars_sq_means = np.zeros_like(data[0], dtype=float) - vars_means = np.zeros_like(data[0], dtype=float) - means = np.zeros_like(data[0], dtype=float) - - result = [] - for sample in data: - # Update step - vars_means = _ew_update(sample, vars_means, alpha) - vars_sq_means = _ew_update(sample**2, vars_sq_means, alpha) - means = _ew_update(sample, means, alpha) - # Get step - varis = vars_sq_means - vars_means ** 2 - y = ((sample - means) / (varis**0.5)) - y[np.isnan(y)] = 0.0 - result.append(y) - - result = np.stack(result, axis=0) - result = np.moveaxis(result, 0, axis_idx) - axis_arr_out = replace(axis_arr_in, data=result) - - -class AdaptiveStandardScalerSettings(ez.Settings): - time_constant: float = 1.0 - axis: Optional[str] = None - - -class AdaptiveStandardScaler(GenAxisArray): - SETTINGS: AdaptiveStandardScalerSettings - - def construct_generator(self): - self.STATE.gen = scaler_np( - time_constant=self.SETTINGS.time_constant, - axis=self.SETTINGS.axis - ) diff --git a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/signalinjector.py b/extensions/ezmsg-sigproc/src/ezmsg/sigproc/signalinjector.py deleted file mode 100644 index f792fea2..00000000 --- a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/signalinjector.py +++ /dev/null @@ -1,67 +0,0 @@ -import typing - -import ezmsg.core as ez - -from dataclasses import replace -from ezmsg.util.messages.axisarray import AxisArray - -import numpy as np -import numpy.typing as npt - - -class SignalInjectorSettings(ez.Settings): - time_dim: str = 'time' # Input signal needs a time dimension with units in sec. - frequency: typing.Optional[float] = None # Hz - amplitude: float = 1.0 - mixing_seed: typing.Optional[int] = None - - -class SignalInjectorState(ez.State): - cur_shape: typing.Optional[typing.Tuple[int, ...]] = None - cur_frequency: typing.Optional[float] = None - cur_amplitude: float - mixing: npt.NDArray - - -class SignalInjector(ez.Unit): - SETTINGS: SignalInjectorSettings - STATE: SignalInjectorState - - INPUT_FREQUENCY = ez.InputStream(typing.Optional[float]) - INPUT_AMPLITUDE = ez.InputStream(float) - INPUT_SIGNAL = ez.InputStream(AxisArray) - OUTPUT_SIGNAL = ez.OutputStream(AxisArray) - - async def initialize(self) -> None: - self.STATE.cur_frequency = self.SETTINGS.frequency - self.STATE.cur_amplitude = self.SETTINGS.amplitude - self.STATE.mixing = np.array([]) - - @ez.subscriber(INPUT_FREQUENCY) - async def on_frequency(self, msg: typing.Optional[float]) -> None: - self.STATE.cur_frequency = msg - - @ez.subscriber(INPUT_AMPLITUDE) - async def on_amplitude(self, msg: float) -> None: - self.STATE.cur_amplitude = msg - - @ez.subscriber(INPUT_SIGNAL) - @ez.publisher(OUTPUT_SIGNAL) - async def inject(self, msg: AxisArray) -> typing.AsyncGenerator: - - if self.STATE.cur_shape != msg.shape: - self.STATE.cur_shape = msg.shape - rng = np.random.default_rng(self.SETTINGS.mixing_seed) - self.STATE.mixing = rng.random((1, msg.shape2d(self.SETTINGS.time_dim)[1])) - self.STATE.mixing = (self.STATE.mixing * 2.0) - 1.0 - - if self.STATE.cur_frequency is None: - yield self.OUTPUT_SIGNAL, msg - else: - out_msg = replace(msg, data = msg.data.copy()) - t = out_msg.ax(self.SETTINGS.time_dim).values[..., np.newaxis] - signal = np.sin(2 * np.pi * self.STATE.cur_frequency * t) - mixed_signal = signal * self.STATE.mixing * self.STATE.cur_amplitude - with out_msg.view2d(self.SETTINGS.time_dim) as view: - view[...] = view + mixed_signal.astype(view.dtype) - yield self.OUTPUT_SIGNAL, out_msg \ No newline at end of file diff --git a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/slicer.py b/extensions/ezmsg-sigproc/src/ezmsg/sigproc/slicer.py deleted file mode 100644 index f1c9aee7..00000000 --- a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/slicer.py +++ /dev/null @@ -1,98 +0,0 @@ -from dataclasses import replace -import typing - -import numpy as np -import ezmsg.core as ez -from ezmsg.util.messages.axisarray import AxisArray, slice_along_axis -from ezmsg.util.generator import consumer, GenAxisArray - - -""" -Slicer:Select a subset of data along a particular axis. -""" - - -def parse_slice(s: str) -> typing.Tuple[typing.Union[slice, int], ...]: - """ - Parses a string representation of a slice and returns a tuple of slice objects. - * "" -> slice(None, None, None) (take all) - * ":" -> slice(None, None, None) - * '"none"` (case-insensitive) -> slice(None, None, None) - * "{start}:{stop}" or {start}:{stop}:{step} -> slice(start, stop, step) - * "5" (or any integer) -> (5,). Take only that item. - applying this to a ndarray or AxisArray will drop the dimension. - * A comma-separated list of the above -> a tuple of slices | ints - - Args: - s (str): The string representation of the slice. - - Returns: - tuple[slice | int, ...]: A tuple of slice objects and/or ints. - """ - if s.lower() in ["", ":", "none"]: - return (slice(None),) - if "," not in s: - parts = [part.strip() for part in s.split(":")] - if len(parts) == 1: - return (int(parts[0]),) - return (slice(*(int(part.strip()) if part else None for part in parts)),) - l = [parse_slice(_) for _ in s.split(",")] - return tuple([item for sublist in l for item in sublist]) - - -@consumer -def slicer( - selection: str = "", axis: typing.Optional[str] = None -) -> typing.Generator[AxisArray, AxisArray, None]: - axis_arr_in = AxisArray(np.array([]), dims=[""]) - axis_arr_out = AxisArray(np.array([]), dims=[""]) - _slice = None - b_change_dims = False - - while True: - axis_arr_in = yield axis_arr_out - - if axis is None: - axis = axis_arr_in.dims[-1] - axis_idx = axis_arr_in.get_axis_idx(axis) - - if _slice is None: - _slices = parse_slice(selection) - if len(_slices) == 1: - _slice = _slices[0] - b_change_dims = isinstance(_slice, int) - else: - # Multiple slices, but this cannot be done in a single step, so we convert the slices - # to a discontinuous set of integer indexes. - indices = np.arange(axis_arr_in.data.shape[axis_idx]) - indices = np.hstack([indices[_] for _ in _slices]) - _slice = np.s_[indices] - - if b_change_dims: - out_dims = [_ for dim_ix, _ in enumerate(axis_arr_in.dims) if dim_ix != axis_idx] - out_axes = axis_arr_in.axes.copy() - out_axes.pop(axis, None) - else: - out_dims = axis_arr_in.dims - out_axes = axis_arr_in.axes - - axis_arr_out = replace( - axis_arr_in, - dims=out_dims, - axes=out_axes, - data=slice_along_axis(axis_arr_in.data, _slice, axis_idx), - ) - - -class SlicerSettings(ez.Settings): - selection: str = "" - axis: typing.Optional[str] = None - - -class Slicer(GenAxisArray): - SETTINGS: SlicerSettings - - def construct_generator(self): - self.STATE.gen = slicer( - selection=self.SETTINGS.selection, axis=self.SETTINGS.axis - ) diff --git a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/spectral.py b/extensions/ezmsg-sigproc/src/ezmsg/sigproc/spectral.py deleted file mode 100644 index 495b228a..00000000 --- a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/spectral.py +++ /dev/null @@ -1,9 +0,0 @@ -from .spectrum import ( - OptionsEnum, - WindowFunction, - SpectralTransform, - SpectralOutput, - SpectrumSettings, - SpectrumState, - Spectrum -) diff --git a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/spectrogram.py b/extensions/ezmsg-sigproc/src/ezmsg/sigproc/spectrogram.py deleted file mode 100644 index 5ce12636..00000000 --- a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/spectrogram.py +++ /dev/null @@ -1,68 +0,0 @@ -import typing - -import numpy as np - -import ezmsg.core as ez -from ezmsg.util.messages.axisarray import AxisArray -from ezmsg.util.generator import consumer, GenAxisArray # , compose -from ezmsg.util.messages.modify import modify_axis -from ezmsg.sigproc.window import windowing -from ezmsg.sigproc.spectrum import ( - spectrum, - WindowFunction, SpectralTransform, SpectralOutput -) - - -@consumer -def spectrogram( - window_dur: typing.Optional[float] = None, - window_shift: typing.Optional[float] = None, - window: WindowFunction = WindowFunction.HANNING, - transform: SpectralTransform = SpectralTransform.REL_DB, - output: SpectralOutput = SpectralOutput.POSITIVE -) -> typing.Generator[typing.Optional[AxisArray], AxisArray, None]: - - # We cannot use `compose` because `windowing` returns a list of axisarray objects, - # even though the length is always exactly 1 for the settings used here. - # pipeline = compose( - f_win = windowing(axis="time", newaxis="step", window_dur=window_dur, window_shift=window_shift) - f_spec = spectrum(axis="time", window=window, transform=transform, output=output) - f_modify = modify_axis(name_map={"step": "time"}) - # ) - - # State variables - axis_arr_in = AxisArray(np.array([]), dims=[""]) - axis_arr_out: typing.Optional[AxisArray] = None - - while True: - axis_arr_in = yield axis_arr_out - - # axis_arr_out = pipeline(axis_arr_in) - axis_arr_out = None - wins = f_win.send(axis_arr_in) - if len(wins): - specs = f_spec.send(wins[0]) - if specs is not None: - axis_arr_out = f_modify.send(specs) - - -class SpectrogramSettings(ez.Settings): - window_dur: typing.Optional[float] = None # window duration in seconds - window_shift: typing.Optional[float] = None # window step in seconds. If None, window_shift == window_dur - # See SpectrumSettings for details of following settings: - window: WindowFunction = WindowFunction.HAMMING - transform: SpectralTransform = SpectralTransform.REL_DB - output: SpectralOutput = SpectralOutput.POSITIVE - - -class Spectrogram(GenAxisArray): - SETTINGS: SpectrogramSettings - - def construct_generator(self): - self.STATE.gen = spectrogram( - window_dur=self.SETTINGS.window_dur, - window_shift=self.SETTINGS.window_shift, - window=self.SETTINGS.window, - transform=self.SETTINGS.transform, - output=self.SETTINGS.output - ) diff --git a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/spectrum.py b/extensions/ezmsg-sigproc/src/ezmsg/sigproc/spectrum.py deleted file mode 100644 index 44ee9e07..00000000 --- a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/spectrum.py +++ /dev/null @@ -1,158 +0,0 @@ -from dataclasses import replace -import enum -from typing import Optional, Generator, AsyncGenerator - -import numpy as np -import ezmsg.core as ez -from ezmsg.util.messages.axisarray import AxisArray, slice_along_axis -from ezmsg.util.generator import consumer, GenAxisArray - - -class OptionsEnum(enum.Enum): - @classmethod - def options(cls): - return list(map(lambda c: c.value, cls)) - - -class WindowFunction(OptionsEnum): - NONE = "None (Rectangular)" - HAMMING = "Hamming" - HANNING = "Hanning" - BARTLETT = "Bartlett" - BLACKMAN = "Blackman" - - -WINDOWS = { - WindowFunction.NONE: np.ones, - WindowFunction.HAMMING: np.hamming, - WindowFunction.HANNING: np.hanning, - WindowFunction.BARTLETT: np.bartlett, - WindowFunction.BLACKMAN: np.blackman, -} - - -class SpectralTransform(OptionsEnum): - RAW_COMPLEX = "Complex FFT Output" - REAL = "Real Component of FFT" - IMAG = "Imaginary Component of FFT" - REL_POWER = "Relative Power" - REL_DB = "Log Power (Relative dB)" - - -class SpectralOutput(OptionsEnum): - FULL = "Full Spectrum" - POSITIVE = "Positive Frequencies" - NEGATIVE = "Negative Frequencies" - - -@consumer -def spectrum( - axis: Optional[str] = None, - out_axis: Optional[str] = "freq", - window: WindowFunction = WindowFunction.HANNING, - transform: SpectralTransform = SpectralTransform.REL_DB, - output: SpectralOutput = SpectralOutput.POSITIVE -) -> Generator[AxisArray, AxisArray, None]: - - # State variables - axis_arr_in = AxisArray(np.array([]), dims=[""]) - axis_arr_out = AxisArray(np.array([]), dims=[""]) - - axis_name = axis - axis_idx = None - n_time = None - - while True: - axis_arr_in = yield axis_arr_out - - if axis_name is None: - axis_name = axis_arr_in.dims[0] - - # Initial setup - if n_time is None or axis_idx is None or axis_arr_in.data.shape[axis_idx] != n_time: - axis_idx = axis_arr_in.get_axis_idx(axis_name) - _axis = axis_arr_in.get_axis(axis_name) - n_time = axis_arr_in.data.shape[axis_idx] - freqs = np.fft.fftshift(np.fft.fftfreq(n_time, d=_axis.gain), axes=-1) - window = WINDOWS[window](n_time) - window = window.reshape([1] * axis_idx + [len(window),] + [1] * (axis_arr_in.data.ndim-2)) - if (transform != SpectralTransform.RAW_COMPLEX and - not (transform == SpectralTransform.REAL or transform == SpectralTransform.IMAG)): - scale = np.sum(window ** 2.0) * _axis.gain - axis_offset = freqs[0] - if output == SpectralOutput.POSITIVE: - axis_offset = freqs[n_time // 2] - freq_axis = AxisArray.Axis( - unit="Hz", gain=1.0 / (_axis.gain * n_time), offset=axis_offset - ) - if out_axis is None: - out_axis = axis_name - new_dims = axis_arr_in.dims[:axis_idx] + [out_axis, ] + axis_arr_in.dims[axis_idx + 1:] - - f_transform = lambda x: x - if transform != SpectralTransform.RAW_COMPLEX: - if transform == SpectralTransform.REAL: - f_transform = lambda x: x.real - elif transform == SpectralTransform.IMAG: - f_transform = lambda x: x.imag - else: - f1 = lambda x: (2.0 * (np.abs(x) ** 2.0)) / scale - if transform == SpectralTransform.REL_DB: - f_transform = lambda x: 10 * np.log10(f1(x)) - else: - f_transform = f1 - - new_axes = {**axis_arr_in.axes, **{out_axis: freq_axis}} - if out_axis != axis_name: - new_axes.pop(axis_name, None) - - spec = np.fft.fft(axis_arr_in.data * window, axis=axis_idx) / n_time - spec = np.fft.fftshift(spec, axes=axis_idx) - spec = f_transform(spec) - - if output == SpectralOutput.POSITIVE: - spec = slice_along_axis(spec, slice(n_time // 2, None), axis_idx) - - elif output == SpectralOutput.NEGATIVE: - spec = slice_along_axis(spec, slice(None, n_time // 2), axis_idx) - - axis_arr_out = replace(axis_arr_in, data=spec, dims=new_dims, axes=new_axes) - - -class SpectrumSettings(ez.Settings): - axis: Optional[str] = None - # n: Optional[int] = None # n parameter for fft - out_axis: Optional[str] = "freq" # If none; don't change dim name - window: WindowFunction = WindowFunction.HAMMING - transform: SpectralTransform = SpectralTransform.REL_DB - output: SpectralOutput = SpectralOutput.POSITIVE - - -class SpectrumState(ez.State): - gen: Generator - cur_settings: SpectrumSettings - - -class Spectrum(GenAxisArray): - SETTINGS: SpectrumSettings - STATE: SpectrumState - - INPUT_SETTINGS = ez.InputStream(SpectrumSettings) - - def initialize(self) -> None: - self.STATE.cur_settings = self.SETTINGS - super().initialize() - - @ez.subscriber(INPUT_SETTINGS) - async def on_settings(self, msg: SpectrumSettings): - self.STATE.cur_settings = msg - self.construct_generator() - - def construct_generator(self): - self.STATE.gen = spectrum( - axis=self.STATE.cur_settings.axis, - out_axis=self.STATE.cur_settings.out_axis, - window=self.STATE.cur_settings.window, - transform=self.STATE.cur_settings.transform, - output=self.STATE.cur_settings.output - ) diff --git a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/synth.py b/extensions/ezmsg-sigproc/src/ezmsg/sigproc/synth.py deleted file mode 100644 index cf55c698..00000000 --- a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/synth.py +++ /dev/null @@ -1,510 +0,0 @@ -import asyncio -from collections import deque -from dataclasses import dataclass, replace, field -import time -from typing import Optional, Generator, AsyncGenerator, Union - -import numpy as np -import ezmsg.core as ez -from ezmsg.util.generator import consumer, GenAxisArray -from ezmsg.util.messages.axisarray import AxisArray - -from .butterworthfilter import ButterworthFilter, ButterworthFilterSettings - - -# CLOCK -- generate events at a specified rate # -def clock( - dispatch_rate: Optional[float] -) -> Generator[ez.Flag, None, None]: - n_dispatch = -1 - t_0 = time.time() - while True: - if dispatch_rate is not None: - n_dispatch += 1 - t_next = t_0 + n_dispatch / dispatch_rate - time.sleep(max(0, t_next - time.time())) - yield ez.Flag() - - -async def aclock( - dispatch_rate: Optional[float] -) -> AsyncGenerator[ez.Flag, None]: - t_0 = time.time() - n_dispatch = -1 - while True: - if dispatch_rate is not None: - n_dispatch += 1 - t_next = t_0 + n_dispatch / dispatch_rate - await asyncio.sleep(t_next - time.time()) - yield ez.Flag() - - -class ClockSettings(ez.Settings): - # Message dispatch rate (Hz), or None (fast as possible) - dispatch_rate: Optional[float] - - -class ClockState(ez.State): - cur_settings: ClockSettings - gen: AsyncGenerator - - -class Clock(ez.Unit): - SETTINGS: ClockSettings - STATE: ClockState - - INPUT_SETTINGS = ez.InputStream(ClockSettings) - OUTPUT_CLOCK = ez.OutputStream(ez.Flag) - - def initialize(self) -> None: - self.STATE.cur_settings = self.SETTINGS - self.construct_generator() - - def construct_generator(self): - self.STATE.gen = aclock(self.STATE.cur_settings.dispatch_rate) - - @ez.subscriber(INPUT_SETTINGS) - async def on_settings(self, msg: ClockSettings) -> None: - self.STATE.cur_settings = msg - self.construct_generator() - - @ez.publisher(OUTPUT_CLOCK) - async def generate(self) -> AsyncGenerator: - while True: - out = await self.STATE.gen.__anext__() - if out: - yield self.OUTPUT_CLOCK, out - - -# COUNTER - Generate incrementing integer. fs and dispatch_rate parameters combine to give many options. # -async def acounter( - n_time: int, # Number of samples to output per block - fs: Optional[float], # Sampling rate of signal output in Hz - n_ch: int = 1, # Number of channels to synthesize - - # Message dispatch rate (Hz), 'realtime' or None (fast as possible) - # Note: if dispatch_rate is a float then time offsets will be synthetic and the - # system will run faster or slower than wall clock time. - dispatch_rate: Optional[Union[float, str]] = None, - - # If set to an integer, counter will rollover at this number. - mod: Optional[int] = None, -) -> AsyncGenerator[AxisArray, None]: - - # TODO: Adapt this to use ezmsg.util.rate? - - counter_start: int = 0 # next sample's first value - - b_realtime = False - b_manual_dispatch = False - b_ext_clock = False - if dispatch_rate is not None: - if isinstance(dispatch_rate, str): - if dispatch_rate.lower() == "realtime": - b_realtime = True - elif dispatch_rate.lower() == "ext_clock": - b_ext_clock = True - else: - b_manual_dispatch = True - - n_sent: int = 0 # It is convenient to know how many samples we have sent. - clock_zero: float = time.time() # time associated with first sample - - while True: - # 1. Sleep, if necessary, until we are at the end of the current block - if b_realtime: - n_next = n_sent + n_time - t_next = clock_zero + n_next / fs - await asyncio.sleep(t_next - time.time()) - elif b_manual_dispatch: - n_disp_next = 1 + n_sent / n_time - t_disp_next = clock_zero + n_disp_next / dispatch_rate - await asyncio.sleep(t_disp_next - time.time()) - - # 2. Prepare counter data. - block_samp = np.arange(counter_start, counter_start + n_time)[:, np.newaxis] - if mod is not None: - block_samp %= mod - block_samp = np.tile(block_samp, (1, n_ch)) - - # 3. Prepare offset - the time associated with block_samp[0] - if b_realtime: - offset = t_next - n_time / fs - elif b_ext_clock: - offset = time.time() - else: - # Purely synthetic. - offset = n_sent / fs - # offset += clock_zero # ?? - - # 4. yield output - yield AxisArray( - block_samp, - dims=["time", "ch"], - axes={"time": AxisArray.Axis.TimeAxis(fs=fs, offset=offset)}, - ) - - # 5. Update state for next iteration (after next yield) - counter_start = block_samp[-1, 0] + 1 # do not % mod - n_sent += n_time - - -class CounterSettings(ez.Settings): - """ - TODO: Adapt this to use ezmsg.util.rate? - NOTE: This module uses asyncio.sleep to delay appropriately in realtime mode. - This method of sleeping/yielding execution priority has quirky behavior with - sub-millisecond sleep periods which may result in unexpected behavior (e.g. - fs = 2000, n_time = 1, realtime = True -- may result in ~1400 msgs/sec) - """ - - n_time: int # Number of samples to output per block - fs: float # Sampling rate of signal output in Hz - n_ch: int = 1 # Number of channels to synthesize - - # Message dispatch rate (Hz), 'realtime', 'ext_clock', or None (fast as possible) - # Note: if dispatch_rate is a float then time offsets will be synthetic and the - # system will run faster or slower than wall clock time. - dispatch_rate: Optional[Union[float, str]] = None - - # If set to an integer, counter will rollover - mod: Optional[int] = None - - -class CounterState(ez.State): - gen: AsyncGenerator[AxisArray, Optional[ez.Flag]] - cur_settings: CounterSettings - new_generator: asyncio.Event - - -class Counter(ez.Unit): - """Generates monotonically increasing counter""" - - SETTINGS: CounterSettings - STATE: CounterState - - INPUT_CLOCK = ez.InputStream(ez.Flag) - INPUT_SETTINGS = ez.InputStream(CounterSettings) - OUTPUT_SIGNAL = ez.OutputStream(AxisArray) - - async def initialize(self) -> None: - self.STATE.new_generator = asyncio.Event() - self.validate_settings(self.SETTINGS) - - @ez.subscriber(INPUT_SETTINGS) - async def on_settings(self, msg: CounterSettings) -> None: - self.validate_settings(msg) - - def validate_settings(self, settings: CounterSettings) -> None: - if isinstance( - settings.dispatch_rate, str - ) and self.SETTINGS.dispatch_rate not in ["realtime", "ext_clock"]: - raise ValueError(f"Unknown dispatch_rate: {self.SETTINGS.dispatch_rate}") - self.STATE.cur_settings = settings - self.construct_generator() - - def construct_generator(self): - self.STATE.gen = acounter( - self.STATE.cur_settings.n_time, - self.STATE.cur_settings.fs, - n_ch=self.STATE.cur_settings.n_ch, - dispatch_rate=self.STATE.cur_settings.dispatch_rate, - mod=self.STATE.cur_settings.mod - ) - self.STATE.new_generator.set() - - @ez.subscriber(INPUT_CLOCK) - @ez.publisher(OUTPUT_SIGNAL) - async def on_clock(self, clock: ez.Flag): - if self.STATE.cur_settings.dispatch_rate == 'ext_clock': - out = await self.STATE.gen.__anext__() - yield self.OUTPUT_SIGNAL, out - - @ez.publisher(OUTPUT_SIGNAL) - async def run_generator(self) -> AsyncGenerator: - while True: - - await self.STATE.new_generator.wait() - self.STATE.new_generator.clear() - - if self.STATE.cur_settings.dispatch_rate == 'ext_clock': - continue - - while not self.STATE.new_generator.is_set(): - out = await self.STATE.gen.__anext__() - yield self.OUTPUT_SIGNAL, out - - -@consumer -def sin( - axis: Optional[str] = "time", - freq: float = 1.0, # Oscillation frequency in Hz - amp: float = 1.0, # Amplitude - phase: float = 0.0, # Phase offset (in radians) -) -> Generator[AxisArray, AxisArray, None]: - axis_arr_in = AxisArray(np.array([]), dims=[""]) - axis_arr_out = AxisArray(np.array([]), dims=[""]) - - ang_freq = 2.0 * np.pi * freq - - while True: - axis_arr_in = yield axis_arr_out - # axis_arr_in is expected to be sample counts - - axis_name = axis - if axis_name is None: - axis_name = axis_arr_in.dims[0] - - w = (ang_freq * axis_arr_in.get_axis(axis_name).gain) * axis_arr_in.data - out_data = amp * np.sin(w + phase) - axis_arr_out = replace(axis_arr_in, data=out_data) - - -class SinGeneratorSettings(ez.Settings): - time_axis: Optional[str] = "time" - freq: float = 1.0 # Oscillation frequency in Hz - amp: float = 1.0 # Amplitude - phase: float = 0.0 # Phase offset (in radians) - - -class SinGenerator(GenAxisArray): - SETTINGS: SinGeneratorSettings - - def construct_generator(self): - self.STATE.gen = sin( - axis=self.SETTINGS.time_axis, - freq=self.SETTINGS.freq, - amp=self.SETTINGS.amp, - phase=self.SETTINGS.phase - ) - - -class OscillatorSettings(ez.Settings): - n_time: int # Number of samples to output per block - fs: float # Sampling rate of signal output in Hz - n_ch: int = 1 # Number of channels to output per block - dispatch_rate: Optional[Union[float, str]] = None # (Hz) | 'realtime' | 'ext_clock' - freq: float = 1.0 # Oscillation frequency in Hz - amp: float = 1.0 # Amplitude - phase: float = 0.0 # Phase offset (in radians) - sync: bool = False # Adjust `freq` to sync with sampling rate - - -class Oscillator(ez.Collection): - SETTINGS: OscillatorSettings - - INPUT_CLOCK = ez.InputStream(ez.Flag) - OUTPUT_SIGNAL = ez.OutputStream(AxisArray) - - COUNTER = Counter() - SIN = SinGenerator() - - def configure(self) -> None: - # Calculate synchronous settings if necessary - freq = self.SETTINGS.freq - mod = None - if self.SETTINGS.sync: - period = 1.0 / self.SETTINGS.freq - mod = round(period * self.SETTINGS.fs) - freq = 1.0 / (mod / self.SETTINGS.fs) - - self.COUNTER.apply_settings( - CounterSettings( - n_time=self.SETTINGS.n_time, - fs=self.SETTINGS.fs, - n_ch=self.SETTINGS.n_ch, - dispatch_rate=self.SETTINGS.dispatch_rate, - mod=mod, - ) - ) - - self.SIN.apply_settings( - SinGeneratorSettings( - freq=freq, amp=self.SETTINGS.amp, phase=self.SETTINGS.phase - ) - ) - - def network(self) -> ez.NetworkDefinition: - return ( - (self.INPUT_CLOCK, self.COUNTER.INPUT_CLOCK), - (self.COUNTER.OUTPUT_SIGNAL, self.SIN.INPUT_SIGNAL), - (self.SIN.OUTPUT_SIGNAL, self.OUTPUT_SIGNAL), - ) - - -class RandomGeneratorSettings(ez.Settings): - loc: float = 0.0 - scale: float = 1.0 - - -class RandomGenerator(ez.Unit): - SETTINGS: RandomGeneratorSettings - - INPUT_SIGNAL = ez.InputStream(AxisArray) - OUTPUT_SIGNAL = ez.OutputStream(AxisArray) - - @ez.subscriber(INPUT_SIGNAL) - @ez.publisher(OUTPUT_SIGNAL) - async def generate(self, msg: AxisArray) -> AsyncGenerator: - random_data = np.random.normal( - size=msg.shape, loc=self.SETTINGS.loc, scale=self.SETTINGS.scale - ) - - yield self.OUTPUT_SIGNAL, replace(msg, data=random_data) - - -class NoiseSettings(ez.Settings): - n_time: int # Number of samples to output per block - fs: float # Sampling rate of signal output in Hz - n_ch: int = 1 # Number of channels to output - dispatch_rate: Optional[ - Union[float, str] - ] = None # (Hz), 'realtime', or 'ext_clock' - loc: float = 0.0 # DC offset - scale: float = 1.0 # Scale (in standard deviations) - - -WhiteNoiseSettings = NoiseSettings - - -class WhiteNoise(ez.Collection): - SETTINGS: NoiseSettings - - INPUT_CLOCK = ez.InputStream(ez.Flag) - OUTPUT_SIGNAL = ez.OutputStream(AxisArray) - - COUNTER = Counter() - RANDOM = RandomGenerator() - - def configure(self) -> None: - self.RANDOM.apply_settings( - RandomGeneratorSettings(loc=self.SETTINGS.loc, scale=self.SETTINGS.scale) - ) - - self.COUNTER.apply_settings( - CounterSettings( - n_time=self.SETTINGS.n_time, - fs=self.SETTINGS.fs, - n_ch=self.SETTINGS.n_ch, - dispatch_rate=self.SETTINGS.dispatch_rate, - mod=None, - ) - ) - - def network(self) -> ez.NetworkDefinition: - return ( - (self.INPUT_CLOCK, self.COUNTER.INPUT_CLOCK), - (self.COUNTER.OUTPUT_SIGNAL, self.RANDOM.INPUT_SIGNAL), - (self.RANDOM.OUTPUT_SIGNAL, self.OUTPUT_SIGNAL), - ) - - -PinkNoiseSettings = NoiseSettings - - -class PinkNoise(ez.Collection): - SETTINGS: PinkNoiseSettings - - INPUT_CLOCK = ez.InputStream(ez.Flag) - OUTPUT_SIGNAL = ez.OutputStream(AxisArray) - - WHITE_NOISE = WhiteNoise() - FILTER = ButterworthFilter() - - def configure(self) -> None: - self.WHITE_NOISE.apply_settings(self.SETTINGS) - self.FILTER.apply_settings( - ButterworthFilterSettings( - axis="time", order=1, cutoff=self.SETTINGS.fs * 0.01 # Hz - ) - ) - - def network(self) -> ez.NetworkDefinition: - return ( - (self.INPUT_CLOCK, self.WHITE_NOISE.INPUT_CLOCK), - (self.WHITE_NOISE.OUTPUT_SIGNAL, self.FILTER.INPUT_SIGNAL), - (self.FILTER.OUTPUT_SIGNAL, self.OUTPUT_SIGNAL), - ) - - -class AddState(ez.State): - queue_a: "asyncio.Queue[AxisArray]" = field(default_factory=asyncio.Queue) - queue_b: "asyncio.Queue[AxisArray]" = field(default_factory=asyncio.Queue) - - -class Add(ez.Unit): - """Add two signals together. Assumes compatible/similar axes/dimensions.""" - - STATE: AddState - - INPUT_SIGNAL_A = ez.InputStream(AxisArray) - INPUT_SIGNAL_B = ez.InputStream(AxisArray) - OUTPUT_SIGNAL = ez.OutputStream(AxisArray) - - @ez.subscriber(INPUT_SIGNAL_A) - async def on_a(self, msg: AxisArray) -> None: - self.STATE.queue_a.put_nowait(msg) - - @ez.subscriber(INPUT_SIGNAL_B) - async def on_b(self, msg: AxisArray) -> None: - self.STATE.queue_b.put_nowait(msg) - - @ez.publisher(OUTPUT_SIGNAL) - async def output(self) -> AsyncGenerator: - while True: - a = await self.STATE.queue_a.get() - b = await self.STATE.queue_b.get() - - yield (self.OUTPUT_SIGNAL, replace(a, data=a.data + b.data)) - - -class EEGSynthSettings(ez.Settings): - fs: float = 500.0 # Hz - n_time: int = 100 - alpha_freq: float = 10.5 # Hz - n_ch: int = 8 - - -class EEGSynth(ez.Collection): - SETTINGS: EEGSynthSettings - - OUTPUT_SIGNAL = ez.OutputStream(AxisArray) - - CLOCK = Clock() - NOISE = PinkNoise() - OSC = Oscillator() - ADD = Add() - - def configure(self) -> None: - self.CLOCK.apply_settings( - ClockSettings(dispatch_rate=self.SETTINGS.fs / self.SETTINGS.n_time) - ) - - self.OSC.apply_settings( - OscillatorSettings( - n_time=self.SETTINGS.n_time, - fs=self.SETTINGS.fs, - n_ch=self.SETTINGS.n_ch, - dispatch_rate="ext_clock", - freq=self.SETTINGS.alpha_freq, - ) - ) - - self.NOISE.apply_settings( - PinkNoiseSettings( - n_time=self.SETTINGS.n_time, - fs=self.SETTINGS.fs, - n_ch=self.SETTINGS.n_ch, - dispatch_rate="ext_clock", - scale=5.0, - ) - ) - - def network(self) -> ez.NetworkDefinition: - return ( - (self.CLOCK.OUTPUT_CLOCK, self.OSC.INPUT_CLOCK), - (self.CLOCK.OUTPUT_CLOCK, self.NOISE.INPUT_CLOCK), - (self.OSC.OUTPUT_SIGNAL, self.ADD.INPUT_SIGNAL_A), - (self.NOISE.OUTPUT_SIGNAL, self.ADD.INPUT_SIGNAL_B), - (self.ADD.OUTPUT_SIGNAL, self.OUTPUT_SIGNAL), - ) diff --git a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/window.py b/extensions/ezmsg-sigproc/src/ezmsg/sigproc/window.py deleted file mode 100644 index 3857aced..00000000 --- a/extensions/ezmsg-sigproc/src/ezmsg/sigproc/window.py +++ /dev/null @@ -1,246 +0,0 @@ -from dataclasses import replace -import traceback -from typing import AsyncGenerator, Optional, Tuple, List, Generator - -import ezmsg.core as ez -import numpy as np -import numpy.typing as npt - -from ezmsg.util.messages.axisarray import AxisArray, slice_along_axis, sliding_win_oneaxis -from ezmsg.util.generator import consumer - - -@consumer -def windowing( - axis: Optional[str] = None, - newaxis: Optional[str] = None, - window_dur: Optional[float] = None, - window_shift: Optional[float] = None, - zero_pad_until: str = "input" -) -> Generator[AxisArray, List[AxisArray], None]: - """ - Window function that generates windows of data from an input `AxisArray`. - :param axis: The axis along which to segment windows. - If None, defaults to the first dimension of the first seen AxisArray. - :param newaxis: Optional new axis for the output. If None, no new axes will be added. - If a string, windows will be stacked in a new axis with key `newaxis`, immediately preceding the windowed axis. - :param window_dur: The duration of the window in seconds. - If None, the function acts as a passthrough and all other parameters are ignored. - :param window_shift: The shift of the window in seconds. - If None (default), windowing operates in "1:1 mode", where each input yields exactly one most-recent window. - :param zero_pad_until: Determines how the function initializes the buffer. - Can be one of "input" (default), "full", "shift", or "none". If `window_shift` is None then this field is - ignored and "input" is always used. - "input" (default) initializes the buffer with the input then prepends with zeros to the window size. - The first input will always yield at least one output. - "shift" fills the buffer until `window_shift`. - No outputs will be yielded until at least `window_shift` data has been seen. - "none" does not pad the buffer. No outputs will be yielded until at least `window_dur` data has been seen. - :return: - A (primed) generator that accepts .send(an AxisArray object) and yields a list of windowed - AxisArray objects. The list will always be length-1 if `newaxis` is not None or `window_shift` is None. - """ - # TODO: The return should be an AxisArray. i.e., always add a new axis. The Unit can do a multi-yield-per-pub - # if the parameterization does not expect a newaxis. - - if window_shift is None and zero_pad_until != "input": - ez.logger.warning("`zero_pad_until` must be 'input' if `window_shift` is None. " - f"Ignoring received argument value: {zero_pad_until}") - zero_pad_until = "input" - elif window_shift is not None and zero_pad_until == "input": - ez.logger.warning("windowing is non-deterministic with `zero_pad_until='input'` as it depends on the size " - "of the first input. We recommend using 'shift' when `window_shift` is float-valued.") - axis_arr_in = AxisArray(np.array([]), dims=[""]) - axis_arr_out = [AxisArray(np.array([]), dims=[""])] - - # State variables - prev_samp_shape: Optional[Tuple[int, ...]] = None - prev_fs: Optional[float] = None - buffer: Optional[npt.NDArray] = None - window_samples: Optional[int] = None - window_shift_samples: Optional[int] = None - shift_deficit: int = 0 # Number of incoming samples to ignore. Only relevant when shift > window. - newaxis_warn_flag: bool = False - mod_ax: Optional[str] = None # The key of the modified axis in the output's .axes - out_template: Optional[AxisArray] = None # Template for building return values. - - while True: - axis_arr_in = yield axis_arr_out - - if window_dur is None: - axis_arr_out = [axis_arr_in] - continue - - if axis is None: - axis = axis_arr_in.dims[0] - axis_idx = axis_arr_in.get_axis_idx(axis) - axis_info = axis_arr_in.get_axis(axis) - fs = 1.0 / axis_info.gain - - if (not newaxis_warn_flag) and newaxis is not None and newaxis in axis_arr_in.dims: - ez.logger.warning(f"newaxis {newaxis} present in input dims and will be ignored.") - newaxis_warn_flag = True - b_newaxis = newaxis is not None and newaxis not in axis_arr_in.dims - - samp_shape = axis_arr_in.data.shape[:axis_idx] + axis_arr_in.data.shape[axis_idx + 1:] - window_samples = int(window_dur * fs) - b_1to1 = window_shift is None - if not b_1to1: - window_shift_samples = int(window_shift * fs) - - # If buffer unset or input stats changed, create a new buffer - if buffer is None or samp_shape != prev_samp_shape or fs != prev_fs: - if zero_pad_until == "none": - req_samples = window_samples - elif zero_pad_until == "shift" and not b_1to1: - req_samples = window_shift_samples - else: # i.e. zero_pad_until == "input" - req_samples = axis_arr_in.data.shape[axis_idx] - n_zero = max(0, window_samples - req_samples) - buffer_shape = axis_arr_in.data.shape[:axis_idx] + (n_zero,) + axis_arr_in.data.shape[axis_idx + 1:] - buffer = np.zeros(buffer_shape) - prev_samp_shape = samp_shape - prev_fs = fs - - # Add new data to buffer. - # Currently we just concatenate the new time samples and clip the output - # np.roll actually returns a copy, and there's no way to construct a - # rolling view of the data. In current numpy implementations, np.concatenate - # is generally faster than np.roll and slicing anyway, but this could still - # be a performance bottleneck for large memory arrays. - buffer = np.concatenate((buffer, axis_arr_in.data), axis=axis_idx) - # Note: if we ever move to using a circular buffer without copies then we need to create copies somewhere, - # because currently the outputs are merely views into the buffer. - - # Create a vector of buffer timestamps to track axis `offset` in output(s) - buffer_offset = np.arange(buffer.shape[axis_idx]).astype(float) - # Adjust so first _new_ sample at index 0 - buffer_offset -= buffer_offset[-axis_arr_in.data.shape[axis_idx]] - # Convert form indices to 'units' (probably seconds). - buffer_offset *= axis_info.gain - buffer_offset += axis_info.offset - - if not b_1to1 and shift_deficit > 0: - n_skip = min(buffer.shape[axis_idx], shift_deficit) - if n_skip > 0: - buffer = slice_along_axis(buffer, np.s_[n_skip:], axis_idx) - buffer_offset = buffer_offset[n_skip:] - shift_deficit -= n_skip - - # Prepare reusable parts of output - if out_template is None: - out_dims = axis_arr_in.dims - if newaxis is None: - out_axes = { - **axis_arr_in.axes, - axis: replace(axis_info, offset=0.0) # offset modified below. - } - mod_ax = axis - else: - out_dims = out_dims[:axis_idx] + [newaxis] + out_dims[axis_idx:] - out_axes = { - **axis_arr_in.axes, - newaxis: AxisArray.Axis( - unit=axis_info.unit, - gain=0.0 if b_1to1 else axis_info.gain * window_shift_samples, - offset=0.0 # offset modified below - ) - } - mod_ax = newaxis - out_template = replace(axis_arr_in, data=np.zeros([0 for _ in out_dims]), dims=out_dims) - - # Generate outputs. - axis_arr_out: List[AxisArray] = [] - if b_1to1: - # one-to-one mode -- Each send yields exactly one window containing only the most recent samples. - buffer = slice_along_axis(buffer, np.s_[-window_samples:], axis_idx) - axis_arr_out.append(replace( - out_template, - data=np.expand_dims(buffer, axis=axis_idx) if b_newaxis else buffer, - axes={ - **out_axes, - mod_ax: replace(out_axes[mod_ax], offset=buffer_offset[-window_samples]) - } - )) - elif buffer.shape[axis_idx] >= window_samples: - # Deterministic window shifts. - win_view = sliding_win_oneaxis(buffer, window_samples, axis_idx) - win_view = slice_along_axis(win_view, np.s_[::window_shift_samples], axis_idx) - offset_view = sliding_win_oneaxis(buffer_offset, window_samples, 0)[::window_shift_samples] - # Place in output - if b_newaxis: - axis_arr_out.append(replace( - out_template, - data=win_view, - axes={**out_axes, mod_ax: replace(out_axes[mod_ax], offset=offset_view[0, 0])} - )) - else: - for win_ix in range(win_view.shape[axis_idx]): - axis_arr_out.append(replace( - out_template, - data=slice_along_axis(win_view, win_ix, axis_idx), - axes={ - **out_axes, - mod_ax: replace(out_axes[mod_ax], offset=offset_view[win_ix, 0]) - } - )) - - # Drop expired beginning of buffer and update shift_deficit - multi_shift = window_shift_samples * win_view.shape[axis_idx] - shift_deficit = max(0, multi_shift - buffer.shape[axis_idx]) - buffer = slice_along_axis(buffer, np.s_[multi_shift:], axis_idx) - - -class WindowSettings(ez.Settings): - axis: Optional[str] = None - newaxis: Optional[str] = None # new axis for output. No new axes if None - window_dur: Optional[float] = None # Sec. passthrough if None - window_shift: Optional[float] = None # Sec. Use "1:1 mode" if None - zero_pad_until: str = "full" # "full", "shift", "input", "none" - - -class WindowState(ez.State): - cur_settings: WindowSettings - gen: Generator - - -class Window(ez.Unit): - STATE: WindowState - SETTINGS: WindowSettings - - INPUT_SIGNAL = ez.InputStream(AxisArray) - OUTPUT_SIGNAL = ez.OutputStream(AxisArray) - INPUT_SETTINGS = ez.InputStream(WindowSettings) - - def initialize(self) -> None: - self.STATE.cur_settings = self.SETTINGS - self.construct_generator() - - @ez.subscriber(INPUT_SETTINGS) - async def on_settings(self, msg: WindowSettings) -> None: - self.STATE.cur_settings = msg - self.construct_generator() - - def construct_generator(self): - self.STATE.gen = windowing( - axis=self.STATE.cur_settings.axis, - newaxis=self.STATE.cur_settings.newaxis, - window_dur=self.STATE.cur_settings.window_dur, - window_shift=self.STATE.cur_settings.window_shift, - zero_pad_until=self.STATE.cur_settings.zero_pad_until - ) - - @ez.subscriber(INPUT_SIGNAL) - @ez.publisher(OUTPUT_SIGNAL) - async def on_signal(self, msg: AxisArray) -> AsyncGenerator: - try: - # TODO: Refactor window generator so it always returns an axis array. - # Then, if the configuration is such that a new "win" axis is not expected, - # then iterate over the "win" axis -- dropping the "win" axis in the process. - out_msgs = self.STATE.gen.send(msg) - for out_msg in out_msgs: - yield self.OUTPUT_SIGNAL, out_msg - except (StopIteration, GeneratorExit): - ez.logger.debug(f"Window closed in {self.address}") - except Exception: - ez.logger.info(traceback.format_exc()) diff --git a/extensions/ezmsg-sigproc/tests/__init__.py b/extensions/ezmsg-sigproc/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/extensions/ezmsg-sigproc/tests/resources/xform.csv b/extensions/ezmsg-sigproc/tests/resources/xform.csv deleted file mode 100644 index f6048cc1..00000000 --- a/extensions/ezmsg-sigproc/tests/resources/xform.csv +++ /dev/null @@ -1,65 +0,0 @@ --0.195501,-1.257677,-0.436903,-0.068509,-0.671182,-0.387884,-0.296285,-0.262393,-0.263331,-0.475408,-0.365876,-0.251207,-0.179991,-0.159654,-0.109226,-0.135206,-0.117753,-0.059841,-0.262114,-0.174046,-0.114994,-0.090672,-0.055225,-0.047486,-0.047646,-0.026847,-0.006476,-0.152276,-0.104683,-0.062207,-0.035597,-0.015471,0.002781,0.012203,0.015279,0.024096,-0.082653,-0.042888,-0.012286,0.006775,0.025409,0.027636,0.033160,0.041450,0.042943,-0.017661,0.004703,0.009999,0.044475,0.045220,0.045648,0.046286,0.059643,0.057485,0.045044,0.045947,0.017319,0.039612,0.045284,0.045781,0.062467,0.045600,0.058191,0.065717 -0.033983,0.885561,0.103925,-0.007080,-0.053048,-0.045225,-0.043081,-0.042049,-0.042632,-0.029549,-0.034032,-0.032580,-0.030646,-0.031250,-0.028898,-0.030480,-0.029030,-0.022174,-0.013848,-0.019439,-0.023085,-0.023570,-0.023357,-0.022973,-0.023548,-0.020069,-0.016523,-0.008978,-0.013680,-0.017310,-0.018876,-0.019541,-0.019602,-0.019142,-0.016594,-0.012651,-0.007791,-0.012045,-0.014804,-0.016653,-0.016566,-0.016863,-0.016154,-0.014861,-0.012124,-0.009175,-0.011730,-0.013632,-0.014355,-0.015188,-0.014422,-0.014460,-0.013823,-0.011898,-0.014394,-0.014409,-0.009639,-0.012776,-0.014375,-0.014405,-0.011687,-0.010822,-0.011201,-0.011655 -0.199034,1.290887,0.456944,0.123896,0.623118,0.460833,0.412317,0.395232,0.411033,0.378779,0.371490,0.317215,0.276710,0.274296,0.239023,0.263075,0.252082,0.186013,0.186026,0.192716,0.197310,0.191628,0.177414,0.172525,0.178929,0.150482,0.121101,0.111782,0.126306,0.135093,0.136740,0.134546,0.128782,0.123558,0.107323,0.078789,0.076739,0.090759,0.098379,0.104966,0.097780,0.099523,0.093671,0.082537,0.064393,0.062127,0.070966,0.082342,0.075251,0.080169,0.075256,0.075358,0.067164,0.055621,0.075190,0.075052,0.052109,0.065236,0.075042,0.075152,0.051456,0.049630,0.047865,0.049165 --0.076106,-1.106559,-0.323809,-0.080550,-0.364634,-0.221163,-0.202513,-0.249226,-0.493946,-0.101098,-0.126624,-0.060316,-0.025616,-0.041323,-0.020275,-0.102358,-0.214556,-0.255223,0.072450,0.075685,0.060558,0.066616,0.067050,0.055700,0.004597,-0.058710,-0.108523,0.099976,0.115709,0.135773,0.126631,0.116923,0.108968,0.080735,0.029289,-0.008305,0.139992,0.160777,0.168707,0.153628,0.151420,0.140758,0.119169,0.103329,0.077181,0.157004,0.172250,0.168074,0.161458,0.173143,0.162083,0.160724,0.139543,0.119718,0.164112,0.162236,0.170808,0.180523,0.162621,0.160797,0.139517,0.180778,0.177599,0.158813 --0.014248,-0.112092,0.013111,-0.051622,0.952951,-0.032658,-0.029367,-0.030444,-0.041509,-0.023601,-0.024207,-0.018606,-0.015060,-0.015580,-0.012888,-0.017560,-0.021706,-0.020060,-0.006491,-0.006607,-0.007416,-0.006855,-0.006094,-0.006320,-0.008785,-0.010004,-0.010610,-0.001525,-0.001545,-0.001102,-0.001543,-0.001823,-0.001852,-0.002768,-0.004115,-0.004260,0.001970,0.002179,0.002160,0.001216,0.001500,0.000967,0.000358,0.000256,0.000066,0.003474,0.003694,0.002955,0.003064,0.003310,0.003091,0.003029,0.002562,0.002303,0.003179,0.003108,0.004587,0.004360,0.003124,0.003042,0.003347,0.005155,0.005122,0.004275 --0.034498,-0.264581,0.110588,-0.033197,-0.126051,0.906549,-0.083931,-0.080998,-0.086100,-0.075319,-0.074640,-0.063706,-0.055581,-0.055323,-0.048254,-0.053643,-0.052366,-0.039539,-0.035890,-0.037718,-0.039099,-0.038030,-0.035327,-0.034461,-0.036159,-0.030967,-0.025477,-0.021160,-0.024263,-0.026135,-0.026689,-0.026428,-0.025417,-0.024622,-0.021728,-0.016252,-0.014098,-0.017015,-0.018677,-0.020241,-0.018898,-0.019348,-0.018352,-0.016252,-0.012765,-0.011351,-0.013158,-0.015538,-0.014336,-0.015252,-0.014339,-0.014373,-0.012941,-0.010730,-0.014306,-0.014297,-0.009397,-0.012121,-0.014289,-0.014328,-0.009756,-0.008972,-0.008707,-0.009159 -0.011972,-0.133293,0.162901,0.068595,-0.076269,-0.064963,0.940215,-0.053462,-0.035867,-0.053960,-0.054639,-0.053236,-0.050339,-0.049296,-0.045518,-0.042631,-0.031345,-0.015922,-0.035804,-0.041100,-0.043192,-0.043469,-0.041870,-0.040154,-0.037153,-0.026419,-0.016623,-0.026564,-0.032983,-0.038325,-0.038977,-0.038487,-0.037299,-0.034132,-0.026337,-0.017303,-0.025239,-0.031665,-0.035234,-0.036139,-0.035152,-0.034705,-0.031813,-0.028384,-0.022281,-0.026174,-0.030299,-0.032615,-0.031913,-0.034009,-0.032016,-0.031947,-0.028902,-0.024619,-0.032151,-0.031997,-0.026609,-0.031185,-0.032001,-0.031888,-0.025617,-0.028075,-0.027931,-0.026907 -0.008668,0.007533,0.059137,0.060961,-0.006107,-0.009003,-0.008446,0.995061,0.009418,-0.010349,-0.009776,-0.012158,-0.012818,-0.011961,-0.011808,-0.007789,-0.000337,0.005449,-0.012335,-0.014124,-0.014249,-0.014608,-0.014245,-0.013342,-0.010533,-0.005122,-0.000492,-0.011012,-0.013517,-0.015875,-0.015760,-0.015289,-0.014686,-0.012728,-0.008565,-0.004596,-0.012316,-0.015001,-0.016371,-0.016084,-0.015758,-0.015227,-0.013595,-0.012038,-0.009319,-0.013362,-0.015152,-0.015654,-0.015271,-0.016311,-0.015326,-0.015256,-0.013612,-0.011641,-0.015441,-0.015327,-0.014084,-0.015789,-0.015342,-0.015241,-0.012692,-0.014936,-0.014798,-0.013803 --0.017031,0.010816,-0.001079,0.025690,0.000032,0.000614,0.001493,0.003131,1.009146,-0.004028,-0.001637,-0.001564,-0.001383,-0.000675,-0.000483,0.001178,0.004167,0.005884,-0.005429,-0.004044,-0.002693,-0.002462,-0.001941,-0.001554,-0.000286,0.001504,0.002955,-0.004555,-0.004112,-0.003877,-0.003213,-0.002654,-0.002180,-0.001354,-0.000102,0.000864,-0.004512,-0.004321,-0.003989,-0.003286,-0.002952,-0.002649,-0.002051,-0.001566,-0.000964,-0.003913,-0.003890,-0.003663,-0.002961,-0.003218,-0.002957,-0.002913,-0.002205,-0.001795,-0.003017,-0.002957,-0.003712,-0.003540,-0.002977,-0.002924,-0.002211,-0.003501,-0.003224,-0.002637 -0.055873,0.071689,0.035993,0.029785,0.033217,0.018181,0.013992,0.013826,0.020040,1.020235,0.015822,0.009376,0.005628,0.005108,0.002630,0.005687,0.007924,0.007094,0.008110,0.003783,0.001331,0.000098,-0.001405,-0.001413,-0.000156,0.000873,0.001584,0.003157,0.000315,-0.002324,-0.003364,-0.004033,-0.004614,-0.004257,-0.002855,-0.001928,-0.000724,-0.003321,-0.005069,-0.005661,-0.006391,-0.006241,-0.005865,-0.005701,-0.004859,-0.004048,-0.005613,-0.005900,-0.007265,-0.007667,-0.007336,-0.007332,-0.007307,-0.006541,-0.007360,-0.007352,-0.005931,-0.007404,-0.007330,-0.007308,-0.007239,-0.007493,-0.007983,-0.007865 --0.008562,-0.107466,0.044298,-0.020238,-0.049759,-0.037091,-0.033591,-0.032892,-0.036577,-0.028589,0.970982,-0.024742,-0.021595,-0.021695,-0.018962,-0.021549,-0.021875,-0.017283,-0.012641,-0.013778,-0.014720,-0.014368,-0.013452,-0.013219,-0.014242,-0.012685,-0.010916,-0.007079,-0.008448,-0.009270,-0.009682,-0.009741,-0.009477,-0.009393,-0.008592,-0.006691,-0.004315,-0.005538,-0.006299,-0.007120,-0.006686,-0.006949,-0.006722,-0.006026,-0.004810,-0.003413,-0.004144,-0.005152,-0.004881,-0.005174,-0.004884,-0.004910,-0.004543,-0.003778,-0.004854,-0.004868,-0.002706,-0.003842,-0.004859,-0.004889,-0.003279,-0.002606,-0.002582,-0.002925 --0.061336,-0.311523,0.080550,-0.070684,-0.144231,-0.102713,-0.091219,-0.089422,-0.102876,-0.083684,-0.081511,0.933504,-0.056172,-0.056039,-0.047641,-0.056120,-0.058554,-0.046885,-0.036371,-0.035963,-0.036450,-0.034703,-0.031401,-0.030835,-0.034227,-0.031233,-0.027495,-0.019349,-0.020579,-0.020635,-0.020875,-0.020529,-0.019471,-0.019535,-0.018635,-0.014732,-0.009851,-0.010906,-0.011467,-0.013019,-0.011442,-0.012187,-0.011916,-0.010359,-0.008045,-0.005254,-0.005874,-0.008097,-0.006452,-0.006862,-0.006406,-0.006472,-0.005608,-0.004326,-0.006317,-0.006355,-0.002177,-0.003840,-0.006349,-0.006439,-0.002611,-0.000795,-0.000381,-0.001299 -0.141146,0.270618,0.109347,0.160956,0.113357,0.064571,0.052712,0.056054,0.091159,0.059088,0.050589,0.029374,1.017301,0.017334,0.009454,0.023671,0.038190,0.039137,0.014598,0.004024,-0.000568,-0.004052,-0.007771,-0.006885,0.000366,0.007862,0.013479,0.000021,-0.007980,-0.015891,-0.017613,-0.018429,-0.019199,-0.016054,-0.008568,-0.003435,-0.012774,-0.020610,-0.025396,-0.025529,-0.027139,-0.025891,-0.023251,-0.021626,-0.017573,-0.022217,-0.027161,-0.027431,-0.030195,-0.032088,-0.030418,-0.030296,-0.028542,-0.025152,-0.030642,-0.030469,-0.027918,-0.032159,-0.030449,-0.030244,-0.028476,-0.032493,-0.033412,-0.031577 -0.010888,0.143138,-0.058942,0.027597,0.066139,0.049321,0.044693,0.043809,0.048870,0.037888,0.038526,0.032847,0.028669,1.028823,0.025196,0.028680,0.029193,0.023136,0.016653,0.018205,0.019495,0.019034,0.017832,0.017533,0.018927,0.016906,0.014593,0.009285,0.011119,0.012219,0.012786,0.012880,0.012542,0.012453,0.011422,0.008921,0.005613,0.007246,0.008268,0.009379,0.008812,0.009169,0.008883,0.007971,0.006370,0.004433,0.005406,0.006753,0.006412,0.006795,0.006416,0.006452,0.005983,0.004977,0.006375,0.006395,0.003499,0.005014,0.006382,0.006423,0.004302,0.003372,0.003348,0.003821 -0.000792,-0.000394,0.000069,-0.001095,0.000035,-0.000005,-0.000048,-0.000120,-0.000377,0.000198,0.000090,0.000081,0.000069,0.000038,1.000027,-0.000043,-0.000172,-0.000248,0.000246,0.000182,0.000121,0.000110,0.000086,0.000069,0.000015,-0.000063,-0.000126,0.000203,0.000182,0.000169,0.000139,0.000114,0.000093,0.000057,0.000004,-0.000038,0.000197,0.000187,0.000171,0.000140,0.000125,0.000112,0.000086,0.000065,0.000039,0.000168,0.000166,0.000156,0.000124,0.000135,0.000124,0.000122,0.000091,0.000074,0.000127,0.000124,0.000158,0.000149,0.000125,0.000123,0.000091,0.000147,0.000135,0.000109 --0.012057,-0.188372,-0.040525,-0.168873,-0.064565,-0.040505,-0.037078,-0.044114,-0.082202,-0.020445,-0.024538,-0.013705,-0.007873,-0.010312,-0.006624,0.980368,-0.036851,-0.042241,0.008908,0.009095,0.006544,0.007512,0.007718,0.006021,-0.001981,-0.011374,-0.018665,0.014068,0.016161,0.019025,0.017523,0.016008,0.014825,0.010519,0.002811,-0.002576,0.020641,0.023541,0.024575,0.022095,0.021822,0.020141,0.016881,0.014586,0.010824,0.023346,0.025512,0.024676,0.023682,0.025410,0.023776,0.023563,0.020384,0.017505,0.024092,0.023802,0.025555,0.026799,0.023864,0.023578,0.020620,0.027069,0.026575,0.023632 -0.211316,0.387293,0.189807,0.256655,0.158309,0.086878,0.070012,0.076286,0.132742,0.079651,0.067146,0.035502,0.017845,0.018170,0.006956,0.029031,1.053091,0.057212,0.015446,-0.001061,-0.008080,-0.013323,-0.018578,-0.016916,-0.005350,0.007884,0.018092,-0.004864,-0.017830,-0.030521,-0.033108,-0.034170,-0.035066,-0.029712,-0.017042,-0.007754,-0.023874,-0.036631,-0.044334,-0.044565,-0.046777,-0.044785,-0.040269,-0.037212,-0.030066,-0.038116,-0.046191,-0.046928,-0.050860,-0.054075,-0.051211,-0.051010,-0.047785,-0.041952,-0.051577,-0.051285,-0.046721,-0.053787,-0.051259,-0.050925,-0.047168,-0.053792,-0.055107,-0.052098 -0.005133,0.026803,0.009398,0.024220,0.009343,0.005487,0.004855,0.005820,0.011277,0.003191,0.003432,0.001670,0.000731,0.001023,0.000446,0.002329,0.004802,1.005628,-0.001077,-0.001433,-0.001285,-0.001488,-0.001587,-0.001350,-0.000215,0.001179,0.002273,-0.001927,-0.002448,-0.003042,-0.002923,-0.002765,-0.002635,-0.002025,-0.000859,-0.000004,-0.002988,-0.003597,-0.003881,-0.003613,-0.003611,-0.003381,-0.002907,-0.002562,-0.001955,-0.003550,-0.003973,-0.003913,-0.003862,-0.004133,-0.003880,-0.003851,-0.003407,-0.002940,-0.003923,-0.003884,-0.003953,-0.004259,-0.003890,-0.003851,-0.003392,-0.004261,-0.004226,-0.003831 -0.032234,0.032144,0.007848,-0.003773,0.018178,0.010819,0.008298,0.007068,0.005779,0.013527,0.010431,0.007541,0.005684,0.005044,0.003685,0.004007,0.002871,0.000853,1.008093,0.005784,0.004139,0.003502,0.002505,0.002217,0.001958,0.000943,0.000011,0.005040,0.003911,0.002921,0.002165,0.001565,0.001012,0.000583,0.000152,-0.000399,0.003276,0.002382,0.001637,0.001068,0.000531,0.000419,0.000128,-0.000223,-0.000476,0.001581,0.001102,0.000979,-0.000005,0.000059,-0.000032,-0.000057,-0.000562,-0.000659,-0.000005,-0.000040,0.000688,0.000195,-0.000020,-0.000044,-0.000696,-0.000021,-0.000381,-0.000683 -0.133270,0.223517,0.120863,0.147983,0.092012,0.049366,0.039140,0.042594,0.075120,0.047071,0.038707,0.019735,0.009185,0.009177,0.002527,0.015324,0.029190,0.031686,0.009723,0.999120,-0.005646,-0.008885,-0.012157,-0.011225,-0.004598,0.003159,0.009179,-0.002406,-0.010602,-0.018518,-0.020328,-0.021139,-0.021794,-0.018738,-0.011250,-0.005646,-0.013784,-0.021768,-0.026651,-0.027070,-0.028474,-0.027364,-0.024744,-0.022939,-0.018612,-0.022582,-0.027604,-0.028217,-0.030794,-0.032719,-0.031010,-0.030901,-0.029088,-0.025556,-0.031214,-0.031053,-0.027836,-0.032322,-0.031031,-0.030844,-0.028584,-0.032214,-0.033094,-0.031449 --0.186144,-0.387450,-0.116954,-0.204600,-0.165774,-0.098363,-0.081573,-0.084990,-0.129157,-0.088781,-0.077635,-0.048949,-0.032250,-0.032142,-0.020905,-0.039521,-0.057086,-0.055645,-0.025785,-0.012643,0.993063,-0.002330,0.002919,0.002023,-0.007216,-0.015408,-0.021232,-0.004615,0.004922,0.014544,0.016701,0.017854,0.019078,0.015395,0.006712,0.001421,0.012772,0.022099,0.027851,0.027828,0.030148,0.028541,0.025497,0.023916,0.019561,0.025229,0.031106,0.031037,0.034833,0.037003,0.035113,0.034959,0.033126,0.029337,0.035391,0.035185,0.032782,0.037627,0.035159,0.034902,0.033632,0.038636,0.039880,0.037577 -0.069051,0.129732,-0.175562,-0.175398,0.097004,0.077655,0.067024,0.053046,0.010000,0.084095,0.074120,0.070535,0.065478,0.061282,0.055484,0.046231,0.021671,-0.003367,0.065525,0.064985,0.061343,1.060418,0.056075,0.052521,0.044258,0.024852,0.007820,0.050000,0.054936,0.059515,0.057330,0.054338,0.050903,0.043992,0.030261,0.016120,0.047287,0.053313,0.055667,0.053738,0.051056,0.049358,0.043690,0.037798,0.028505,0.045106,0.049398,0.051101,0.047109,0.050471,0.047193,0.046938,0.040557,0.034131,0.047584,0.047166,0.044316,0.048558,0.047257,0.046919,0.036786,0.044766,0.043356,0.039837 -0.121117,0.522381,0.018959,0.267769,0.217700,0.143291,0.126114,0.132041,0.189943,0.109180,0.107093,0.076820,0.058468,0.060408,0.047222,0.071046,0.093749,0.089469,0.027715,0.022534,0.022997,0.019266,1.014841,0.016201,0.028756,0.037550,0.043172,0.003536,-0.000797,-0.006656,-0.005983,-0.005358,-0.005549,-0.000516,0.008519,0.012213,-0.015102,-0.020172,-0.022720,-0.019493,-0.021225,-0.018684,-0.015010,-0.013603,-0.010491,-0.025174,-0.028688,-0.026362,-0.028061,-0.030018,-0.028274,-0.027990,-0.025245,-0.022350,-0.028707,-0.028358,-0.031972,-0.033584,-0.028406,-0.028006,-0.027771,-0.036354,-0.036678,-0.032690 --0.011755,0.068098,-0.026114,0.034118,0.026992,0.020772,0.019709,0.020819,0.028237,0.011797,0.014291,0.012103,0.010591,0.011301,0.010007,0.012901,0.015768,0.014808,0.001903,0.003870,0.005666,0.005703,0.005698,1.005922,0.007622,0.008376,0.008708,-0.000292,0.000919,0.001633,0.002485,0.003039,0.003335,0.004036,0.004710,0.004523,-0.001715,-0.000820,-0.000065,0.001040,0.001117,0.001536,0.001949,0.002004,0.001865,-0.001611,-0.001158,-0.000388,0.000125,0.000060,0.000131,0.000186,0.000628,0.000566,0.000061,0.000123,-0.001786,-0.000976,0.000099,0.000164,-0.000078,-0.001623,-0.001374,-0.000651 --0.000719,0.002019,-0.000732,0.001470,0.000703,0.000557,0.000550,0.000617,0.000946,0.000214,0.000336,0.000282,0.000247,0.000282,0.000253,0.000366,0.000509,0.000522,-0.000075,0.000010,0.000089,0.000096,0.000108,0.000123,1.000196,0.000256,0.000297,-0.000112,-0.000068,-0.000043,-0.000005,0.000023,0.000041,0.000078,0.000124,0.000140,-0.000147,-0.000118,-0.000089,-0.000044,-0.000034,-0.000016,0.000008,0.000020,0.000030,-0.000129,-0.000116,-0.000090,-0.000060,-0.000068,-0.000060,-0.000057,-0.000030,-0.000023,-0.000063,-0.000060,-0.000128,-0.000102,-0.000061,-0.000058,-0.000049,-0.000119,-0.000106,-0.000073 --0.110743,-0.255859,-0.061551,-0.131415,-0.109422,-0.066501,-0.055869,-0.058068,-0.086383,-0.058289,-0.052075,-0.033997,-0.023370,-0.023445,-0.016185,-0.028172,-0.039313,-0.037846,-0.017046,-0.009598,-0.006579,-0.003778,-0.000531,-0.001091,-0.007065,0.988004,-0.015403,-0.003377,0.002019,0.007607,0.008711,0.009298,0.010005,0.007664,0.002383,-0.000595,0.007614,0.012955,0.016210,0.015916,0.017340,0.016270,0.014381,0.013483,0.010996,0.015181,0.018571,0.018294,0.020511,0.021803,0.020678,0.020573,0.019421,0.017220,0.020861,0.020724,0.019829,0.022493,0.020714,0.020544,0.019973,0.023334,0.024040,0.022487 -0.072502,0.171781,0.078326,0.131030,0.066146,0.036848,0.030618,0.034757,0.063918,0.029531,0.026522,0.013371,0.006163,0.006955,0.002449,0.013210,0.026032,0.029347,0.001415,-0.004199,-0.005879,-0.007856,-0.009568,-0.008514,-0.002508,0.004681,1.010291,-0.006258,-0.011264,-0.016350,-0.016815,-0.016741,-0.016676,-0.013674,-0.007219,-0.002443,-0.014158,-0.019312,-0.022223,-0.021669,-0.022283,-0.021170,-0.018729,-0.017002,-0.013458,-0.019554,-0.022906,-0.023004,-0.023996,-0.025578,-0.024139,-0.024015,-0.022011,-0.019193,-0.024348,-0.024170,-0.022962,-0.025755,-0.024178,-0.023989,-0.021758,-0.025721,-0.026013,-0.024212 -0.211166,0.256492,0.144794,0.111657,0.118608,0.063311,0.047932,0.047496,0.070913,0.072343,0.055533,0.031625,0.017842,0.015872,0.006872,0.018196,0.026836,0.024509,0.028613,0.011964,0.002506,-0.002135,-0.007699,-0.007669,-0.002979,0.001301,0.004380,1.010661,-0.000400,-0.010609,-0.014614,-0.017153,-0.019304,-0.017886,-0.012338,-0.008434,-0.003701,-0.013816,-0.020605,-0.022966,-0.025667,-0.025123,-0.023619,-0.022856,-0.019408,-0.016220,-0.022320,-0.023571,-0.028704,-0.030302,-0.028979,-0.028966,-0.028772,-0.025701,-0.029066,-0.029035,-0.023284,-0.029090,-0.028953,-0.028869,-0.028303,-0.029226,-0.031077,-0.030658 -0.316095,0.240441,0.179691,0.021456,0.131739,0.066421,0.045285,0.038094,0.040073,0.097339,0.067741,0.040076,0.023646,0.018401,0.007459,0.013470,0.011157,0.001955,0.054190,0.028848,0.012075,0.005744,-0.002791,-0.004328,-0.004228,-0.006422,-0.008586,0.030347,1.015399,0.002386,-0.005100,-0.010388,-0.014826,-0.016575,-0.015163,-0.014338,0.013758,0.000982,-0.008432,-0.014209,-0.018692,-0.019343,-0.020029,-0.021049,-0.019344,-0.003227,-0.010553,-0.013092,-0.021958,-0.022803,-0.022295,-0.022465,-0.025119,-0.023122,-0.022146,-0.022361,-0.012220,-0.019872,-0.022179,-0.022304,-0.024373,-0.020024,-0.023333,-0.025315 --0.243677,-0.295614,-0.141445,-0.100347,-0.141494,-0.078198,-0.059912,-0.057659,-0.077928,-0.089650,-0.069598,-0.042774,-0.026965,-0.024220,-0.013617,-0.024867,-0.031287,-0.025747,-0.039555,-0.020860,-0.009789,-0.004521,0.002182,0.002594,-0.001493,-0.003847,-0.005184,-0.017901,-0.006128,1.004706,0.009478,0.012699,0.015525,0.014836,0.010382,0.007742,-0.001728,0.008841,0.016149,0.019084,0.022413,0.022043,0.021088,0.020956,0.018241,0.012445,0.018733,0.019972,0.026191,0.027554,0.026486,0.026506,0.027094,0.024456,0.026531,0.026551,0.020372,0.026366,0.026447,0.026399,0.026978,0.026929,0.029172,0.029183 -0.001270,0.009737,-0.004070,0.001222,0.004639,0.003439,0.003089,0.002981,0.003169,0.002772,0.002747,0.002345,0.002046,0.002036,0.001776,0.001974,0.001927,0.001455,0.001321,0.001388,0.001439,0.001400,0.001300,0.001268,0.001331,0.001140,0.000938,0.000779,0.000893,0.000962,1.000982,0.000973,0.000935,0.000906,0.000800,0.000598,0.000519,0.000626,0.000687,0.000745,0.000695,0.000712,0.000675,0.000598,0.000470,0.000418,0.000484,0.000572,0.000528,0.000561,0.000528,0.000529,0.000476,0.000395,0.000527,0.000526,0.000346,0.000446,0.000526,0.000527,0.000359,0.000330,0.000320,0.000337 -0.039315,0.127310,-0.027848,0.018595,0.061465,0.042876,0.037363,0.035795,0.038824,0.037816,0.035253,0.028490,0.023849,0.023380,0.019644,0.022537,0.022244,0.016603,0.018223,0.016887,0.016131,0.015179,0.013427,0.013003,0.013859,0.011864,0.009707,0.010413,0.010372,0.009988,0.009635,1.009140,0.008413,0.008059,0.007217,0.005263,0.006158,0.006201,0.006068,0.006270,0.005388,0.005550,0.005180,0.004328,0.003187,0.003648,0.003715,0.004460,0.003296,0.003547,0.003268,0.003272,0.002570,0.001932,0.003260,0.003246,0.002107,0.002526,0.003256,0.003267,0.001378,0.001275,0.000915,0.000999 --0.106226,0.058640,-0.011542,0.148489,-0.002146,0.002606,0.008151,0.017793,0.052617,-0.025184,-0.010606,-0.009576,-0.008181,-0.003951,-0.002622,0.006889,0.024245,0.034323,-0.032519,-0.023856,-0.015603,-0.014117,-0.010906,-0.008631,-0.001282,0.009101,0.017514,-0.026977,-0.024012,-0.022315,-0.018278,-0.014911,0.987934,-0.007245,-0.000051,0.005481,-0.026366,-0.024923,-0.022749,-0.018526,-0.016487,-0.014714,-0.011242,-0.008424,-0.005009,-0.022498,-0.022166,-0.020770,-0.016494,-0.017952,-0.016463,-0.016204,-0.012048,-0.009740,-0.016813,-0.016459,-0.021123,-0.019917,-0.016581,-0.016271,-0.012127,-0.019711,-0.018022,-0.014574 --0.044721,-0.046099,-0.025382,-0.011821,-0.023081,-0.012429,-0.009212,-0.008531,-0.010771,-0.015460,-0.011566,-0.007061,-0.004393,-0.003786,-0.001998,-0.003548,-0.004070,-0.002898,-0.007515,-0.004030,-0.001858,-0.000921,0.000304,0.000443,0.000008,-0.000089,-0.000081,-0.003770,-0.001642,0.000274,0.001226,0.001884,0.002451,1.002489,0.001948,0.001638,-0.001058,0.000813,0.002143,0.002797,0.003420,0.003421,0.003366,0.003420,0.003043,0.001460,0.002556,0.002833,0.004025,0.004215,0.004076,0.004088,0.004315,0.003925,0.004071,0.004087,0.002838,0.003921,0.004065,0.004067,0.004266,0.003997,0.004433,0.004554 -0.112911,0.194765,0.093861,0.119918,0.081522,0.044916,0.035970,0.038468,0.064677,0.042701,0.035532,0.019441,0.010379,0.010277,0.004461,0.015019,0.025974,0.027200,0.010341,0.001575,-0.002423,-0.005157,-0.008032,-0.007354,-0.001987,0.003911,0.008412,-0.000417,-0.007038,-0.013464,-0.014999,-0.015742,-0.016383,-0.014012,0.991840,-0.003944,-0.010091,-0.016517,-0.020479,-0.020829,-0.022084,-0.021188,-0.019158,-0.017836,-0.014528,-0.017506,-0.021539,-0.021953,-0.024205,-0.025709,-0.024382,-0.024297,-0.022963,-0.020222,-0.024545,-0.024420,-0.021931,-0.025497,-0.024400,-0.024251,-0.022703,-0.025566,-0.026332,-0.025030 -0.009691,0.054357,0.021647,0.052769,0.018293,0.010501,0.009321,0.011514,0.023449,0.005598,0.006243,0.002619,0.000726,0.001391,0.000261,0.004308,0.009780,0.011816,-0.003078,-0.003799,-0.003434,-0.003843,-0.003993,-0.003453,-0.000979,0.002172,0.004668,-0.004653,-0.005793,-0.007082,-0.006801,-0.006431,-0.006115,-0.004754,-0.002151,0.999809,-0.006830,-0.008168,-0.008784,-0.008195,-0.008155,-0.007655,-0.006593,-0.005800,-0.004419,-0.007967,-0.008900,-0.008792,-0.008631,-0.009238,-0.008669,-0.008607,-0.007598,-0.006545,-0.008765,-0.008677,-0.008795,-0.009481,-0.008692,-0.008606,-0.007523,-0.009439,-0.009347,-0.008477 -0.215650,0.265640,0.146419,0.115138,0.122755,0.065877,0.050072,0.049633,0.073772,0.074746,0.057632,0.033077,0.018901,0.016905,0.007626,0.019305,0.028191,0.025695,0.029540,0.012597,0.002991,-0.001756,-0.007459,-0.007429,-0.002584,0.001767,0.004880,0.011040,-0.000205,-0.010600,-0.014666,-0.017247,-0.019445,-0.017985,-0.012324,-0.008381,0.996283,-0.014005,-0.020908,-0.023280,-0.026048,-0.025477,-0.023939,-0.023174,-0.019681,-0.016522,-0.022728,-0.023969,-0.029208,-0.030835,-0.029488,-0.029473,-0.029275,-0.026157,-0.029580,-0.029546,-0.023756,-0.029646,-0.029463,-0.029375,-0.028837,-0.029825,-0.031711,-0.031261 -0.009416,0.122676,0.002323,0.081735,0.046895,0.031502,0.028655,0.031475,0.049582,0.019685,0.021547,0.015140,0.011379,0.012472,0.009822,0.016506,0.024194,0.024870,0.000840,0.001091,0.002502,0.001928,0.001478,0.002135,0.006029,0.009590,0.012150,-0.003525,-0.003997,-0.005000,-0.004213,-0.003542,-0.003131,-0.001272,0.001834,0.003517,-0.007474,0.991653,-0.008555,-0.007184,-0.007236,-0.006403,-0.005053,-0.004299,-0.003079,-0.009046,-0.009757,-0.009031,-0.008720,-0.009377,-0.008761,-0.008659,-0.007402,-0.006405,-0.008911,-0.008779,-0.010303,-0.010463,-0.008809,-0.008673,-0.007962,-0.011030,-0.010832,-0.009419 --0.027691,0.014494,-0.002690,0.038481,-0.000910,0.000416,0.001883,0.004396,0.013428,-0.006752,-0.002963,-0.002665,-0.002280,-0.001181,-0.000816,0.001640,0.006152,0.008806,-0.008548,-0.006302,-0.004162,-0.003773,-0.002932,-0.002338,-0.000434,0.002278,0.004479,-0.007067,-0.006305,-0.005870,-0.004823,-0.003947,-0.003205,-0.001951,-0.000074,0.001378,-0.006889,-0.006522,0.994038,-0.004869,-0.004336,-0.003876,-0.002972,-0.002234,-0.001337,-0.005876,-0.005795,-0.005439,-0.004325,-0.004706,-0.004317,-0.004249,-0.003166,-0.002560,-0.004407,-0.004316,-0.005513,-0.005208,-0.004347,-0.004267,-0.003177,-0.005145,-0.004706,-0.003811 -0.021928,-0.138925,-0.036445,-0.166099,-0.038908,-0.024350,-0.023959,-0.032141,-0.070146,-0.003005,-0.010086,-0.002942,0.000504,-0.002511,-0.000601,-0.012659,-0.030552,-0.038363,0.018301,0.016265,0.012215,0.012473,0.011560,0.009588,0.001643,-0.008712,-0.016957,0.019610,0.020655,0.022532,0.020347,0.018269,0.016525,0.011885,0.003849,-0.002171,0.023914,0.025969,0.026305,1.023402,0.022544,0.020820,0.017325,0.014655,0.010597,0.024730,0.026435,0.025608,0.023611,0.025392,0.023674,0.023445,0.019794,0.016806,0.024007,0.023690,0.025896,0.026734,0.023769,0.023471,0.019747,0.026640,0.025791,0.022655 --0.032558,-0.121117,0.017046,-0.033489,-0.055543,-0.038205,-0.033475,-0.033025,-0.039662,-0.032037,-0.030449,-0.023903,-0.019593,-0.019492,-0.016139,-0.019868,-0.021562,-0.017734,-0.013349,-0.012249,-0.011920,-0.011057,-0.009651,-0.009506,-0.011072,-0.010558,-0.009681,-0.006622,-0.006337,-0.005698,-0.005580,-0.005355,-0.004912,-0.005114,-0.005319,-0.004392,-0.002532,-0.002225,-0.001992,-0.002421,0.998245,-0.002069,-0.002143,-0.001731,-0.001260,-0.000241,-0.000041,-0.000716,0.000096,0.000094,0.000130,0.000102,0.000300,0.000475,0.000171,0.000150,0.001225,0.001045,0.000149,0.000108,0.001259,0.002056,0.002287,0.001894 --0.007865,-0.035708,0.015290,-0.000503,-0.017860,-0.013122,-0.011622,-0.010936,-0.010675,-0.011345,-0.010838,-0.009266,-0.008079,-0.007923,-0.006888,-0.007381,-0.006711,-0.004616,-0.005984,-0.005998,-0.005961,-0.005768,-0.005296,-0.005109,-0.005142,-0.004116,-0.003104,-0.003748,-0.004103,-0.004319,-0.004284,-0.004152,-0.003929,-0.003681,-0.003070,-0.002140,-0.002734,-0.003105,-0.003278,-0.003380,-0.003133,0.996853,-0.002908,-0.002532,-0.001944,-0.002237,-0.002483,-0.002779,-0.002489,-0.002660,-0.002489,-0.002486,-0.002167,-0.001790,-0.002494,-0.002483,-0.001922,-0.002273,-0.002485,-0.002482,-0.001719,-0.001822,-0.001737,-0.001704 --0.133871,-0.119434,-0.179024,-0.135274,-0.043149,-0.011537,-0.004450,-0.008881,-0.039453,-0.018707,-0.009186,0.006389,0.014195,0.014338,0.018426,0.007643,-0.006765,-0.014510,0.003503,0.016184,0.022386,0.025474,0.027978,0.026729,0.020806,0.010809,0.002427,0.010332,0.020660,0.030071,0.032493,0.033445,0.033867,0.030524,0.021643,0.013544,0.019449,0.029378,0.035474,0.036860,0.037893,0.037005,1.033961,0.031265,0.025288,0.027894,0.034095,0.035884,0.038366,0.040749,0.038601,0.038515,0.036183,0.031557,0.038785,0.038629,0.032745,0.038826,0.038595,0.038430,0.034234,0.037334,0.038259,0.036897 --0.017153,0.015003,-0.004095,0.025566,0.002099,0.002261,0.003005,0.004566,0.010510,-0.002757,-0.000324,-0.000364,-0.000289,0.000416,0.000501,0.002203,0.005090,0.006531,-0.004757,-0.003267,-0.001856,-0.001627,-0.001141,-0.000776,0.000490,0.002132,0.003436,-0.004114,-0.003556,-0.003235,-0.002547,-0.001987,-0.001529,-0.000735,0.000415,0.001237,-0.004150,-0.003851,-0.003453,-0.002714,-0.002398,-0.002092,-0.001528,0.998903,-0.000591,-0.003554,-0.003463,-0.003182,-0.002488,-0.002715,-0.002483,-0.002439,-0.001769,-0.001425,-0.002542,-0.002483,-0.003361,-0.003106,-0.002504,-0.002451,-0.001845,-0.003133,-0.002855,-0.002267 --0.096197,-0.007534,0.014350,0.117456,-0.028810,-0.017873,-0.011193,-0.002481,0.025640,-0.037247,-0.024903,-0.021715,-0.018813,-0.015215,-0.012589,-0.005763,0.009071,0.020280,-0.034955,-0.028110,-0.021502,-0.020032,-0.016765,-0.014672,-0.008872,0.000956,0.009195,-0.027158,-0.025357,-0.024351,-0.021092,-0.018217,-0.015617,-0.011403,-0.004742,0.001080,-0.025143,-0.024576,-0.023121,-0.019921,-0.017926,-0.016536,-0.013409,-0.010594,0.993025,-0.021326,-0.021445,-0.020789,-0.016953,-0.018375,-0.016928,-0.016718,-0.012925,-0.010511,-0.017214,-0.016916,-0.019761,-0.019345,-0.017016,-0.016764,-0.012305,-0.018494,-0.017024,-0.014230 --0.164224,-0.360383,-0.151049,-0.246774,-0.144180,-0.081658,-0.067519,-0.074301,-0.128787,-0.069295,-0.061116,-0.033505,-0.018083,-0.019072,-0.009206,-0.030063,-0.053451,-0.057709,-0.010307,0.002108,0.006550,0.010840,0.014968,0.013227,0.001977,-0.010684,-0.020406,0.007168,0.017454,0.027822,0.029284,0.029625,0.029975,0.024644,0.012836,0.004396,0.023795,0.034181,0.040235,0.039581,0.041215,0.039177,0.034809,0.031882,0.025477,1.035465,0.042143,0.042344,0.045047,0.047965,0.045339,0.045121,0.041767,0.036573,0.045714,0.045405,0.042705,0.048302,0.045405,0.045063,0.041502,0.048514,0.049363,0.046169 --0.340834,-0.423019,-0.215142,-0.166379,-0.198201,-0.108234,-0.082848,-0.081138,-0.115560,-0.122523,-0.095145,-0.056653,-0.034212,-0.030796,-0.015927,-0.033430,-0.045462,-0.039742,-0.050791,-0.024369,-0.009144,-0.001685,0.007512,0.007727,0.000790,-0.004562,-0.008160,-0.020851,-0.003704,0.012146,0.018637,0.022877,0.026551,0.024785,0.016976,0.011874,0.002322,0.017899,0.028475,0.032312,0.036817,0.036054,0.034074,0.033315,0.028558,0.022418,1.031767,0.033576,0.042026,0.044312,0.042456,0.042451,0.042611,0.038226,0.042569,0.042548,0.033739,0.042601,0.042412,0.042300,0.042204,0.043155,0.046196,0.045754 --0.004707,0.002464,-0.000457,0.006540,-0.000155,0.000071,0.000320,0.000747,0.002282,-0.001148,-0.000504,-0.000453,-0.000388,-0.000201,-0.000139,0.000279,0.001046,0.001497,-0.001453,-0.001071,-0.000707,-0.000641,-0.000498,-0.000397,-0.000074,0.000387,0.000761,-0.001201,-0.001072,-0.000998,-0.000820,-0.000671,-0.000545,-0.000332,-0.000013,0.000234,-0.001171,-0.001108,-0.001013,-0.000828,-0.000737,-0.000659,-0.000505,-0.000380,-0.000227,-0.000999,-0.000985,0.999076,-0.000735,-0.000800,-0.000734,-0.000722,-0.000538,-0.000435,-0.000749,-0.000733,-0.000937,-0.000885,-0.000739,-0.000725,-0.000540,-0.000875,-0.000800,-0.000648 -0.197188,0.350909,0.156605,0.209245,0.147682,0.082629,0.066649,0.070818,0.116435,0.077843,0.065403,0.036986,0.020878,0.020688,0.010253,0.028805,0.047695,0.049143,0.019778,0.004706,-0.002140,-0.006927,-0.012042,-0.010915,-0.001540,0.008394,0.015893,0.000389,-0.010904,-0.021926,-0.024552,-0.025851,-0.027013,-0.022953,-0.012979,-0.005971,-0.016721,-0.027687,-0.034456,-0.034978,-0.037221,-0.035647,-0.032186,-0.030009,-0.024468,-0.029695,-0.036579,-0.037170,0.958869,-0.043685,-0.041437,-0.041288,-0.039054,-0.034427,-0.041720,-0.041505,-0.037456,-0.043483,-0.041470,-0.041211,-0.038772,-0.043766,-0.045107,-0.042830 -0.082289,0.146952,0.065509,0.087867,0.061789,0.034579,0.027904,0.029669,0.048830,0.032518,0.027343,0.015455,0.008718,0.008647,0.004284,0.012066,0.020007,0.020634,0.008207,0.001918,-0.000929,-0.002929,-0.005063,-0.004586,-0.000648,0.003531,0.006686,0.000106,-0.004614,-0.009222,-0.010312,-0.010849,-0.011329,-0.009620,-0.005432,-0.002487,-0.007052,-0.011638,-0.014466,-0.014675,-0.015610,-0.014947,-0.013492,-0.012575,-0.010249,-0.012473,-0.015353,-0.015597,-0.017246,0.981682,-0.017374,-0.017311,-0.016367,-0.014426,-0.017493,-0.017402,-0.015717,-0.018237,-0.017388,-0.017279,-0.016249,-0.018355,-0.018913,-0.017953 -0.013229,-0.006924,0.001285,-0.018383,0.000435,-0.000199,-0.000900,-0.002100,-0.006415,0.003226,0.001416,0.001273,0.001089,0.000564,0.000390,-0.000783,-0.002939,-0.004207,0.004084,0.003011,0.001988,0.001803,0.001401,0.001117,0.000207,-0.001088,-0.002140,0.003376,0.003012,0.002804,0.002304,0.001886,0.001531,0.000932,0.000035,-0.000658,0.003291,0.003116,0.002848,0.002326,0.002071,0.001852,0.001420,0.001067,0.000639,0.002807,0.002768,0.002598,0.002066,0.002248,1.002062,0.002030,0.001512,0.001223,0.002106,0.002062,0.002634,0.002488,0.002077,0.002038,0.001518,0.002458,0.002248,0.001821 -0.000215,-0.000112,0.000021,-0.000298,0.000007,-0.000003,-0.000015,-0.000034,-0.000104,0.000052,0.000023,0.000021,0.000018,0.000009,0.000006,-0.000013,-0.000048,-0.000068,0.000066,0.000049,0.000032,0.000029,0.000023,0.000018,0.000003,-0.000018,-0.000035,0.000055,0.000049,0.000045,0.000037,0.000031,0.000025,0.000015,0.000001,-0.000011,0.000053,0.000051,0.000046,0.000038,0.000034,0.000030,0.000023,0.000017,0.000010,0.000046,0.000045,0.000042,0.000034,0.000036,0.000033,1.000033,0.000025,0.000020,0.000034,0.000033,0.000043,0.000040,0.000034,0.000033,0.000025,0.000040,0.000036,0.000030 -0.076730,0.016093,-0.028876,-0.105508,0.029918,0.020336,0.014448,0.006599,-0.018736,0.035338,0.025301,0.022770,0.020228,0.017153,0.014706,0.008627,-0.004888,-0.015654,0.032187,0.027163,0.021958,0.020809,0.017997,0.016091,0.010927,0.001682,-0.006138,0.025013,0.024237,0.024013,0.021403,0.018989,0.016735,0.012910,0.006567,0.000823,0.023364,0.023607,0.022804,0.020249,0.018524,0.017318,0.014424,0.011748,0.008120,0.020407,0.020961,0.020647,0.017440,0.018839,0.017432,0.017251,1.013802,0.011354,0.017684,0.017421,0.019221,0.019368,0.017504,0.017282,0.012970,0.018367,0.017149,0.014721 --0.067883,-0.179687,-0.071735,-0.134698,-0.069150,-0.039544,-0.033325,-0.037621,-0.067580,-0.030657,-0.028259,-0.015067,-0.007789,-0.008714,-0.004106,-0.015165,-0.028284,-0.031421,-0.001481,0.003589,0.004857,0.006767,0.008414,0.007332,0.001131,-0.006082,-0.011668,0.006341,0.010990,0.015811,0.016089,0.015896,0.015762,0.012679,0.006220,0.001557,0.014285,0.019144,0.021834,0.021073,0.021643,0.020473,0.017997,0.016285,0.012830,0.019499,0.022686,0.022638,0.023486,0.025049,0.023624,0.023491,0.021432,1.018680,0.023842,0.023655,0.022842,0.025416,0.023668,0.023470,0.021309,0.025493,0.025723,0.023818 -0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,1.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000 -0.023043,0.041007,0.018301,0.024452,0.017258,0.009656,0.007789,0.008276,0.013607,0.009097,0.007643,0.004322,0.002440,0.002418,0.001198,0.003366,0.005574,0.005743,0.002311,0.000550,-0.000250,-0.000809,-0.001407,-0.001276,-0.000180,0.000981,0.001857,0.000045,-0.001274,-0.002562,-0.002869,-0.003021,-0.003157,-0.002682,-0.001517,-0.000698,-0.001954,-0.003235,-0.004026,-0.004088,-0.004350,-0.004166,-0.003761,-0.003507,-0.002859,-0.003470,-0.004275,-0.004344,-0.004807,-0.005105,-0.004842,-0.004825,-0.004564,-0.004023,-0.004875,0.995150,-0.004377,-0.005081,-0.004846,-0.004816,-0.004531,-0.005115,-0.005271,-0.005005 --0.002908,-0.022630,0.009454,-0.002892,-0.010770,-0.007986,-0.007175,-0.006928,-0.007377,-0.006426,-0.006374,-0.005440,-0.004746,-0.004726,-0.004122,-0.004586,-0.004484,-0.003391,-0.003054,-0.003214,-0.003335,-0.003244,-0.003014,-0.002941,-0.003089,-0.002649,-0.002184,-0.001798,-0.002064,-0.002225,-0.002274,-0.002253,-0.002167,-0.002101,-0.001857,-0.001391,-0.001195,-0.001445,-0.001587,-0.001723,-0.001609,-0.001648,-0.001564,-0.001386,-0.001089,-0.000961,-0.001116,-0.001320,-0.001219,-0.001296,-0.001219,-0.001222,-0.001101,-0.000913,-0.001216,-0.001215,0.999205,-0.001028,-0.001215,-0.001218,-0.000829,-0.000759,-0.000737,-0.000777 -0.024587,0.188567,-0.078816,0.023659,0.089837,0.066603,0.059818,0.057728,0.061363,0.053680,0.053196,0.045404,0.039613,0.039429,0.034391,0.038232,0.037321,0.028179,0.025579,0.026882,0.027866,0.027104,0.025178,0.024561,0.025771,0.022070,0.018158,0.015081,0.017292,0.018626,0.019021,0.018836,0.018114,0.017548,0.015485,0.011583,0.010048,0.012127,0.013311,0.014426,0.013469,0.013789,0.013080,0.011583,0.009097,0.008090,0.009378,0.011074,0.010217,0.010870,0.010219,0.010244,0.009223,0.007647,0.010196,0.010190,0.006697,1.008639,0.010184,0.010212,0.006953,0.006395,0.006206,0.006527 -0.000382,-0.000200,0.000037,-0.000531,0.000013,-0.000006,-0.000026,-0.000061,-0.000185,0.000093,0.000041,0.000037,0.000031,0.000016,0.000011,-0.000023,-0.000085,-0.000121,0.000118,0.000087,0.000057,0.000052,0.000040,0.000032,0.000006,-0.000031,-0.000062,0.000098,0.000087,0.000081,0.000067,0.000054,0.000044,0.000027,0.000001,-0.000019,0.000095,0.000090,0.000082,0.000067,0.000060,0.000053,0.000041,0.000031,0.000018,0.000081,0.000080,0.000075,0.000060,0.000065,0.000060,0.000059,0.000044,0.000035,0.000061,0.000060,0.000076,0.000072,1.000060,0.000059,0.000044,0.000071,0.000065,0.000053 -0.060214,0.078322,0.039094,0.033212,0.036131,0.019798,0.015269,0.015141,0.022091,0.021881,0.017159,0.010151,0.006081,0.005538,0.002849,0.006226,0.008755,0.007905,0.008648,0.003988,0.001364,0.000031,-0.001587,-0.001583,-0.000176,0.001001,0.001819,0.003289,0.000213,-0.002649,-0.003758,-0.004468,-0.005084,-0.004670,-0.003106,-0.002067,-0.000930,-0.003748,-0.005638,-0.006259,-0.007042,-0.006869,-0.006441,-0.006248,-0.005314,-0.004526,-0.006227,-0.006531,-0.007993,-0.008439,-0.008071,-0.008065,-0.008015,-0.007169,-0.008098,-0.008088,-0.006567,-0.008163,-0.008065,0.991961,-0.007941,-0.008259,-0.008782,-0.008636 --0.155078,-0.303781,-0.131579,-0.194155,-0.124791,-0.070225,-0.057319,-0.061948,-0.104557,-0.063042,-0.054149,-0.030190,-0.016703,-0.017022,-0.008336,-0.025133,-0.043113,-0.045498,-0.013038,-0.001241,0.003629,0.007517,0.011487,0.010273,0.001495,-0.008109,-0.015426,0.002752,0.011993,0.021146,0.022915,0.023635,0.024296,0.020308,0.011035,0.004460,0.017178,0.026314,0.031807,0.031774,0.033442,0.031908,0.028579,0.026412,0.021324,0.027735,0.033534,0.033881,0.036752,0.039084,0.037008,0.036852,0.034486,0.030300,0.037288,0.037065,0.034155,0.039131,0.037050,0.036795,1.034252,0.039344,0.040289,0.037969 -0.013185,0.023464,0.010471,0.013991,0.009875,0.005525,0.004457,0.004735,0.007785,0.005205,0.004373,0.002473,0.001396,0.001383,0.000686,0.001926,0.003189,0.003286,0.001322,0.000315,-0.000143,-0.000463,-0.000805,-0.000730,-0.000103,0.000561,0.001063,0.000026,-0.000729,-0.001466,-0.001642,-0.001729,-0.001806,-0.001535,-0.000868,-0.000399,-0.001118,-0.001851,-0.002304,-0.002339,-0.002489,-0.002384,-0.002152,-0.002007,-0.001636,-0.001986,-0.002446,-0.002485,-0.002750,-0.002921,-0.002771,-0.002761,-0.002611,-0.002302,-0.002790,-0.002775,-0.002504,-0.002907,-0.002773,-0.002756,-0.002592,0.997074,-0.003016,-0.002864 -0.272415,0.285370,0.330009,0.268459,0.109711,0.041066,0.025125,0.033002,0.093169,0.052265,0.033063,0.000290,-0.016682,-0.017122,-0.026580,-0.004580,0.023024,0.035441,0.001040,-0.023721,-0.035838,-0.042151,-0.047684,-0.045461,-0.033679,-0.015336,-0.000175,-0.015424,-0.035169,-0.053324,-0.058072,-0.060078,-0.061180,-0.054893,-0.038206,-0.023524,-0.034740,-0.053700,-0.065397,-0.067917,-0.070262,-0.068476,-0.062782,-0.057973,-0.047013,-0.051966,-0.063794,-0.066873,-0.072093,-0.076555,-0.072554,-0.072386,-0.068195,-0.059605,-0.072915,-0.072619,-0.061914,-0.073348,-0.072550,-0.072226,-0.064994,-0.071045,0.927042,-0.070280 --0.011982,-0.171579,0.070532,-0.034405,-0.078998,-0.058951,-0.053475,-0.052512,-0.058895,-0.045023,-0.045926,-0.039151,-0.034174,-0.034398,-0.030077,-0.034331,-0.035112,-0.027972,-0.019582,-0.021520,-0.023141,-0.022605,-0.021200,-0.020864,-0.022601,-0.020286,-0.017604,-0.010833,-0.013053,-0.014384,-0.015100,-0.015244,-0.014868,-0.014808,-0.013646,-0.010711,-0.006452,-0.008416,-0.009659,-0.011027,-0.010369,-0.010813,-0.010505,-0.009442,-0.007563,-0.005079,-0.006245,-0.007867,-0.007502,-0.007944,-0.007506,-0.007552,-0.007031,-0.005853,-0.007454,-0.007481,-0.003977,-0.005798,-0.007464,-0.007517,-0.005023,-0.003839,-0.003826,0.995575 -0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000 diff --git a/extensions/ezmsg-sigproc/tests/test_affine_transform.py b/extensions/ezmsg-sigproc/tests/test_affine_transform.py deleted file mode 100644 index cb75fa60..00000000 --- a/extensions/ezmsg-sigproc/tests/test_affine_transform.py +++ /dev/null @@ -1,82 +0,0 @@ -from pathlib import Path - -import numpy as np -from ezmsg.util.messages.axisarray import AxisArray - -from ezmsg.sigproc.affinetransform import affine_transform, common_rereference - - -def test_affine_generator(): - n_times = 13 - n_chans = 64 - in_dat = np.arange(n_times * n_chans).reshape(n_times, n_chans) - axis_arr_in = AxisArray(in_dat, dims=["time", "ch"]) - - gen = affine_transform(weights=np.eye(n_chans), axis="ch") - ax_arr_out = gen.send(axis_arr_in) - assert ax_arr_out.data.shape == in_dat.shape - assert np.allclose(ax_arr_out.data, in_dat) - assert not np.may_share_memory(ax_arr_out.data, in_dat) - - # Test with weights from a CSV file. - csv_path = Path(__file__).parent / "resources" / "xform.csv" - weights = np.loadtxt(csv_path, delimiter=",") - expected_out = in_dat @ weights.T - # Same result: expected_out = np.vstack([(step[None, :] * weights).sum(axis=1) for step in in_dat]) - - gen = affine_transform(weights=csv_path, axis="ch", right_multiply=False) - ax_arr_out = gen.send(axis_arr_in) - assert np.array_equal(ax_arr_out.data, expected_out) - - # Try again as str, not Path - gen = affine_transform(weights=str(csv_path), axis="ch", right_multiply=False) - ax_arr_out = gen.send(axis_arr_in) - assert np.array_equal(ax_arr_out.data, expected_out) - - # Try again as direct ndarray - gen = affine_transform(weights=weights, axis="ch", right_multiply=False) - ax_arr_out = gen.send(axis_arr_in) - assert np.array_equal(ax_arr_out.data, expected_out) - - # One more time, but we pre-transpose the weights and do not override right_multiply - gen = affine_transform(weights=weights.T, axis="ch", right_multiply=True) - ax_arr_out = gen.send(axis_arr_in) - assert np.array_equal(ax_arr_out.data, expected_out) - - -def test_common_rereference(): - n_times = 300 - n_chans = 64 - in_dat = np.arange(n_times * n_chans).reshape(n_times, n_chans) - axis_arr_in = AxisArray(in_dat, dims=["time", "ch"]) - - gen = common_rereference(mode="mean", axis="ch", include_current=True) - axis_arr_out = gen.send(axis_arr_in) - assert np.array_equal( - axis_arr_out.data, - axis_arr_in.data - np.mean(axis_arr_in.data, axis=1, keepdims=True), - ) - - # Use a slow deliberate way of calculating the CAR uniquely for each channel, excluding itself. - # common_rereference uses a faster way of doing this, but we test against something intuitive. - expected_out = [] - for ch_ix in range(n_chans): - idx = np.arange(n_chans) - idx = np.hstack((idx[:ch_ix], idx[ch_ix + 1 :])) - expected_out.append( - axis_arr_in.data[..., ch_ix] - np.mean(axis_arr_in.data[..., idx], axis=1) - ) - expected_out = np.stack(expected_out).T - - gen = common_rereference(mode="mean", axis="ch", include_current=False) - axis_arr_out = gen.send(axis_arr_in) # 41 us - assert np.allclose(axis_arr_out.data, expected_out) - - # Instead of CAR, we could use affine_transform with weights that reproduce CAR. - # However, this method is 30x slower than above. (Actual difference varies depending on data shape). - if False: - weights = -np.ones((n_chans, n_chans)) / (n_chans - 1) - np.fill_diagonal(weights, 1) - gen = affine_transform(weights=weights, axis="ch") - axis_arr_out = gen.send(axis_arr_in) - assert np.allclose(axis_arr_out.data, expected_out) diff --git a/extensions/ezmsg-sigproc/tests/test_aggregate.py b/extensions/ezmsg-sigproc/tests/test_aggregate.py deleted file mode 100644 index f3c7beff..00000000 --- a/extensions/ezmsg-sigproc/tests/test_aggregate.py +++ /dev/null @@ -1,65 +0,0 @@ -from functools import partial - -import numpy as np -import pytest -from ezmsg.util.messages.axisarray import AxisArray - -from ezmsg.sigproc.aggregate import ranged_aggregate, AggregationFunction - - -@pytest.mark.parametrize("agg_func", [AggregationFunction.MEAN, AggregationFunction.MEDIAN, AggregationFunction.STD]) -def test_aggregate(agg_func: AggregationFunction): - n_chans = 20 - n_freqs = 100 - data_dur = 30.0 - fs = 1024.0 - bands = [(5.0, 20.0), (30.0, 50.0)] - targ_ax = "freq" - agg_func = AggregationFunction.MEAN - - n_samples = int(data_dur * fs) - data = np.arange(n_samples * n_chans * n_freqs).reshape(n_samples, n_chans, n_freqs) - n_msgs = int(data_dur / 2) - - offset = 0 - messages = [] - for arr in np.array_split(data, n_samples // n_msgs): - messages.append( - AxisArray( - arr, - dims=["time", "ch", "freq"], - axes={ - "time": AxisArray.Axis.TimeAxis(fs=fs, offset=offset), - "freq": AxisArray.Axis(gain=1.0, offset=0.0, unit="Hz") - } - ) - ) - offset += arr.shape[0] / fs - - gen = ranged_aggregate(axis=targ_ax, bands=bands, operation=agg_func) - results = [gen.send(_) for _ in messages] - - assert all([type(_) is AxisArray for _ in results]) - - # Check output axis - for res in results: - ax = res.axes[targ_ax] - assert ax.offset == np.mean(bands[0]) - if len(bands) > 1: - assert ax.gain == np.mean(bands[1]) - np.mean(bands[0]) - assert ax.unit == messages[0].axes[targ_ax].unit - - # Check data - targ_ax = messages[0].axes[targ_ax] - targ_ax_vec = targ_ax.offset + np.arange(data.shape[-1]) * targ_ax.gain - agg_func = { - AggregationFunction.MEAN: partial(np.mean, axis=-1, keepdims=True), - AggregationFunction.MEDIAN: partial(np.median, axis=-1, keepdims=True), - AggregationFunction.STD: partial(np.std, axis=-1, keepdims=True) - }[agg_func] - expected_data = np.concatenate([ - agg_func(data[..., np.logical_and(targ_ax_vec >= start, targ_ax_vec <= stop)]) - for (start, stop) in bands - ], axis=-1) - received_data = AxisArray.concatenate(*results, dim="time").data - assert np.allclose(received_data, expected_data) diff --git a/extensions/ezmsg-sigproc/tests/test_bandpower.py b/extensions/ezmsg-sigproc/tests/test_bandpower.py deleted file mode 100644 index 4b75f48e..00000000 --- a/extensions/ezmsg-sigproc/tests/test_bandpower.py +++ /dev/null @@ -1,55 +0,0 @@ -import numpy as np -from ezmsg.util.messages.axisarray import AxisArray -from ezmsg.sigproc.bandpower import bandpower, SpectrogramSettings - -from util import create_messages_with_periodic_signal - - -def _debug_plot(result): - import matplotlib.pyplot as plt - - t_vec = result.axes["time"].offset + np.arange(result.data.shape[0]) * result.axes["time"].gain - plt.plot(t_vec, result.data[..., 0]) - - -def test_bandpower(): - win_dur = 1.0 - fs = 1000. - bands = [(9, 11), (70, 90), (134, 136)] - - sin_params = [ - {"f": 10.0, "a": 3.0, "dur": 4.0, "offset": 1.0}, - {"f": 10.0, "a": 1.0, "dur": 3.0, "offset": 5.0}, - {"f": 135.0, "a": 4.0, "dur": 4.0, "offset": 1.0}, - {"f": 135.0, "a": 2.0, "dur": 3.0, "offset": 5.0}, - ] - messages = create_messages_with_periodic_signal( - sin_params=sin_params, - fs=fs, - msg_dur=0.4, - win_step_dur=None # The spectrogram will do the windowing - ) - - gen = bandpower( - spectrogram_settings=SpectrogramSettings( - window_dur=win_dur, - window_shift=0.1, - ), - bands=bands - ) - results = [gen.send(msg) for msg in messages] - - result = AxisArray.concatenate(*results, dim="time") - # _debug_plot(result) - - # Check the amplitudes at the midpoints of each of our sinusoids. - t_vec = result.axes["time"].offset + np.arange(result.data.shape[0]) * result.axes["time"].gain - mags = [] - for s_p in sin_params[:2]: - ix = np.argmin(np.abs(t_vec - (s_p["offset"] + s_p["dur"]/2))) - mags.append(result.data[ix, 0, 0]) - for s_p in sin_params[2:]: - ix = np.argmin(np.abs(t_vec - (s_p["offset"] + s_p["dur"]/2))) - mags.append(result.data[ix, 2, 0]) - # The sorting of the measured magnitudes should match the sorting of the parameter magnitudes. - assert np.array_equal(np.argsort(mags), np.argsort([_["a"] for _ in sin_params])) diff --git a/extensions/ezmsg-sigproc/tests/test_butter.py b/extensions/ezmsg-sigproc/tests/test_butter.py deleted file mode 100644 index b77a3285..00000000 --- a/extensions/ezmsg-sigproc/tests/test_butter.py +++ /dev/null @@ -1,137 +0,0 @@ -# from typing import Optional -import numpy as np -import pytest -import scipy.signal -from ezmsg.util.messages.axisarray import AxisArray - -from ezmsg.sigproc.butterworthfilter import ( - ButterworthFilterSettings as LegacyButterSettings, -) -from ezmsg.sigproc.butterworthfilter import butter -from ezmsg.sigproc.butterworthfilter import ButterworthFilterSettings as LegacyButterSettings - - -@pytest.mark.parametrize( - "cutoff, cuton", - [ - (30.0, None), # lowpass - (None, 30.0), # highpass - (45.0, 30.0), # bandpass - (30.0, 45.0), # bandstop - ], -) -@pytest.mark.parametrize("order", [2, 4, 8]) -def test_butterworth_legacy_filter_settings(cutoff: float, cuton: float, order: int): - """ - Test the butterworth legacy filter settings generation of btype and Wn. - We test them explicitly because we assume they are correct when used in our later settings. - - Parameters: - cutoff (float): The cutoff frequency for the filter. Can be None for highpass filters. - cuton (float): The cuton frequency for the filter. Can be None for lowpass filters. - If cuton is larger than cutoff we assume bandstop. - order (int): The order of the filter. - """ - settings_obj = LegacyButterSettings( - axis="time", fs=500, order=order, cuton=cuton, cutoff=cutoff - ) - btype, Wn = settings_obj.filter_specs() - if cuton is None: - assert btype == "lowpass" - assert Wn == cutoff - elif cutoff is None: - assert btype == "highpass" - assert Wn == cuton - elif cuton <= cutoff: - assert btype == "bandpass" - assert Wn == (cuton, cutoff) - else: - assert btype == "bandstop" - assert Wn == (cutoff, cuton) - - -@pytest.mark.parametrize( - "cutoff, cuton", - [ - (30.0, None), # lowpass - (None, 30.0), # highpass - (45.0, 30.0), # bandpass - (30.0, 45.0), # bandstop - ], -) -@pytest.mark.parametrize("order", [0, 2, 5, 8]) # 0 = passthrough -# All fs entries must be greater than 2x the largest of cutoff | cuton -@pytest.mark.parametrize("fs", [200.0]) -@pytest.mark.parametrize("n_chans", [3]) -@pytest.mark.parametrize("n_dims, time_ax", [(1, 0), (3, 0), (3, 1), (3, 2)]) -@pytest.mark.parametrize("coef_type", ["ba", "sos"]) -def test_butterworth( - cutoff: float, - cuton: float, - order: int, - fs: float, - n_chans: int, - n_dims: int, - time_ax: int, - coef_type: str, -): - dur = 2.0 - n_freqs = 5 - n_splits = 4 - - n_times = int(dur * fs) - if n_dims == 1: - dat_shape = [n_times] - dat_dims = ["time"] - other_axes = {} - else: - dat_shape = [n_freqs, n_chans] - dat_shape.insert(time_ax, n_times) - dat_dims = ["freq", "ch"] - dat_dims.insert(time_ax, "time") - other_axes = {"freq": AxisArray.Axis(unit="Hz"), "ch": AxisArray.Axis()} - in_dat = np.arange(np.prod(dat_shape), dtype=float).reshape(*dat_shape) - - # Calculate Expected Result - btype, Wn = LegacyButterSettings( - axis="time", fs=500, order=order, cuton=cuton, cutoff=cutoff - ).filter_specs() - coefs = scipy.signal.butter(order, Wn, btype=btype, output=coef_type, fs=fs) - tmp_dat = np.moveaxis(in_dat, time_ax, -1) - if coef_type == "ba": - if order == 0: - # butter does not return correct coefs under these conditions; Set manually. - coefs = (np.array([1.0, 0.0]),) * 2 - zi = scipy.signal.lfilter_zi(*coefs) - if n_dims == 3: - zi = np.tile(zi[None, None, :], (n_freqs, n_chans, 1)) - out_dat, _ = scipy.signal.lfilter(*coefs, tmp_dat, zi=zi) - elif coef_type == "sos": - zi = scipy.signal.sosfilt_zi(coefs) - if n_dims == 3: - zi = np.tile(zi[:, None, None, :], (1, n_freqs, n_chans, 1)) - out_dat, _ = scipy.signal.sosfilt(coefs, tmp_dat, zi=zi) - out_dat = np.moveaxis(out_dat, -1, time_ax) - - # Split the data into multiple messages - n_seen = 0 - messages = [] - for split_dat in np.array_split(in_dat, n_splits, axis=time_ax): - _time_axis = AxisArray.Axis.TimeAxis(fs=fs, offset=n_seen / fs) - messages.append( - AxisArray(split_dat, dims=dat_dims, axes={**other_axes, "time": _time_axis}) - ) - n_seen += split_dat.shape[time_ax] - - # Test axis_name `None` when target axis idx is 0. - axis_name = "time" if time_ax != 0 else None - gen = butter( - axis=axis_name, - order=order, - cuton=cuton, - cutoff=cutoff, - coef_type=coef_type, - ) - - result = np.concatenate([gen.send(_).data for _ in messages], axis=time_ax) - assert np.allclose(result, out_dat) \ No newline at end of file diff --git a/extensions/ezmsg-sigproc/tests/test_butterworth.py b/extensions/ezmsg-sigproc/tests/test_butterworth.py deleted file mode 100644 index 5967c89e..00000000 --- a/extensions/ezmsg-sigproc/tests/test_butterworth.py +++ /dev/null @@ -1,142 +0,0 @@ -import os -import json - -import pytest -import numpy as np - -import ezmsg.core as ez -from ezmsg.util.messages.axisarray import AxisArray -from ezmsg.util.messagegate import MessageGate, MessageGateSettings -from ezmsg.util.messagelogger import MessageLogger, MessageLoggerSettings -from ezmsg.util.messagecodec import message_log -from ezmsg.sigproc.synth import WhiteNoise, WhiteNoiseSettings -from ezmsg.sigproc.butterworthfilter import ButterworthFilter, ButterworthFilterSettings - -from util import get_test_fn -from ezmsg.util.terminate import TerminateOnTimeout as TerminateTest -from ezmsg.util.terminate import TerminateOnTimeoutSettings as TerminateTestSettings - -from typing import Optional, List - - -class ButterworthSystemSettings(ez.Settings): - noise_settings: WhiteNoiseSettings - gate_settings: MessageGateSettings - butter_settings: ButterworthFilterSettings - log_settings: MessageLoggerSettings - term_settings: TerminateTestSettings - - -class ButterworthSystem(ez.Collection): - NOISE = WhiteNoise() - GATE = MessageGate() - BUTTER = ButterworthFilter() - LOG = MessageLogger() - TERM = TerminateTest() - - SETTINGS: ButterworthSystemSettings - - def configure(self) -> None: - self.NOISE.apply_settings(self.SETTINGS.noise_settings) - self.GATE.apply_settings(self.SETTINGS.gate_settings) - self.BUTTER.apply_settings(self.SETTINGS.butter_settings) - self.LOG.apply_settings(self.SETTINGS.log_settings) - self.TERM.apply_settings(self.SETTINGS.term_settings) - - def network(self) -> ez.NetworkDefinition: - return ( - (self.NOISE.OUTPUT_SIGNAL, self.GATE.INPUT), - (self.GATE.OUTPUT, self.BUTTER.INPUT_SIGNAL), - (self.BUTTER.OUTPUT_SIGNAL, self.LOG.INPUT_MESSAGE), - (self.LOG.OUTPUT_MESSAGE, self.TERM.INPUT), - ) - - -@pytest.mark.parametrize( - "cutoff, cuton", - [ - (30.0, None), # lowpass - (None, 30.0), # highpass - (45.0, 30.0), # bandpass - (30.0, 45.0), # bandstop - ], -) -def test_butterworth_system( - cutoff: float, cuton: float, test_name: Optional[str] = None -): - in_fs = 128.0 - block_size = 128 - - # in_fs / block_size = 1 second of data - seconds_of_data = 10.0 - num_msgs = int((in_fs / block_size) * seconds_of_data) - - test_filename = get_test_fn(test_name) - ez.logger.info(test_filename) - - settings = ButterworthSystemSettings( - noise_settings=WhiteNoiseSettings( - n_time=block_size, - fs=in_fs, - dispatch_rate=None, - ), - gate_settings=MessageGateSettings( - start_open=True, default_open=False, default_after=num_msgs - ), - butter_settings=ButterworthFilterSettings(order=5, cutoff=cutoff, cuton=cuton), - log_settings=MessageLoggerSettings(output=test_filename), - term_settings=TerminateTestSettings(time=1.0), - ) - - system = ButterworthSystem(settings) - - ez.run(SYSTEM = system) - - messages: List[AxisArray] = [] - for msg in message_log(test_filename): - messages.append(msg) - - os.remove(test_filename) - - ez.logger.info(f"Analyzing recording of { len( messages ) } messages...") - - data = np.concatenate([msg.data for msg in messages], axis=0) - - # Assert that graph has correct values - freqs = np.fft.fftfreq(data.size, d=(1 / in_fs)) - fft_vals = np.log10(np.abs(np.fft.fft(data, axis=0))) - all_vals = list(zip(freqs, fft_vals)) - - specs = settings.butter_settings.filter_specs() - assert specs is not None - btype, cut = specs - ez.logger.info(f"Testing {btype}...") - - if btype == "lowpass": - zeroed_values = [val[1] for val in all_vals if val[0] > cutoff] - white_values = [val[1] for val in all_vals if not val[0] > cutoff] - if btype == "highpass": - zeroed_values = [val[1] for val in all_vals if val[0] < cuton] - white_values = [val[1] for val in all_vals if not val[0] < cuton] - if btype == "bandpass": - zeroed_values = [ - val[1] for val in all_vals if val[0] < cuton or val[0] > cutoff - ] - white_values = [ - val[1] for val in all_vals if not val[0] < cuton and not val[0] > cutoff - ] - if btype == "bandstop": - zeroed_values = [ - val[1] for val in all_vals if val[0] < cuton and val[0] > cutoff - ] - white_values = [ - val[1] for val in all_vals if not val[0] < cuton or not val[0] > cutoff - ] - - assert np.mean(zeroed_values) < np.mean(white_values) - - ez.logger.info("Test Complete.") - - -if __name__ == "__main__": - test_butterworth_system(20, None, test_name="test_butterworth_system") diff --git a/extensions/ezmsg-sigproc/tests/test_downsample.py b/extensions/ezmsg-sigproc/tests/test_downsample.py deleted file mode 100644 index 222f8e0d..00000000 --- a/extensions/ezmsg-sigproc/tests/test_downsample.py +++ /dev/null @@ -1,129 +0,0 @@ -import os -import json - -import pytest -import numpy as np - -import ezmsg.core as ez -from ezmsg.util.messages.axisarray import AxisArray -from ezmsg.util.messagegate import MessageGate, MessageGateSettings -from ezmsg.util.messagelogger import MessageLogger, MessageLoggerSettings -from ezmsg.util.messagecodec import message_log -from ezmsg.sigproc.downsample import Downsample, DownsampleSettings -from ezmsg.sigproc.synth import Counter, CounterSettings - -from util import get_test_fn -from ezmsg.util.terminate import TerminateOnTimeout as TerminateTest -from ezmsg.util.terminate import TerminateOnTimeoutSettings as TerminateTestSettings -from ezmsg.util.debuglog import DebugLog - -from typing import Optional, List - - -class DownsampleSystemSettings(ez.Settings): - num_msgs: int - counter_settings: CounterSettings - down_settings: DownsampleSettings - log_settings: MessageLoggerSettings - term_settings: TerminateTestSettings - - -class DownsampleSystem(ez.Collection): - COUNT = Counter() - GATE = MessageGate() - DOWN = Downsample() - LOG = MessageLogger() - TERM = TerminateTest() - - DEBUG = DebugLog() - - SETTINGS: DownsampleSystemSettings - - def configure(self) -> None: - self.COUNT.apply_settings(self.SETTINGS.counter_settings) - self.GATE.apply_settings( - MessageGateSettings( - start_open=True, - default_open=False, - default_after=self.SETTINGS.num_msgs, - ) - ) - self.DOWN.apply_settings(self.SETTINGS.down_settings) - self.LOG.apply_settings(self.SETTINGS.log_settings) - self.TERM.apply_settings(self.SETTINGS.term_settings) - - def network(self) -> ez.NetworkDefinition: - return ( - (self.COUNT.OUTPUT_SIGNAL, self.GATE.INPUT), - # ( self.COUNT.OUTPUT_SIGNAL, self.DEBUG.INPUT ), - (self.GATE.OUTPUT, self.DOWN.INPUT_SIGNAL), - # ( self.GATE.OUTPUT, self.DEBUG.INPUT ), - (self.DOWN.OUTPUT_SIGNAL, self.LOG.INPUT_MESSAGE), - # ( self.DOWN.OUTPUT_SIGNAL, self.DEBUG.INPUT ), - (self.LOG.OUTPUT_MESSAGE, self.TERM.INPUT), - # ( self.LOG.OUTPUT_MESSAGE, self.DEBUG.INPUT ), - ) - - -@pytest.mark.parametrize("block_size", [1, 5, 10, 20]) -@pytest.mark.parametrize("factor", [1, 2, 3]) -def test_downsample_system( - block_size: int, factor: int, test_name: Optional[str] = None -): - in_fs = 19.0 - num_msgs = int(4.0 / (block_size / in_fs)) # Ensure 4 seconds of data - - test_filename = get_test_fn(test_name) - ez.logger.info(test_filename) - - settings = DownsampleSystemSettings( - num_msgs=num_msgs, - counter_settings=CounterSettings( - n_time=block_size, fs=in_fs, dispatch_rate=20.0, - ), - down_settings=DownsampleSettings(factor=factor), - log_settings=MessageLoggerSettings(output=test_filename), - term_settings=TerminateTestSettings(time=1.0), - ) - - system = DownsampleSystem(settings) - - ez.run(SYSTEM = system) - - messages: List[AxisArray] = [_ for _ in message_log(test_filename)] - os.remove(test_filename) - ez.logger.info(f"Analyzing recording of { len( messages ) } messages...") - - # Check fs - out_fs = in_fs / factor - assert np.allclose( - np.array([1 / msg.axes["time"].gain for msg in messages]), - np.ones(len(messages,)) * out_fs - ) - - # Check data - time_ax_idx = messages[0].get_axis_idx("time") - data = np.concatenate([_.data for _ in messages], axis=time_ax_idx) - expected_data = np.arange(data.shape[time_ax_idx]) * factor - assert np.array_equal(data, expected_data[:, None]) - - # Grab first sample from each message. We will use their values to get the offsets. - # This works because the input is Counter and we validated it above. - first_samps = [np.take(msg.data, [0], axis=time_ax_idx) for msg in messages] - - # Check that the shape of each message is the same -- the set of shapes will be reduced to a single item. - assert len(set([_.shape for _ in first_samps])) == 1 - - # Check offsets - first_samps = np.concatenate(first_samps, axis=time_ax_idx) - expected_offsets = first_samps.squeeze() / out_fs / factor - assert np.allclose( - np.array([msg.axes["time"].offset for msg in messages]), - expected_offsets - ) - - ez.logger.info("Test Complete.") - - -if __name__ == "__main__": - test_downsample_system(10, 2, test_name="test_window_system") diff --git a/extensions/ezmsg-sigproc/tests/test_sampler.py b/extensions/ezmsg-sigproc/tests/test_sampler.py deleted file mode 100644 index fb3601be..00000000 --- a/extensions/ezmsg-sigproc/tests/test_sampler.py +++ /dev/null @@ -1,162 +0,0 @@ -import os -from typing import Optional, List - -import numpy as np - -import ezmsg.core as ez -from ezmsg.util.messagecodec import message_log -from ezmsg.util.messages.axisarray import AxisArray -from ezmsg.util.messagelogger import MessageLogger, MessageLoggerSettings -from ezmsg.sigproc.sampler import ( - Sampler, SamplerSettings, - TriggerGenerator, TriggerGeneratorSettings, - SampleTriggerMessage, SampleMessage, - sampler -) -from ezmsg.sigproc.synth import Oscillator, OscillatorSettings -from ezmsg.util.terminate import TerminateOnTotal, TerminateOnTotalSettings -from ezmsg.util.debuglog import DebugLog - -from util import get_test_fn - - -def test_sampler_gen(): - data_dur = 10.0 - chunk_period = 0.1 - fs = 500. - n_chans = 3 - - # The sampler is a bit complicated as it requires 2 different inputs: signal and triggers - # Prepare signal data - n_data = int(data_dur * fs) - data = np.arange(n_chans *n_data).reshape(n_chans, n_data) - offsets = np.arange(n_data) / fs - n_chunks = int(np.ceil(data_dur / chunk_period)) - n_per_chunk = int(np.ceil(n_data / n_chunks)) - signal_msgs = [ - AxisArray( - data=data[:, ix * n_per_chunk:(ix + 1) * n_per_chunk], - dims=["ch", "time"], - axes={"time": AxisArray.Axis.TimeAxis(fs=fs, offset=offsets[ix * n_per_chunk])} - ) - for ix in range(n_chunks) - ] - # Prepare triggers - n_trigs = 7 - trig_ts = np.linspace(0.1, data_dur - 1.0, n_trigs) + np.random.randn(n_trigs) / fs - period = (-0.01, 0.74) - trigger_msgs = [ - SampleTriggerMessage( - timestamp=_ts, - period=period, - value=["Start", "Stop"][_ix % 2] - ) - for _ix, _ts in enumerate(trig_ts) - ] - # Mix the messages and sort by time - msg_ts = [_.axes["time"].offset for _ in signal_msgs] + [_.timestamp for _ in trigger_msgs] - mix_msgs = signal_msgs + trigger_msgs - mix_msgs = [mix_msgs[_] for _ in np.argsort(msg_ts)] - - # Create the sample-generator - period_dur = period[1] - period[0] - buffer_dur = 2 * max(period_dur, period[1]) - gen = sampler(buffer_dur, axis="time", period=None, value=None, estimate_alignment=True) - - # Run the messages through the generator and collect samples. - samples = [] - for msg in mix_msgs: - samples.extend(gen.send(msg)) - - assert len(samples) == n_trigs - # Check sample data size - assert all([_.sample.data.shape == (n_chans, int(fs * period_dur)) for _ in samples]) - # Compare the sample window slice against the trigger timestamps - latencies = [_.sample.axes["time"].offset - (_.trigger.timestamp + _.trigger.period[0]) for _ in samples] - assert all([0 <= _ < 1 / fs for _ in latencies]) - # Check the sample trigger value matches the trigger input. - assert all([_.trigger.value == ["Start", "Stop"][ix % 2] for ix, _ in enumerate(samples)]) - - -class SamplerSystemSettings(ez.Settings): - # num_msgs: int - osc_settings: OscillatorSettings - trigger_settings: TriggerGeneratorSettings - sampler_settings: SamplerSettings - log_settings: MessageLoggerSettings - term_settings: TerminateOnTotalSettings - - -class SamplerSystem(ez.Collection): - SETTINGS: SamplerSystemSettings - - OSC = Oscillator() - TRIGGER = TriggerGenerator() - SAMPLER = Sampler() - LOG = MessageLogger() - DEBUG = DebugLog() - TERM = TerminateOnTotal() - - def configure(self) -> None: - self.OSC.apply_settings(self.SETTINGS.osc_settings) - self.LOG.apply_settings(self.SETTINGS.log_settings) - self.SAMPLER.apply_settings(self.SETTINGS.sampler_settings) - self.TRIGGER.apply_settings(self.SETTINGS.trigger_settings) - self.TERM.apply_settings(self.SETTINGS.term_settings) - - def network(self) -> ez.NetworkDefinition: - return ( - (self.OSC.OUTPUT_SIGNAL, self.SAMPLER.INPUT_SIGNAL), - (self.SAMPLER.OUTPUT_SAMPLE, self.LOG.INPUT_MESSAGE), - (self.LOG.OUTPUT_MESSAGE, self.TERM.INPUT_MESSAGE), - # Trigger branch - (self.TRIGGER.OUTPUT_TRIGGER, self.SAMPLER.INPUT_TRIGGER), - # Debug branches - (self.TRIGGER.OUTPUT_TRIGGER, self.DEBUG.INPUT), - (self.SAMPLER.OUTPUT_SAMPLE, self.DEBUG.INPUT), - ) - - -def test_sampler_system(test_name: Optional[str] = None): - freq = 40.0 - period = (0.5, 1.5) - n_msgs = 4 - - sample_dur = (period[1] - period[0]) - publish_period = sample_dur * 2.0 - - test_filename = get_test_fn(test_name) - ez.logger.info(test_filename) - - settings = SamplerSystemSettings( - osc_settings=OscillatorSettings( - n_time=2, # Number of samples to output per block - fs=freq, # Sampling rate of signal output in Hz - dispatch_rate="realtime", - freq=2.0, # Oscillation frequency in Hz - amp=1.0, # Amplitude - phase=0.0, # Phase offset (in radians) - sync=True, # Adjust `freq` to sync with sampling rate - ), - trigger_settings=TriggerGeneratorSettings( - period=period, - prewait=0.5, - publish_period=publish_period - ), - sampler_settings=SamplerSettings(buffer_dur=publish_period + 1.0), - log_settings=MessageLoggerSettings(output=test_filename), - term_settings=TerminateOnTotalSettings(total=n_msgs), - ) - - system = SamplerSystem(settings) - - ez.run(SYSTEM=system) - messages: List[SampleTriggerMessage] = [_ for _ in message_log(test_filename)] - os.remove(test_filename) - ez.logger.info(f"Analyzing recording of {len(messages)} messages...") - assert len(messages) == n_msgs - assert all([_.sample.data.shape == (int(freq * sample_dur), 1) for _ in messages]) - # Test the sample window slice vs the trigger timestamps - latencies = [_.sample.axes["time"].offset - (_.trigger.timestamp + _.trigger.period[0]) for _ in messages] - assert all([0 <= _ < 1/freq for _ in latencies]) - # Given that the input is a pure sinusoid, we could test that the signal has expected characteristics. diff --git a/extensions/ezmsg-sigproc/tests/test_scaler.py b/extensions/ezmsg-sigproc/tests/test_scaler.py deleted file mode 100644 index 33cb3df4..00000000 --- a/extensions/ezmsg-sigproc/tests/test_scaler.py +++ /dev/null @@ -1,119 +0,0 @@ -import os -from typing import Optional, List -from dataclasses import field -import numpy as np - -from ezmsg.util.messages.axisarray import AxisArray - -from ezmsg.sigproc.scaler import scaler, scaler_np - -# For test system -from util import get_test_fn -from ezmsg.sigproc.scaler import AdaptiveStandardScalerSettings, AdaptiveStandardScaler -import ezmsg.core as ez -from ezmsg.sigproc.synth import Counter, CounterSettings -from ezmsg.util.terminate import TerminateOnTotalSettings, TerminateOnTotal -from ezmsg.util.messagelogger import MessageLogger, MessageLoggerSettings -from ezmsg.util.messagecodec import message_log - - -def test_adaptive_standard_scaler_river(): - try: - # Test data values taken from river: - # https://github.com/online-ml/river/blob/main/river/preprocessing/scale.py#L511-L536C17 - data = np.array([5.278, 5.050, 6.550, 7.446, 9.472, 10.353, 11.784, 11.173]) - expected_result = np.array([0.0, -0.816, 0.812, 0.695, 0.754, 0.598, 0.651, 0.124]) - - test_input = AxisArray(np.tile(data, (2, 1)), dims=["ch", "time"], axes={"time": AxisArray.Axis()}) - - # The River example used alpha = 0.6 - # tau = -gain / np.log(1 - alpha) and here we're using gain = 1.0 - tau = 1.0914 - _scaler = scaler(time_constant=tau, axis="time") - output = _scaler.send(test_input) - assert np.allclose(output.data[0], expected_result, atol=1e-3) - - _scaler_np = scaler_np(time_constant=tau, axis="time") - output = _scaler_np.send(test_input) - assert np.allclose(output.data[0], expected_result, atol=1e-3) - except ModuleNotFoundError: - pass # Do not fail if river not installed. - - -class ScalerTestSystemSettings(ez.Settings): - counter_settings: CounterSettings - scaler_settings: AdaptiveStandardScalerSettings - log_settings: MessageLoggerSettings - term_settings: TerminateOnTotalSettings = field(default_factory=TerminateOnTotalSettings) - - -class ScalerTestSystem(ez.Collection): - SETTINGS: ScalerTestSystemSettings - - COUNTER = Counter() - SCALER = AdaptiveStandardScaler() - LOG = MessageLogger() - TERM = TerminateOnTotal() - - def configure(self) -> None: - self.COUNTER.apply_settings(self.SETTINGS.counter_settings) - self.SCALER.apply_settings(self.SETTINGS.scaler_settings) - self.LOG.apply_settings(self.SETTINGS.log_settings) - self.TERM.apply_settings(self.SETTINGS.term_settings) - - def network(self) -> ez.NetworkDefinition: - return ( - (self.COUNTER.OUTPUT_SIGNAL, self.SCALER.INPUT_SIGNAL), - (self.SCALER.OUTPUT_SIGNAL, self.LOG.INPUT_MESSAGE), - (self.LOG.OUTPUT_MESSAGE, self.TERM.INPUT_MESSAGE) - ) - - -def test_scaler_system( - tau: float = 1.0, - fs: float = 10.0, - duration: float = 2.0, - test_name: Optional[str] = None, -): - """ - For this test, we assume that Counter and scaler_np are functioning properly. - The purpose of this test is exclusively to test that the AdaptiveStandardScaler and AdaptiveStandardScalerSettings - generated classes are wrapping scaler_np and exposing its parameters. - This test passing should only be considered a success if test_adaptive_standard_scaler_river also passed. - """ - block_size: int = 4 - test_filename = get_test_fn(test_name) - ez.logger.info(test_filename) - settings = ScalerTestSystemSettings( - counter_settings=CounterSettings( - n_time=block_size, - fs=fs, - n_ch=1, - dispatch_rate=duration, # Simulation duration in 1.0 seconds - mod=None, - ), - scaler_settings=AdaptiveStandardScalerSettings( - time_constant=tau, - axis="time" - ), - log_settings=MessageLoggerSettings( - output=test_filename, - ), - term_settings=TerminateOnTotalSettings( - total=int(duration * fs / block_size), - ) - ) - system = ScalerTestSystem(settings) - ez.run(SYSTEM=system) - - # Collect result - messages: List[AxisArray] = [_ for _ in message_log(test_filename)] - os.remove(test_filename) - - data = np.concatenate([_.data for _ in messages]).squeeze() - - expected_input = AxisArray(np.arange(len(data))[None, :], - dims=["ch", "time"], axes={"time": AxisArray.Axis(gain=1/fs, offset=0.0)}) - _scaler = scaler_np(time_constant=tau, axis="time") - expected_output = _scaler.send(expected_input) - assert np.allclose(expected_output.data.squeeze(), data) diff --git a/extensions/ezmsg-sigproc/tests/test_slicer.py b/extensions/ezmsg-sigproc/tests/test_slicer.py deleted file mode 100644 index 6af370af..00000000 --- a/extensions/ezmsg-sigproc/tests/test_slicer.py +++ /dev/null @@ -1,67 +0,0 @@ -import numpy as np -from ezmsg.util.messages.axisarray import AxisArray - -from ezmsg.sigproc.slicer import slicer, parse_slice - - -def test_parse_slice(): - assert parse_slice("") == (slice(None),) - assert parse_slice(":") == (slice(None),) - assert parse_slice("NONE") == (slice(None),) - assert parse_slice("none") == (slice(None),) - assert parse_slice("0") == (0,) - assert parse_slice("10") == (10,) - assert parse_slice(":-1") == (slice(None, -1),) - assert parse_slice("0:3") == (slice(0, 3),) - assert parse_slice("::2") == (slice(None, None, 2),) - assert parse_slice("0,1") == (0, 1) - assert parse_slice("4:64, 68:100") == (slice(4, 64), slice(68, 100)) - - -def test_slicer_generator(): - n_times = 13 - n_chans = 255 - in_dat = np.arange(n_times * n_chans).reshape(n_times, n_chans) - axis_arr_in = AxisArray(in_dat, dims=["time", "ch"]) - - gen = slicer(selection=":2", axis="ch") - ax_arr_out = gen.send(axis_arr_in) - assert ax_arr_out.data.shape == (n_times, 2) - assert np.array_equal(ax_arr_out.data, in_dat[:, :2]) - assert np.may_share_memory(ax_arr_out.data, in_dat) - - gen = slicer(selection="::3", axis="ch") - ax_arr_out = gen.send(axis_arr_in) - assert ax_arr_out.data.shape == (n_times, n_chans // 3) - assert np.array_equal(ax_arr_out.data, in_dat[:, ::3]) - assert np.may_share_memory(ax_arr_out.data, in_dat) - - gen = slicer(selection="4:64", axis="ch") - ax_arr_out = gen.send(axis_arr_in) - assert ax_arr_out.data.shape == (n_times, 60) - assert np.array_equal(ax_arr_out.data, in_dat[:, 4:64]) - assert np.may_share_memory(ax_arr_out.data, in_dat) - - # Discontiguous slices leads to a copy - gen = slicer(selection="1, 3:5", axis="ch") - ax_arr_out = gen.send(axis_arr_in) - assert np.array_equal(ax_arr_out.data, axis_arr_in.data[:, [1, 3, 4]]) - assert not np.may_share_memory(ax_arr_out.data, in_dat) - - -def test_slicer_gen_drop_dim(): - n_times = 50 - n_chans = 10 - in_dat = np.arange(n_times * n_chans).reshape(n_times, n_chans) - axis_arr_in = AxisArray( - in_dat, - dims=["time", "ch"], - axes={ - "time": AxisArray.Axis.TimeAxis(fs=100.0, offset=0.1), - } - ) - - gen = slicer(selection="5", axis="ch") - ax_arr_out = gen.send(axis_arr_in) - assert ax_arr_out.data.shape == (n_times,) - assert np.array_equal(ax_arr_out.data, axis_arr_in.data[:, 5]) diff --git a/extensions/ezmsg-sigproc/tests/test_spectrogram.py b/extensions/ezmsg-sigproc/tests/test_spectrogram.py deleted file mode 100644 index c0df6918..00000000 --- a/extensions/ezmsg-sigproc/tests/test_spectrogram.py +++ /dev/null @@ -1,80 +0,0 @@ -import typing - -import numpy as np - -from ezmsg.util.messages.axisarray import AxisArray -from ezmsg.sigproc.spectrum import WindowFunction, SpectralTransform, SpectralOutput -from ezmsg.sigproc.spectrogram import spectrogram - -from util import create_messages_with_periodic_signal - - -def _debug_plot( - ax_arr: AxisArray, - sin_params: typing.List[typing.Dict[str, float]] = None -): - import matplotlib.pyplot as plt - - t_ix = ax_arr.get_axis_idx("time") - t_vec = ax_arr.axes["time"].offset + np.arange(ax_arr.data.shape[t_ix] * ax_arr.axes["time"].gain) - t_vec -= ax_arr.axes["time"].gain / 2 - f_ix = ax_arr.get_axis_idx("freq") - f_vec = ax_arr.axes["freq"].offset + np.arange(ax_arr.data.shape[f_ix] * ax_arr.axes["freq"].gain) - f_vec -= ax_arr.axes["freq"].gain / 2 - plt.imshow( - ax_arr.data[..., 0].T, - origin="lower", - aspect="auto", - extent=(t_vec[0], t_vec[-1], f_vec[0], f_vec[-1]) - ) - plt.xlabel("Time") - plt.ylabel("Frequency") - - if sin_params is not None: - for s_p in sin_params: - xx = (s_p.get("offset", 0.0) + t_vec[0], s_p.get("offset", 0.0) + t_vec[0] + s_p["dur"]) - yy = (s_p["f"], s_p["f"]) - plt.plot(xx, yy, linestyle="--", color="r", linewidth=1.0) - - -def test_spectrogram(): - win_dur = 1.0 - win_step_dur = 0.5 - fs = 1000. - seg_dur = 5.0 - sin_params = [ - {"f": 10.0, "dur": seg_dur, "offset": 0.0}, - {"f": 20.0, "dur": seg_dur, "offset": 0.0}, - {"f": 70.0, "dur": seg_dur, "offset": 0.0}, - {"f": 14.0, "dur": seg_dur, "offset": seg_dur}, - {"f": 35.0, "dur": seg_dur, "offset": seg_dur}, - {"f": 300.0, "dur": seg_dur, "offset": seg_dur}, - ] - messages = create_messages_with_periodic_signal( - sin_params=sin_params, - fs=fs, - msg_dur=0.4, - win_step_dur=None # The spectrogram will do the windowing - ) - - gen = spectrogram( - window_dur=win_dur, - window_shift=win_step_dur, - window=WindowFunction.HANNING, - transform=SpectralTransform.REL_DB, - output=SpectralOutput.POSITIVE, - ) - - results = [gen.send(msg) for msg in messages] - results = [_ for _ in results if _ is not None] # Drop None - - # Check that the windows span the expected times. - expected_t_span = 2 * seg_dur - data_t_span = (results[-1].axes["time"].offset + win_step_dur) - results[0].axes["time"].offset - assert np.abs(expected_t_span - data_t_span) < 1e-9 - all_deltas = np.diff([_.axes["time"].offset for _ in results]) - assert np.allclose(all_deltas, win_step_dur + np.zeros((len(results) - 1))) - - # result = AxisArray.concatenate(*results, dim="time") - # TODO: Check spectral peaks change over time. _debug_plot is useful if not automatic. - # _debug_plot(result, sin_params=sin_params) diff --git a/extensions/ezmsg-sigproc/tests/test_spectrum.py b/extensions/ezmsg-sigproc/tests/test_spectrum.py deleted file mode 100644 index 467252dd..00000000 --- a/extensions/ezmsg-sigproc/tests/test_spectrum.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest -import numpy as np - -from ezmsg.util.messages.axisarray import AxisArray, slice_along_axis -from ezmsg.sigproc.spectrum import spectrum, SpectralTransform, SpectralOutput, WindowFunction -from util import get_test_fn, create_messages_with_periodic_signal - - -def _debug_plot_welch(raw: AxisArray, result: AxisArray, welch_db: bool = True): - import scipy.signal - import matplotlib.pyplot as plt - - fig, ax = plt.subplots(2, 1) - - t_ax = raw.axes["time"] - t_vec = np.arange(raw.data.shape[raw.get_axis_idx("time")]) * t_ax.gain + t_ax.offset - ch0_raw = raw.data[..., :, 0] - if ch0_raw.ndim > 1: - # For multi-win inputs - ch0_raw = ch0_raw[0] - ax[0].plot(t_vec, ch0_raw) - ax[0].set_xlabel("Time (s)") - - f_ax = result.axes["freq"] - f_vec = np.arange(result.data.shape[result.get_axis_idx("freq")]) * f_ax.gain + f_ax.offset - ch0_spec = result.data[..., :, 0] - if ch0_spec.ndim > 1: - ch0_spec = ch0_spec[0] - ax[1].plot(f_vec, ch0_spec, label="calculated", linewidth=2.0) - ax[1].set_xlabel("Frequency (Hz)") - - f, Pxx = scipy.signal.welch(ch0_raw, fs=1 / raw.axes["time"].gain, window="hamming", nperseg=len(ch0_raw)) - if welch_db: - Pxx = 10 * np.log10(Pxx) - ax[1].plot(f, Pxx, label="welch", color="tab:orange", linestyle="--") - ax[1].set_ylabel("dB" if welch_db else "V**2/Hz") - ax[1].legend() - - plt.tight_layout() - plt.show() - - -@pytest.mark.parametrize("window", [WindowFunction.HANNING, WindowFunction.HAMMING]) -@pytest.mark.parametrize("transform", [SpectralTransform.REL_DB, SpectralTransform.REL_POWER]) -@pytest.mark.parametrize("output", [SpectralOutput.POSITIVE, SpectralOutput.NEGATIVE, SpectralOutput.FULL]) -def test_spectrum_gen_multiwin( - window: WindowFunction, - transform: SpectralTransform, - output: SpectralOutput -): - win_dur = 1.0 - win_step_dur = 0.5 - fs = 1000.0 - sin_params = [ - {"a": 1.0, "f": 10.0, "p": 0.0, "dur": 20.0}, - {"a": 0.5, "f": 20.0, "p": np.pi / 7, "dur": 20.0}, - {"a": 0.2, "f": 200.0, "p": np.pi / 11, "dur": 20.0}, - ] - win_len = int(win_dur * fs) - - messages = create_messages_with_periodic_signal( - sin_params=sin_params, - fs=fs, - msg_dur=win_dur, - win_step_dur=win_step_dur - ) - input_multiwin = AxisArray.concatenate(*messages, dim="win") - input_multiwin.axes["win"] = AxisArray.Axis.TimeAxis(offset=0, fs=1/win_step_dur) - - gen = spectrum(axis="time", window=window, transform=transform, output=output) - result = gen.send(input_multiwin) - # _debug_plot_welch(input_multiwin, result, welch_db=True) - assert isinstance(result, AxisArray) - assert "time" not in result.dims - assert "time" not in result.axes - assert "ch" in result.dims - assert "win" in result.dims - assert "freq" in result.axes - assert result.axes["freq"].gain == 1 / win_dur - assert "freq" in result.dims - fax_ix = result.get_axis_idx("freq") - f_len = win_len if output == SpectralOutput.FULL else win_len // 2 - assert result.data.shape[fax_ix] == f_len - f_vec = result.axes["freq"].gain * np.arange(f_len) + result.axes["freq"].offset - if output == SpectralOutput.NEGATIVE: - f_vec = np.abs(f_vec) - for s_p in sin_params: - f_ix = np.argmin(np.abs(f_vec - s_p["f"])) - peak_inds = np.argmax(slice_along_axis(result.data, slice(f_ix-3, f_ix+3), axis=fax_ix), axis=fax_ix) - assert np.all(peak_inds == 3) - - -@pytest.mark.parametrize("window", [WindowFunction.HANNING, WindowFunction.HAMMING]) -@pytest.mark.parametrize("transform", [SpectralTransform.REL_DB, SpectralTransform.REL_POWER]) -@pytest.mark.parametrize("output", [SpectralOutput.POSITIVE, SpectralOutput.NEGATIVE, SpectralOutput.FULL]) -def test_spectrum_gen( - window: WindowFunction, - transform: SpectralTransform, - output: SpectralOutput -): - win_dur = 1.0 - win_step_dur = 0.5 - fs = 1000.0 - sin_params = [ - {"a": 1.0, "f": 10.0, "p": 0.0, "dur": 20.0}, - {"a": 0.5, "f": 20.0, "p": np.pi / 7, "dur": 20.0}, - {"a": 0.2, "f": 200.0, "p": np.pi / 11, "dur": 20.0}, - ] - messages = create_messages_with_periodic_signal( - sin_params=sin_params, - fs=fs, - msg_dur=win_dur, - win_step_dur=win_step_dur - ) - gen = spectrum(axis="time", window=window, transform=transform, output=output) - results = [gen.send(msg) for msg in messages] - - assert "freq" in results[0].dims - assert "ch" in results[0].dims - assert "win" not in results[0].dims - # _debug_plot_welch(messages[0], results[0], welch_db=True) diff --git a/extensions/ezmsg-sigproc/tests/test_synth.py b/extensions/ezmsg-sigproc/tests/test_synth.py deleted file mode 100644 index 4a238999..00000000 --- a/extensions/ezmsg-sigproc/tests/test_synth.py +++ /dev/null @@ -1,302 +0,0 @@ -import asyncio -from dataclasses import field -import os -import time -import typing - -import numpy as np -import pytest - -import ezmsg.core as ez -from ezmsg.util.messages.axisarray import AxisArray -from ezmsg.util.messagelogger import MessageLogger, MessageLoggerSettings -from ezmsg.util.messagecodec import message_log -from ezmsg.util.terminate import TerminateOnTotalSettings, TerminateOnTotal -from util import get_test_fn -from ezmsg.sigproc.synth import ( - clock, aclock, Clock, ClockSettings, - acounter, Counter, CounterSettings, - sin, SinGenerator, SinGeneratorSettings -) - - -# TEST CLOCK -@pytest.mark.parametrize("dispatch_rate", [None, 2.0, 20.0]) -def test_clock_gen(dispatch_rate: typing.Optional[float]): - run_time = 1.0 - n_target = int(np.ceil(dispatch_rate * run_time)) if dispatch_rate else 100 - gen = clock(dispatch_rate=dispatch_rate) - result = [] - t_start = time.time() - while len(result) < n_target: - result.append(next(gen)) - t_elapsed = time.time() - t_start - assert all([_ == ez.Flag() for _ in result]) - if dispatch_rate is not None: - assert (run_time - 1 / dispatch_rate) < t_elapsed < (run_time + 0.1) - else: - assert t_elapsed < (n_target * 1e-4) # 100 usec per iteration is pretty generous - - -@pytest.mark.parametrize("dispatch_rate", [None, 2.0, 20.0]) -@pytest.mark.asyncio -async def test_aclock_agen(dispatch_rate: typing.Optional[float]): - run_time = 1.0 - n_target = int(np.ceil(dispatch_rate * run_time)) if dispatch_rate else 100 - agen = aclock(dispatch_rate=dispatch_rate) - result = [] - t_start = time.time() - while len(result) < n_target: - new_result = await agen.__anext__() - result.append(new_result) - t_elapsed = time.time() - t_start - assert all([_ == ez.Flag() for _ in result]) - if dispatch_rate: - assert (run_time - 1 / dispatch_rate) < t_elapsed < (run_time + 0.1) - else: - assert t_elapsed < (n_target * 1e-4) # 100 usec per iteration is pretty generous - - -class ClockTestSystemSettings(ez.Settings): - clock_settings: ClockSettings - log_settings: MessageLoggerSettings - term_settings: TerminateOnTotalSettings = field(default_factory=TerminateOnTotalSettings) - - -class ClockTestSystem(ez.Collection): - SETTINGS: ClockTestSystemSettings - - CLOCK = Clock() - LOG = MessageLogger() - TERM = TerminateOnTotal() - - def configure(self) -> None: - self.CLOCK.apply_settings(self.SETTINGS.clock_settings) - self.LOG.apply_settings(self.SETTINGS.log_settings) - self.TERM.apply_settings(self.SETTINGS.term_settings) - - def network(self) -> ez.NetworkDefinition: - return ( - (self.CLOCK.OUTPUT_CLOCK, self.LOG.INPUT_MESSAGE), - (self.LOG.OUTPUT_MESSAGE, self.TERM.INPUT_MESSAGE) - ) - - -@pytest.mark.parametrize("dispatch_rate", [None, 2.0, 20.0]) -def test_clock_system( - dispatch_rate: typing.Optional[float], - test_name: typing.Optional[str] = None, -): - run_time = 1.0 - n_target = int(np.ceil(dispatch_rate * run_time)) if dispatch_rate else 100 - test_filename = get_test_fn(test_name) - ez.logger.info(test_filename) - settings = ClockTestSystemSettings( - clock_settings=ClockSettings(dispatch_rate=dispatch_rate), - log_settings=MessageLoggerSettings(output=test_filename), - term_settings=TerminateOnTotalSettings(total=n_target) - ) - system = ClockTestSystem(settings) - ez.run(SYSTEM=system) - - # Collect result - messages: typing.List[AxisArray] = [_ for _ in message_log(test_filename)] - os.remove(test_filename) - - assert all([_ == ez.Flag() for _ in messages]) - assert len(messages) >= n_target - - - -@pytest.mark.parametrize("block_size", [1, 20]) -@pytest.mark.parametrize("fs", [10.0, 1000.0]) -@pytest.mark.parametrize("n_ch", [3]) -@pytest.mark.parametrize("dispatch_rate", [None, "realtime", "ext_clock", 2.0, 20.0]) # "ext_clock" needs a separate test -@pytest.mark.parametrize("mod", [2 ** 3, None]) -@pytest.mark.asyncio -async def test_acounter( - block_size: int, - fs: float, - n_ch: int, - dispatch_rate: typing.Optional[typing.Union[float, str]], - mod: typing.Optional[int] -): - target_dur = 2.6 # 2.6 seconds per test - if dispatch_rate is None: - # No sleep / wait - chunk_dur = 0.1 - elif isinstance(dispatch_rate, str): - if dispatch_rate == "realtime": - chunk_dur = block_size / fs - elif dispatch_rate == "ext_clock": - # No sleep / wait - chunk_dur = 0.1 - else: - # Note: float dispatch_rate will yield different number of samples than expected by target_dur and fs - chunk_dur = 1. / dispatch_rate - target_messages = int(target_dur / chunk_dur) - - # Run generator - agen = acounter(block_size, fs, n_ch=n_ch, dispatch_rate=dispatch_rate, mod=mod) - messages = [await agen.__anext__() for _ in range(target_messages)] - - # Test contents of individual messages - for ax_arr in messages: - assert type(ax_arr) is AxisArray - assert ax_arr.data.shape == (block_size, n_ch) - assert "time" in ax_arr.axes - assert ax_arr.axes["time"].gain == 1 / fs - - agg = AxisArray.concatenate(*messages, dim="time") - - target_samples = block_size * target_messages - expected_data = np.arange(target_samples) - if mod is not None: - expected_data = expected_data % mod - assert np.array_equal(agg.data[:, 0], expected_data) - - offsets = np.array([m.axes["time"].offset for m in messages]) - expected_offsets = np.arange(target_messages) * block_size / fs - if dispatch_rate == "realtime" or dispatch_rate == "ext_clock": - expected_offsets += offsets[0] # offsets are in real-time - atol = 0.002 - else: - # Offsets are synthetic. - atol = 1.e-8 - assert np.allclose(offsets[2:], expected_offsets[2:], atol=atol) - - -class CounterTestSystemSettings(ez.Settings): - counter_settings: CounterSettings - log_settings: MessageLoggerSettings - term_settings: TerminateOnTotalSettings = field(default_factory=TerminateOnTotalSettings) - - -class CounterTestSystem(ez.Collection): - SETTINGS: CounterTestSystemSettings - - COUNTER = Counter() - LOG = MessageLogger() - TERM = TerminateOnTotal() - - def configure(self) -> None: - self.COUNTER.apply_settings(self.SETTINGS.counter_settings) - self.LOG.apply_settings(self.SETTINGS.log_settings) - self.TERM.apply_settings(self.SETTINGS.term_settings) - - def network(self) -> ez.NetworkDefinition: - return ( - (self.COUNTER.OUTPUT_SIGNAL, self.LOG.INPUT_MESSAGE), - (self.LOG.OUTPUT_MESSAGE, self.TERM.INPUT_MESSAGE) - ) - - -# Integration Test. -# General functionality of acounter verified above. Here we only need to test a couple configs. -@pytest.mark.parametrize( - "block_size, fs, dispatch_rate, mod", - [ - (1, 10.0, None, None), - (20, 1000.0, "realtime", None), - (1, 1000.0, 2.0, 2**3), - (10, 10.0, 20.0, 2**3), - # No test for ext_clock because that requires a different system - # (20, 10.0, "ext_clock", None), - ] -) -def test_counter_system( - block_size: int, - fs: float, - dispatch_rate: typing.Optional[typing.Union[float, str]], - mod: typing.Optional[int], - test_name: typing.Optional[str] = None, -): - n_ch = 3 - target_dur = 2.6 # 2.6 seconds per test - if dispatch_rate is None: - # No sleep / wait - chunk_dur = 0.1 - elif isinstance(dispatch_rate, str): - if dispatch_rate == "realtime": - chunk_dur = block_size / fs - else: - # Note: float dispatch_rate will yield different number of samples than expected by target_dur and fs - chunk_dur = 1. / dispatch_rate - target_messages = int(target_dur / chunk_dur) - - test_filename = get_test_fn(test_name) - ez.logger.info(test_filename) - settings = CounterTestSystemSettings( - counter_settings=CounterSettings( - n_time=block_size, - fs=fs, - n_ch=n_ch, - dispatch_rate=dispatch_rate, - mod=mod, - ), - log_settings=MessageLoggerSettings( - output=test_filename, - ), - term_settings=TerminateOnTotalSettings( - total=target_messages, - ) - ) - system = CounterTestSystem(settings) - ez.run(SYSTEM=system) - - # Collect result - messages: typing.List[AxisArray] = [_ for _ in message_log(test_filename)] - os.remove(test_filename) - - if dispatch_rate is None: - # The number of messages depends on how fast the computer is - target_messages = len(messages) - # This should be an equivalence assertion (==) but the use of TerminateOnTotal does - # not guarantee that MessageLogger will exit before an additional message is received. - # Let's just clip the last message if we exceed the target messages. - if len(messages) > target_messages: - messages = messages[:target_messages] - assert len(messages) == target_messages - - # Just do one quick data check - agg = AxisArray.concatenate(*messages, dim="time") - target_samples = block_size * target_messages - expected_data = np.arange(target_samples) - if mod is not None: - expected_data = expected_data % mod - assert np.array_equal(agg.data[:, 0], expected_data) - - -# TEST SIN # -def test_sin_gen( - freq: float = 1.0, - amp: float = 1.0, - phase: float = 0.0 -): - axis: typing.Optional[str] = "time" - srate = max(4.0 * freq, 1000.0) - sim_dur = 30.0 - n_samples = int(srate * sim_dur) - n_msgs = min(n_samples, 10) - axis_idx = 0 - - messages = [] - for split_dat in np.array_split(np.arange(n_samples)[:, None], n_msgs, axis=axis_idx): - _time_axis = AxisArray.Axis.TimeAxis(fs=srate, offset=float(split_dat[0, 0])) - messages.append( - AxisArray(split_dat, dims=["time", "ch"], axes={"time": _time_axis}) - ) - - def f_test(t): return amp * np.sin(2 * np.pi * freq * t + phase) - - gen = sin(axis=axis, freq=freq, amp=amp, phase=phase) - results = [] - for msg in messages: - res = gen.send(msg) - assert np.allclose(res.data, f_test(msg.data / srate)) - results.append(res) - concat_ax_arr = AxisArray.concatenate(*results, dim="time") - assert np.allclose(concat_ax_arr.data, f_test(np.arange(n_samples) / srate)[:, None]) - - -# TODO: test SinGenerator in a system. diff --git a/extensions/ezmsg-sigproc/tests/test_window.py b/extensions/ezmsg-sigproc/tests/test_window.py deleted file mode 100644 index a92ed606..00000000 --- a/extensions/ezmsg-sigproc/tests/test_window.py +++ /dev/null @@ -1,316 +0,0 @@ -from dataclasses import field, replace - -import os -import json - -import pytest -import numpy as np -from numpy.lib.stride_tricks import sliding_window_view -import ezmsg.core as ez - -from ezmsg.util.messages.axisarray import AxisArray -from ezmsg.util.messagegate import MessageGate, MessageGateSettings -from ezmsg.util.messagelogger import MessageLogger, MessageLoggerSettings -from ezmsg.util.messagecodec import message_log -from ezmsg.sigproc.synth import Counter, CounterSettings -from ezmsg.sigproc.window import Window, WindowSettings, windowing - -from util import get_test_fn -from ezmsg.util.terminate import TerminateOnTimeout as TerminateTest -from ezmsg.util.terminate import TerminateOnTimeoutSettings as TerminateTestSettings -from ezmsg.util.debuglog import DebugLog - -from typing import Optional, Dict, Any, List, Tuple - - -def calculate_expected_results(orig, fs, win_shift, zero_pad, msg_block_size, shift_len, win_len, nchans, data_len, n_msgs, win_ax): - # For the calculation, we assume time_ax is last then transpose if necessary at the end. - expected = orig.copy() - tvec = np.arange(orig.shape[1]) / fs - # Prepend the data with zero-padding, if necessary. - if win_shift is None or zero_pad == "input": - n_cut = msg_block_size - elif zero_pad == "shift": - n_cut = shift_len - else: # "none" -- no buffer needed - n_cut = win_len - n_keep = win_len - n_cut - if n_keep > 0: - expected = np.concatenate((np.zeros((nchans, win_len))[..., -n_keep:], expected), axis=-1) - tvec = np.hstack(((np.arange(-win_len, 0) / fs)[-n_keep:], tvec)) - # Moving window -- assumes step size of 1 - expected = sliding_window_view(expected, win_len, axis=-1) - tvec = sliding_window_view(tvec, win_len) - # Mimic win_shift - if win_shift is None: - # 1:1 mode. Each input (block) yields a new output. - # If the window length is smaller than the block size then we only the tail of each block. - first = max(min(msg_block_size, data_len) - win_len, 0) - if tvec[::msg_block_size].shape[0] < n_msgs: - expected = np.concatenate((expected[:, first::msg_block_size], expected[:, -1:]), axis=1) - tvec = np.hstack((tvec[first::msg_block_size, 0], tvec[-1:, 0])) - else: - expected = expected[:, first::msg_block_size] - tvec = tvec[first::msg_block_size, 0] - else: - expected = expected[:, ::shift_len] - tvec = tvec[::shift_len, 0] - # Transpose to put time_ax and win_ax in the correct locations. - if win_ax == 0: - expected = np.moveaxis(expected, 0, -1) - - return expected, tvec - - -def test_window_gen_nodur(): - """ - Test window generator method when window_dur is None. Should be a simple pass through. - """ - nchans = 64 - data_len = 20 - data = np.arange(nchans * data_len, dtype=float).reshape((nchans, data_len)) - test_msg = AxisArray( - data=data, - dims=["ch", "time"], - axes={"time": AxisArray.Axis.TimeAxis(fs=500., offset=0.)} - ) - gen = windowing(window_dur=None) - result = gen.send(test_msg) - assert result[0] is test_msg - assert np.shares_memory(result[0].data, test_msg.data) - - -@pytest.mark.parametrize("msg_block_size", [1, 5, 10, 20, 60]) -@pytest.mark.parametrize("newaxis", [None, "win"]) -@pytest.mark.parametrize("win_dur", [0.2, 1.0]) -@pytest.mark.parametrize("win_shift", [None, 0.1, 1.0]) -@pytest.mark.parametrize("zero_pad", ["input", "shift", "none"]) -@pytest.mark.parametrize("fs", [10.0, 500.0]) -@pytest.mark.parametrize("time_ax", [0, 1]) -def test_window_generator( - msg_block_size: int, - newaxis: Optional[str], - win_dur: float, - win_shift: Optional[float], - zero_pad: str, - fs: float, - time_ax: int -): - nchans = 3 - - shift_len = int(win_shift * fs) if win_shift is not None else None - win_len = int(win_dur * fs) - data_len = 2 * win_len - if win_shift is not None: - data_len += shift_len - 1 - data = np.arange(nchans * data_len, dtype=float).reshape((nchans, data_len)) - # Below, we transpose the individual messages if time_ax == 0. - tvec = np.arange(data_len) / fs - - n_msgs = int(np.ceil(data_len / msg_block_size)) - - # Instantiate the generator function - gen = windowing(axis="time", newaxis=newaxis, window_dur=win_dur, window_shift=win_shift, zero_pad_until=zero_pad) - - # Create inputs and send them to the generator, collecting the results along the way. - test_msg = AxisArray( - data[..., ()], - dims=["ch", "time"] if time_ax == 1 else ["time", "ch"], - axes={"time": AxisArray.Axis.TimeAxis(fs=fs, offset=0.)} - ) - results = [] - for msg_ix in range(n_msgs): - msg_data = data[..., msg_ix * msg_block_size:(msg_ix+1) * msg_block_size] - if time_ax == 0: - msg_data = np.ascontiguousarray(msg_data.T) - test_msg = replace(test_msg, data=msg_data, axes={ - "time": AxisArray.Axis.TimeAxis(fs=fs, offset=tvec[msg_ix * msg_block_size]) - }) - wins = gen.send(test_msg) - results.extend(wins) - - # Check each return value's metadata (offsets checked at end) - for msg in results: - assert msg.axes["time"].gain == 1/fs - if newaxis is None: - assert msg.dims == test_msg.dims - else: - assert msg.dims == test_msg.dims[:time_ax] + [newaxis] + test_msg.dims[time_ax:] - assert newaxis in msg.axes - assert msg.axes[newaxis].gain == 0.0 if win_shift is None else shift_len / fs - - # Post-process the results to yield a single data array and a single vector of offsets. - win_ax = time_ax - if newaxis is None: - result = np.stack([_.data for _ in results], axis=time_ax) - # np.stack creates new axis before target axis. - time_ax += 1 - offsets = np.array([_.axes["time"].offset for _ in results]) - else: - # win_ax already in data; replaced time_ax, time_ax moved to end. - result = np.concatenate([_.data for _ in results], axis=win_ax) - time_ax = result.ndim - 1 - offsets = np.hstack([ - _.axes[newaxis].offset + _.axes[newaxis].gain * np.arange(_.data.shape[win_ax]) - for _ in results - ]) - - # Calculate the expected results for comparison. - expected, tvec = calculate_expected_results(data, fs, win_shift, zero_pad, msg_block_size, shift_len, win_len, - nchans, data_len, n_msgs, win_ax) - - # Compare results to expected - assert np.array_equal(result, expected) - assert np.allclose(offsets, tvec) - - -class WindowSystemSettings(ez.Settings): - num_msgs: int - counter_settings: CounterSettings - window_settings: WindowSettings - log_settings: MessageLoggerSettings - term_settings: TerminateTestSettings = field(default_factory=TerminateTestSettings) - - -class WindowSystem(ez.Collection): - COUNTER = Counter() - GATE = MessageGate() - WIN = Window() - LOG = MessageLogger() - TERM = TerminateTest() - - DEBUG = DebugLog() - - SETTINGS: WindowSystemSettings - - def configure(self) -> None: - self.COUNTER.apply_settings(self.SETTINGS.counter_settings) - self.GATE.apply_settings( - MessageGateSettings( - start_open=True, - default_open=False, - default_after=self.SETTINGS.num_msgs, - ) - ) - self.WIN.apply_settings(self.SETTINGS.window_settings) - self.LOG.apply_settings(self.SETTINGS.log_settings) - self.TERM.apply_settings(self.SETTINGS.term_settings) - - def network(self) -> ez.NetworkDefinition: - return ( - (self.COUNTER.OUTPUT_SIGNAL, self.GATE.INPUT), - # ( self.COUNTER.OUTPUT_SIGNAL, self.DEBUG.INPUT ), - (self.GATE.OUTPUT, self.WIN.INPUT_SIGNAL), - # ( self.GATE.OUTPUT, self.DEBUG.INPUT ), - (self.WIN.OUTPUT_SIGNAL, self.LOG.INPUT_MESSAGE), - # ( self.WIN.OUTPUT_SIGNAL, self.DEBUG.INPUT ), - (self.LOG.OUTPUT_MESSAGE, self.TERM.INPUT), - # ( self.LOG.OUTPUT_MESSAGE, self.DEBUG.INPUT ), - ) - - -# It takes >15 minutes to go through the full set of combinations tested for the generator. -# We need only test a subset to assert integration is correct. -@pytest.mark.parametrize("msg_block_size, newaxis, win_dur, win_shift, zero_pad, fs", [ - (1, None, 0.2, None, "input", 10.0), - (20, None, 0.2, None, "input", 10.0), - (1, "step", 0.2, None, "input", 10.0), - (10, "step", 0.2, 1.0, "shift", 500.0), - (20, "step", 1.0, 1.0, "shift", 500.0), - (10, "step", 1.0, 1.0, "none", 500.0), - (20, None, None, None, "input", 10.0), -]) -def test_window_system( - msg_block_size: int, - newaxis: Optional[str], - win_dur: float, - win_shift: Optional[float], - zero_pad: str, - fs: float, - test_name: Optional[str] = None, -): - # Calculate expected dimensions. - win_len = int((win_dur or 1.0) * fs) - shift_len = int(win_shift * fs) if win_shift is not None else msg_block_size - # num_msgs should be the greater value between (2 full windows + a shift) or 4.0 seconds - data_len = max(2 * win_len + shift_len - 1, int(4.0 * fs)) - num_msgs = int(np.ceil(data_len / msg_block_size)) - - test_filename = get_test_fn(test_name) - ez.logger.info(test_filename) - - settings = WindowSystemSettings( - num_msgs=num_msgs, - counter_settings=CounterSettings( - n_time=msg_block_size, - fs=fs, - dispatch_rate=float(num_msgs), # Get through them in about 1 second. - ), - window_settings=WindowSettings( - axis="time", - newaxis=newaxis, - window_dur=win_dur, - window_shift=win_shift, - zero_pad_until=zero_pad - ), - log_settings=MessageLoggerSettings(output=test_filename), - term_settings=TerminateTestSettings(time=1.0), # sec - ) - - system = WindowSystem(settings) - ez.run(SYSTEM=system) - - messages: List[AxisArray] = [_ for _ in message_log(test_filename)] - os.remove(test_filename) - ez.logger.info(f"Analyzing recording of { len( messages ) } messages...") - - # Within a test config, the metadata should not change across messages. - for msg in messages: - # In this test, fs should never change - assert 1.0 / msg.axes["time"].gain == fs - # In this test, we should have consistent dimensions - assert msg.dims == ([newaxis, "time", "ch"] if newaxis else ["time", "ch"]) - # Window should always output the same shape data - assert msg.shape[msg.get_axis_idx("ch")] == 1 # Counter yields only one channel. - assert msg.shape[msg.get_axis_idx("time")] == (msg_block_size if win_dur is None else win_len) - - ez.logger.info("Consistent metadata!") - - # Collect the outputs we want to test - data: List[np.ndarray] = [msg.data for msg in messages] - if newaxis is None: - offsets = np.array([_.axes["time"].offset for _ in messages]) - else: - offsets = np.hstack([ - _.axes[newaxis].offset + _.axes[newaxis].gain * np.arange(_.data.shape[0]) - for _ in messages - ]) - - # If this test was performed in "one-to-one" mode, we should - # have one window output per message pushed to Window - if win_shift is None: - assert len(data) == num_msgs - - # Turn the data into a ndarray. - if newaxis is not None: - data = np.concatenate(data, axis=messages[0].get_axis_idx(newaxis)) - else: - data = np.stack(data, axis=messages[0].get_axis_idx("time")) - - # Calculate the expected results for comparison. - sent_data = np.arange(num_msgs * msg_block_size)[None, :] - expected, tvec = calculate_expected_results(sent_data, fs, win_shift, zero_pad, msg_block_size, shift_len, win_len, - 1, data_len, num_msgs, 0) - - # Compare results to expected - if win_dur is None: - assert np.array_equal(data, sent_data.reshape((num_msgs, msg_block_size, -1))) - else: - assert np.array_equal(data, expected) - assert np.allclose(offsets, tvec) - - ez.logger.info("Test Complete.") - - -if __name__ == "__main__": - test_window_system(5, 0.6, None, test_name="test_window_system") diff --git a/extensions/ezmsg-sigproc/tests/util.py b/extensions/ezmsg-sigproc/tests/util.py deleted file mode 100644 index cddcc1ea..00000000 --- a/extensions/ezmsg-sigproc/tests/util.py +++ /dev/null @@ -1,78 +0,0 @@ -import os -import tempfile -from pathlib import Path -import typing - -import numpy as np -from numpy.lib.stride_tricks import sliding_window_view -from ezmsg.util.messages.axisarray import AxisArray - - -def get_test_fn(test_name: typing.Optional[str] = None, extension: str = "txt") -> Path: - """PYTEST compatible temporary test file creator""" - - # Get current test name if we can.. - if test_name is None: - test_name = os.environ.get("PYTEST_CURRENT_TEST") - if test_name is not None: - test_name = test_name.split(":")[-1].split(" ")[0] - else: - test_name = __name__ - - file_path = Path(tempfile.gettempdir()) - file_path = file_path / Path(f"{test_name}.{extension}") - - # Create the file - with open(file_path, "w"): - pass - - return file_path - - -def create_messages_with_periodic_signal( - sin_params: typing.List[typing.Dict[str, float]] = [ - {"f": 10.0, "dur": 5.0, "offset": 0.0}, - {"f": 20.0, "dur": 5.0, "offset": 0.0}, - {"f": 70.0, "dur": 5.0, "offset": 0.0}, - {"f": 14.0, "dur": 5.0, "offset": 5.0}, - {"f": 35.0, "dur": 5.0, "offset": 5.0}, - {"f": 300.0, "dur": 5.0, "offset": 5.0}, - ], - fs: float = 1000., - msg_dur: float = 1.0, - win_step_dur: typing.Optional[float] = None -) -> typing.List[AxisArray]: - """ - Create a continuous signal with periodic components. The signal will be divided into n segments, - where n is the number of lists in f_sets. Each segment will have sinusoids (of equal amplitude) - at each of the frequencies in the f_set. Each segment will be seg_dur seconds long. - """ - t_end = max([_.get("offset", 0.0) + _["dur"] for _ in sin_params]) - t_vec = np.arange(int(t_end * fs)) / fs - data = np.zeros((len(t_vec),)) - # TODO: each freq should be evaluated independently and the dict should have a "dur" and "offset" value, both in sec - # TODO: Get rid of `win_dur` and replace with `msg_dur` - # TODO: if win_step_dur is not None then we do sliding_window_view - for s_p in sin_params: - offs = s_p.get("offset", 0.0) - b_t = np.logical_and(t_vec >= offs, t_vec <= offs + s_p["dur"]) - data[b_t] += s_p.get("a", 1.) * np.sin(2 * np.pi * s_p["f"] * t_vec[b_t] + s_p.get("p", 0)) - - # How will we split the data into messages? With a rolling window or non-overlapping? - if win_step_dur is not None: - win_step = int(win_step_dur * fs) - data_splits = sliding_window_view(data, (int(msg_dur * fs),), axis=0)[::win_step] - else: - n_msgs = int(t_end / msg_dur) - data_splits = np.array_split(data, n_msgs, axis=0) - - # Create the output messages - offset = 0.0 - messages = [] - for split_dat in data_splits: - _time_axis = AxisArray.Axis.TimeAxis(fs=fs, offset=offset) - messages.append( - AxisArray(split_dat[..., None], dims=["time", "ch"], axes={"time": _time_axis}) - ) - offset += split_dat.shape[0] / fs - return messages diff --git a/extensions/ezmsg-websocket/LICENSE.txt b/extensions/ezmsg-websocket/LICENSE.txt deleted file mode 100644 index a21312ac..00000000 --- a/extensions/ezmsg-websocket/LICENSE.txt +++ /dev/null @@ -1,9 +0,0 @@ -MIT License - -Copyright (c) 2022 Johns Hopkins University Applied Physics Lab - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/extensions/ezmsg-websocket/README.md b/extensions/ezmsg-websocket/README.md deleted file mode 100644 index abb4a64b..00000000 --- a/extensions/ezmsg-websocket/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# ezmsg.websocket - -Websocket server and client units for ezmsg - -## Installation -`pip install ezmsg-websocket` - -## Dependencies -* `websockets` - -## Setup (Development) -1. Install `ezmsg` either using `pip install ezmsg` or set up the repo for development as described in the `ezmsg` readme. -2. `cd` to this directory (`ezmsg-websocket`) and run `pip install -e .` -3. Signal processing modules are available under `import ezmsg.websocket` - - diff --git a/extensions/ezmsg-websocket/examples/ezmsg_websocket.py b/extensions/ezmsg-websocket/examples/ezmsg_websocket.py deleted file mode 100644 index 588766ce..00000000 --- a/extensions/ezmsg-websocket/examples/ezmsg_websocket.py +++ /dev/null @@ -1,107 +0,0 @@ -import json -import math -import time -import asyncio - -import ezmsg.core as ez - -from ezmsg.websocket.units import WebsocketServer, WebsocketClient, WebsocketSettings - -from typing import Any, AsyncGenerator, Dict, Tuple - -# LFO: Low Frequency Oscillator - -class LFOSettings(ez.Settings): - freq: float = 0.2 # Hz, sinus frequency - update_rate: float = 2.0 # Hz, update rate - - -class LFO(ez.Unit): - SETTINGS: LFOSettings - - OUTPUT = ez.OutputStream(float) - - def initialize(self) -> None: - self.start_time = time.time() - - @ez.publisher(OUTPUT) - async def generate(self) -> AsyncGenerator: - while True: - t = time.time() - self.start_time - yield self.OUTPUT, math.sin(2.0 * math.pi * self.SETTINGS.freq * t) - await asyncio.sleep(1.0 / self.SETTINGS.update_rate) - - -class JSONAdapter(ez.Unit): - DICT_INPUT = ez.InputStream(Dict[str, Any]) - JSON_OUTPUT = ez.OutputStream(str) - - @ez.subscriber(DICT_INPUT) - @ez.publisher(JSON_OUTPUT) - async def dict_to_json(self, message: Dict[str, Any]) -> AsyncGenerator: - yield self.JSON_OUTPUT, json.dumps(message) - - JSON_INPUT = ez.InputStream(str) - DICT_OUTPUT = ez.OutputStream(Dict[str, Any]) - - @ez.subscriber(JSON_INPUT) - @ez.publisher(DICT_OUTPUT) - async def json_to_dict(self, message: str) -> AsyncGenerator: - yield self.DICT_OUTPUT, json.loads(message) - - -class DebugOutput(ez.Unit): - INPUT = ez.InputStream(str) - - @ez.subscriber(INPUT) - async def print(self, message: str) -> None: - print("DEBUG:", message) - - -class WebsocketSystemSettings(ez.Settings): - host: str - port: int - - -class WebsocketSystem(ez.Collection): - SETTINGS: WebsocketSystemSettings - - OSC = LFO() - SERVER = WebsocketServer() - JSON = JSONAdapter() - OUT = DebugOutput() - CLIENT = WebsocketClient() - - def configure(self) -> None: - self.OSC.apply_settings(LFOSettings(freq=0.2, update_rate=1.0)) - - self.SERVER.apply_settings( - WebsocketSettings(host=self.SETTINGS.host, port=self.SETTINGS.port) - ) - - self.CLIENT.apply_settings( - WebsocketSettings(host=self.SETTINGS.host, port=self.SETTINGS.port) - ) - - # Define Connections - def network(self) -> ez.NetworkDefinition: - return ( - (self.OSC.OUTPUT, self.JSON.DICT_INPUT), - (self.JSON.JSON_OUTPUT, self.SERVER.INPUT), - (self.CLIENT.OUTPUT, self.CLIENT.INPUT), # Relay - (self.SERVER.OUTPUT, self.JSON.JSON_INPUT), - (self.JSON.DICT_OUTPUT, self.OUT.INPUT), - ) - - def process_components(self) -> Tuple[ez.Component, ...]: - return (self.OSC, self.CLIENT, self.SERVER) - - -if __name__ == "__main__": - host = "127.0.0.1" - port = 5038 - - # Run the websocket system - system = WebsocketSystem() - system.apply_settings(WebsocketSystemSettings(host=host, port=port)) - ez.run(SYSTEM = system) diff --git a/extensions/ezmsg-websocket/poetry.lock b/extensions/ezmsg-websocket/poetry.lock deleted file mode 100644 index b10afdb8..00000000 --- a/extensions/ezmsg-websocket/poetry.lock +++ /dev/null @@ -1,66 +0,0 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. - -[[package]] -name = "ezmsg" -version = "3.3.3" -description = "A simple DAG-based computation model" -optional = false -python-versions = ">=3.8" -files = [ - {file = "ezmsg-3.3.3-py3-none-any.whl", hash = "sha256:62920470d8a692fcd986e980a80e27d0ec3c0a36677d2068ee75b8cf301a0cde"}, - {file = "ezmsg-3.3.3.tar.gz", hash = "sha256:411dd4e027e37bb322bfbcef75264d47134ea64efa2b428522c257d959ca439f"}, -] - -[package.dependencies] -typing-extensions = "*" - -[package.extras] -all-ext = ["ezmsg-sigproc", "ezmsg-websocket", "ezmsg-zmq"] -test = ["numpy", "pytest", "pytest-asyncio", "pytest-cov"] - -[[package]] -name = "typing-extensions" -version = "4.9.0" -description = "Backported and Experimental Type Hints for Python 3.8+" -optional = false -python-versions = ">=3.8" -files = [ - {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, - {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, -] - -[[package]] -name = "websockets" -version = "8.1" -description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" -optional = false -python-versions = ">=3.6.1" -files = [ - {file = "websockets-8.1-cp36-cp36m-macosx_10_6_intel.whl", hash = "sha256:3762791ab8b38948f0c4d281c8b2ddfa99b7e510e46bd8dfa942a5fff621068c"}, - {file = "websockets-8.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:3db87421956f1b0779a7564915875ba774295cc86e81bc671631379371af1170"}, - {file = "websockets-8.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4f9f7d28ce1d8f1295717c2c25b732c2bc0645db3215cf757551c392177d7cb8"}, - {file = "websockets-8.1-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:295359a2cc78736737dd88c343cd0747546b2174b5e1adc223824bcaf3e164cb"}, - {file = "websockets-8.1-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:1d3f1bf059d04a4e0eb4985a887d49195e15ebabc42364f4eb564b1d065793f5"}, - {file = "websockets-8.1-cp36-cp36m-win32.whl", hash = "sha256:2db62a9142e88535038a6bcfea70ef9447696ea77891aebb730a333a51ed559a"}, - {file = "websockets-8.1-cp36-cp36m-win_amd64.whl", hash = "sha256:0e4fb4de42701340bd2353bb2eee45314651caa6ccee80dbd5f5d5978888fed5"}, - {file = "websockets-8.1-cp37-cp37m-macosx_10_6_intel.whl", hash = "sha256:9b248ba3dd8a03b1a10b19efe7d4f7fa41d158fdaa95e2cf65af5a7b95a4f989"}, - {file = "websockets-8.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:ce85b06a10fc65e6143518b96d3dca27b081a740bae261c2fb20375801a9d56d"}, - {file = "websockets-8.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:965889d9f0e2a75edd81a07592d0ced54daa5b0785f57dc429c378edbcffe779"}, - {file = "websockets-8.1-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:751a556205d8245ff94aeef23546a1113b1dd4f6e4d102ded66c39b99c2ce6c8"}, - {file = "websockets-8.1-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:3ef56fcc7b1ff90de46ccd5a687bbd13a3180132268c4254fc0fa44ecf4fc422"}, - {file = "websockets-8.1-cp37-cp37m-win32.whl", hash = "sha256:7ff46d441db78241f4c6c27b3868c9ae71473fe03341340d2dfdbe8d79310acc"}, - {file = "websockets-8.1-cp37-cp37m-win_amd64.whl", hash = "sha256:20891f0dddade307ffddf593c733a3fdb6b83e6f9eef85908113e628fa5a8308"}, - {file = "websockets-8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c1ec8db4fac31850286b7cd3b9c0e1b944204668b8eb721674916d4e28744092"}, - {file = "websockets-8.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:5c01fd846263a75bc8a2b9542606927cfad57e7282965d96b93c387622487485"}, - {file = "websockets-8.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:9bef37ee224e104a413f0780e29adb3e514a5b698aabe0d969a6ba426b8435d1"}, - {file = "websockets-8.1-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:d705f8aeecdf3262379644e4b55107a3b55860eb812b673b28d0fbc347a60c55"}, - {file = "websockets-8.1-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:c8a116feafdb1f84607cb3b14aa1418424ae71fee131642fc568d21423b51824"}, - {file = "websockets-8.1-cp38-cp38-win32.whl", hash = "sha256:e898a0863421650f0bebac8ba40840fc02258ef4714cb7e1fd76b6a6354bda36"}, - {file = "websockets-8.1-cp38-cp38-win_amd64.whl", hash = "sha256:f8a7bff6e8664afc4e6c28b983845c5bc14965030e3fb98789734d416af77c4b"}, - {file = "websockets-8.1.tar.gz", hash = "sha256:5c65d2da8c6bce0fca2528f69f44b2f977e06954c8512a952222cea50dad430f"}, -] - -[metadata] -lock-version = "2.0" -python-versions = "^3.8" -content-hash = "fdb04df5030874bb7abf259f59f88ec0ca4056dd4d0353d3748bd412aa5a6ebd" diff --git a/extensions/ezmsg-websocket/pyproject.toml b/extensions/ezmsg-websocket/pyproject.toml deleted file mode 100644 index c13128cd..00000000 --- a/extensions/ezmsg-websocket/pyproject.toml +++ /dev/null @@ -1,24 +0,0 @@ -[tool.poetry] -name = "ezmsg-websocket" -version = "1.1.2" -description = "Websocket server and client units for ezmsg" -authors = [ - "Milsap, Griffin ", - "Peranich, Preston ", -] -license = "MIT" -readme = "README.md" -packages = [{ include = "ezmsg", from = "src" }] -classifiers = [ - "Programming Language :: Python :: 3", - "Operating System :: OS Independent", -] - -[tool.poetry.dependencies] -python = "^3.8" -ezmsg = "^3.3.0" -websockets = "^8.1" - -[build-system] -requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" diff --git a/extensions/ezmsg-websocket/src/ezmsg/websocket/__init__.py b/extensions/ezmsg-websocket/src/ezmsg/websocket/__init__.py deleted file mode 100644 index 5da10165..00000000 --- a/extensions/ezmsg-websocket/src/ezmsg/websocket/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -import importlib.metadata - - -__version__ = importlib.metadata.version("ezmsg-websocket") diff --git a/extensions/ezmsg-websocket/src/ezmsg/websocket/units.py b/extensions/ezmsg-websocket/src/ezmsg/websocket/units.py deleted file mode 100644 index 6d9e25ca..00000000 --- a/extensions/ezmsg-websocket/src/ezmsg/websocket/units.py +++ /dev/null @@ -1,158 +0,0 @@ -import asyncio -import ssl - -from dataclasses import field - -import websockets.server -import websockets.exceptions -from websockets.legacy.client import connect, WebSocketClientProtocol - -import ezmsg.core as ez - -from typing import Optional, Union, AsyncGenerator - - -class WebsocketSettings(ez.Settings): - host: str - port: int - cert_path: Optional[str] = None - - -class WebsocketState(ez.State): - incoming_queue: "asyncio.Queue[Union[str,bytes]]" = field( - default_factory=asyncio.Queue - ) - outgoing_queue: "asyncio.Queue[Union[str,bytes]]" = field( - default_factory=asyncio.Queue - ) - - -class WebsocketServer(ez.Unit): - - """ - Receives arbitrary content from outside world - and injects it into system in a DataArray - """ - - SETTINGS: WebsocketSettings - STATE: WebsocketState - - INPUT = ez.InputStream(bytes) - OUTPUT = ez.OutputStream(bytes) - - @ez.task - async def start_server(self): - ez.logger.info( - f"Starting WS Input Server @ ws://{self.SETTINGS.host}:{self.SETTINGS.port}" - ) - - async def connection( - websocket: websockets.server.WebSocketServerProtocol, path - ): - async def loop(mode): - try: - if mode == "rx": - while True: - data = await websocket.recv() - self.STATE.incoming_queue.put_nowait(data) - elif mode == "tx": - while True: - data = await self.STATE.outgoing_queue.get() - await websocket.send(data) - except websockets.exceptions.ConnectionClosedOK: - pass - except asyncio.CancelledError: - pass - except Exception as e: - print("Error in websocket server:", e) - pass - finally: - ... - - await asyncio.wait([loop(mode="tx"), loop(mode="rx")]) - - try: - if self.SETTINGS.cert_path: - ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - ssl_context.load_cert_chain(self.SETTINGS.cert_path) - else: - ssl_context = None - - server = await websockets.server.serve( - connection, self.SETTINGS.host, self.SETTINGS.port, ssl=ssl_context - ) - - await server.wait_closed() - - finally: - ... - - @ez.publisher(OUTPUT) - async def publish_incoming(self): - while True: - data = await self.STATE.incoming_queue.get() - yield self.OUTPUT, data - - @ez.subscriber(INPUT) - async def transmit_outgoing(self, message: bytes): - self.STATE.outgoing_queue.put_nowait(message) - - -class WebsocketClient(ez.Unit): - SETTINGS: WebsocketSettings - STATE: WebsocketState - - INPUT = ez.InputStream(bytes) - OUTPUT = ez.OutputStream(bytes) - - async def rx_from(self, websocket: WebSocketClientProtocol): - # await incoming data from websocket and post them - # to incoming queue for publication within ezmsg - async for message in websocket: - self.STATE.incoming_queue.put_nowait(message) - - async def tx_to(self, websocket: WebSocketClientProtocol): - # await messages from subscription within ezmsg - # and post them to outgoing websocket - while True: - message = await self.STATE.outgoing_queue.get() - await websocket.send(message) - - @ez.task - async def connection(self): - if self.SETTINGS.cert_path: - prefix = "wss" - else: - prefix = "ws" - uri = f"{prefix}://{self.SETTINGS.host}:{self.SETTINGS.port}" - websocket = None - for attempt in range(10): - try: - websocket = await connect(uri) - break - except: - await asyncio.sleep(0.5) - - if websocket is None: - raise Exception(f"Could not connect to {uri}") - - receive_task = asyncio.ensure_future(self.rx_from(websocket)) - transmit_task = asyncio.ensure_future(self.tx_to(websocket)) - done, pending = await asyncio.wait( - [receive_task, transmit_task], return_when=asyncio.FIRST_COMPLETED - ) - - for task in pending: - task.cancel() - - await websocket.close() - - @ez.publisher(OUTPUT) - async def receive(self) -> AsyncGenerator: - while True: - message = await self.STATE.incoming_queue.get() - yield self.OUTPUT, message - - @ez.subscriber(INPUT) - async def transmit(self, message: bytes) -> None: - self.STATE.outgoing_queue.put_nowait(message) diff --git a/extensions/ezmsg-zmq/LICENSE.txt b/extensions/ezmsg-zmq/LICENSE.txt deleted file mode 100644 index a21312ac..00000000 --- a/extensions/ezmsg-zmq/LICENSE.txt +++ /dev/null @@ -1,9 +0,0 @@ -MIT License - -Copyright (c) 2022 Johns Hopkins University Applied Physics Lab - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/extensions/ezmsg-zmq/README.md b/extensions/ezmsg-zmq/README.md deleted file mode 100644 index edac228b..00000000 --- a/extensions/ezmsg-zmq/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# ezmsg.zmq - -Zero-MQ pub/sub units for ezmsg - -## Installation -`pip install ezmsg-sigproc` - -## Dependencies -* `ezmsg` -* `pyzmq` - -## Setup (Development) -1. Install `ezmsg` either using `pip install ezmsg` or set up the repo for development as described in the `ezmsg` readme. -2. `cd` to this directory (`ezmsg-zmq`) and run `pip install -e .` -3. Signal processing modules are available under `import ezmsg.zmq` \ No newline at end of file diff --git a/extensions/ezmsg-zmq/poetry.lock b/extensions/ezmsg-zmq/poetry.lock deleted file mode 100644 index c31aa79d..00000000 --- a/extensions/ezmsg-zmq/poetry.lock +++ /dev/null @@ -1,215 +0,0 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. - -[[package]] -name = "cffi" -version = "1.16.0" -description = "Foreign Function Interface for Python calling C code." -optional = false -python-versions = ">=3.8" -files = [ - {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, - {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, - {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, - {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, - {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, - {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, - {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, - {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, - {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, - {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, - {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, - {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, - {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, - {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, - {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, - {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, - {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, - {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, - {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, - {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, - {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, - {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, - {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, - {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, - {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, - {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, - {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, -] - -[package.dependencies] -pycparser = "*" - -[[package]] -name = "ezmsg" -version = "3.3.3" -description = "A simple DAG-based computation model" -optional = false -python-versions = ">=3.8" -files = [ - {file = "ezmsg-3.3.3-py3-none-any.whl", hash = "sha256:62920470d8a692fcd986e980a80e27d0ec3c0a36677d2068ee75b8cf301a0cde"}, - {file = "ezmsg-3.3.3.tar.gz", hash = "sha256:411dd4e027e37bb322bfbcef75264d47134ea64efa2b428522c257d959ca439f"}, -] - -[package.dependencies] -typing-extensions = "*" - -[package.extras] -all-ext = ["ezmsg-sigproc", "ezmsg-websocket", "ezmsg-zmq"] -test = ["numpy", "pytest", "pytest-asyncio", "pytest-cov"] - -[[package]] -name = "pycparser" -version = "2.21" -description = "C parser in Python" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, - {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, -] - -[[package]] -name = "pyzmq" -version = "25.1.2" -description = "Python bindings for 0MQ" -optional = false -python-versions = ">=3.6" -files = [ - {file = "pyzmq-25.1.2-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:e624c789359f1a16f83f35e2c705d07663ff2b4d4479bad35621178d8f0f6ea4"}, - {file = "pyzmq-25.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:49151b0efece79f6a79d41a461d78535356136ee70084a1c22532fc6383f4ad0"}, - {file = "pyzmq-25.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9a5f194cf730f2b24d6af1f833c14c10f41023da46a7f736f48b6d35061e76e"}, - {file = "pyzmq-25.1.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:faf79a302f834d9e8304fafdc11d0d042266667ac45209afa57e5efc998e3872"}, - {file = "pyzmq-25.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f51a7b4ead28d3fca8dda53216314a553b0f7a91ee8fc46a72b402a78c3e43d"}, - {file = "pyzmq-25.1.2-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:0ddd6d71d4ef17ba5a87becf7ddf01b371eaba553c603477679ae817a8d84d75"}, - {file = "pyzmq-25.1.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:246747b88917e4867e2367b005fc8eefbb4a54b7db363d6c92f89d69abfff4b6"}, - {file = "pyzmq-25.1.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:00c48ae2fd81e2a50c3485de1b9d5c7c57cd85dc8ec55683eac16846e57ac979"}, - {file = "pyzmq-25.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5a68d491fc20762b630e5db2191dd07ff89834086740f70e978bb2ef2668be08"}, - {file = "pyzmq-25.1.2-cp310-cp310-win32.whl", hash = "sha256:09dfe949e83087da88c4a76767df04b22304a682d6154de2c572625c62ad6886"}, - {file = "pyzmq-25.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:fa99973d2ed20417744fca0073390ad65ce225b546febb0580358e36aa90dba6"}, - {file = "pyzmq-25.1.2-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:82544e0e2d0c1811482d37eef297020a040c32e0687c1f6fc23a75b75db8062c"}, - {file = "pyzmq-25.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:01171fc48542348cd1a360a4b6c3e7d8f46cdcf53a8d40f84db6707a6768acc1"}, - {file = "pyzmq-25.1.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc69c96735ab501419c432110016329bf0dea8898ce16fab97c6d9106dc0b348"}, - {file = "pyzmq-25.1.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3e124e6b1dd3dfbeb695435dff0e383256655bb18082e094a8dd1f6293114642"}, - {file = "pyzmq-25.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7598d2ba821caa37a0f9d54c25164a4fa351ce019d64d0b44b45540950458840"}, - {file = "pyzmq-25.1.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:d1299d7e964c13607efd148ca1f07dcbf27c3ab9e125d1d0ae1d580a1682399d"}, - {file = "pyzmq-25.1.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4e6f689880d5ad87918430957297c975203a082d9a036cc426648fcbedae769b"}, - {file = "pyzmq-25.1.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cc69949484171cc961e6ecd4a8911b9ce7a0d1f738fcae717177c231bf77437b"}, - {file = "pyzmq-25.1.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9880078f683466b7f567b8624bfc16cad65077be046b6e8abb53bed4eeb82dd3"}, - {file = "pyzmq-25.1.2-cp311-cp311-win32.whl", hash = "sha256:4e5837af3e5aaa99a091302df5ee001149baff06ad22b722d34e30df5f0d9097"}, - {file = "pyzmq-25.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:25c2dbb97d38b5ac9fd15586e048ec5eb1e38f3d47fe7d92167b0c77bb3584e9"}, - {file = "pyzmq-25.1.2-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:11e70516688190e9c2db14fcf93c04192b02d457b582a1f6190b154691b4c93a"}, - {file = "pyzmq-25.1.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:313c3794d650d1fccaaab2df942af9f2c01d6217c846177cfcbc693c7410839e"}, - {file = "pyzmq-25.1.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b3cbba2f47062b85fe0ef9de5b987612140a9ba3a9c6d2543c6dec9f7c2ab27"}, - {file = "pyzmq-25.1.2-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fc31baa0c32a2ca660784d5af3b9487e13b61b3032cb01a115fce6588e1bed30"}, - {file = "pyzmq-25.1.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02c9087b109070c5ab0b383079fa1b5f797f8d43e9a66c07a4b8b8bdecfd88ee"}, - {file = "pyzmq-25.1.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:f8429b17cbb746c3e043cb986328da023657e79d5ed258b711c06a70c2ea7537"}, - {file = "pyzmq-25.1.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:5074adeacede5f810b7ef39607ee59d94e948b4fd954495bdb072f8c54558181"}, - {file = "pyzmq-25.1.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:7ae8f354b895cbd85212da245f1a5ad8159e7840e37d78b476bb4f4c3f32a9fe"}, - {file = "pyzmq-25.1.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b264bf2cc96b5bc43ce0e852be995e400376bd87ceb363822e2cb1964fcdc737"}, - {file = "pyzmq-25.1.2-cp312-cp312-win32.whl", hash = "sha256:02bbc1a87b76e04fd780b45e7f695471ae6de747769e540da909173d50ff8e2d"}, - {file = "pyzmq-25.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:ced111c2e81506abd1dc142e6cd7b68dd53747b3b7ae5edbea4578c5eeff96b7"}, - {file = "pyzmq-25.1.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:7b6d09a8962a91151f0976008eb7b29b433a560fde056ec7a3db9ec8f1075438"}, - {file = "pyzmq-25.1.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:967668420f36878a3c9ecb5ab33c9d0ff8d054f9c0233d995a6d25b0e95e1b6b"}, - {file = "pyzmq-25.1.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5edac3f57c7ddaacdb4d40f6ef2f9e299471fc38d112f4bc6d60ab9365445fb0"}, - {file = "pyzmq-25.1.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:0dabfb10ef897f3b7e101cacba1437bd3a5032ee667b7ead32bbcdd1a8422fe7"}, - {file = "pyzmq-25.1.2-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:2c6441e0398c2baacfe5ba30c937d274cfc2dc5b55e82e3749e333aabffde561"}, - {file = "pyzmq-25.1.2-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:16b726c1f6c2e7625706549f9dbe9b06004dfbec30dbed4bf50cbdfc73e5b32a"}, - {file = "pyzmq-25.1.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:a86c2dd76ef71a773e70551a07318b8e52379f58dafa7ae1e0a4be78efd1ff16"}, - {file = "pyzmq-25.1.2-cp36-cp36m-win32.whl", hash = "sha256:359f7f74b5d3c65dae137f33eb2bcfa7ad9ebefd1cab85c935f063f1dbb245cc"}, - {file = "pyzmq-25.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:55875492f820d0eb3417b51d96fea549cde77893ae3790fd25491c5754ea2f68"}, - {file = "pyzmq-25.1.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b8c8a419dfb02e91b453615c69568442e897aaf77561ee0064d789705ff37a92"}, - {file = "pyzmq-25.1.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8807c87fa893527ae8a524c15fc505d9950d5e856f03dae5921b5e9aa3b8783b"}, - {file = "pyzmq-25.1.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5e319ed7d6b8f5fad9b76daa0a68497bc6f129858ad956331a5835785761e003"}, - {file = "pyzmq-25.1.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:3c53687dde4d9d473c587ae80cc328e5b102b517447456184b485587ebd18b62"}, - {file = "pyzmq-25.1.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:9add2e5b33d2cd765ad96d5eb734a5e795a0755f7fc49aa04f76d7ddda73fd70"}, - {file = "pyzmq-25.1.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:e690145a8c0c273c28d3b89d6fb32c45e0d9605b2293c10e650265bf5c11cfec"}, - {file = "pyzmq-25.1.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:00a06faa7165634f0cac1abb27e54d7a0b3b44eb9994530b8ec73cf52e15353b"}, - {file = "pyzmq-25.1.2-cp37-cp37m-win32.whl", hash = "sha256:0f97bc2f1f13cb16905a5f3e1fbdf100e712d841482b2237484360f8bc4cb3d7"}, - {file = "pyzmq-25.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6cc0020b74b2e410287e5942e1e10886ff81ac77789eb20bec13f7ae681f0fdd"}, - {file = "pyzmq-25.1.2-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:bef02cfcbded83473bdd86dd8d3729cd82b2e569b75844fb4ea08fee3c26ae41"}, - {file = "pyzmq-25.1.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e10a4b5a4b1192d74853cc71a5e9fd022594573926c2a3a4802020360aa719d8"}, - {file = "pyzmq-25.1.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8c5f80e578427d4695adac6fdf4370c14a2feafdc8cb35549c219b90652536ae"}, - {file = "pyzmq-25.1.2-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5dde6751e857910c1339890f3524de74007958557593b9e7e8c5f01cd919f8a7"}, - {file = "pyzmq-25.1.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea1608dd169da230a0ad602d5b1ebd39807ac96cae1845c3ceed39af08a5c6df"}, - {file = "pyzmq-25.1.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:0f513130c4c361201da9bc69df25a086487250e16b5571ead521b31ff6b02220"}, - {file = "pyzmq-25.1.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:019744b99da30330798bb37df33549d59d380c78e516e3bab9c9b84f87a9592f"}, - {file = "pyzmq-25.1.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2e2713ef44be5d52dd8b8e2023d706bf66cb22072e97fc71b168e01d25192755"}, - {file = "pyzmq-25.1.2-cp38-cp38-win32.whl", hash = "sha256:07cd61a20a535524906595e09344505a9bd46f1da7a07e504b315d41cd42eb07"}, - {file = "pyzmq-25.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb7e49a17fb8c77d3119d41a4523e432eb0c6932187c37deb6fbb00cc3028088"}, - {file = "pyzmq-25.1.2-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:94504ff66f278ab4b7e03e4cba7e7e400cb73bfa9d3d71f58d8972a8dc67e7a6"}, - {file = "pyzmq-25.1.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6dd0d50bbf9dca1d0bdea219ae6b40f713a3fb477c06ca3714f208fd69e16fd8"}, - {file = "pyzmq-25.1.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:004ff469d21e86f0ef0369717351073e0e577428e514c47c8480770d5e24a565"}, - {file = "pyzmq-25.1.2-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c0b5ca88a8928147b7b1e2dfa09f3b6c256bc1135a1338536cbc9ea13d3b7add"}, - {file = "pyzmq-25.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c9a79f1d2495b167119d02be7448bfba57fad2a4207c4f68abc0bab4b92925b"}, - {file = "pyzmq-25.1.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:518efd91c3d8ac9f9b4f7dd0e2b7b8bf1a4fe82a308009016b07eaa48681af82"}, - {file = "pyzmq-25.1.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:1ec23bd7b3a893ae676d0e54ad47d18064e6c5ae1fadc2f195143fb27373f7f6"}, - {file = "pyzmq-25.1.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:db36c27baed588a5a8346b971477b718fdc66cf5b80cbfbd914b4d6d355e44e2"}, - {file = "pyzmq-25.1.2-cp39-cp39-win32.whl", hash = "sha256:39b1067f13aba39d794a24761e385e2eddc26295826530a8c7b6c6c341584289"}, - {file = "pyzmq-25.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:8e9f3fabc445d0ce320ea2c59a75fe3ea591fdbdeebec5db6de530dd4b09412e"}, - {file = "pyzmq-25.1.2-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a8c1d566344aee826b74e472e16edae0a02e2a044f14f7c24e123002dcff1c05"}, - {file = "pyzmq-25.1.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:759cfd391a0996345ba94b6a5110fca9c557ad4166d86a6e81ea526c376a01e8"}, - {file = "pyzmq-25.1.2-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7c61e346ac34b74028ede1c6b4bcecf649d69b707b3ff9dc0fab453821b04d1e"}, - {file = "pyzmq-25.1.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4cb8fc1f8d69b411b8ec0b5f1ffbcaf14c1db95b6bccea21d83610987435f1a4"}, - {file = "pyzmq-25.1.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:3c00c9b7d1ca8165c610437ca0c92e7b5607b2f9076f4eb4b095c85d6e680a1d"}, - {file = "pyzmq-25.1.2-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:df0c7a16ebb94452d2909b9a7b3337940e9a87a824c4fc1c7c36bb4404cb0cde"}, - {file = "pyzmq-25.1.2-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:45999e7f7ed5c390f2e87ece7f6c56bf979fb213550229e711e45ecc7d42ccb8"}, - {file = "pyzmq-25.1.2-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ac170e9e048b40c605358667aca3d94e98f604a18c44bdb4c102e67070f3ac9b"}, - {file = "pyzmq-25.1.2-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1b604734bec94f05f81b360a272fc824334267426ae9905ff32dc2be433ab96"}, - {file = "pyzmq-25.1.2-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:a793ac733e3d895d96f865f1806f160696422554e46d30105807fdc9841b9f7d"}, - {file = "pyzmq-25.1.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0806175f2ae5ad4b835ecd87f5f85583316b69f17e97786f7443baaf54b9bb98"}, - {file = "pyzmq-25.1.2-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ef12e259e7bc317c7597d4f6ef59b97b913e162d83b421dd0db3d6410f17a244"}, - {file = "pyzmq-25.1.2-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ea253b368eb41116011add00f8d5726762320b1bda892f744c91997b65754d73"}, - {file = "pyzmq-25.1.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b9b1f2ad6498445a941d9a4fee096d387fee436e45cc660e72e768d3d8ee611"}, - {file = "pyzmq-25.1.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:8b14c75979ce932c53b79976a395cb2a8cd3aaf14aef75e8c2cb55a330b9b49d"}, - {file = "pyzmq-25.1.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:889370d5174a741a62566c003ee8ddba4b04c3f09a97b8000092b7ca83ec9c49"}, - {file = "pyzmq-25.1.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a18fff090441a40ffda8a7f4f18f03dc56ae73f148f1832e109f9bffa85df15"}, - {file = "pyzmq-25.1.2-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:99a6b36f95c98839ad98f8c553d8507644c880cf1e0a57fe5e3a3f3969040882"}, - {file = "pyzmq-25.1.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4345c9a27f4310afbb9c01750e9461ff33d6fb74cd2456b107525bbeebcb5be3"}, - {file = "pyzmq-25.1.2-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:3516e0b6224cf6e43e341d56da15fd33bdc37fa0c06af4f029f7d7dfceceabbc"}, - {file = "pyzmq-25.1.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:146b9b1f29ead41255387fb07be56dc29639262c0f7344f570eecdcd8d683314"}, - {file = "pyzmq-25.1.2.tar.gz", hash = "sha256:93f1aa311e8bb912e34f004cf186407a4e90eec4f0ecc0efd26056bf7eda0226"}, -] - -[package.dependencies] -cffi = {version = "*", markers = "implementation_name == \"pypy\""} - -[[package]] -name = "typing-extensions" -version = "4.9.0" -description = "Backported and Experimental Type Hints for Python 3.8+" -optional = false -python-versions = ">=3.8" -files = [ - {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, - {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, -] - -[metadata] -lock-version = "2.0" -python-versions = "^3.8" -content-hash = "6c99bae1ef5cd4d37f0d7a633854c3851c31e323948fc09fb44825921af80326" diff --git a/extensions/ezmsg-zmq/pyproject.toml b/extensions/ezmsg-zmq/pyproject.toml deleted file mode 100644 index 32400153..00000000 --- a/extensions/ezmsg-zmq/pyproject.toml +++ /dev/null @@ -1,21 +0,0 @@ -[tool.poetry] -name = "ezmsg-zmq" -version = "1.1.5" -description = "Zero-MQ pub/sub units for ezmsg" -authors = [ - "Milsap, Griffin ", - "Peranich, Preston ", -] -license = "MIT" -readme = "README.md" -packages = [{ include = "ezmsg", from = "src" }] - -[tool.poetry.dependencies] -python = "^3.8" -pyzmq = "^25.1.2" -ezmsg = "^3.3.3" - - -[build-system] -requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" diff --git a/extensions/ezmsg-zmq/src/ezmsg/zmq/__init__.py b/extensions/ezmsg-zmq/src/ezmsg/zmq/__init__.py deleted file mode 100644 index 090677cf..00000000 --- a/extensions/ezmsg-zmq/src/ezmsg/zmq/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -import importlib.metadata - - -__version__ = importlib.metadata.version("ezmsg-zmq") diff --git a/extensions/ezmsg-zmq/src/ezmsg/zmq/units.py b/extensions/ezmsg-zmq/src/ezmsg/zmq/units.py deleted file mode 100644 index 2cb1de93..00000000 --- a/extensions/ezmsg-zmq/src/ezmsg/zmq/units.py +++ /dev/null @@ -1,218 +0,0 @@ -import asyncio - -from dataclasses import dataclass -from pickle import PickleBuffer - -import zmq -import zmq.asyncio -from zmq.utils.monitor import parse_monitor_message - -from typing import AsyncGenerator - -import ezmsg.core as ez - -POLL_TIME = 0.1 -STARTUP_WAIT_TIME = 0.1 - - -class ZeroCopyBytes(bytes): - def __reduce_ex__(self, protocol): - if protocol >= 5: - return type(self)._reconstruct, (PickleBuffer(self),), None - else: - # PickleBuffer is forbidden with pickle protocols <= 4. - return type(self)._reconstruct, (bytes(self),) - - @classmethod - def _reconstruct(cls, obj): - with memoryview(obj) as m: - # Get a handle over the original buffer object - obj = m.obj - if isinstance(obj, cls): - # Original buffer object is a ZeroCopyBytes, return it - # as-is. - return obj - else: - return cls(obj) - - -@dataclass -class ZMQMessage: - data: bytes - - -class ZMQSenderSettings(ez.Settings): - write_addr: str - zmq_topic: str - multipart: bool = False - wait_for_sub: bool = True - - -class ZMQSenderState(ez.State): - context: zmq.asyncio.Context - socket: zmq.asyncio.Socket - monitor: zmq.asyncio.Socket - - -class ZMQSenderUnit(ez.Unit): - """ - Represents a node in a Labgraph graph that subscribes to messages in a - Labgraph topic and forwards them by writing to a ZMQ socket. - - Args: - write_addr: The address to which ZMQ data should be written. - zmq_topic: The ZMQ topic being sent. - """ - - INPUT = ez.InputStream(ZMQMessage) - - SETTINGS: ZMQSenderSettings - STATE: ZMQSenderState - - def initialize(self) -> None: - self.STATE.context = zmq.asyncio.Context() - self.STATE.socket = self.STATE.context.socket(zmq.PUB) - self.STATE.monitor = self.STATE.socket.get_monitor_socket() - ez.logger.debug(f"{self}:binding to {self.SETTINGS.write_addr}") - self.STATE.socket.bind(self.SETTINGS.write_addr) - self.has_subscribers = False - - def shutdown(self) -> None: - self.STATE.monitor.close() - self.STATE.socket.close() - self.STATE.context.term() - - @ez.task - async def _socket_monitor(self) -> None: - while True: - monitor_result = await self.STATE.monitor.poll(100, zmq.POLLIN) - if monitor_result: - data = await self.STATE.monitor.recv_multipart() - evt = parse_monitor_message(data) - - event = evt["event"] - - if event == zmq.EVENT_ACCEPTED: - ez.logger.debug(f"{self}:subscriber joined") - self.has_subscribers = True - elif event in ( - zmq.EVENT_DISCONNECTED, - zmq.EVENT_MONITOR_STOPPED, - zmq.EVENT_CLOSED, - ): - break - - @ez.subscriber(INPUT) - async def zmq_subscriber(self, message: ZMQMessage) -> None: - while self.SETTINGS.wait_for_sub and not self.has_subscribers: - await asyncio.sleep(STARTUP_WAIT_TIME) - if self.SETTINGS.multipart is True: - await self.STATE.socket.send_multipart( - (bytes(self.SETTINGS.zmq_topic, "UTF-8"), message.data), - flags=zmq.NOBLOCK, - ) - else: - await self.STATE.socket.send( - b"".join((bytes(self.SETTINGS.zmq_topic, "UTF-8"), message.data)), - flags=zmq.NOBLOCK, - ) - - -class ZMQPollerSettings(ez.Settings): - read_addr: str - zmq_topic: str - poll_time: float = POLL_TIME - multipart: bool = False - - -class ZMQPollerState(ez.State): - context: zmq.asyncio.Context - socket: zmq.asyncio.Socket - monitor: zmq.asyncio.Socket - poller: zmq.Poller - - -class ZMQPollerUnit(ez.Unit): - """ - Represents a node in the graph which polls data from ZMQ. - Data polled from ZMQ are subsequently pushed to the rest of the - graph as a ZMQMessage. - - Args: - read_addr: The address from which ZMQ data should be polled. - zmq_topic: The ZMQ topic being polled. - timeout: - The maximum amount of time (in seconds) that should be - spent polling a ZMQ socket each time. Defaults to - FOREVER_POLL_TIME if not specified. - exit_condition: - An optional ZMQ event code specifying the event which, - if encountered by the monitor, should signal the termination - of this particular node's activity. - """ - - OUTPUT = ez.OutputStream(ZMQMessage) - SETTINGS: ZMQPollerSettings - STATE: ZMQPollerState - - def initialize(self) -> None: - self.STATE.context = zmq.asyncio.Context() - self.STATE.socket = self.STATE.context.socket(zmq.SUB) - self.STATE.monitor = self.STATE.socket.get_monitor_socket() - self.STATE.socket.connect(self.SETTINGS.read_addr) - self.STATE.socket.subscribe(self.SETTINGS.zmq_topic) - - self.STATE.poller = zmq.Poller() - self.STATE.poller.register(self.STATE.socket, zmq.POLLIN) - - self.socket_open = False - - def shutdown(self) -> None: - self.STATE.monitor.close() - self.STATE.socket.close() - self.STATE.context.term() - - @ez.task - async def socket_monitor(self) -> None: - while True: - monitor_result = await self.STATE.monitor.poll(100, zmq.POLLIN) - if monitor_result: - data = await self.STATE.monitor.recv_multipart() - evt = parse_monitor_message(data) - - event = evt["event"] - - if event == zmq.EVENT_CONNECTED: - self.socket_open = True - elif event == zmq.EVENT_CLOSED: - was_open = self.socket_open - self.socket_open = False - # ZMQ seems to be sending spurious CLOSED event when we - # try to connect before the source is running. Only give up - # if we were previously connected. If we give up now, we - # will never unblock zmq_publisher. - if was_open: - break - elif event in ( - zmq.EVENT_DISCONNECTED, - zmq.EVENT_MONITOR_STOPPED, - ): - self.socket_open = False - break - - @ez.publisher(OUTPUT) - async def zmq_publisher(self) -> AsyncGenerator: - # Wait for socket connection - while not self.socket_open: - await asyncio.sleep(POLL_TIME) - - while self.socket_open: - poll_result = await self.STATE.socket.poll( - self.SETTINGS.poll_time * 1000, zmq.POLLIN - ) - if poll_result: - if self.SETTINGS.multipart is True: - _, data = await self.STATE.socket.recv_multipart() - else: - data = await self.STATE.socket.recv() - yield self.OUTPUT, ZMQMessage(data) diff --git a/poetry.lock b/poetry.lock index e88893eb..c6522ae9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,44 @@ # This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +[[package]] +name = "alabaster" +version = "0.7.13" +description = "A configurable sidebar-enabled Sphinx theme" +optional = false +python-versions = ">=3.6" +files = [ + {file = "alabaster-0.7.13-py3-none-any.whl", hash = "sha256:1ee19aca801bbabb5ba3f5f258e4422dfa86f82f3e9cefb0859b283cdd7f62a3"}, + {file = "alabaster-0.7.13.tar.gz", hash = "sha256:a27a4a084d5e690e16e01e03ad2b2e552c61a65469419b907243193de1a84ae2"}, +] + +[[package]] +name = "babel" +version = "2.15.0" +description = "Internationalization utilities" +optional = false +python-versions = ">=3.8" +files = [ + {file = "Babel-2.15.0-py3-none-any.whl", hash = "sha256:08706bdad8d0a3413266ab61bd6c34d0c28d6e1e7badf40a2cebe67644e2e1fb"}, + {file = "babel-2.15.0.tar.gz", hash = "sha256:8daf0e265d05768bc6c7a314cf1321e9a123afc328cc635c18622a2f30a04413"}, +] + +[package.dependencies] +pytz = {version = ">=2015.7", markers = "python_version < \"3.9\""} + +[package.extras] +dev = ["freezegun (>=1.0,<2.0)", "pytest (>=6.0)", "pytest-cov"] + +[[package]] +name = "certifi" +version = "2024.6.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.6.2-py3-none-any.whl", hash = "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56"}, + {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"}, +] + [[package]] name = "cffi" version = "1.16.0" @@ -64,6 +103,105 @@ files = [ [package.dependencies] pycparser = "*" +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + [[package]] name = "colorama" version = "0.4.6" @@ -142,6 +280,17 @@ tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.1 [package.extras] toml = ["tomli"] +[[package]] +name = "docutils" +version = "0.20.1" +description = "Docutils -- Python Documentation Utilities" +optional = false +python-versions = ">=3.7" +files = [ + {file = "docutils-0.20.1-py3-none-any.whl", hash = "sha256:96f387a2c5562db4476f09f13bbab2192e764cac08ebbf3a34a95d9b1e4a59d6"}, + {file = "docutils-0.20.1.tar.gz", hash = "sha256:f08a4e276c3a1583a86dce3e34aba3fe04d02bba2dd51ed16106244e8a923e3b"}, +] + [[package]] name = "exceptiongroup" version = "1.2.0" @@ -160,7 +309,7 @@ test = ["pytest (>=6)"] name = "ezmsg-sigproc" version = "1.2.2" description = "Timeseries signal processing implementations in ezmsg" -optional = true +optional = false python-versions = ">=3.8" files = [ {file = "ezmsg-sigproc-1.2.2.tar.gz", hash = "sha256:0ce5f8bdcfbc32be8085f2edec7e464afc754aaa7b0b2dae25ac28e18a623bd1"}, @@ -272,6 +421,47 @@ files = [ {file = "hsluv-5.0.4.tar.gz", hash = "sha256:2281f946427a882010042844a38c7bbe9e0d0aaf9d46babe46366ed6f169b72e"}, ] +[[package]] +name = "idna" +version = "3.7" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, + {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, +] + +[[package]] +name = "imagesize" +version = "1.4.1" +description = "Getting image size from png/jpeg/jpeg2000/gif file" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "imagesize-1.4.1-py2.py3-none-any.whl", hash = "sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b"}, + {file = "imagesize-1.4.1.tar.gz", hash = "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a"}, +] + +[[package]] +name = "importlib-metadata" +version = "7.1.0" +description = "Read metadata from Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "importlib_metadata-7.1.0-py3-none-any.whl", hash = "sha256:30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570"}, + {file = "importlib_metadata-7.1.0.tar.gz", hash = "sha256:b78938b926ee8d5f020fc4772d487045805a55ddbad2ecf21c6d60938dc7fcd2"}, +] + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +perf = ["ipython"] +testing = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-perf (>=0.9.2)", "pytest-ruff (>=0.2.1)"] + [[package]] name = "iniconfig" version = "2.0.0" @@ -283,6 +473,23 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] +[[package]] +name = "jinja2" +version = "3.1.4" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"}, + {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + [[package]] name = "kiwisolver" version = "1.4.5" @@ -396,6 +603,75 @@ files = [ {file = "kiwisolver-1.4.5.tar.gz", hash = "sha256:e57e563a57fb22a142da34f38acc2fc1a5c864bc29ca1517a88abc963e60d6ec"}, ] +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + [[package]] name = "mccabe" version = "0.7.0" @@ -503,6 +779,20 @@ files = [ {file = "pyflakes-2.5.0.tar.gz", hash = "sha256:491feb020dca48ccc562a8c0cbe8df07ee13078df59813b83959cbdada312ea3"}, ] +[[package]] +name = "pygments" +version = "2.18.0" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"}, + {file = "pygments-2.18.0.tar.gz", hash = "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199"}, +] + +[package.extras] +windows-terminal = ["colorama (>=0.4.6)"] + [[package]] name = "pyqt6" version = "6.6.1" @@ -621,6 +911,17 @@ pytest = ">=4.6" [package.extras] testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"] +[[package]] +name = "pytz" +version = "2024.1" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, +] + [[package]] name = "pyzmq" version = "25.1.2" @@ -762,11 +1063,32 @@ packaging = "*" [package.extras] test = ["pytest (>=6,!=7.0.0,!=7.0.1)", "pytest-cov (>=3.0.0)", "pytest-qt"] +[[package]] +name = "requests" +version = "2.32.3" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.8" +files = [ + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + [[package]] name = "scipy" version = "1.9.3" description = "Fundamental algorithms for scientific computing in Python" -optional = true +optional = false python-versions = ">=3.8" files = [ {file = "scipy-1.9.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1884b66a54887e21addf9c16fb588720a8309a57b2e258ae1c7986d4444d3bc0"}, @@ -800,6 +1122,174 @@ dev = ["flake8", "mypy", "pycodestyle", "typing_extensions"] doc = ["matplotlib (>2)", "numpydoc", "pydata-sphinx-theme (==0.9.0)", "sphinx (!=4.1.0)", "sphinx-panels (>=0.5.2)", "sphinx-tabs"] test = ["asv", "gmpy2", "mpmath", "pytest", "pytest-cov", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] +[[package]] +name = "snowballstemmer" +version = "2.2.0" +description = "This package provides 29 stemmers for 28 languages generated from Snowball algorithms." +optional = false +python-versions = "*" +files = [ + {file = "snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"}, + {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"}, +] + +[[package]] +name = "sphinx" +version = "7.1.2" +description = "Python documentation generator" +optional = false +python-versions = ">=3.8" +files = [ + {file = "sphinx-7.1.2-py3-none-any.whl", hash = "sha256:d170a81825b2fcacb6dfd5a0d7f578a053e45d3f2b153fecc948c37344eb4cbe"}, + {file = "sphinx-7.1.2.tar.gz", hash = "sha256:780f4d32f1d7d1126576e0e5ecc19dc32ab76cd24e950228dcf7b1f6d3d9e22f"}, +] + +[package.dependencies] +alabaster = ">=0.7,<0.8" +babel = ">=2.9" +colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} +docutils = ">=0.18.1,<0.21" +imagesize = ">=1.3" +importlib-metadata = {version = ">=4.8", markers = "python_version < \"3.10\""} +Jinja2 = ">=3.0" +packaging = ">=21.0" +Pygments = ">=2.13" +requests = ">=2.25.0" +snowballstemmer = ">=2.0" +sphinxcontrib-applehelp = "*" +sphinxcontrib-devhelp = "*" +sphinxcontrib-htmlhelp = ">=2.0.0" +sphinxcontrib-jsmath = "*" +sphinxcontrib-qthelp = "*" +sphinxcontrib-serializinghtml = ">=1.1.5" + +[package.extras] +docs = ["sphinxcontrib-websupport"] +lint = ["docutils-stubs", "flake8 (>=3.5.0)", "flake8-simplify", "isort", "mypy (>=0.990)", "ruff", "sphinx-lint", "types-requests"] +test = ["cython", "filelock", "html5lib", "pytest (>=4.6)"] + +[[package]] +name = "sphinx-rtd-theme" +version = "2.0.0" +description = "Read the Docs theme for Sphinx" +optional = false +python-versions = ">=3.6" +files = [ + {file = "sphinx_rtd_theme-2.0.0-py2.py3-none-any.whl", hash = "sha256:ec93d0856dc280cf3aee9a4c9807c60e027c7f7b461b77aeffed682e68f0e586"}, + {file = "sphinx_rtd_theme-2.0.0.tar.gz", hash = "sha256:bd5d7b80622406762073a04ef8fadc5f9151261563d47027de09910ce03afe6b"}, +] + +[package.dependencies] +docutils = "<0.21" +sphinx = ">=5,<8" +sphinxcontrib-jquery = ">=4,<5" + +[package.extras] +dev = ["bump2version", "sphinxcontrib-httpdomain", "transifex-client", "wheel"] + +[[package]] +name = "sphinxcontrib-applehelp" +version = "1.0.4" +description = "sphinxcontrib-applehelp is a Sphinx extension which outputs Apple help books" +optional = false +python-versions = ">=3.8" +files = [ + {file = "sphinxcontrib-applehelp-1.0.4.tar.gz", hash = "sha256:828f867945bbe39817c210a1abfd1bc4895c8b73fcaade56d45357a348a07d7e"}, + {file = "sphinxcontrib_applehelp-1.0.4-py3-none-any.whl", hash = "sha256:29d341f67fb0f6f586b23ad80e072c8e6ad0b48417db2bde114a4c9746feb228"}, +] + +[package.extras] +lint = ["docutils-stubs", "flake8", "mypy"] +test = ["pytest"] + +[[package]] +name = "sphinxcontrib-devhelp" +version = "1.0.2" +description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp document." +optional = false +python-versions = ">=3.5" +files = [ + {file = "sphinxcontrib-devhelp-1.0.2.tar.gz", hash = "sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4"}, + {file = "sphinxcontrib_devhelp-1.0.2-py2.py3-none-any.whl", hash = "sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e"}, +] + +[package.extras] +lint = ["docutils-stubs", "flake8", "mypy"] +test = ["pytest"] + +[[package]] +name = "sphinxcontrib-htmlhelp" +version = "2.0.1" +description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files" +optional = false +python-versions = ">=3.8" +files = [ + {file = "sphinxcontrib-htmlhelp-2.0.1.tar.gz", hash = "sha256:0cbdd302815330058422b98a113195c9249825d681e18f11e8b1f78a2f11efff"}, + {file = "sphinxcontrib_htmlhelp-2.0.1-py3-none-any.whl", hash = "sha256:c38cb46dccf316c79de6e5515e1770414b797162b23cd3d06e67020e1d2a6903"}, +] + +[package.extras] +lint = ["docutils-stubs", "flake8", "mypy"] +test = ["html5lib", "pytest"] + +[[package]] +name = "sphinxcontrib-jquery" +version = "4.1" +description = "Extension to include jQuery on newer Sphinx releases" +optional = false +python-versions = ">=2.7" +files = [ + {file = "sphinxcontrib-jquery-4.1.tar.gz", hash = "sha256:1620739f04e36a2c779f1a131a2dfd49b2fd07351bf1968ced074365933abc7a"}, + {file = "sphinxcontrib_jquery-4.1-py2.py3-none-any.whl", hash = "sha256:f936030d7d0147dd026a4f2b5a57343d233f1fc7b363f68b3d4f1cb0993878ae"}, +] + +[package.dependencies] +Sphinx = ">=1.8" + +[[package]] +name = "sphinxcontrib-jsmath" +version = "1.0.1" +description = "A sphinx extension which renders display math in HTML via JavaScript" +optional = false +python-versions = ">=3.5" +files = [ + {file = "sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8"}, + {file = "sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178"}, +] + +[package.extras] +test = ["flake8", "mypy", "pytest"] + +[[package]] +name = "sphinxcontrib-qthelp" +version = "1.0.3" +description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp document." +optional = false +python-versions = ">=3.5" +files = [ + {file = "sphinxcontrib-qthelp-1.0.3.tar.gz", hash = "sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72"}, + {file = "sphinxcontrib_qthelp-1.0.3-py2.py3-none-any.whl", hash = "sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6"}, +] + +[package.extras] +lint = ["docutils-stubs", "flake8", "mypy"] +test = ["pytest"] + +[[package]] +name = "sphinxcontrib-serializinghtml" +version = "1.1.5" +description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)." +optional = false +python-versions = ">=3.5" +files = [ + {file = "sphinxcontrib-serializinghtml-1.1.5.tar.gz", hash = "sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952"}, + {file = "sphinxcontrib_serializinghtml-1.1.5-py2.py3-none-any.whl", hash = "sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd"}, +] + +[package.extras] +lint = ["docutils-stubs", "flake8", "mypy"] +test = ["pytest"] + [[package]] name = "tomli" version = "2.0.1" @@ -822,6 +1312,23 @@ files = [ {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, ] +[[package]] +name = "urllib3" +version = "2.2.1" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, + {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + [[package]] name = "vispy" version = "0.14.1" @@ -959,6 +1466,21 @@ files = [ {file = "websockets-12.0.tar.gz", hash = "sha256:81df9cbcbb6c260de1e007e58c011bfebe2dafc8435107b0537f393dd38c8b1b"}, ] +[[package]] +name = "zipp" +version = "3.19.2" +description = "Backport of pathlib-compatible object wrapper for zip files" +optional = false +python-versions = ">=3.8" +files = [ + {file = "zipp-3.19.2-py3-none-any.whl", hash = "sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c"}, + {file = "zipp-3.19.2.tar.gz", hash = "sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19"}, +] + +[package.extras] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] + [extras] all-ext = ["ezmsg-sigproc", "ezmsg-vispy", "ezmsg-websocket", "ezmsg-zmq"] sigproc = ["ezmsg-sigproc"] @@ -969,4 +1491,4 @@ zmq = ["ezmsg-zmq"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "9bb31dfcc40f477db41113f8a4859623995ebe56953c2eba70531968d1b346ea" +content-hash = "ba3e62a6e7a812d508784cb7cd9cb627d05db306fda4a5ee44be0a1f56848e7d" diff --git a/pyproject.toml b/pyproject.toml index ea27bdb8..18049e08 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,12 @@ pytest-cov = "*" numpy = "^1.24.2" flake8 = "*" + +[tool.poetry.group.docs.dependencies] +sphinx = "<7.2" +sphinx-rtd-theme = "^2.0.0" +ezmsg-sigproc = { version = "*", source = "pypi" } + [tool.poetry.extras] sigproc = ["ezmsg-sigproc"] websocket = ["ezmsg-websocket"] diff --git a/src/ezmsg/core/backend.py b/src/ezmsg/core/backend.py index 29a1da5d..1752e1f5 100644 --- a/src/ezmsg/core/backend.py +++ b/src/ezmsg/core/backend.py @@ -167,6 +167,27 @@ def run( force_single_process: bool = False, **components_kwargs: Component, ) -> None: + """ + Begin execution of a set of :obj:`Component` s. + + `The old method` :obj:`run_system` `has been deprecated and uses` ``run()`` `instead.` + + Args: + components: represents the nodes in the directed acyclic graph. It is a dictionary which contains the + ``Components`` to be run mapped to string names. On initialization, ``ezmsg`` will call ``initialize()`` + for each :obj:`Unit` and ``configure()`` for each :obj:`Collection`, if defined. + root_name: + connections: represents the edges is a ``NetworkDefinition`` which connects + ``OutputStreams`` to ``InputStreams``. On initialization, ``ezmsg`` will create a directed acyclic graph + using the contents of this parameter. + process_components: a list of ``Components`` which should live in their own process. + backend_process: is currently under development. + graph_address: the hostname and port of the graph server which ``ezmsg`` should connect to. + If not defined, ``ezmsg`` will start a new graph server at 127.0.0.1:25978. + force_single_process: run all ``Components`` in one process. + This is necessary when running ``ezmsg`` in a notebook. + components_kwargs: + """ # FIXME: This function is the last major re-implementation needed to make this # codebase more maintainable. graph_service = GraphService(graph_address) diff --git a/src/ezmsg/core/backendprocess.py b/src/ezmsg/core/backendprocess.py index 487e78c3..e185cc2f 100644 --- a/src/ezmsg/core/backendprocess.py +++ b/src/ezmsg/core/backendprocess.py @@ -41,10 +41,17 @@ class Complete(Exception): + """ + A type of ``Exception`` which signals to ``ezmsg`` that the function can be shut down gracefully. + If all functions in all :obj:`Units` raise ``Complete``, the entire pipeline will terminate execution. + """ pass class NormalTermination(Exception): + """ + A type of ``Exception`` which signals to ``ezmsg`` that the pipeline can be shut down gracefully. + """ pass @@ -294,32 +301,28 @@ async def perf_publish(stream: Stream, obj: Any) -> None: ) pub_fn = perf_publish if hasattr(task, TIMEIT_ATTR) else publish + + call_fn = lambda _: task(unit) + signature = inspect.signature(task) + if len(signature.parameters) == 1: + call_fn = lambda _: task(unit) + elif len(signature.parameters) == 2: + call_fn = lambda msg: task(unit, msg) + else: + logger.error(f'Incompatible call signature on task: {task.__name__}') @wraps(task) async def wrapped_task(msg: Any = None) -> None: try: - # If we don't sub or pub anything, we are a simple task - if not hasattr(task, SUBSCRIBES_ATTR) and not hasattr( - task, PUBLISHES_ATTR - ): - await task(unit) - - # No subscriptions; only publications... - elif not hasattr(task, SUBSCRIBES_ATTR): - async for stream, obj in task(unit): + result = call_fn(msg) + if inspect.isasyncgen(result): + async for stream, obj in result: + if obj and getattr(task, ZERO_COPY_ATTR, False) and obj is msg: + obj = deepcopy(obj) await pub_fn(stream, obj) - # Subscribers need to be called with a message - else: - if not getattr(task, ZERO_COPY_ATTR): - msg = deepcopy(msg) - if hasattr(task, PUBLISHES_ATTR): - async for stream, obj in task(unit, msg): - if getattr(task, ZERO_COPY_ATTR) and obj is msg: - obj = deepcopy(obj) - await pub_fn(stream, obj) - else: - await task(unit, msg) + elif asyncio.iscoroutine(result): + await result except Complete: logger.info(f"{task_address} Complete") diff --git a/src/ezmsg/core/collection.py b/src/ezmsg/core/collection.py index 969df212..7de4dd0c 100644 --- a/src/ezmsg/core/collection.py +++ b/src/ezmsg/core/collection.py @@ -34,7 +34,9 @@ def __init__( class Collection(Component, metaclass=CollectionMeta): - """Collections can contain subunits and connect them together""" + """ + Connects :obj:`Unit` s together by defining a graph which connects :obj:`OutputStream` s to :obj:`InputStream` s. + """ def __init__(self, *args, settings: typing.Optional[Settings] = None, **kwargs): super(Collection, self).__init__(*args, settings=settings, **kwargs) @@ -44,11 +46,24 @@ def __init__(self, *args, settings: typing.Optional[Settings] = None, **kwargs): setattr(self, comp_name, comp) def configure(self) -> None: - """This is where to percolate apply_settings to subnodes""" + """ + A lifecycle hook that runs when the :obj:`Collection` is instantiated. + This is the best place to call ``Unit.apply_settings()`` on each member :obj:`Unit` of the :obj:`Collection`. + """ ... def network(self) -> NetworkDefinition: + """ + Override this method and have the definition return a :obj:`NetworkDefinition` which defines how + :obj:`InputStream` and :obj:`OutputStream` from member :obj:`Unit` s will be connected. + """ return () def process_components(self) -> typing.Collection[Component]: + """ + Override this method and have the definition return a tuple which contains :obj:`Unit` and :obj:`Collection` + which should run in their own processes. + + Return: the :obj:`Collection`. + """ return (self,) diff --git a/src/ezmsg/core/command.py b/src/ezmsg/core/command.py index 81cfa8b8..9c25e971 100644 --- a/src/ezmsg/core/command.py +++ b/src/ezmsg/core/command.py @@ -1,4 +1,5 @@ import os +import sys import base64 import asyncio import argparse @@ -97,7 +98,7 @@ async def run_command(cmd: str, graph_address: Address, shm_address: Address) -> elif cmd == "start": popen = subprocess.Popen( - ["python", "-m", "ezmsg.core", "serve", f"--address={graph_address}"] + [sys.executable, "-m", "ezmsg.core", "serve", f"--address={graph_address}"] ) while True: diff --git a/src/ezmsg/core/component.py b/src/ezmsg/core/component.py index 0989853d..d0f187ff 100644 --- a/src/ezmsg/core/component.py +++ b/src/ezmsg/core/component.py @@ -72,6 +72,10 @@ def __init__( class Component(Addressable, metaclass=ComponentMeta): + """ + Metaclass which :obj:`Unit` and :obj:`Collection` inherit from. + """ + SETTINGS: Settings STATE: State @@ -96,21 +100,21 @@ def __init__(self, *args, settings: Optional[Settings] = None, **kwargs): for stream_name, stream in self.streams.items(): setattr(self, stream_name, stream) - try: - if settings is None: - # settings not supplied as a kwarg. Try to build it. - if len(args) > 0 and type(args[0]) == self.__class__.__settings_type__: - settings = args[0] - elif len(args) > 0 or len(kwargs) > 0: - settings = self.__class__.__settings_type__(*args, **kwargs) - else: + if settings is None: + # settings not supplied as a kwarg. Try to build it. + if len(args) > 0 and type(args[0]) == self.__class__.__settings_type__: + settings = args[0] + elif len(args) > 0 or len(kwargs) > 0: + settings = self.__class__.__settings_type__(*args, **kwargs) + else: + try: # If we weren't supplied settings, we will try to # instantiate the settings type from annotations settings = self.__class__.__settings_type__() - except TypeError: - # We couldn't instantiate settings with default value - # We will rely on late configuration via apply_settings - pass + except TypeError: + # We couldn't instantiate settings with default value + # We will rely on late configuration via apply_settings + pass if settings is not None: self.apply_settings(settings) @@ -132,6 +136,13 @@ def _check_state(self) -> None: ) def apply_settings(self, settings: Settings) -> None: + """ + Update the ``Component``‘s ``Settings`` object. + + Args: + settings: An instance of the class-specific ``Settings``. + + """ self.SETTINGS = settings self._settings_applied = True diff --git a/src/ezmsg/core/settings.py b/src/ezmsg/core/settings.py index d35f3e41..9bc6203b 100644 --- a/src/ezmsg/core/settings.py +++ b/src/ezmsg/core/settings.py @@ -28,4 +28,38 @@ def __new__( class Settings(ABC, metaclass=SettingsMeta): + """ + To pass parameters into a :obj:`Component`, inherit from ``Settings``. + + .. code-block:: python + + class YourSettings(Settings): + setting1: int + setting2: float + + To use, declare the ``Settings`` object for a ``Component`` as a member variable called (all-caps!) ``SETTINGS``. ``ezmsg`` will monitor the variable called ``SETTINGS`` in the background, so it is important to name it correctly. + + .. code-block:: python + + class YourUnit(Unit): + + SETTINGS: YourSettings + + A ``Unit`` can accept a ``Settings`` object as a parameter on instantiation. + + .. code-block:: python + + class YourCollection(Collection): + + YOUR_UNIT = YourUnit( + YourSettings( + setting1: int, + setting2: float + ) + ) + + .. note:: + ``Settings`` uses type hints to define member variables, but does not enforce type checking. + + """ ... diff --git a/src/ezmsg/core/state.py b/src/ezmsg/core/state.py index 2cf48ac7..b4937d29 100644 --- a/src/ezmsg/core/state.py +++ b/src/ezmsg/core/state.py @@ -24,6 +24,33 @@ def __new__( class State(ABC, metaclass=StateMeta): """ States are mutable dataclasses that are instantiated by the Unit in its home process. - """ + To track a mutable state for a ``Component``, inherit from ``State``. + + .. code-block:: python + + class YourState(State): + state1: int + state2: float + + To use, declare the ``State`` object for a ``Component`` as a member variable called (all-caps!) ``STATE``. + ``ezmsg`` will monitor the variable called ``STATE`` in the background, so it is important to name it correctly. + + Member functions can then access and mutate ``STATE`` as needed during function execution. + It is recommended to initialize state values inside the ``initialize()`` or ``configure()`` lifecycle hooks if + defaults are not defined. + + .. code-block:: python + + class YourUnit(Unit): + + STATE: YourState + + def initialize(self): + this.STATE.state1 = 0 + this.STATE.state2 = 0.0 + + .. note:: + ``State`` uses type hints to define member variables, but does not enforce type checking. + """ ... diff --git a/src/ezmsg/core/stream.py b/src/ezmsg/core/stream.py index 92671b44..7b5e8414 100644 --- a/src/ezmsg/core/stream.py +++ b/src/ezmsg/core/stream.py @@ -5,6 +5,10 @@ class Stream(Addressable): + """ + + """ + msg_type: Type def __init__(self, msg_type: Type): @@ -17,11 +21,17 @@ def __repr__(self) -> str: class InputStream(Stream): + """ + Can be added to any ``Component`` as a member variable. Methods may subscribe to it. + """ def __repr__(self) -> str: return f"Input{super().__repr__()}()" class OutputStream(Stream): + """ + Can be added to any ``Component`` as a member variable. Methods may publish to it. + """ host: Optional[str] port: Optional[int] num_buffers: int diff --git a/src/ezmsg/core/unit.py b/src/ezmsg/core/unit.py index 45d12aef..05bc5d73 100644 --- a/src/ezmsg/core/unit.py +++ b/src/ezmsg/core/unit.py @@ -59,7 +59,11 @@ def __init__( class Unit(Component, metaclass=UnitMeta): - """Units can subscribe, publish, and have tasks""" + """ + Represents a single step in the graph. + Units can subscribe, publish, and have tasks. + To create a ``Unit``, inherit from the ``Unit`` class. + """ def __init__(self, *args, settings: Optional[Settings] = None, **kwargs): super(Unit, self).__init__(*args, settings=settings, **kwargs) @@ -85,16 +89,42 @@ async def setup(self): self._check_state() async def initialize(self) -> None: - """This is called from within the same process this unit will live""" + """ + Runs when the ``Unit`` is instantiated. + This is called from within the same process this unit will live. + This lifecycle hook can be overridden. It can be run as ``async`` functions by simply adding the + ``async`` keyword when overriding. + """ pass async def shutdown(self) -> None: - """This is called from within the same process this unit will live""" + """ + Runs when the ``Unit`` terminates. + This is called from within the same process this unit will live. + This lifecycle hook can be overridden. It can be run as ``async`` functions by simply adding the + ``async`` keyword when overriding. + """ pass def publisher(stream: OutputStream): - """A decorator for a method that publishes to a stream in the task/messaging thread""" + """ + A decorator for a method that publishes to a stream in the task/messaging thread. + An async function will yield messages on the designated :obj:`OutputStream`. + + .. code-block:: python + + from typing import AsyncGenerator + + OUTPUT = OutputStream(ez.Message) + + @publisher(OUTPUT) + async def send_message(self) -> AsyncGenerator: + message = Message() + yield(OUTPUT, message) + + A function can have both ``@subscriber`` and ``@publisher`` decorators. + """ if not isinstance(stream, OutputStream): raise ValueError(f"Cannot publish to object of type {type(stream)}") @@ -109,7 +139,22 @@ def pub_factory(func): def subscriber(stream: InputStream, zero_copy: bool = False): - """A decorator for a method that subscribes to a stream in the task/messaging thread""" + """ + A decorator for a method that subscribes to a stream in the task/messaging thread. + An async function will run once per message received from the :obj:`InputStream` it subscribes to. + + Example: + + .. code-block:: python + + INPUT = ez.InputStream(Message) + + @subscriber(INPUT) + async def print_message(self, message: Message) -> None: + print(message) + + A function can have both ``@subscriber`` and ``@publisher`` decorators. + """ if not isinstance(stream, InputStream): raise ValueError(f"Cannot subscribe to object of type {type(stream)}") @@ -126,12 +171,18 @@ def sub_factory(func): def main(func: Callable): - """A decorator for a function that runs as the main thread. A Unit may only have one of these.""" + """ + A decorator which designates this function to run as the main thread for this :obj:`Unit`. + A :obj:`Unit` may only have one of these. + """ setattr(func, MAIN_ATTR, True) return func def timeit(func: Callable): + """ + ``ezmsg`` will log the amount of time this function takes to execute. + """ setattr(func, TIMEIT_ATTR, True) @functools.wraps(func) @@ -148,17 +199,24 @@ def wrapper(self, *args, **kwargs): def thread(func: Callable): - """A decorator for a function that runs in a background thread""" + """ + A decorator which designates this function to run as a background thread for this `:obj:`Unit`. + """ setattr(func, THREAD_ATTR, True) return func def task(func: Callable): - """A decorator for a function that runs as a task in the task/messaging thread""" + """ + A decorator which designates this function to run as a task in the task/messaging thread. + """ setattr(func, TASK_ATTR, True) return func def process(func: Callable): + """ + A decorator which designates this function to run in its own process. + """ setattr(func, PROCESS_ATTR, True) return func diff --git a/src/ezmsg/util/debuglog.py b/src/ezmsg/util/debuglog.py index a3757b22..6ad3ccdc 100644 --- a/src/ezmsg/util/debuglog.py +++ b/src/ezmsg/util/debuglog.py @@ -4,15 +4,31 @@ class DebugLogSettings(ez.Settings): - name: str = "DEBUG" # Useful name for logger - max_length: Optional[int] = 400 # No limit if `None`` + """ + ``Settings`` class associated with :obj:`DebugLog` + + Args: + name: Useful name for the logger. The name is included in the logstring so that if multiple DebugLogs + are used in one pipeline, their messages can be differentiated. + max_length: Sets a maximum number of chars which will be printed from the message. + If the message is longer, the log message will be truncated. + """ + name: str = "DEBUG" + max_length: Optional[int] = 400 class DebugLog(ez.Unit): + """ + Logs messages that pass through. + """ + SETTINGS: DebugLogSettings INPUT = ez.InputStream(Any) + """Send messages to log here.""" + OUTPUT = ez.OutputStream(Any) + """Send messages back out to continue through the graph.""" @ez.subscriber(INPUT, zero_copy=True) @ez.publisher(OUTPUT) diff --git a/src/ezmsg/util/generator.py b/src/ezmsg/util/generator.py index 1d3500d8..72801296 100644 --- a/src/ezmsg/util/generator.py +++ b/src/ezmsg/util/generator.py @@ -94,6 +94,7 @@ class GenAxisArray(ez.Unit): INPUT_SIGNAL = ez.InputStream(AxisArray) OUTPUT_SIGNAL = ez.OutputStream(AxisArray) + INPUT_SETTINGS = ez.InputStream(ez.Settings) def initialize(self) -> None: self.construct_generator() @@ -102,6 +103,11 @@ def initialize(self) -> None: def construct_generator(self): raise NotImplementedError + @ez.subscriber(INPUT_SETTINGS) + async def on_settings(self, msg: ez.Settings) -> None: + self.apply_settings(msg) + self.construct_generator() + @ez.subscriber(INPUT_SIGNAL) @ez.publisher(OUTPUT_SIGNAL) async def on_message(self, message: AxisArray) -> AsyncGenerator: diff --git a/src/ezmsg/util/messagegate.py b/src/ezmsg/util/messagegate.py index 9149e224..94a95e84 100644 --- a/src/ezmsg/util/messagegate.py +++ b/src/ezmsg/util/messagegate.py @@ -6,14 +6,23 @@ @dataclass class GateMessage: + """Send this message to ``INPUT_GATE`` to open or close the gate.""" open: bool class MessageGateSettings(ez.Settings): + """ + Settings for :obj:`MessageGate` unit. + + Args: + start_open: sets the gate's initial state to allow messages to flow through or be discarded. ``True`` will + allow messages to flow through initially, ``False`` will discard messages initially. + default_open: sets the gate's behavior after the `default_after` number of messages have flowed through. + ``True`` will allow messages to flow through, ``False`` will discard messages. + default_after: sets the number of messages after which the `default_open` state will be applied. + """ start_open: bool = False default_open: bool = False - - # Automatically change back to default state after X messages default_after: typing.Optional[int] = None @@ -23,13 +32,25 @@ class MessageGateState(ez.State): class MessageGate(ez.Unit): + """ + Blocks ``Messages`` from continuing through the system. + Can be set as open, closed, open after n messages, or closed after n messages. + """ + SETTINGS: MessageGateSettings STATE: MessageGateState INPUT_GATE = ez.InputStream(GateMessage) + """ + Stop or start message flow. If ``GateMessage.open == True``, messages will flow through. + If ``GateMessage.open == False``, messages will be discarded. + """ INPUT = ez.InputStream(typing.Any) + """Messages which will flow through or be discarded, depending on gate status.""" + OUTPUT = ez.OutputStream(typing.Any) + """Publishes messages which flow through.""" def initialize(self) -> None: self.STATE.gate_open = self.SETTINGS.start_open diff --git a/src/ezmsg/util/messagelogger.py b/src/ezmsg/util/messagelogger.py index 3ad3f9e4..52c9c45c 100644 --- a/src/ezmsg/util/messagelogger.py +++ b/src/ezmsg/util/messagelogger.py @@ -17,6 +17,13 @@ def log_object(obj: Any) -> str: class MessageLoggerSettings(ez.Settings): + """ + Settings for :obj:`MessageLogger` Unit. + + Args: + output: :py:class:`pathlib.Path` for a file where the messages will be logged. + If the file path already exists, the existing file will be truncated to 0 length. + """ output: Optional[Path] = None @@ -25,15 +32,41 @@ class MessageLoggerState(ez.State): class MessageLogger(ez.Unit): + """ + Logs all messages it receives to a file. + File path can be set in ``SETTINGS`` or set dynamically by passing a + :py:class:`pathlib.Path` to ``INPUT_START``. + """ + SETTINGS: MessageLoggerSettings STATE: MessageLoggerState INPUT_START = ez.InputStream(Path) + """ + Pass a :py:class:`pathlib.Path` + to begin logging messages to that path. If the file path already exists, the existing + file will be truncated to 0 length. If the file is already open, nothing will happen. + """ + INPUT_STOP = ez.InputStream(Path) + """ + Pass a :py:class:`pathlib.Path` + to stop logging messages to that path. + """ + INPUT_MESSAGE = ez.InputStream(Any) + """Pass a piece of data to log it to every open file which the ``MessageLogger`` is using.""" + OUTPUT_MESSAGE = ez.OutputStream(Any) + """Messages which are sent to ``INPUT_MESSAGE`` will pass through and be published on ``OUTPUT_MESSAGE``.""" + OUTPUT_START = ez.OutputStream(Path) + """If a file passed to ``INPUT_START`` is successfully opened, its path will be published to + ``OUTPUT_START``, otherwise ``None``.""" + OUTPUT_STOP = ez.OutputStream(Path) + """If a file passed to ``INPUT_STOP`` is successfully closed, its path will be published to + ``OUTPUT_STOP``, otherwise ``None``.""" def open_file(self, filepath: Path) -> Optional[Path]: """Returns file path if file successfully opened, otherwise None""" diff --git a/src/ezmsg/util/messagequeue.py b/src/ezmsg/util/messagequeue.py index 0f030cfa..bffac8a8 100644 --- a/src/ezmsg/util/messagequeue.py +++ b/src/ezmsg/util/messagequeue.py @@ -5,6 +5,13 @@ class MessageQueueSettings(ez.Settings): + """ + Settings for :obj:`MessageQueue` class. + + Args: + maxsize: The maximum number of items which the queue will hold. + leaky: Whether the queue will drop new messages when it reaches its maxsize, or whether it will wait for space to open for them. + """ maxsize: int = 0 leaky: bool = False log_above_n: Optional[int] = None @@ -17,11 +24,18 @@ class MessageQueueState(ez.State): class MessageQueue(ez.Unit): + """ + Place between two other ``Units`` to induce backpressure. + """ + SETTINGS: MessageQueueSettings STATE: MessageQueueState INPUT = ez.InputStream(Any) + """Send messages to queue here.""" + OUTPUT = ez.OutputStream(Any) + """Subscribe to pull messages out of the queue.""" def initialize(self): self.STATE.leaky = self.SETTINGS.leaky diff --git a/src/ezmsg/util/messagereplay.py b/src/ezmsg/util/messagereplay.py index 355d2f3c..97e1b094 100644 --- a/src/ezmsg/util/messagereplay.py +++ b/src/ezmsg/util/messagereplay.py @@ -13,6 +13,15 @@ @dataclass class ReplayStatusMessage: + """ + Message which gives the status of a file replay. + + Args: + filename: The name of the file currently being replayed. + idx: The line number of the message that was just published. + total: Number of messages in the file. + done: Whether the file has finished replaying. + """ filename: Path idx: int total: int @@ -21,13 +30,26 @@ class ReplayStatusMessage: @dataclass class FileReplayMessage: + """ + Add a file to the queue. + + Args: + filename: The path of the file to replay. + rate: in Hertz at which the messages will be published. + 0 = realtime (if timestamps in file) + If not specified, messages will publish as fast as possible. + """ filename: typing.Optional[Path] = None - - # 0 = realtime (if timestamps in file), None = as fast as possible rate: typing.Optional[float] = None # Hz class MessageReplaySettings(ez.Settings, FileReplayMessage): + """ + Settings for :obj:`MesssageReplay` Unit. + + Args: + progress: will use tqdm to indicate progress through the file. tqdm must be installed. + """ progress: bool = False @@ -38,17 +60,37 @@ class MessageReplayState(ez.State): class MessageReplay(ez.Unit): + """ + Stream messages from files created by :obj:`MessageLogger`. + Stores a queue of files to stream and streams from them in order. + """ + SETTINGS: MessageReplaySettings STATE: MessageReplayState INPUT_FILE = ez.InputStream(FileReplayMessage) - INPUT_PAUSED = ez.InputStream(bool) # Pause state; True = paused, False = running - INPUT_STOP = ez.InputStream(bool) # True = clear queue + """Add a new file to the queue.""" + + INPUT_PAUSED = ez.InputStream(bool) + """Send ``True`` to pause the stream, ``False`` to restart the stream.""" + + INPUT_STOP = ez.InputStream(bool) + """ + Stop the stream. Send ``True`` to also clear the queue. + Send ``False`` to reset to the beginning of the current file. + """ OUTPUT_MESSAGE = ez.OutputStream(typing.Any) + """The output on which the messages from the files will be streamed.""" + OUTPUT_TOTAL = ez.OutputStream(int) + """ + Publishes an integer total of messages which have been published on OUTPUT_MESSAGE from a single file. + Resets when a file completes. + """ OUTPUT_REPLAY_STATUS = ez.OutputStream(ReplayStatusMessage) + """Publishes status messages.""" async def initialize(self) -> None: self.STATE.replay_files = asyncio.Queue() @@ -167,10 +209,17 @@ class MessageCollectorState(ez.State): class MessageCollector(ez.Unit): + """ + Collects ``Messages`` into a local list. + """ + STATE: MessageCollectorState INPUT_MESSAGE = ez.InputStream(typing.Any) + """Send messages here to be collected.""" + OUTPUT_MESSAGE = ez.OutputStream(typing.Any) + """Messages will pass straight through after being recorded and be published here.""" @ez.subscriber(INPUT_MESSAGE) @ez.publisher(OUTPUT_MESSAGE) @@ -180,4 +229,9 @@ async def on_message(self, msg: typing.Any) -> typing.AsyncGenerator: @property def messages(self) -> typing.List[typing.Any]: + """ + Access the list of messages. + + :return: A list of messages which have been collected. + """ return self.STATE.messages diff --git a/src/ezmsg/util/messages/axisarray.py b/src/ezmsg/util/messages/axisarray.py index 6d603a3a..6d0b5f2c 100644 --- a/src/ezmsg/util/messages/axisarray.py +++ b/src/ezmsg/util/messages/axisarray.py @@ -12,10 +12,14 @@ from ezmsg.core.util import either_dict_or_kwargs # TODO: Typehinting is all wrong in this and -# concatenate/transpose should probably not be staticmethods +# concatenate/transpose should probably not be staticmethods + @dataclass class AxisArray: + """ + A lightweight message class comprising a numpy ndarray and its metadata. + """ data: npt.NDArray dims: typing.List[str] axes: typing.Dict[str, "AxisArray.Axis"] = field(default_factory=dict) @@ -206,22 +210,24 @@ def transpose( def slice_along_axis(in_arr: npt.NDArray, sl: typing.Union[slice, int], axis: int) -> npt.NDArray: """ Slice the input array along a specified axis using the given slice object or integer index. + Integer arguments to `sl` will cause the sliced dimension to be dropped. + Use `slice(my_int, my_int+1, None)` to keep the sliced dimension. Parameters: - in_arr (npt.NDArray): The input array to be sliced. - sl (Union[slice, int]): The slice object or integer index to use for slicing. - axis (int): The axis along which to slice the array. + in_arr: The input array to be sliced. + sl: The slice object or integer index to use for slicing. + axis: The axis along which to slice the array. Returns: - npt.NDArray: The sliced array (view). + The sliced array (view). Raises: ValueError: If the axis value is invalid for the input array. """ if axis < -in_arr.ndim or axis >= in_arr.ndim: raise ValueError(f"Invalid axis value {axis} for input array with {in_arr.ndim} dimensions.") - if axis < 0: - axis = in_arr.ndim + axis + if -in_arr.ndim <= axis < 0: + axis = in_arr.ndim + axis all_slice = (slice(None),) * axis + (sl,) + (slice(None),) * (in_arr.ndim - axis - 1) return in_arr[all_slice] @@ -229,23 +235,36 @@ def slice_along_axis(in_arr: npt.NDArray, sl: typing.Union[slice, int], axis: in def sliding_win_oneaxis(in_arr: npt.NDArray, nwin: int, axis: int) -> npt.NDArray: """ Generates a view of an array using a sliding window of specified length along a specified axis of the input array. - This is a slightly optimized version of nps.sliding_window_view with a few important differences. - Because we only accept a single nwin and a single axis, we can skip some checks. - The new `win` axis precedes immediately the original target axis, unlike sliding_window_view where the - target axis is moved to the end of the output. + This is a slightly optimized version of nps.sliding_window_view with a few important differences: - Parameters: - in_arr (npt.NDArray): The input array. - nwin (int): The size of the sliding window. - axis (int): The axis along which the sliding window will be applied. + - This only accepts a single nwin and a single axis, thus we can skip some checks. + - The new `win` axis precedes immediately the original target axis, unlike sliding_window_view where the + target axis is moved to the end of the output. + + Combine this with slice_along_axis(..., sl=slice(None, None, step), axis=axis) to step the window + by more than 1 sample at a time. + + Args: + in_arr: The input array. + nwin: The size of the sliding window. + axis: The axis along which the sliding window will be applied. Returns: - npt.NDArray: A view to the input array with the sliding window applied. + A view to the input array with the sliding window applied. + + Note: There is a known edge case when nwin == shape[axis] + 1. While this should raise + an error because the window is larger than the input, the implementation ends up + returning a 0-length window. We could check for this but this function is intended + to have minimal latency so we have decided to skip the checks and deal with the + support issues as they arise. """ + if -in_arr.ndim <= axis < 0: + axis = in_arr.ndim + axis out_strides = in_arr.strides[:axis] + (in_arr.strides[axis],) * 2 + in_arr.strides[axis+1:] out_shape = in_arr.shape[:axis] + (in_arr.shape[axis]-(nwin-1),) + (nwin,) + in_arr.shape[axis+1:] return nps.as_strided(in_arr, strides=out_strides, shape=out_shape, writeable=False) + def _as2d( in_arr: npt.NDArray, axis: int = 0 ) -> typing.Tuple[npt.NDArray, typing.Tuple[int]]: diff --git a/src/ezmsg/util/terminate.py b/src/ezmsg/util/terminate.py index 69ab52eb..bc3fa91c 100644 --- a/src/ezmsg/util/terminate.py +++ b/src/ezmsg/util/terminate.py @@ -9,8 +9,15 @@ class TerminateOnTimeoutSettings(ez.Settings): - time: float = 2.0 # Terminate if no message has been received in this time (sec) - poll_rate: float = 4.0 # Probably no good reason to mess with this (Hz) + """ + Settings for :obj:`TerminateOnTimeout` Unit. + + Args: + time: Terminate if no message has been received in this time (sec) + poll_rate: Hz. + """ + time: float = 2.0 + poll_rate: float = 4.0 class TerminateOnTimeoutState(ez.State): @@ -18,10 +25,15 @@ class TerminateOnTimeoutState(ez.State): class TerminateOnTimeout(ez.Unit): + """ + End a pipeline execution when a certain amount of time has passed without receiving a message. + """ + SETTINGS: TerminateOnTimeoutSettings STATE: TerminateOnTimeoutState INPUT = ez.InputStream(Any) + """Send messages here.""" @ez.subscriber(INPUT) async def keepalive(self, _: Any) -> None: @@ -40,6 +52,12 @@ async def poll_terminate(self) -> None: class TerminateOnTotalSettings(ez.Settings): + """ + Settings for :obj:`TerminateOnTotal` Unit. + + Args: + total: The total number of messages to terminate after. + """ total: Optional[int] = None @@ -49,11 +67,21 @@ class TerminateOnTotalState(ez.State): class TerminateOnTotal(ez.Unit): + """ + End a pipeline execution once a certain number of messages have been received. + """ + SETTINGS: TerminateOnTotalSettings STATE: TerminateOnTotalState INPUT_MESSAGE = ez.InputStream(Any) + """Send messages here.""" + INPUT_TOTAL = ez.InputStream(int) + """ + Change the total number of messages to terminate after. + If this number has already been reached, termination will occur immediately. + """ def initialize(self) -> None: self.STATE.total = self.SETTINGS.total diff --git a/tests/messages/test_axisarray.py b/tests/messages/test_axisarray.py index 0fe4dfb3..94137b3c 100644 --- a/tests/messages/test_axisarray.py +++ b/tests/messages/test_axisarray.py @@ -3,7 +3,7 @@ from dataclasses import dataclass, field -from ezmsg.util.messages.axisarray import AxisArray, shape2d +from ezmsg.util.messages.axisarray import AxisArray, shape2d, slice_along_axis, sliding_win_oneaxis from typing import Generator, List @@ -125,3 +125,66 @@ def test_sel(): aa_idx = aa.isel(dim0=-1) # index slice of last index assert aa_idx.data == data[-1] + +@pytest.mark.parametrize("axis", [0, 1, 2, -1, 3, -4]) +@pytest.mark.parametrize("sl", [ + 3, + slice(None, None, 2), + slice(2, 4, None), + slice(-3, -1, None), + slice(3, 10, None) +]) +def test_slice_along_axis(axis: int, sl): + dims = [4, 5, 6] + data = np.arange(np.prod(dims)).reshape(dims) + + if axis >= len(dims) or axis < -len(dims): + with pytest.raises(ValueError): + res = slice_along_axis(data, sl=sl, axis=axis) + return + + res = slice_along_axis(data, sl=sl, axis=axis) + if isinstance(sl, int): + assert res.ndim == len(dims) - 1 + else: + assert res.ndim == len(dims) + + if axis in [0, -len(dims)]: + expected = data[sl] + elif axis in [1, 1 - len(dims)]: + expected = data[:, sl] + elif axis in [2, 2 - len(dims)]: + expected = data[:, :, sl] + assert np.array_equal(res, expected) + assert np.shares_memory(res, expected) + + +@pytest.mark.parametrize("nwin", [0, 3, 8]) +@pytest.mark.parametrize("axis", [0, 1, 2, -1, 3, -4]) +def test_sliding_win_oneaxis(nwin: int, axis: int): + import numpy.lib.stride_tricks as nps + + dims = [4, 5, 6] + data = np.arange(np.prod(dims)).reshape(dims) + + if axis < -len(dims) or axis >= len(dims): + with pytest.raises(IndexError): + sliding_win_oneaxis(data, nwin, axis) + return + + if nwin > dims[axis]: + with pytest.raises(ValueError): + sliding_win_oneaxis(data, nwin, axis) + return + + res = sliding_win_oneaxis(data, nwin, axis) + + if nwin == 0: + assert res.size == 0 + return + + expected = nps.sliding_window_view(data, nwin, axis) + dest_ax = axis if axis >= 0 else len(dims) + axis + expected = np.moveaxis(expected, -1, dest_ax + 1) + assert np.array_equal(res, expected) + assert np.shares_memory(res, expected) diff --git a/tests/test_perf.py b/tests/test_perf.py index efa4cacc..4c5bc20d 100644 --- a/tests/test_perf.py +++ b/tests/test_perf.py @@ -66,7 +66,7 @@ async def publish(self) -> AsyncGenerator: break yield self.OUTPUT, LoadTestSample( - _timestamp=time.perf_counter(), + _timestamp=time.time(), counter=self.counter, dynamic_data=np.zeros( int(self.SETTINGS.dynamic_size // 8), dtype=np.float32 @@ -101,7 +101,7 @@ async def receive(self, sample: LoadTestSample) -> None: f"{sample.counter - self.STATE.counter-1} samples skipped!" ) self.STATE.received_data.append( - (sample._timestamp, time.perf_counter(), sample.counter) + (sample._timestamp, time.time(), sample.counter) ) self.STATE.counter = sample.counter