Skip to content

Commit

Permalink
release 6.660.25006
Browse files Browse the repository at this point in the history
  • Loading branch information
klahnakoski committed Jan 7, 2025
2 parents cb4fd3f + 327ce53 commit 17b39a0
Show file tree
Hide file tree
Showing 15 changed files with 287 additions and 153 deletions.
6 changes: 3 additions & 3 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]

steps:
- uses: actions/checkout@v3
Expand Down Expand Up @@ -62,7 +62,7 @@ jobs:
env:
CI: true
run: |
python -m unittest discover tests -v
python -m unittest discover . -v
coverage:
if: github.ref == 'refs/heads/dev'
Expand Down Expand Up @@ -113,6 +113,6 @@ jobs:
pip install --no-deps -r tests/requirements.lock
pip install coverage coveralls
pip install .
coverage run --rcfile=packaging/coverage.ini -m unittest discover tests
coverage run --rcfile=packaging/coverage.ini -m unittest discover .
coverage report --rcfile=packaging/coverage.ini
coveralls
20 changes: 7 additions & 13 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,31 +1,25 @@

# More Threads!


|Branch | Status |
|------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|master | [![Build Status](https://github.com/klahnakoski/mo-threads/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/klahnakoski/mo-threads/actions/workflows/build.yml) |
|dev | [![Build Status](https://app.travis-ci.com/klahnakoski/mo-threads.svg?branch=dev)](https://travis-ci.com/github/klahnakoski/mo-threads) [![Coverage Status](https://coveralls.io/repos/github/klahnakoski/mo-threads/badge.svg?branch=dev)](https://coveralls.io/github/klahnakoski/mo-threads?branch=dev) ← child thread coverage is missing |
[![PyPI Latest Release](https://img.shields.io/pypi/v/mo-threads.svg)](https://pypi.org/project/mo-threads/)
[![Build Status](https://github.com/klahnakoski/mo-threads/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/klahnakoski/mo-threads/actions/workflows/build.yml)
[![Coverage Status](https://coveralls.io/repos/github/klahnakoski/mo-threads/badge.svg?branch=dev)](https://coveralls.io/github/klahnakoski/mo-threads?branch=dev)
[![Downloads](https://static.pepy.tech/badge/mo-threads/month)](https://pepy.tech/project/mo-threads)

## Module `threads`

The main benefits over Python's threading library is:

1. **Multi-threaded queues do not use serialization** - Serialization is
great in the general case, where you may also be communicating between
processes, but it is a needless overhead for single-process multi-threading.
It is left to the programmer to ensure the messages put on the queue are
not changed, which is not ominous demand.
2. **Shutdown order is deterministic and explicit** - Python's threading
1. **Shutdown order is deterministic and explicit** - Python's threading
library is missing strict conventions for controlled and orderly shutdown.
Each thread can shutdown on its own terms, but is expected to do so expediently.
* All threads are required to accept a `please_stop` signal; are
expected to test it in a timely manner; and expected to exit when signalled.
* All threads have a parent - The parent is responsible for ensuring their children get the `please_stop` signal, and are dead, before stopping themselves. This responsibility is baked into the thread spawning process,
so you need not deal with it unless you want.
3. Uses [**Signals**](#signal-class) to simplify logical
2. Uses [**Signals**](#signal-class) to simplify logical
dependencies among multiple threads, events, and timeouts.
4. **Logging and Profiling is Integrated** - Logging and exception handling
3. **Logging and Profiling is Integrated** - Logging and exception handling
is seamlessly integrated: This means logs are centrally handled, and thread
safe. Parent threads have access to uncaught child thread exceptions, and
the cProfiler properly aggregates results from the multiple threads.
Expand Down
6 changes: 3 additions & 3 deletions mo_threads/commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,8 @@ def __init__(
if debug:
name = f"{name} (using {process.name})"
self.name = name
self.stdout = Queue("stdout for " + name, max=max_stdout)
self.stderr = Queue("stderr for " + name, max=max_stdout)
self.stdout = Queue(f"stdout for {name}", max=max_stdout)
self.stderr = Queue(f"stderr for {name}", max=max_stdout)
self.stderr_thread = Thread.run(f"{name} stderr", _stderr_relay, process.stderr, self.stderr).release()
# stdout_thread IS CONSIDERED THE LIFETIME OF THE COMMAND
self.worker_thread = Thread.run(f"{name} worker", self._worker, process.stdout, self.stdout).release()
Expand Down Expand Up @@ -211,7 +211,7 @@ def get_or_create_process(self, *, params, bufsize, cwd, debug, env, name, shell

# WAIT FOR START
try:
process.stdin.add("cd " + cmd_escape(cwd))
process.stdin.add(f"cd {cmd_escape(cwd)}")
process.stdin.add(LAST_RETURN_CODE)
start_timeout = Till(seconds=START_TIMEOUT)
while not start_timeout:
Expand Down
10 changes: 3 additions & 7 deletions mo_threads/pools.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from mo_threads.queues import Queue
from mo_threads.threads import Thread
from mo_threads.threads import join_all_threads, PLEASE_STOP, current_thread
from mo_threads.threads import join_all_threads, PLEASE_STOP


class ThreadPool:
Expand All @@ -13,7 +13,7 @@ def __init__(self, num_threads, name=None):

def __enter__(self):
self.workers = [
Thread.run(f"{self.name}-{i}", worker, self)
Thread.run(f"{self.name}-worker-{i}", worker, self)
for i in range(self.num_threads)
]
return self
Expand All @@ -29,15 +29,11 @@ def run(self, name, target, *args, **kwargs):


def worker(pool, please_stop):
this = current_thread()
this_name = this.name
while not please_stop:
thread = pool.queue.pop(till=please_stop)
if thread is PLEASE_STOP:
break
this.threading_thread.name = thread.name
try:
thread.start().join(till=please_stop)
except Exception as cause:
except Exception:
pass
this.threading_thread.name = this_name
10 changes: 5 additions & 5 deletions mo_threads/processes.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,13 +76,13 @@ def __init__(

self.debug = debug or DEBUG
self.name = f"{name} ({self.process_id})"
self.stopped = Signal("stopped signal for " + strings.quote(name))
self.please_stop = Signal("please stop for " + strings.quote(name))
self.stopped = Signal(f"stopped signal for {strings.quote(name)}")
self.please_stop = Signal(f"please stop for {strings.quote(name)}")
self.second_last_stdin = None
self.last_stdin = None
self.stdin = Queue("stdin for process " + strings.quote(name), silent=not self.debug)
self.stdout = Queue("stdout for process " + strings.quote(name), silent=not self.debug)
self.stderr = Queue("stderr for process " + strings.quote(name), silent=not self.debug)
self.stdin = Queue(f"stdin for process {strings.quote(name)}", silent=not self.debug)
self.stdout = Queue(f"stdout for process {strings.quote(name)}", silent=not self.debug)
self.stderr = Queue(f"stderr for process {strings.quote(name)}", silent=not self.debug)
self.timeout = timeout
self.monitor_period = 0.5

Expand Down
2 changes: 1 addition & 1 deletion mo_threads/python.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ def __init__(self, name, config, parent_thread=None):
if line == '{"out":"ok"}':
break
logger.info("waiting to start python: {line}", line=line)
self.lock = Lock("wait for response from " + name)
self.lock = Lock(f"wait for response from {name}")
self.stop_error = None
self.done = DONE
self.response = None
Expand Down
Loading

0 comments on commit 17b39a0

Please sign in to comment.