Run publish in parallel

astral-sh · Dec 14, 2023 · 9190854 · 9190854
1 parent af4454f
commit 9190854
Show file tree

Hide file tree

Showing 3 changed files with 50 additions and 67 deletions.
diff --git a/src/packse/publish.py b/src/packse/publish.py
@@ -5,6 +5,8 @@
 import subprocess
 import textwrap
 import time
+from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import wait as wait_for_futures
 from pathlib import Path
 
 from packse.error import (
@@ -36,13 +38,42 @@ def publish(
     logger.info("Publishing %s target%s...", len(targets), s)
     for target in sorted(targets):
         logger.info("Publishing '%s'...", target.name)
-        for distfile in sorted(target.iterdir()):
-            publish_package_distribution_with_retries(
-                distfile,
-                skip_existing=skip_existing,
-                dry_run=dry_run,
-                max_attempts=MAX_ATTEMPTS if retry_on_rate_limit else 1,
+
+    # Publish each directory
+    with ThreadPoolExecutor(thread_name_prefix="packse-scenario-") as executor:
+        futures = [
+            executor.submit(
+                publish_package_distributions,
+                target,
+                skip_existing,
+                dry_run,
+                retry_on_rate_limit,
             )
+            for target in targets
+        ]
+
+        wait_for_futures(futures)
+
+    results = [future.result() for future in futures]
+    for result in sorted(results):
+        print(result)
+
+
+def publish_package_distributions(
+    target: Path, skip_existing: bool, dry_run: bool, retry_on_rate_limit: bool
+) -> str:
+    """
+    Publish a directory of package distribution files.
+    """
+    for distfile in sorted(target.iterdir()):
+        publish_package_distribution_with_retries(
+            distfile,
+            skip_existing=skip_existing,
+            dry_run=dry_run,
+            max_attempts=MAX_ATTEMPTS if retry_on_rate_limit else 1,
+        )
+
+    return target.name
 
 
 def publish_package_distribution_with_retries(
@@ -51,6 +82,9 @@ def publish_package_distribution_with_retries(
     dry_run: bool,
     max_attempts: int,
 ):
+    """
+    Publish a package distribution file with retries and error handling.
+    """
     attempts = 0
     retry_time = RETRY_TIME
     while attempts < max_attempts:
@@ -78,7 +112,7 @@ def publish_package_distribution_with_retries(
 
 def publish_package_distribution(target: Path, dry_run: bool) -> None:
     """
-    Publish package distribution.
+    Publish a package distribution file.
     """
     command = ["twine", "upload", "-r", "testpypi", str(target.resolve())]
     if dry_run:

diff --git a/tests/__snapshots__/test_publish.ambr b/tests/__snapshots__/test_publish.ambr
@@ -2,16 +2,7 @@
 # name: test_publish_example_dry_run
   dict({
     'exit_code': 0,
-    'stderr': '''
-      INFO:packse.publish:Publishing 1 target...
-      INFO:packse.publish:Publishing 'example-0611cb74'...
-      INFO:packse.publish:Published 'example_0611cb74-0.0.0.tar.gz'
-      INFO:packse.publish:Published 'example_0611cb74_a-1.0.0-py3-none-any.whl'
-      INFO:packse.publish:Published 'example_0611cb74_a-1.0.0.tar.gz'
-      INFO:packse.publish:Published 'example_0611cb74_b-1.0.0-py3-none-any.whl'
-      INFO:packse.publish:Published 'example_0611cb74_b-1.0.0.tar.gz'
-
-    ''',
+    'stderr': '<not included>',
     'stdout': '''
       Would execute: twine upload -r testpypi [DISTDIR]/example_0611cb74-0.0.0.tar.gz
       Would execute: twine upload -r testpypi [DISTDIR]/example_0611cb74_a-1.0.0-py3-none-any.whl
@@ -25,74 +16,31 @@
 # name: test_publish_example_twine_fails_with_already_exists
   dict({
     'exit_code': 1,
-    'stderr': '''
-      INFO:packse.publish:Publishing 1 target...
-      INFO:packse.publish:Publishing 'example-0611cb74'...
-      Publish for 'example_0611cb74-0.0.0.tar.gz' already exists
-
-    ''',
+    'stderr': '<not included>',
     'stdout': '',
   })
 # ---
 # name: test_publish_example_twine_fails_with_rate_limit
   dict({
     'exit_code': 1,
-    'stderr': '''
-      INFO:packse.publish:Publishing 1 target...
-      INFO:packse.publish:Publishing 'example-0611cb74'...
-      Publish of 'example_0611cb74-0.0.0.tar.gz' failed due to rate limits
-
-    ''',
+    'stderr': '<not included>',
     'stdout': '',
   })
 # ---
 # name: test_publish_example_twine_fails_with_unknown_error
   dict({
     'exit_code': 1,
-    'stderr': '''
-      INFO:packse.publish:Publishing 1 target...
-      INFO:packse.publish:Publishing 'example-0611cb74'...
-      Publishing example_0611cb74-0.0.0.tar.gz with twine failed:
-          <twine error message>
-
-
-    ''',
+    'stderr': '<not included>',
     'stdout': '',
   })
 # ---
 # name: test_publish_example_twine_succeeds
   dict({
     'exit_code': 0,
-    'stderr': '''
-      INFO:packse.publish:Publishing 1 target...
-      INFO:packse.publish:Publishing 'example-0611cb74'...
-      DEBUG:packse.publish:Published example_0611cb74-0.0.0.tar.gz in [TIME]:
-
-          <twine happy message>
-
-      INFO:packse.publish:Published 'example_0611cb74-0.0.0.tar.gz'
-      DEBUG:packse.publish:Published example_0611cb74_a-1.0.0-py3-none-any.whl in [TIME]:
-
-          <twine happy message>
-
-      INFO:packse.publish:Published 'example_0611cb74_a-1.0.0-py3-none-any.whl'
-      DEBUG:packse.publish:Published example_0611cb74_a-1.0.0.tar.gz in [TIME]:
-
-          <twine happy message>
-
-      INFO:packse.publish:Published 'example_0611cb74_a-1.0.0.tar.gz'
-      DEBUG:packse.publish:Published example_0611cb74_b-1.0.0-py3-none-any.whl in [TIME]:
-
-          <twine happy message>
-
-      INFO:packse.publish:Published 'example_0611cb74_b-1.0.0-py3-none-any.whl'
-      DEBUG:packse.publish:Published example_0611cb74_b-1.0.0.tar.gz in [TIME]:
-
-          <twine happy message>
-
-      INFO:packse.publish:Published 'example_0611cb74_b-1.0.0.tar.gz'
+    'stderr': '<not included>',
+    'stdout': '''
+      example-0611cb74
 
     ''',
-    'stdout': '',
   })
 # ---
diff --git a/tests/test_publish.py b/tests/test_publish.py
@@ -131,8 +131,9 @@ def test_publish_example_twine_fails_with_rate_limit(
 
     assert (
         snapshot_command(
-            ["publish", scenario_dist, "-v"],
+            ["publish", scenario_dist],
             extra_filters=[(re.escape(str(scenario_dist.resolve())), "[DISTDIR]")],
+            stderr=False,
         )
         == snapshot
     )