Unified task runner scripts

.github/workflows/build_publish.yml

@@ -18,10 +18,10 @@ jobs:
         with:
           poetry-version: 1.2.2
       - run: poetry install
-      - run: poetry run poe test
-      - run: poetry run poe typecheck
-      - run: poetry run poe codestyle
-      - run: poetry run poe liccheck
+      - run: poetry run poe tests_unit
+      - run: poetry run poe checks_codestyle
+      - run: poetry run poe checks_typing
+      - run: poetry run poe checks_license
 
   build:
     needs: [test]

.github/workflows/goth-nightly.yml

@@ -86,8 +86,8 @@ jobs:
         env:
           GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          poetry run poe goth-assets
-          poetry run poe goth-tests
+          poetry run poe tests_integration_assets
+          poetry run poe tests_integration
 
       - name: Upload test logs
         uses: actions/upload-artifact@v2

.github/workflows/goth.yml

@@ -57,8 +57,8 @@ jobs:
         env:
           GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          poetry run poe goth-assets
-          poetry run poe goth-tests
+          poetry run poe tests_integration_assets
+          poetry run poe tests_integration
 
       - name: Upload test logs
         uses: actions/upload-artifact@v2

.github/workflows/tests.yml

@@ -50,11 +50,11 @@ jobs:
       - run: poetry install -E docs
         if: ${{ steps.extended-checks-sphinx.outputs.ENABLE }}
 
-      - run: poetry run poe test
-      - run: poetry run poe typecheck
-      - run: poetry run poe codestyle
+      - run: poetry run poe tests_unit
+      - run: poetry run poe checks_codestyle
+      - run: poetry run poe checks_typing
         if: ${{ steps.extended-checks.outputs.ENABLE }}
-      - run: poetry run poe liccheck
+      - run: poetry run poe checks_license
         if: ${{ steps.extended-checks.outputs.ENABLE }}
       - run: poetry run poe sphinx -W
         if: ${{ steps.extended-checks-sphinx.outputs.ENABLE }}

.gitignore

@@ -8,7 +8,7 @@ __pycache__/
 *.log
 *.png
 .coverage
-.requirements.txt
+requirements.txt
 
 tests/goth_tests/assets/
 

README.md

@@ -68,7 +68,7 @@ All of the project's dependencies will be installed to that virtual environment.
 Declarations of project tasks can be found in `pyproject.toml`.
 
 ```bash
-poetry run poe test
+poetry run poe tests_unit
 ```
 
 ### Running `goth` integration tests
@@ -96,23 +96,23 @@ poetry install -E integration-tests
 Finally, generate goth's default assets:
 
 ```bash
-poetry run poe goth-assets
+poetry run poe tests_integration_assets
 ```
 
 #### Running the tests
 
 Once you have the environment set up, to run all the integration tests, use:
 
 ```bash
-poetry run poe goth-tests
+poetry run poe tests_integration
 ```
 
 ### Contributing
 
 It is recommended to run unit tests and static code analysis before committing changes.
 
 ```bash
-poetry run poe check
+poetry run poe checks
 ```
 
 You can clean up the artifacts created during the test runs with:

docs/sphinx/conf.py

@@ -12,6 +12,7 @@
 #
 import os
 import sys
+from typing import List
 
 sys.path.insert(0, os.path.abspath("../../"))
 
@@ -56,7 +57,7 @@
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = []
+html_static_path: List[str] = []
 
 #   This removes the `yapapi.log` docstrings.
 #   There are two reasons:

examples/blender/blender.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
-from datetime import datetime, timedelta
 import pathlib
 import sys
+from datetime import datetime, timedelta
 
 from yapapi import Golem, Task, WorkContext
 from yapapi.payload import vm
@@ -31,7 +31,8 @@ async def main(
         min_mem_gib=0.5,
         # only run on provider nodes that have more than 2gb of storage space available
         min_storage_gib=2.0,
-        # only run on provider nodes which a certain number of CPU threads (logical CPU cores) available
+        # only run on provider nodes which a certain number of CPU threads (logical CPU cores)
+        #  available
         min_cpu_threads=min_cpu_threads,
     )
 

examples/blender/start_stop_blender.py

@@ -2,14 +2,14 @@
 
 """Other version of the blender example.
 
-Instead of using Golem in the default way - as a context manager - we directly call `Golem.start()` and `Golem.stop()`.
-This way of using Golem might be more convenient for some specific use cases (although doesn't change a lot
-in the blender example).
+Instead of using Golem in the default way - as a context manager - we directly call `Golem.start()`
+and `Golem.stop()`. This way of using Golem might be more convenient for some specific use cases
+(although doesn't change a lot in the blender example).
 """
 
-from datetime import datetime, timedelta
 import pathlib
 import sys
+from datetime import datetime, timedelta
 
 from yapapi import Golem, Task, WorkContext
 from yapapi.payload import vm

examples/custom-usage-counter/custom_usage_counter.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python3
 import asyncio
-from dataclasses import dataclass
-from datetime import datetime, timedelta
-from decimal import Decimal
 import pathlib
 import sys
+from datetime import datetime, timedelta
+from decimal import Decimal
+
+from dataclasses import dataclass
 
 from yapapi import Golem
 from yapapi.ctx import ActivityUsage
@@ -101,6 +102,8 @@ def print_instances():
         type=int,
         help="How long should the the service run (in seconds, default: %(default)s)",
     )
+    now = datetime.now().strftime("%Y-%m-%d_%H.%M.%S")
+    parser.set_defaults(log_file=f"custom-counters-yapapi-{now}.log")
     args = parser.parse_args()
     enable_default_logger(
         log_file=args.log_file,

examples/custom_runtime/custom_runtime.py

@@ -1,4 +1,5 @@
 import asyncio
+
 from dataclasses import dataclass
 
 from yapapi import Golem

examples/external-api-request/external_api_request.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
 import asyncio
 import base64
-from datetime import datetime
 import pathlib
 import sys
+from datetime import datetime
 
 from yapapi import Golem
 from yapapi.payload import vm
@@ -46,7 +46,7 @@ async def run(self):
         future_result = script.run(
             "/bin/sh",
             "-c",
-            f"GOLEM_PRICE=`curl -X 'GET' \
+            "GOLEM_PRICE=`curl -X 'GET' \
                     'https://api.coingecko.com/api/v3/simple/price?ids=golem&vs_currencies=usd' \
                     -H 'accept: application/json' | jq .golem.usd`; \
                 echo ---; \

examples/http-proxy/http_proxy.py

@@ -1,12 +1,10 @@
 #!/usr/bin/env python3
-"""
-a simple http proxy example
-"""
+"""A simple http proxy example."""
 import asyncio
-from datetime import datetime, timedelta, timezone
 import pathlib
 import shlex
 import sys
+from datetime import datetime, timedelta, timezone
 
 from yapapi import Golem
 from yapapi.contrib.service.http_proxy import HttpProxyService, LocalHttpProxy
@@ -123,7 +121,8 @@ def still_starting():
         await proxy.run()
 
         print(
-            f"{TEXT_COLOR_CYAN}Local HTTP server listening on:\nhttp://localhost:{port}{TEXT_COLOR_DEFAULT}"
+            f"{TEXT_COLOR_CYAN}Local HTTP server listening on:\n"
+            f"http://localhost:{port}{TEXT_COLOR_DEFAULT}"
         )
 
         # wait until Ctrl-C

examples/low-level-api/list-offers.py

@@ -1,10 +1,10 @@
 #!/usr/bin/env python3
 import asyncio
-from asyncio import TimeoutError
-from datetime import datetime, timezone
 import json
 import pathlib
 import sys
+from asyncio import TimeoutError
+from datetime import datetime, timezone
 
 from yapapi import props as yp
 from yapapi.config import ApiConfig

examples/market-strategy/market_strategy.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
-from collections import defaultdict
 import itertools
 import pathlib
 import sys
+from collections import defaultdict
 
 from yapapi import Golem, Task, WorkContext
 from yapapi.payload import vm
@@ -21,11 +21,13 @@
 
 
 class FastestProviderStrategy(MarketStrategy):
-    """Strategy that ignores all offer parameters (including pricing) and just selects the fastest provider.
+    """Strategy that ignores all offer parameters (including pricing) and just selects the fastest \
+    provider.
 
     Decision algorithm:
     * we always try any new provider
-    * if there are no new providers, we select the one with shortest average execution time in past runs
+    * if there are no new providers, we select the one with shortest average execution time in past
+      runs
     """
 
     def __init__(self):
@@ -45,7 +47,8 @@ async def score_offer(self, offer):
             #   Non-draft offers are not shown to limit the number of lines printed
             if previous_runs:
                 print(
-                    f"Scored known provider: {provider_id}: {score} ({len(previous_runs)} runs, avg time {avg_time})"
+                    f"Scored known provider: {provider_id}: "
+                    f"{score} ({len(previous_runs)} runs, avg time {avg_time})"
                 )
             else:
                 print(f"Found new provider: {provider_id}, default score {SCORE_TRUSTED}")

examples/scan/scan.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
-from datetime import datetime
 import logging
 import pathlib
 import re
 import sys
+from datetime import datetime
 from typing import List
 
 from yapapi import Golem, Task, WorkContext

examples/simple-service-poc/simple_service.py

@@ -1,13 +1,11 @@
 #!/usr/bin/env python3
-"""
-the requestor agent controlling and interacting with the "simple service"
-"""
+"""The requestor agent controlling and interacting with the "simple service"."""
 import asyncio
-from datetime import datetime, timedelta, timezone
 import pathlib
 import random
 import string
 import sys
+from datetime import datetime, timedelta, timezone
 
 from yapapi import Golem
 from yapapi.payload import vm
@@ -20,7 +18,6 @@
     TEXT_COLOR_CYAN,
     TEXT_COLOR_DEFAULT,
     TEXT_COLOR_MAGENTA,
-    TEXT_COLOR_RED,
     TEXT_COLOR_YELLOW,
     build_parser,
     format_usage,
@@ -55,7 +52,7 @@ async def get_payload():
         )
 
     async def start(self):
-        """handler responsible for starting the service."""
+        """Handle starting the service."""
 
         # perform the initialization of the Service
         async for script in super().start():
@@ -148,7 +145,7 @@ async def main(
 
         def print_instances():
             print(
-                f"instances: "
+                "instances: "
                 + str(
                     [
                         f"{s.name}: {s.state.value}"

examples/simple-service-poc/simple_service/simple_service.py

@@ -1,23 +1,24 @@
 #!/usr/local/bin/python
-"""
-a very basic "stub" that exposes a few commands of an imagined, very simple CLI-based
-service that is able to accumulate some linear, time-based values and present it stats
+"""A very basic "stub" that exposes a few commands of an imagined, very simple CLI-based service.
+
+The Service is able to accumulate some linear, time-based values and present it stats
 (characteristics of the statistical distribution of the data collected so far) or provide
 distribution and time-series plots of the collected data.
 
 [ part of the VM image that's deployed by the runtime on the Provider's end. ]
 """
 import argparse
 import contextlib
-from datetime import datetime
 import enum
 import json
-import matplotlib.pyplot as plt
-import numpy
-from pathlib import Path
 import random
 import sqlite3
 import string
+from datetime import datetime
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import numpy
 
 DB_PATH = Path(__file__).absolute().parent / "service.db"
 PLOT_PATH = Path("/golem/out").absolute()

examples/simple-service-poc/simple_service/simulate_observations.py

@@ -1,6 +1,5 @@
 #!/usr/local/bin/python
-"""
-the "hello world" service here just adds randomized numbers with normal distribution
+"""The "hello world" service here just adds randomized numbers with normal distribution.
 
 in a real-world example, this could be e.g. a thermometer connected to the provider's
 machine providing its inputs into the database or some other piece of information
@@ -10,9 +9,9 @@
 [ part of the VM image that's deployed by the runtime on the Provider's end. ]
 """
 import os
-from pathlib import Path
 import random
 import time
+from pathlib import Path
 
 MU = 14
 SIGMA = 3

examples/simple-service-poc/simple_service/simulate_observations_ctl.py

@@ -1,6 +1,5 @@
 #!/usr/local/bin/python
-"""
-a helper, control script that starts and stops our example `simulate_observations` service
+"""a helper, control script that starts and stops our example `simulate_observations` service.
 
 [ part of the VM image that's deployed by the runtime on the Provider's end. ]
 """