From b5f3b3ea7242f37594cac3658a2ca4815231c728 Mon Sep 17 00:00:00 2001 From: approxit Date: Mon, 16 Jan 2023 14:07:54 +0100 Subject: [PATCH 01/20] Unified task runner scripts --- examples/blender/blender.py | 2 +- examples/blender/start_stop_blender.py | 2 +- .../custom_usage_counter.py | 7 +- examples/custom_runtime/custom_runtime.py | 1 + .../external_api_request.py | 2 +- examples/http-proxy/http_proxy.py | 2 +- examples/low-level-api/list-offers.py | 4 +- examples/market-strategy/market_strategy.py | 2 +- examples/scan/scan.py | 2 +- examples/simple-service-poc/simple_service.py | 2 +- .../simple_service/simple_service.py | 9 +- .../simple_service/simulate_observations.py | 2 +- examples/ssh/ssh.py | 2 +- examples/utils/__init__.py | 5 +- examples/webapp/http/app.py | 1 + examples/webapp/webapp.py | 2 +- .../webapp_fileupload/http-file-upload/app.py | 13 +- .../webapp_fileupload/webapp_fileupload.py | 2 +- examples/yacat/yacat.py | 4 +- pyproject.toml | 123 +++++++++++------- tests/conftest.py | 1 + tests/contrib/service/test_chunk.py | 3 +- tests/drone/drone.py | 2 +- tests/engine/test_debit_note_intervals.py | 5 +- tests/engine/test_engine.py | 7 +- tests/executor/test_smartq.py | 3 +- tests/factories/context.py | 3 +- tests/factories/events.py | 3 +- tests/factories/golem.py | 2 +- tests/factories/network.py | 3 +- tests/factories/rest/__init__.py | 3 +- tests/factories/rest/market.py | 1 + tests/factories/rest/payment.py | 3 +- tests/goth_tests/conftest.py | 3 +- .../test_agreement_termination/requestor.py | 2 +- .../test_agreement_termination.py | 7 +- .../test_async_task_generation/requestor.py | 2 +- .../test_async_task_generation.py | 3 +- .../test_concurrent_executors.py | 3 +- .../test_instance_restart/requestor.py | 9 +- .../test_instance_restart.py | 3 +- .../requestor_agent.py | 2 +- .../test_mid_agreement_payments.py | 5 +- .../test_multiactivity_agreement/requestor.py | 2 +- .../test_multiactivity_agreement.py | 7 +- tests/goth_tests/test_power_outage.py | 3 +- .../test_recycle_ip/ssh_recycle_ip.py | 2 +- .../test_recycle_ip/test_recycle_ip.py | 7 +- .../test_renegotiate_proposal/requestor.py | 3 +- .../test_renegotiate_proposal.py | 3 +- tests/goth_tests/test_resubscription.py | 13 +- tests/goth_tests/test_run_blender.py | 9 +- .../test_run_custom_usage_counter.py | 5 +- tests/goth_tests/test_run_scan.py | 5 +- tests/goth_tests/test_run_simple_service.py | 5 +- tests/goth_tests/test_run_ssh.py | 7 +- tests/goth_tests/test_run_webapp.py | 7 +- tests/goth_tests/test_run_yacat.py | 11 +- tests/payload/test_payload.py | 2 +- tests/payload/test_vm.py | 3 +- tests/props/test_base.py | 2 +- tests/props/test_builder.py | 2 +- tests/rest/test_activity.py | 3 +- tests/rest/test_allocation.py | 1 + tests/rest/test_repeat_on_error.py | 9 +- tests/script/test_script.py | 5 +- tests/services/test_cluster.py | 3 +- tests/services/test_service_runner.py | 5 +- tests/storage/test_gftp.py | 7 +- tests/strategy/test_base.py | 1 + .../test_decrease_score_for_unconfirmed.py | 1 + tests/strategy/test_default_strategies.py | 5 +- tests/test_add_event_consumer.py | 2 +- tests/test_agreements_pool.py | 7 +- tests/test_async_wrapper.py | 3 +- tests/test_config.py | 1 + tests/test_ctx.py | 7 +- tests/test_network.py | 5 +- tests/test_payment_platforms.py | 3 +- tests/test_yapapi.py | 1 + yapapi/__init__.py | 5 +- yapapi/agreements_pool.py | 5 +- yapapi/config.py | 5 +- yapapi/contrib/service/http_proxy.py | 7 +- yapapi/contrib/service/socket_proxy.py | 3 +- yapapi/ctx.py | 5 +- yapapi/engine.py | 15 ++- yapapi/events.py | 5 +- yapapi/executor/__init__.py | 5 +- yapapi/executor/_smartq.py | 14 +- yapapi/executor/task.py | 15 +-- yapapi/golem.py | 10 +- yapapi/invoice_manager.py | 5 +- yapapi/log.py | 11 +- yapapi/network.py | 14 +- yapapi/payload/package.py | 1 + yapapi/payload/vm.py | 15 +-- yapapi/props/__init__.py | 3 +- yapapi/props/base.py | 3 +- yapapi/props/builder.py | 7 +- yapapi/props/com.py | 3 +- yapapi/props/inf.py | 5 +- yapapi/rest/activity.py | 9 +- yapapi/rest/common.py | 3 +- yapapi/rest/configuration.py | 1 + yapapi/rest/market.py | 7 +- yapapi/rest/payment.py | 7 +- yapapi/script/__init__.py | 14 +- yapapi/script/capture.py | 3 +- yapapi/script/command.py | 21 +-- yapapi/services/cluster.py | 13 +- yapapi/services/service.py | 7 +- yapapi/services/service_runner.py | 12 +- yapapi/services/service_state.py | 3 +- yapapi/storage/__init__.py | 5 +- yapapi/storage/gftp.py | 13 +- yapapi/storage/webdav.py | 9 +- yapapi/strategy/base.py | 5 +- yapapi/strategy/dummy.py | 5 +- yapapi/strategy/least_expensive.py | 5 +- yapapi/utils.py | 4 +- 121 files changed, 380 insertions(+), 361 deletions(-) diff --git a/examples/blender/blender.py b/examples/blender/blender.py index 377fe091d..74a3ab1f7 100755 --- a/examples/blender/blender.py +++ b/examples/blender/blender.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -from datetime import datetime, timedelta import pathlib import sys +from datetime import datetime, timedelta from yapapi import Golem, Task, WorkContext from yapapi.payload import vm diff --git a/examples/blender/start_stop_blender.py b/examples/blender/start_stop_blender.py index 94dc2b4cc..8fd588a0b 100755 --- a/examples/blender/start_stop_blender.py +++ b/examples/blender/start_stop_blender.py @@ -7,9 +7,9 @@ in the blender example). """ -from datetime import datetime, timedelta import pathlib import sys +from datetime import datetime, timedelta from yapapi import Golem, Task, WorkContext from yapapi.payload import vm diff --git a/examples/custom-usage-counter/custom_usage_counter.py b/examples/custom-usage-counter/custom_usage_counter.py index b5ad1ef70..7593d1050 100755 --- a/examples/custom-usage-counter/custom_usage_counter.py +++ b/examples/custom-usage-counter/custom_usage_counter.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 import asyncio -from dataclasses import dataclass -from datetime import datetime, timedelta -from decimal import Decimal import pathlib import sys +from datetime import datetime, timedelta +from decimal import Decimal + +from dataclasses import dataclass from yapapi import Golem from yapapi.ctx import ActivityUsage diff --git a/examples/custom_runtime/custom_runtime.py b/examples/custom_runtime/custom_runtime.py index a5fd42885..8d57b0e4f 100644 --- a/examples/custom_runtime/custom_runtime.py +++ b/examples/custom_runtime/custom_runtime.py @@ -1,4 +1,5 @@ import asyncio + from dataclasses import dataclass from yapapi import Golem diff --git a/examples/external-api-request/external_api_request.py b/examples/external-api-request/external_api_request.py index fbc32c2db..86b8a8955 100644 --- a/examples/external-api-request/external_api_request.py +++ b/examples/external-api-request/external_api_request.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 import asyncio import base64 -from datetime import datetime import pathlib import sys +from datetime import datetime from yapapi import Golem from yapapi.payload import vm diff --git a/examples/http-proxy/http_proxy.py b/examples/http-proxy/http_proxy.py index 582d49712..63a8e909a 100644 --- a/examples/http-proxy/http_proxy.py +++ b/examples/http-proxy/http_proxy.py @@ -3,10 +3,10 @@ a simple http proxy example """ import asyncio -from datetime import datetime, timedelta, timezone import pathlib import shlex import sys +from datetime import datetime, timedelta, timezone from yapapi import Golem from yapapi.contrib.service.http_proxy import HttpProxyService, LocalHttpProxy diff --git a/examples/low-level-api/list-offers.py b/examples/low-level-api/list-offers.py index 45000dfd8..33f41bac6 100755 --- a/examples/low-level-api/list-offers.py +++ b/examples/low-level-api/list-offers.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 import asyncio -from asyncio import TimeoutError -from datetime import datetime, timezone import json import pathlib import sys +from asyncio import TimeoutError +from datetime import datetime, timezone from yapapi import props as yp from yapapi.config import ApiConfig diff --git a/examples/market-strategy/market_strategy.py b/examples/market-strategy/market_strategy.py index 24fca61c8..ed904c8ae 100755 --- a/examples/market-strategy/market_strategy.py +++ b/examples/market-strategy/market_strategy.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 -from collections import defaultdict import itertools import pathlib import sys +from collections import defaultdict from yapapi import Golem, Task, WorkContext from yapapi.payload import vm diff --git a/examples/scan/scan.py b/examples/scan/scan.py index 69b415d91..e58684657 100755 --- a/examples/scan/scan.py +++ b/examples/scan/scan.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 -from datetime import datetime import logging import pathlib import re import sys +from datetime import datetime from typing import List from yapapi import Golem, Task, WorkContext diff --git a/examples/simple-service-poc/simple_service.py b/examples/simple-service-poc/simple_service.py index 20fd72c9d..a92e7704f 100755 --- a/examples/simple-service-poc/simple_service.py +++ b/examples/simple-service-poc/simple_service.py @@ -3,11 +3,11 @@ the requestor agent controlling and interacting with the "simple service" """ import asyncio -from datetime import datetime, timedelta, timezone import pathlib import random import string import sys +from datetime import datetime, timedelta, timezone from yapapi import Golem from yapapi.payload import vm diff --git a/examples/simple-service-poc/simple_service/simple_service.py b/examples/simple-service-poc/simple_service/simple_service.py index c52f71b83..2025e588f 100644 --- a/examples/simple-service-poc/simple_service/simple_service.py +++ b/examples/simple-service-poc/simple_service/simple_service.py @@ -9,15 +9,16 @@ """ import argparse import contextlib -from datetime import datetime import enum import json -import matplotlib.pyplot as plt -import numpy -from pathlib import Path import random import sqlite3 import string +from datetime import datetime +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy DB_PATH = Path(__file__).absolute().parent / "service.db" PLOT_PATH = Path("/golem/out").absolute() diff --git a/examples/simple-service-poc/simple_service/simulate_observations.py b/examples/simple-service-poc/simple_service/simulate_observations.py index 430807dc1..c2f04fba7 100644 --- a/examples/simple-service-poc/simple_service/simulate_observations.py +++ b/examples/simple-service-poc/simple_service/simulate_observations.py @@ -10,9 +10,9 @@ [ part of the VM image that's deployed by the runtime on the Provider's end. ] """ import os -from pathlib import Path import random import time +from pathlib import Path MU = 14 SIGMA = 3 diff --git a/examples/ssh/ssh.py b/examples/ssh/ssh.py index a747cf8b9..f8ee85e11 100755 --- a/examples/ssh/ssh.py +++ b/examples/ssh/ssh.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 import asyncio -from datetime import datetime, timedelta import pathlib import random import string import sys +from datetime import datetime, timedelta from yapapi import Golem from yapapi.contrib.service.socket_proxy import SocketProxy, SocketProxyService diff --git a/examples/utils/__init__.py b/examples/utils/__init__.py index 2a6563527..3864dbd20 100644 --- a/examples/utils/__init__.py +++ b/examples/utils/__init__.py @@ -1,10 +1,11 @@ """Utilities for yapapi example scripts.""" import argparse import asyncio -import colorama # type: ignore +import tempfile from datetime import datetime, timezone from pathlib import Path -import tempfile + +import colorama # type: ignore from yapapi import Golem, NoPaymentAccountError from yapapi import __version__ as yapapi_version diff --git a/examples/webapp/http/app.py b/examples/webapp/http/app.py index 557931658..777e43608 100644 --- a/examples/webapp/http/app.py +++ b/examples/webapp/http/app.py @@ -1,4 +1,5 @@ import argparse + from flask import Flask, redirect, render_template, request, url_for from flask_sqlalchemy import SQLAlchemy diff --git a/examples/webapp/webapp.py b/examples/webapp/webapp.py index ad08af7ab..091937983 100755 --- a/examples/webapp/webapp.py +++ b/examples/webapp/webapp.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 import asyncio -from datetime import datetime, timedelta import pathlib import sys +from datetime import datetime, timedelta from yapapi import Golem from yapapi.contrib.service.http_proxy import HttpProxyService, LocalHttpProxy diff --git a/examples/webapp_fileupload/http-file-upload/app.py b/examples/webapp_fileupload/http-file-upload/app.py index 627909d89..9055e1dae 100644 --- a/examples/webapp_fileupload/http-file-upload/app.py +++ b/examples/webapp_fileupload/http-file-upload/app.py @@ -1,15 +1,8 @@ import argparse -from flask import ( - Flask, - Request, - redirect, - render_template, - request, - send_from_directory, - url_for, -) -from flask_sqlalchemy import SQLAlchemy from pathlib import Path + +from flask import Flask, Request, redirect, render_template, request, send_from_directory, url_for +from flask_sqlalchemy import SQLAlchemy from werkzeug.datastructures import FileStorage from werkzeug.utils import secure_filename diff --git a/examples/webapp_fileupload/webapp_fileupload.py b/examples/webapp_fileupload/webapp_fileupload.py index 35f73df20..2462f3458 100755 --- a/examples/webapp_fileupload/webapp_fileupload.py +++ b/examples/webapp_fileupload/webapp_fileupload.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 import asyncio -from datetime import datetime, timedelta import pathlib import sys +from datetime import datetime, timedelta from yapapi import Golem from yapapi.contrib.service.http_proxy import HttpProxyService, LocalHttpProxy diff --git a/examples/yacat/yacat.py b/examples/yacat/yacat.py index 4239c9d02..ca226ee30 100755 --- a/examples/yacat/yacat.py +++ b/examples/yacat/yacat.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 import argparse import asyncio -from datetime import datetime, timedelta import math -from pathlib import Path import sys +from datetime import datetime, timedelta +from pathlib import Path from tempfile import gettempdir from typing import AsyncIterable, List, Optional diff --git a/pyproject.toml b/pyproject.toml index e60094b71..aac03fe1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,8 @@ -[build-system] -requires = ["poetry_core>=1.0.0"] -build-backend = "poetry.core.masonry.api" - [tool.poetry] name = "yapapi" version = "0.11.0-dev" description = "High-level Python API for the New Golem" -authors = ["Przemysław K. Rekucki ", "GolemFactory "] +authors = ["GolemFactory "] license = "LGPL-3.0-or-later" classifiers = [ "Development Status :: 3 - Alpha", @@ -18,8 +14,13 @@ repository = "https://github.com/golemfactory/yapapi" documentation = "https://yapapi.readthedocs.io/en/stable/api.html" readme = "README.md" +[build-system] +requires = ["poetry_core>=1.0.0"] +build-backend = "poetry.core.masonry.api" + [tool.poetry.dependencies] python = "^3.6.2" + setuptools = [ { version = "*", python = ">=3.7" }, { version = "^59.0.0", python = ">=3.6.2, <3.7" }, @@ -65,7 +66,7 @@ sphinx-rtd-theme = { version = "^1.0.0", optional = true } integration-tests = ['goth', 'pytest', 'pytest-asyncio', 'pexpect'] docs = ['sphinx', 'sphinx-autodoc-typehints', 'sphinx-rtd-theme'] -[tool.poetry.dev-dependencies] +[tool.poetry.group.dev.dependencies] pip = [ { version = "*", python = ">=3.7" }, { version = "^21.0.0", python = ">=3.6.2, <3.7" }, @@ -74,26 +75,50 @@ black = "^21.7b0" factory-boy = "^3.2.0" isort = "^5.10.1" liccheck = "^0.4.7" -mypy = "^0.782" +mypy = "^0.942" poethepoet = "^0.8.0" pytest = "^6.2" pytest-asyncio = "^0.14" pytest-cov = "^2.11" pytest-rerunfailures = "^10.1" +autoflake = "^1" +flake8 = "^5" +flake8-docstrings = "^1.6" +Flake8-pyproject = "^1.2.2" -[tool.isort] -profile = "black" -force_sort_within_sections = true -known_golem = "goth, ya_activity, ya_market, ya_net, ya_payment" -sections = ["FUTURE","THIRDPARTY","GOLEM", "FIRSTPARTY","LOCALFOLDER"] -default_section = "THIRDPARTY" # STDLIB section will be cobined with THIRDPARTY section - -[tool.black] -line-length = 100 -target-version = ['py36'] +[tool.poe.tasks] +checks = {sequence = ["checks_codestyle", "checks_typing", "checks_license"], help = "Run all available code checks"} +checks_codestyle = {sequence = ["_checks_codestyle_flake8", "_checks_codestyle_isort", "_checks_codestyle_black"], help = "Run only code style checks"} +_checks_codestyle_flake8 = "flake8 yapapi tests" +_checks_codestyle_isort = "isort --check-only --diff ." +_checks_codestyle_black = "black --check --diff ." +checks_typing = {cmd = "mypy --install-types --non-interactive --ignore-missing-imports --check-untyped-defs --warn-unused-ignores --show-error-codes .", help = "Run only code typing checks" } +checks_license = {sequence = ["_checks_license_export", "_checks_license_verify"], help = "Run only license compatibility checks"} +_checks_license_export = "poetry export -f requirements.txt -o .requirements.txt" +_checks_license_verify = "liccheck -r .requirements.txt" +format = {sequence = ["_format_autoflake", "_format_isort", "_format_black"], help = "Run code auto formatting"} +_format_autoflake = "autoflake ." +_format_isort = "isort ." +_format_black = "black ." +tests = {sequence = ["tests_unit", "tests_integration"], help = "Run all available tests"} +tests_unit = {cmd = "pytest --cov=yapapi --cov-report html --cov-report term -sv --ignore tests/goth_tests", help = "Run only unit tests"} +tests_integration = {cmd = "pytest -svx tests/goth_tests --config-override docker-compose.build-environment.use-prerelease=true --config-path tests/goth_tests/assets/goth-config-testing.yml --ssh-verify-connection --reruns 3 --only-rerun AssertionError --only-rerun TimeoutError --only-rerun goth.runner.exceptions.TemporalAssertionError --only-rerun urllib.error.URLError --only-rerun goth.runner.exceptions.CommandError", help = "Run only integration tests"} +tests_integration_assets = {cmd = "python -m goth create-assets tests/goth_tests/assets", help = "Generate assets files for integration tests"} +clean = {cmd = "rm -rf .coverage .requirements.txt dist md handbook build", help = "Clean all development related files" } +clean_examples = {sequence = ["clean_examples_blender", "clean_examples_hello", "clean_examples_http_proxy", "clean_examples_simple_service", "clean_examples_ssh", "clean_examples_webapp", "clean_examples_yacat"], help = "Clean all example related files"} +clean_examples_blender = {cmd = "rm -f examples/blender/*.png examples/blender/*.log", help = "Clean \"blender\" example related files"} +clean_examples_hello = {cmd = "rm -f examples/hello-world/*.log", help = "Clean \"hello\" example related files"} +clean_examples_http_proxy = {cmd = "rm -f examples/http-proxy/*.log", help = "Clean \"http_proxy\" example related files"} +clean_examples_simple_service = {cmd = "rm -f examples/simple-service-poc/*.png examples/simple-service-poc/*.log", help= "Clean example \"simple_service\" related files"} +clean_examples_ssh = {cmd = "rm -f examples/ssh/*.log", help = "Clean \"ssh\" example related files"} +clean_examples_webapp = {cmd = "rm -f examples/webapp/*.log", help = "Clean \"webapp\" example related files"} +clean_examples_yacat = {cmd = "rm -f examples/yacat/*.log", help = "Clean \"yacat\" example related files"} +sphinx = {cmd = "sphinx-build docs/sphinx/ build -E", help = "Build Sphinx docs"} -[tool.blacken-docs] -line-length = 80 +[tool.liccheck.authorized_packages] +ya-client-activity = "0.1.0" +ya-client-payment = "0.1.0" +ya-market = "0.1.0" [tool.liccheck] authorized_licenses = [ @@ -103,41 +128,49 @@ authorized_licenses = [ "apache 2", "apache", "Apache Software", + "Apache Software License", "MPL-2.0", + "Mozilla Public License 2.0 (MPL 2.0)", "MIT", "LGPL", "LGPL-3.0-or-later", + "GNU Lesser General Public License v3 or later (LGPLv3+)", + "GNU Lesser General Public License v3 (LGPLv3)", "Python Software Foundation", "ISC", + "ISC License (ISCL)", ] unauthorized_licenses = [ "gpl v3" ] -[tool.liccheck.authorized_packages] -ya-client-activity = "0.1.0" -ya-client-payment = "0.1.0" -ya-market = "0.1.0" +[tool.autoflake] +recursive = true +in-place = true +remove-all-unused-imports = true +ignore-init-module-imports = true -[tool.poe.tasks] -test = "pytest --cov=yapapi --ignore tests/goth_tests" -goth-assets = "python -m goth create-assets tests/goth_tests/assets" -goth-tests = "pytest -svx tests/goth_tests --config-override docker-compose.build-environment.use-prerelease=true --config-path tests/goth_tests/assets/goth-config-testing.yml --ssh-verify-connection --reruns 3 --only-rerun AssertionError --only-rerun TimeoutError --only-rerun goth.runner.exceptions.TemporalAssertionError --only-rerun urllib.error.URLError --only-rerun goth.runner.exceptions.CommandError" -typecheck = "mypy --check-untyped-defs --no-implicit-optional --show-error-codes ." -_codestyle_isort = "isort --check-only --diff ." -_codestyle_black = "black --check --diff ." -codestyle = ["_codestyle_isort", "_codestyle_black"] -_liccheck_export = "poetry export -f requirements.txt -o .requirements.txt" -_liccheck_verify = "liccheck -r .requirements.txt" -liccheck = ["_liccheck_export", "_liccheck_verify"] -check = ["test", "typecheck", "codestyle", "liccheck"] -clean = "rm -rf .coverage .requirements.txt dist md handbook build" -clean_blender = "rm -f examples/blender/*.png examples/blender/*.log" -clean_hello = "rm -f examples/hello-world/*.log" -clean_http_proxy = "rm -f examples/http-proxy/*.log" -clean_simple_service = "rm -f examples/simple-service-poc/*.png examples/simple-service-poc/*.log" -clean_ssh = "rm -f examples/ssh/*.log" -clean_webapp = "rm -f examples/webapp/*.log" -clean_yacat = "rm -f examples/yacat/*.log" -clean_examples = ["clean_blender", "clean_hello", "clean_http_proxy", "clean_simple_service", "clean_ssh", "clean_webapp", "clean_yacat"] -sphinx = "sphinx-build docs/sphinx/ build -E" +[tool.flake8] +max-line-length = 100 +extend-ignore = [ + "E203", # See https://github.com/PyCQA/pycodestyle/issues/373 + "E231", # black ignores this rule when formatting + "D105", # No docs for magic method + "D104", # No docs for public package + "D107", # No docs for __init__ + "D202", # We prefer whitelines after docstrings +] + +[tool.isort] +profile = "black" +py_version = 36 +known_golem = "goth, ya_activity, ya_market, ya_net, ya_payment" +sections = ["FUTURE", "STDLIB", "THIRDPARTY", "GOLEM", "FIRSTPARTY", "LOCALFOLDER"] +line_length = 100 + +[tool.black] +line-length = 100 +target-version = ['py36'] + +[tool.pytest.ini_options] +asyncio_mode = "auto" diff --git a/tests/conftest.py b/tests/conftest.py index b347c5c48..74463035b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ import os + import pytest from yapapi.config import ApiConfig diff --git a/tests/contrib/service/test_chunk.py b/tests/contrib/service/test_chunk.py index da15f7d2c..c3986b3a1 100644 --- a/tests/contrib/service/test_chunk.py +++ b/tests/contrib/service/test_chunk.py @@ -1,6 +1,7 @@ -import pytest import secrets +import pytest + from yapapi.contrib.service.chunk import chunks diff --git a/tests/drone/drone.py b/tests/drone/drone.py index 33afeb17c..7d38364cb 100755 --- a/tests/drone/drone.py +++ b/tests/drone/drone.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 import asyncio -from datetime import datetime, timedelta import pathlib import random import sys +from datetime import datetime, timedelta examples_dir = pathlib.Path(__file__).resolve().parents[2] / "examples" sys.path.append(str(examples_dir)) diff --git a/tests/engine/test_debit_note_intervals.py b/tests/engine/test_debit_note_intervals.py index 52a919750..f03ae2c8a 100644 --- a/tests/engine/test_debit_note_intervals.py +++ b/tests/engine/test_debit_note_intervals.py @@ -1,10 +1,11 @@ -from datetime import datetime, timedelta import functools -import pytest import re import sys +from datetime import datetime, timedelta from unittest import mock +import pytest + from tests.factories.rest.market import AgreementFactory from tests.factories.rest.payment import DebitNoteFactory from yapapi.engine import _Engine diff --git a/tests/engine/test_engine.py b/tests/engine/test_engine.py index 17bb143e1..5964659f3 100644 --- a/tests/engine/test_engine.py +++ b/tests/engine/test_engine.py @@ -1,13 +1,14 @@ """Unit tests for `yapapi.engine` module.""" -import pytest from unittest.mock import Mock +import pytest + +import yapapi.engine +import yapapi.rest from tests.factories.golem import GolemFactory from yapapi import Golem from yapapi.config import ApiConfig -import yapapi.engine from yapapi.engine import Job -import yapapi.rest @pytest.mark.parametrize( diff --git a/tests/executor/test_smartq.py b/tests/executor/test_smartq.py index f8fb5e687..a6314ec6d 100644 --- a/tests/executor/test_smartq.py +++ b/tests/executor/test_smartq.py @@ -1,9 +1,10 @@ import asyncio import itertools import logging -import pytest from random import randint +import pytest + from yapapi.executor._smartq import SmartQueue diff --git a/tests/factories/context.py b/tests/factories/context.py index b4f9f77c4..cb5cc76f1 100644 --- a/tests/factories/context.py +++ b/tests/factories/context.py @@ -1,6 +1,7 @@ -import factory from unittest import mock +import factory + from yapapi.ctx import WorkContext diff --git a/tests/factories/events.py b/tests/factories/events.py index 64a53c8bc..8adfe9a26 100644 --- a/tests/factories/events.py +++ b/tests/factories/events.py @@ -1,6 +1,7 @@ -import factory from unittest import mock +import factory + from yapapi.events import AgreementConfirmed, AgreementEvent, AgreementRejected diff --git a/tests/factories/golem.py b/tests/factories/golem.py index e4b620c8e..3df7efda8 100644 --- a/tests/factories/golem.py +++ b/tests/factories/golem.py @@ -1,7 +1,7 @@ import factory -from tests.factories.config import ApiConfigFactory import yapapi.golem +from tests.factories.config import ApiConfigFactory class GolemFactory(factory.Factory): diff --git a/tests/factories/network.py b/tests/factories/network.py index ea3c79fdb..7adbcbf70 100644 --- a/tests/factories/network.py +++ b/tests/factories/network.py @@ -1,8 +1,9 @@ import asyncio from concurrent import futures +from unittest import mock + import factory import faker -from unittest import mock from yapapi.network import Network diff --git a/tests/factories/rest/__init__.py b/tests/factories/rest/__init__.py index 180e31b83..f8eb54f35 100644 --- a/tests/factories/rest/__init__.py +++ b/tests/factories/rest/__init__.py @@ -1,6 +1,7 @@ -import factory from unittest import mock +import factory + from ya_market.api.requestor_api import RequestorApi diff --git a/tests/factories/rest/market.py b/tests/factories/rest/market.py index 7790920d0..594f674b9 100644 --- a/tests/factories/rest/market.py +++ b/tests/factories/rest/market.py @@ -1,4 +1,5 @@ import datetime + import factory from ya_market import models as market_models diff --git a/tests/factories/rest/payment.py b/tests/factories/rest/payment.py index 23e61162d..a327a819e 100644 --- a/tests/factories/rest/payment.py +++ b/tests/factories/rest/payment.py @@ -1,7 +1,8 @@ import datetime -import factory from unittest import mock +import factory + from ya_payment import models as payment_models from ya_payment.api.requestor_api import RequestorApi diff --git a/tests/goth_tests/conftest.py b/tests/goth_tests/conftest.py index 3c309770b..ca1e07365 100644 --- a/tests/goth_tests/conftest.py +++ b/tests/goth_tests/conftest.py @@ -1,9 +1,10 @@ import asyncio from datetime import datetime, timezone from pathlib import Path -import pytest from typing import List, cast +import pytest + from goth.configuration import Override from yapapi.payload import vm diff --git a/tests/goth_tests/test_agreement_termination/requestor.py b/tests/goth_tests/test_agreement_termination/requestor.py index 24f7e184b..065b4db48 100755 --- a/tests/goth_tests/test_agreement_termination/requestor.py +++ b/tests/goth_tests/test_agreement_termination/requestor.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 """A requestor script used for testing agreement termination.""" import asyncio -from datetime import timedelta import logging +from datetime import timedelta from yapapi import Golem, Task, WorkContext from yapapi.log import enable_default_logger diff --git a/tests/goth_tests/test_agreement_termination/test_agreement_termination.py b/tests/goth_tests/test_agreement_termination/test_agreement_termination.py index 5d05a6a5a..e5674787d 100644 --- a/tests/goth_tests/test_agreement_termination/test_agreement_termination.py +++ b/tests/goth_tests/test_agreement_termination/test_agreement_termination.py @@ -1,12 +1,13 @@ """A goth test scenario for agreement termination.""" -from functools import partial import logging import os -from pathlib import Path -import pytest import re +from functools import partial +from pathlib import Path from typing import List +import pytest + from goth.configuration import Override, load_yaml from goth.runner import Runner from goth.runner.log import configure_logging diff --git a/tests/goth_tests/test_async_task_generation/requestor.py b/tests/goth_tests/test_async_task_generation/requestor.py index ac3f3e877..996842a96 100755 --- a/tests/goth_tests/test_async_task_generation/requestor.py +++ b/tests/goth_tests/test_async_task_generation/requestor.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 """A requestor script for testing asynchronous generation of input tasks.""" import asyncio -from datetime import timedelta import pathlib import sys +from datetime import timedelta from yapapi import Golem, Task from yapapi.log import enable_default_logger, log_event_repr diff --git a/tests/goth_tests/test_async_task_generation/test_async_task_generation.py b/tests/goth_tests/test_async_task_generation/test_async_task_generation.py index b4b091d36..556e1ba41 100644 --- a/tests/goth_tests/test_async_task_generation/test_async_task_generation.py +++ b/tests/goth_tests/test_async_task_generation/test_async_task_generation.py @@ -1,9 +1,10 @@ import logging import os from pathlib import Path -import pytest from typing import List +import pytest + import goth.configuration from goth.runner import Runner from goth.runner.log import configure_logging diff --git a/tests/goth_tests/test_concurrent_executors/test_concurrent_executors.py b/tests/goth_tests/test_concurrent_executors/test_concurrent_executors.py index 570885350..9eee5f732 100644 --- a/tests/goth_tests/test_concurrent_executors/test_concurrent_executors.py +++ b/tests/goth_tests/test_concurrent_executors/test_concurrent_executors.py @@ -3,9 +3,10 @@ import logging import os from pathlib import Path -import pytest from typing import List +import pytest + import goth.configuration from goth.runner import Runner from goth.runner.log import configure_logging diff --git a/tests/goth_tests/test_instance_restart/requestor.py b/tests/goth_tests/test_instance_restart/requestor.py index 3af138d52..61ba97318 100755 --- a/tests/goth_tests/test_instance_restart/requestor.py +++ b/tests/goth_tests/test_instance_restart/requestor.py @@ -6,16 +6,11 @@ termination before going to the `running` state. """ import asyncio -from datetime import datetime import sys +from datetime import datetime from yapapi import Golem -from yapapi.log import ( # noqa - enable_default_logger, - log_event_repr, - log_summary, - pluralize, -) +from yapapi.log import enable_default_logger, log_event_repr, log_summary, pluralize # noqa from yapapi.payload import vm from yapapi.services import Service diff --git a/tests/goth_tests/test_instance_restart/test_instance_restart.py b/tests/goth_tests/test_instance_restart/test_instance_restart.py index 6bbaaaffc..011d68805 100644 --- a/tests/goth_tests/test_instance_restart/test_instance_restart.py +++ b/tests/goth_tests/test_instance_restart/test_instance_restart.py @@ -3,9 +3,10 @@ import logging import os from pathlib import Path -import pytest from typing import List +import pytest + import goth.configuration from goth.runner import Runner from goth.runner.log import configure_logging diff --git a/tests/goth_tests/test_mid_agreement_payments/requestor_agent.py b/tests/goth_tests/test_mid_agreement_payments/requestor_agent.py index 86abe3e1c..551aa8025 100755 --- a/tests/goth_tests/test_mid_agreement_payments/requestor_agent.py +++ b/tests/goth_tests/test_mid_agreement_payments/requestor_agent.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import asyncio -from datetime import timedelta import logging +from datetime import timedelta from typing import AsyncIterable, List from yapapi import Golem, Task, WorkContext diff --git a/tests/goth_tests/test_mid_agreement_payments/test_mid_agreement_payments.py b/tests/goth_tests/test_mid_agreement_payments/test_mid_agreement_payments.py index b3e077f82..0a88614c3 100644 --- a/tests/goth_tests/test_mid_agreement_payments/test_mid_agreement_payments.py +++ b/tests/goth_tests/test_mid_agreement_payments/test_mid_agreement_payments.py @@ -1,11 +1,12 @@ """A goth test scenario for mid-agreement payments.""" import logging import os -from pathlib import Path -import pytest import re +from pathlib import Path from typing import List, Optional +import pytest + from goth.assertions import EventStream from goth.configuration import Override, load_yaml from goth.runner import Runner diff --git a/tests/goth_tests/test_multiactivity_agreement/requestor.py b/tests/goth_tests/test_multiactivity_agreement/requestor.py index fa0ded930..ef1f1d78e 100755 --- a/tests/goth_tests/test_multiactivity_agreement/requestor.py +++ b/tests/goth_tests/test_multiactivity_agreement/requestor.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 """A requestor script for testing if multiple workers are run for an agreement.""" import asyncio -from datetime import timedelta import logging +from datetime import timedelta from yapapi import Golem, Task from yapapi.log import enable_default_logger, log_event_repr # noqa diff --git a/tests/goth_tests/test_multiactivity_agreement/test_multiactivity_agreement.py b/tests/goth_tests/test_multiactivity_agreement/test_multiactivity_agreement.py index 66a7bbfea..197ce423f 100644 --- a/tests/goth_tests/test_multiactivity_agreement/test_multiactivity_agreement.py +++ b/tests/goth_tests/test_multiactivity_agreement/test_multiactivity_agreement.py @@ -1,12 +1,13 @@ """A goth test scenario for multi-activity agreements.""" -from functools import partial import logging import os -from pathlib import Path -import pytest import re +from functools import partial +from pathlib import Path from typing import List +import pytest + import goth.configuration from goth.runner import Runner from goth.runner.log import configure_logging diff --git a/tests/goth_tests/test_power_outage.py b/tests/goth_tests/test_power_outage.py index 4f069ac26..11429a7d6 100644 --- a/tests/goth_tests/test_power_outage.py +++ b/tests/goth_tests/test_power_outage.py @@ -1,9 +1,10 @@ import logging import os from pathlib import Path -import pytest from typing import List +import pytest + from goth.assertions import EventStream from goth.configuration import Override, load_yaml from goth.runner import Runner diff --git a/tests/goth_tests/test_recycle_ip/ssh_recycle_ip.py b/tests/goth_tests/test_recycle_ip/ssh_recycle_ip.py index 3e43a5517..2d0a1f567 100755 --- a/tests/goth_tests/test_recycle_ip/ssh_recycle_ip.py +++ b/tests/goth_tests/test_recycle_ip/ssh_recycle_ip.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 import asyncio -from datetime import timedelta import random import string +from datetime import timedelta from yapapi import Golem from yapapi.contrib.service.socket_proxy import SocketProxy, SocketProxyService diff --git a/tests/goth_tests/test_recycle_ip/test_recycle_ip.py b/tests/goth_tests/test_recycle_ip/test_recycle_ip.py index ddb8b7415..b85842f92 100644 --- a/tests/goth_tests/test_recycle_ip/test_recycle_ip.py +++ b/tests/goth_tests/test_recycle_ip/test_recycle_ip.py @@ -2,15 +2,16 @@ import asyncio import logging import os -from pathlib import Path -import pexpect -import pytest import re import signal import sys import time +from pathlib import Path from typing import List +import pexpect +import pytest + from goth.configuration import Override, load_yaml from goth.runner import Runner from goth.runner.log import configure_logging diff --git a/tests/goth_tests/test_renegotiate_proposal/requestor.py b/tests/goth_tests/test_renegotiate_proposal/requestor.py index 9cf539498..33bfb2009 100755 --- a/tests/goth_tests/test_renegotiate_proposal/requestor.py +++ b/tests/goth_tests/test_renegotiate_proposal/requestor.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 import asyncio -from asyncio import TimeoutError import datetime +from asyncio import TimeoutError + from typing_extensions import Final import ya_market diff --git a/tests/goth_tests/test_renegotiate_proposal/test_renegotiate_proposal.py b/tests/goth_tests/test_renegotiate_proposal/test_renegotiate_proposal.py index b2f2b71d7..f19bdd016 100644 --- a/tests/goth_tests/test_renegotiate_proposal/test_renegotiate_proposal.py +++ b/tests/goth_tests/test_renegotiate_proposal/test_renegotiate_proposal.py @@ -1,9 +1,10 @@ import logging import os from pathlib import Path -import pytest from typing import List +import pytest + from goth.configuration import Override, load_yaml from goth.runner import Runner from goth.runner.log import configure_logging diff --git a/tests/goth_tests/test_resubscription.py b/tests/goth_tests/test_resubscription.py index 660849a17..dc2ec59be 100644 --- a/tests/goth_tests/test_resubscription.py +++ b/tests/goth_tests/test_resubscription.py @@ -1,14 +1,16 @@ """Test if subscription expiration is handled correctly by Golem""" -import colors -from datetime import timedelta import logging import os -from pathlib import Path -import pytest import time +from datetime import timedelta +from pathlib import Path from typing import Dict, List, Set, Type from unittest.mock import Mock +import colors +import pytest + +import ya_market.api.requestor_api from goth.assertions import EventStream from goth.assertions.monitor import EventMonitor from goth.assertions.operators import eventually @@ -17,13 +19,12 @@ from goth.runner.log import configure_logging from goth.runner.probe import RequestorProbe from ya_market import ApiException -import ya_market.api.requestor_api +import yapapi.rest.market from yapapi import Golem, Task from yapapi.events import Event, JobFinished, JobStarted, SubscriptionCreated from yapapi.log import enable_default_logger from yapapi.payload import vm -import yapapi.rest.market logger = logging.getLogger("goth.test") diff --git a/tests/goth_tests/test_run_blender.py b/tests/goth_tests/test_run_blender.py index 6089356ab..39e95c8f1 100644 --- a/tests/goth_tests/test_run_blender.py +++ b/tests/goth_tests/test_run_blender.py @@ -1,9 +1,10 @@ import logging import os from pathlib import Path -import pytest from typing import List +import pytest + from goth.assertions import EventStream from goth.configuration import Override, load_yaml from goth.runner import Runner @@ -12,11 +13,7 @@ from yapapi.log import SummaryLogger -from .assertions import ( - assert_all_invoices_accepted, - assert_no_errors, - assert_tasks_processed, -) +from .assertions import assert_all_invoices_accepted, assert_no_errors, assert_tasks_processed logger = logging.getLogger("goth.test.run_blender") diff --git a/tests/goth_tests/test_run_custom_usage_counter.py b/tests/goth_tests/test_run_custom_usage_counter.py index 55e09747f..e88c5414d 100644 --- a/tests/goth_tests/test_run_custom_usage_counter.py +++ b/tests/goth_tests/test_run_custom_usage_counter.py @@ -1,11 +1,12 @@ """An integration test scenario that runs custom usage counter example requestor app.""" import logging import os -from pathlib import Path -import pytest import re +from pathlib import Path from typing import List +import pytest + from goth.assertions import EventStream from goth.configuration import Override, load_yaml from goth.runner import Runner diff --git a/tests/goth_tests/test_run_scan.py b/tests/goth_tests/test_run_scan.py index 24c6bb052..b33ed2984 100644 --- a/tests/goth_tests/test_run_scan.py +++ b/tests/goth_tests/test_run_scan.py @@ -2,12 +2,13 @@ import asyncio import logging import os -from pathlib import Path -import pytest import re import signal +from pathlib import Path from typing import List +import pytest + from goth.configuration import Override, load_yaml from goth.runner import Runner from goth.runner.log import configure_logging diff --git a/tests/goth_tests/test_run_simple_service.py b/tests/goth_tests/test_run_simple_service.py index ca9b6113f..03e9902b7 100644 --- a/tests/goth_tests/test_run_simple_service.py +++ b/tests/goth_tests/test_run_simple_service.py @@ -1,11 +1,12 @@ """An integration test scenario that runs the Simple Service example requestor app.""" import logging import os -from pathlib import Path -import pytest import time +from pathlib import Path from typing import List +import pytest + from goth.configuration import Override, load_yaml from goth.runner import Runner from goth.runner.log import configure_logging diff --git a/tests/goth_tests/test_run_ssh.py b/tests/goth_tests/test_run_ssh.py index 36775fca7..ca1d1ec7a 100644 --- a/tests/goth_tests/test_run_ssh.py +++ b/tests/goth_tests/test_run_ssh.py @@ -2,14 +2,15 @@ import asyncio import logging import os -from pathlib import Path -import pexpect -import pytest import re import signal import time +from pathlib import Path from typing import List +import pexpect +import pytest + from goth.configuration import Override, load_yaml from goth.runner import Runner from goth.runner.log import configure_logging diff --git a/tests/goth_tests/test_run_webapp.py b/tests/goth_tests/test_run_webapp.py index 7f422d47d..722e674da 100644 --- a/tests/goth_tests/test_run_webapp.py +++ b/tests/goth_tests/test_run_webapp.py @@ -2,13 +2,14 @@ import asyncio import logging import os -from pathlib import Path -import pytest -import requests import signal import time +from pathlib import Path from typing import List +import pytest +import requests + from goth.configuration import Override, load_yaml from goth.runner import Runner from goth.runner.log import configure_logging diff --git a/tests/goth_tests/test_run_yacat.py b/tests/goth_tests/test_run_yacat.py index 0905a55a5..7ae9055b2 100644 --- a/tests/goth_tests/test_run_yacat.py +++ b/tests/goth_tests/test_run_yacat.py @@ -1,11 +1,12 @@ -from datetime import datetime import logging import math import os +from datetime import datetime from pathlib import Path -import pytest from typing import List +import pytest + from goth.assertions import EventStream from goth.configuration import Override, load_yaml from goth.runner import Runner @@ -14,11 +15,7 @@ from yapapi.log import SummaryLogger -from .assertions import ( - assert_all_invoices_accepted, - assert_no_errors, - assert_tasks_processed, -) +from .assertions import assert_all_invoices_accepted, assert_no_errors, assert_tasks_processed logger = logging.getLogger("goth.test.run_yacat") diff --git a/tests/payload/test_payload.py b/tests/payload/test_payload.py index 5ce561452..34a7b6ac6 100644 --- a/tests/payload/test_payload.py +++ b/tests/payload/test_payload.py @@ -1,5 +1,5 @@ -from dataclasses import dataclass import pytest +from dataclasses import dataclass from yapapi.payload import Payload from yapapi.props import constraint, inf, prop diff --git a/tests/payload/test_vm.py b/tests/payload/test_vm.py index f5f62100d..000b01441 100644 --- a/tests/payload/test_vm.py +++ b/tests/payload/test_vm.py @@ -1,6 +1,7 @@ +from unittest import mock + from dns.exception import DNSException from srvresolver.srv_resolver import SRVRecord # type: ignore -from unittest import mock from yapapi.payload.vm import _FALLBACK_REPO_URL, resolve_repo_srv diff --git a/tests/props/test_base.py b/tests/props/test_base.py index 69b7ca22a..a08f4cb8a 100644 --- a/tests/props/test_base.py +++ b/tests/props/test_base.py @@ -1,5 +1,5 @@ -from dataclasses import dataclass import pytest +from dataclasses import dataclass from yapapi.props import base as props_base diff --git a/tests/props/test_builder.py b/tests/props/test_builder.py index aaf5471d1..645a8c7e4 100644 --- a/tests/props/test_builder.py +++ b/tests/props/test_builder.py @@ -1,5 +1,5 @@ -from dataclasses import dataclass import pytest +from dataclasses import dataclass from yapapi.props import constraint, prop from yapapi.props.builder import AutodecoratingModel, DemandBuilder diff --git a/tests/rest/test_activity.py b/tests/rest/test_activity.py index ee6b04bf3..0769b43eb 100644 --- a/tests/rest/test_activity.py +++ b/tests/rest/test_activity.py @@ -1,7 +1,8 @@ -import pytest from typing import List, Optional, Tuple, Type from unittest.mock import Mock +import pytest + from ya_activity.exceptions import ApiException from yapapi.rest.activity import BatchError, PollingBatch diff --git a/tests/rest/test_allocation.py b/tests/rest/test_allocation.py index 699374ed3..0b217690a 100644 --- a/tests/rest/test_allocation.py +++ b/tests/rest/test_allocation.py @@ -1,4 +1,5 @@ from decimal import Decimal + import pytest from yapapi.config import ApiConfig diff --git a/tests/rest/test_repeat_on_error.py b/tests/rest/test_repeat_on_error.py index 620863b6a..b1b7ba4eb 100644 --- a/tests/rest/test_repeat_on_error.py +++ b/tests/rest/test_repeat_on_error.py @@ -1,16 +1,13 @@ -import aiohttp import asyncio + +import aiohttp import pytest import ya_activity import ya_market import ya_payment -from yapapi.rest.common import ( - SuppressedExceptions, - is_intermittent_error, - repeat_on_error, -) +from yapapi.rest.common import SuppressedExceptions, is_intermittent_error, repeat_on_error @pytest.mark.parametrize( diff --git a/tests/script/test_script.py b/tests/script/test_script.py index a27d8b251..2dbdd075c 100644 --- a/tests/script/test_script.py +++ b/tests/script/test_script.py @@ -1,9 +1,10 @@ -from functools import partial import json -import pytest import sys +from functools import partial from unittest import mock +import pytest + from yapapi.events import CommandExecuted, CommandStdErr, CommandStdOut from yapapi.script import Script diff --git a/tests/services/test_cluster.py b/tests/services/test_cluster.py index 6f82ea09d..d3b5d218c 100644 --- a/tests/services/test_cluster.py +++ b/tests/services/test_cluster.py @@ -1,10 +1,11 @@ import asyncio import itertools -import pytest import sys from unittest import mock from unittest.mock import Mock, patch +import pytest + from tests.factories.golem import GolemFactory from yapapi import Golem from yapapi.services import Service, ServiceRunner diff --git a/tests/services/test_service_runner.py b/tests/services/test_service_runner.py index d057dcf57..e72ce1cf9 100644 --- a/tests/services/test_service_runner.py +++ b/tests/services/test_service_runner.py @@ -1,10 +1,11 @@ import asyncio -import pytest -from statemachine import State import sys from typing import Optional from unittest import mock +import pytest +from statemachine import State + from ya_activity.exceptions import ApiException from yapapi.ctx import WorkContext diff --git a/tests/storage/test_gftp.py b/tests/storage/test_gftp.py index 5ce9e7dda..38bfe05f9 100644 --- a/tests/storage/test_gftp.py +++ b/tests/storage/test_gftp.py @@ -1,11 +1,12 @@ import asyncio -from collections import defaultdict -from pathlib import Path -import pytest import random import tempfile +from collections import defaultdict +from pathlib import Path from typing import List, cast +import pytest + from yapapi.storage import gftp diff --git a/tests/strategy/test_base.py b/tests/strategy/test_base.py index 9484455ac..71bccec14 100644 --- a/tests/strategy/test_base.py +++ b/tests/strategy/test_base.py @@ -1,4 +1,5 @@ from datetime import datetime, timedelta, timezone + import pytest from tests.factories.rest.market import OfferProposalFactory diff --git a/tests/strategy/test_decrease_score_for_unconfirmed.py b/tests/strategy/test_decrease_score_for_unconfirmed.py index 47d296594..7c967db85 100644 --- a/tests/strategy/test_decrease_score_for_unconfirmed.py +++ b/tests/strategy/test_decrease_score_for_unconfirmed.py @@ -1,4 +1,5 @@ import asyncio + import pytest from tests.factories.events import AgreementConfirmedFactory as AgreementConfirmed diff --git a/tests/strategy/test_default_strategies.py b/tests/strategy/test_default_strategies.py index 83d2edfe5..dde511754 100644 --- a/tests/strategy/test_default_strategies.py +++ b/tests/strategy/test_default_strategies.py @@ -1,13 +1,14 @@ from decimal import Decimal from itertools import product -import pytest from unittest.mock import Mock +import pytest + +import yapapi.rest.configuration from tests.factories.golem import GolemFactory from tests.factories.rest.market import OfferProposalFactory from yapapi import Golem from yapapi.props.com import Counter -import yapapi.rest.configuration from yapapi.strategy import ( SCORE_NEUTRAL, SCORE_REJECTED, diff --git a/tests/test_add_event_consumer.py b/tests/test_add_event_consumer.py index 48051ceea..115195071 100644 --- a/tests/test_add_event_consumer.py +++ b/tests/test_add_event_consumer.py @@ -1,5 +1,5 @@ -from attr import asdict import pytest +from attr import asdict from yapapi import Golem, events diff --git a/tests/test_agreements_pool.py b/tests/test_agreements_pool.py index a82ebcaa8..3e35aaa89 100644 --- a/tests/test_agreements_pool.py +++ b/tests/test_agreements_pool.py @@ -1,10 +1,11 @@ -from factories.agreements_pool import BufferedAgreement, BufferedAgreementFactory -from operator import xor -import pytest import random import sys +from operator import xor from unittest import mock +import pytest +from factories.agreements_pool import BufferedAgreement, BufferedAgreementFactory + from yapapi import agreements_pool from yapapi.events import AgreementTerminated diff --git a/tests/test_async_wrapper.py b/tests/test_async_wrapper.py index 40d0429af..9d74b9224 100644 --- a/tests/test_async_wrapper.py +++ b/tests/test_async_wrapper.py @@ -1,7 +1,8 @@ import asyncio -import pytest import time +import pytest + from yapapi.utils import AsyncWrapper diff --git a/tests/test_config.py b/tests/test_config.py index 293ad3405..0b927695d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,4 +1,5 @@ import os + import pytest import yapapi.config diff --git a/tests/test_ctx.py b/tests/test_ctx.py index 26fccfdec..dad07a09e 100644 --- a/tests/test_ctx.py +++ b/tests/test_ctx.py @@ -1,10 +1,11 @@ -import factory -from functools import partial import json -import pytest import sys +from functools import partial from unittest import mock +import factory +import pytest + from yapapi.ctx import WorkContext from yapapi.script import Script diff --git a/tests/test_network.py b/tests/test_network.py index 5b8bf5243..41271e6c1 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -1,8 +1,9 @@ -import pytest -from statemachine.exceptions import TransitionNotAllowed import sys from unittest import mock +import pytest +from statemachine.exceptions import TransitionNotAllowed + from yapapi.network import Network, NetworkError, NetworkState if sys.version_info >= (3, 8): diff --git a/tests/test_payment_platforms.py b/tests/test_payment_platforms.py index 136204783..452a6f667 100644 --- a/tests/test_payment_platforms.py +++ b/tests/test_payment_platforms.py @@ -1,7 +1,8 @@ """Unit tests for code that selects payment platforms based on driver/network specification.""" -import pytest from unittest import mock +import pytest + from ya_payment import RequestorApi from tests.factories.golem import GolemFactory diff --git a/tests/test_yapapi.py b/tests/test_yapapi.py index 540a3cb81..fa4a0cd46 100644 --- a/tests/test_yapapi.py +++ b/tests/test_yapapi.py @@ -1,5 +1,6 @@ import asyncio from pathlib import Path + import toml import yapapi diff --git a/yapapi/__init__.py b/yapapi/__init__.py index 74f933b9f..80fedd6db 100644 --- a/yapapi/__init__.py +++ b/yapapi/__init__.py @@ -1,9 +1,10 @@ """Golem Python API.""" import asyncio -from pathlib import Path -from pkg_resources import get_distribution import sys +from pathlib import Path + import toml +from pkg_resources import get_distribution from yapapi.ctx import ExecOptions, WorkContext from yapapi.engine import NoPaymentAccountError diff --git a/yapapi/agreements_pool.py b/yapapi/agreements_pool.py index e5875417d..3a9200311 100644 --- a/yapapi/agreements_pool.py +++ b/yapapi/agreements_pool.py @@ -1,12 +1,13 @@ -import aiohttp import asyncio -from dataclasses import dataclass import datetime import logging import random import sys from typing import Callable, Dict, NamedTuple, Optional +import aiohttp +from dataclasses import dataclass + from yapapi import events from yapapi.props import Activity, NodeInfo from yapapi.rest.market import Agreement, AgreementDetails, ApiException, OfferProposal diff --git a/yapapi/config.py b/yapapi/config.py index 623fb1155..8e757d871 100644 --- a/yapapi/config.py +++ b/yapapi/config.py @@ -1,8 +1,9 @@ -from dataclasses import dataclass, field -from functools import partial import os +from functools import partial from typing import Optional +from dataclasses import dataclass, field + class MissingConfiguration(Exception): def __init__(self, key: str, description: str): diff --git a/yapapi/contrib/service/http_proxy.py b/yapapi/contrib/service/http_proxy.py index 04cb309f2..7b5489b5b 100644 --- a/yapapi/contrib/service/http_proxy.py +++ b/yapapi/contrib/service/http_proxy.py @@ -11,14 +11,15 @@ yapapi repository. """ import abc -import aiohttp -from aiohttp import client_ws, web import asyncio import logging -from multidict import CIMultiDict import re import traceback from typing import Optional + +import aiohttp +from aiohttp import client_ws, web +from multidict import CIMultiDict from typing_extensions import Final from yapapi.services import Cluster, Service, ServiceState diff --git a/yapapi/contrib/service/socket_proxy.py b/yapapi/contrib/service/socket_proxy.py index f33362431..bf4051f07 100644 --- a/yapapi/contrib/service/socket_proxy.py +++ b/yapapi/contrib/service/socket_proxy.py @@ -10,11 +10,12 @@ yapapi repository. """ import abc -import aiohttp import asyncio import itertools import logging from typing import Dict, Iterator, List, Optional + +import aiohttp from typing_extensions import Final from yapapi.services import Cluster, Service diff --git a/yapapi/ctx.py b/yapapi/ctx.py index ebcecf430..7b0b6b88d 100644 --- a/yapapi/ctx.py +++ b/yapapi/ctx.py @@ -1,9 +1,10 @@ -from dataclasses import dataclass, field -from datetime import datetime, timedelta import enum import logging +from datetime import datetime, timedelta from typing import Any, Awaitable, Callable, Dict, List, Optional, Type +from dataclasses import dataclass, field + try: from typing import Protocol except ImportError: diff --git a/yapapi/engine.py b/yapapi/engine.py index d0a1a67f8..a13a0fa6d 100644 --- a/yapapi/engine.py +++ b/yapapi/engine.py @@ -1,16 +1,14 @@ -import aiohttp import asyncio -from asyncio import CancelledError -from collections import defaultdict import contextlib -from copy import deepcopy -from dataclasses import dataclass -from datetime import datetime, timezone -from decimal import Decimal import itertools import logging import os import sys +from asyncio import CancelledError +from collections import defaultdict +from copy import deepcopy +from datetime import datetime, timezone +from decimal import Decimal from typing import ( AsyncContextManager, Awaitable, @@ -24,6 +22,9 @@ Union, cast, ) + +import aiohttp +from dataclasses import dataclass from typing_extensions import AsyncGenerator, Final from yapapi.config import ApiConfig diff --git a/yapapi/events.py b/yapapi/events.py index 618296584..e85d55c12 100644 --- a/yapapi/events.py +++ b/yapapi/events.py @@ -180,12 +180,13 @@ def event_consumer(event): """ import abc -import attr -from datetime import datetime, timedelta import logging +from datetime import datetime, timedelta from types import TracebackType from typing import List, Optional, Tuple, Type, TypeVar +import attr + # Q: Why `import yapapi` here? # A: Because we want to have typing annotations without circular imports # Q: Why not `if TYPE_CHECKING:`? diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index 83d7bf1a9..ef5d00c3b 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -1,8 +1,8 @@ """An implementation of the new Golem's task executor.""" import asyncio +import sys from asyncio import CancelledError from datetime import datetime, timedelta, timezone -import sys from typing import ( AsyncIterator, Awaitable, @@ -15,15 +15,16 @@ Union, cast, ) + from typing_extensions import AsyncGenerator, Final +import yapapi.utils from yapapi import events from yapapi.ctx import WorkContext from yapapi.engine import Job, _Engine from yapapi.payload import Payload from yapapi.script import Script from yapapi.script.command import Deploy, Start -import yapapi.utils from ._smartq import SmartQueue from .task import Task, TaskStatus diff --git a/yapapi/executor/_smartq.py b/yapapi/executor/_smartq.py index 5756125a9..63a11da66 100644 --- a/yapapi/executor/_smartq.py +++ b/yapapi/executor/_smartq.py @@ -4,19 +4,11 @@ """ import asyncio -from asyncio.locks import Condition, Lock import logging +from asyncio.locks import Condition, Lock from types import TracebackType -from typing import ( - AsyncIterator, - ContextManager, - Dict, - Generic, - Optional, - Set, - Type, - TypeVar, -) +from typing import AsyncIterator, ContextManager, Dict, Generic, Optional, Set, Type, TypeVar + from typing_extensions import AsyncIterable _logger = logging.getLogger("yapapi.executor") diff --git a/yapapi/executor/task.py b/yapapi/executor/task.py index 8ef80d677..07e08d146 100644 --- a/yapapi/executor/task.py +++ b/yapapi/executor/task.py @@ -1,19 +1,8 @@ import asyncio +import itertools from datetime import datetime, timedelta, timezone from enum import Enum, auto -import itertools -from typing import ( - Callable, - ClassVar, - Generic, - Iterator, - Optional, - Set, - Tuple, - Type, - TypeVar, - Union, -) +from typing import Callable, ClassVar, Generic, Iterator, Optional, Set, Tuple, Type, TypeVar, Union try: from typing import Protocol diff --git a/yapapi/golem.py b/yapapi/golem.py index 55979a8c6..717a397fb 100644 --- a/yapapi/golem.py +++ b/yapapi/golem.py @@ -1,8 +1,8 @@ import asyncio -from datetime import datetime, timedelta -from decimal import Decimal import json import sys +from datetime import datetime, timedelta +from decimal import Decimal from typing import ( TYPE_CHECKING, Any, @@ -17,6 +17,7 @@ TypeVar, Union, ) + from typing_extensions import AsyncGenerator from yapapi.config import ApiConfig @@ -39,10 +40,7 @@ from yapapi.props import com from yapapi.script import Script from yapapi.services import Cluster, ServiceType -from yapapi.strategy import ( - DecreaseScoreForUnconfirmedAgreement, - LeastExpensiveLinearPayuMS, -) +from yapapi.strategy import DecreaseScoreForUnconfirmedAgreement, LeastExpensiveLinearPayuMS if TYPE_CHECKING: from yapapi.strategy import BaseMarketStrategy diff --git a/yapapi/invoice_manager.py b/yapapi/invoice_manager.py index 758b23ac2..8b635b5da 100644 --- a/yapapi/invoice_manager.py +++ b/yapapi/invoice_manager.py @@ -1,9 +1,10 @@ +import sys from asyncio import CancelledError -from dataclasses import dataclass from decimal import Decimal -import sys from typing import TYPE_CHECKING, Awaitable, Callable, Dict, Optional, Set +from dataclasses import dataclass + from yapapi import events if TYPE_CHECKING: diff --git a/yapapi/log.py b/yapapi/log.py index 3ab929381..b225d91cd 100644 --- a/yapapi/log.py +++ b/yapapi/log.py @@ -41,18 +41,19 @@ ) ``` """ -from asyncio import CancelledError, get_event_loop -from collections import Counter, defaultdict -from dataclasses import dataclass -from datetime import datetime, timedelta, timezone -from decimal import Decimal import inspect import logging import os import sys import time +from asyncio import CancelledError, get_event_loop +from collections import Counter, defaultdict +from datetime import datetime, timedelta, timezone +from decimal import Decimal from typing import Any, Callable, Dict, List, Optional, Set +from dataclasses import dataclass + if sys.version_info >= (3, 8): from typing import Final else: diff --git a/yapapi/network.py b/yapapi/network.py index b9e79b516..58ead8638 100644 --- a/yapapi/network.py +++ b/yapapi/network.py @@ -1,18 +1,12 @@ import asyncio -from dataclasses import dataclass -from ipaddress import ( - IPv4Address, - IPv4Network, - IPv6Address, - IPv6Network, - ip_address, - ip_network, -) import logging -from statemachine import State, StateMachine # type: ignore +from ipaddress import IPv4Address, IPv4Network, IPv6Address, IPv6Network, ip_address, ip_network from typing import Dict, Optional, Union from urllib.parse import urlparse +from dataclasses import dataclass +from statemachine import State, StateMachine # type: ignore + from ya_net.exceptions import ApiException import yapapi diff --git a/yapapi/payload/package.py b/yapapi/payload/package.py index 1b60dbe1b..81c0882b9 100644 --- a/yapapi/payload/package.py +++ b/yapapi/payload/package.py @@ -1,4 +1,5 @@ import abc + import aiohttp from dataclasses import dataclass diff --git a/yapapi/payload/vm.py b/yapapi/payload/vm.py index 45588aaf6..3899aa026 100644 --- a/yapapi/payload/vm.py +++ b/yapapi/payload/vm.py @@ -1,9 +1,10 @@ -from dataclasses import dataclass -from dns.exception import DNSException -from enum import Enum import logging import sys +from enum import Enum from typing import List, Optional + +from dataclasses import dataclass +from dns.exception import DNSException from typing_extensions import Final if sys.version_info > (3, 8): @@ -22,13 +23,7 @@ from yapapi.props import base as prop_base from yapapi.props import inf from yapapi.props.builder import DemandBuilder, Model -from yapapi.props.inf import ( - INF_CORES, - RUNTIME_VM, - ExeUnitManifestRequest, - ExeUnitRequest, - InfBase, -) +from yapapi.props.inf import INF_CORES, RUNTIME_VM, ExeUnitManifestRequest, ExeUnitRequest, InfBase _DEFAULT_REPO_SRV: Final[str] = "_girepo._tcp.dev.golem.network" _FALLBACK_REPO_URL: Final[str] = "http://girepo.dev.golem.network:8000" diff --git a/yapapi/props/__init__.py b/yapapi/props/__init__.py index be5bc8191..dbe7ed521 100644 --- a/yapapi/props/__init__.py +++ b/yapapi/props/__init__.py @@ -1,8 +1,9 @@ -from dataclasses import dataclass, field from datetime import datetime from decimal import Decimal from typing import Optional +from dataclasses import dataclass, field + from .base import InvalidPropertiesError, Model, constraint, prop diff --git a/yapapi/props/base.py b/yapapi/props/base.py index 66971db4c..25cca942d 100644 --- a/yapapi/props/base.py +++ b/yapapi/props/base.py @@ -11,9 +11,10 @@ else: from typing_extensions import Literal -from dataclasses import MISSING, Field, dataclass, field, fields from datetime import datetime, timezone +from dataclasses import MISSING, Field, dataclass, field, fields + Props = Dict[str, str] PROP_KEY = "key" diff --git a/yapapi/props/builder.py b/yapapi/props/builder.py index 318e5afdd..a24f2081a 100644 --- a/yapapi/props/builder.py +++ b/yapapi/props/builder.py @@ -1,11 +1,12 @@ import abc -from dataclasses import asdict -from datetime import datetime import enum +from datetime import datetime from typing import List -from . import Model +from dataclasses import asdict + from ..rest.market import Market, Subscription +from . import Model from .base import constraint_model_serialize, join_str_constraints diff --git a/yapapi/props/com.py b/yapapi/props/com.py index 11ef9ea21..93671aeb0 100644 --- a/yapapi/props/com.py +++ b/yapapi/props/com.py @@ -1,9 +1,10 @@ """Payment-related properties.""" import abc -from dataclasses import dataclass, field import enum from typing import Any, Dict, List +from dataclasses import dataclass, field + from .base import Model, Props, as_list SCHEME: str = "golem.com.scheme" diff --git a/yapapi/props/inf.py b/yapapi/props/inf.py index ccb2e5c41..fdfd56848 100644 --- a/yapapi/props/inf.py +++ b/yapapi/props/inf.py @@ -1,10 +1,11 @@ """Infrastructural properties.""" -from dataclasses import dataclass -from deprecated import deprecated # type: ignore from enum import Enum from typing import List, Optional +from dataclasses import dataclass +from deprecated import deprecated # type: ignore + from .base import Model, prop INF_MEM: str = "golem.inf.mem.gib" diff --git a/yapapi/rest/activity.py b/yapapi/rest/activity.py index f7a57542a..00c73785b 100644 --- a/yapapi/rest/activity.py +++ b/yapapi/rest/activity.py @@ -1,12 +1,13 @@ import abc -from aiohttp import ClientPayloadError -from aiohttp_sse_client.client import MessageEvent # type: ignore import asyncio -from dataclasses import dataclass -from datetime import datetime, timedelta, timezone import json import logging +from datetime import datetime, timedelta, timezone from typing import Any, AsyncIterator, Dict, List, Optional, Tuple, Type + +from aiohttp import ClientPayloadError +from aiohttp_sse_client.client import MessageEvent # type: ignore +from dataclasses import dataclass from typing_extensions import AsyncContextManager, AsyncIterable from ya_activity import ApiClient, ApiException, RequestorControlApi, RequestorStateApi diff --git a/yapapi/rest/common.py b/yapapi/rest/common.py index 9bf9752aa..1a36a7cc2 100644 --- a/yapapi/rest/common.py +++ b/yapapi/rest/common.py @@ -1,9 +1,10 @@ -import aiohttp import asyncio import functools import logging from typing import Callable, Optional +import aiohttp + import ya_activity import ya_market import ya_payment diff --git a/yapapi/rest/configuration.py b/yapapi/rest/configuration.py index 28d19123a..d6d7ee6b0 100644 --- a/yapapi/rest/configuration.py +++ b/yapapi/rest/configuration.py @@ -1,5 +1,6 @@ import os from typing import Optional + from typing_extensions import Final import ya_activity # type: ignore diff --git a/yapapi/rest/market.py b/yapapi/rest/market.py index b38f46a81..43b6b31b8 100644 --- a/yapapi/rest/market.py +++ b/yapapi/rest/market.py @@ -1,10 +1,11 @@ -import aiohttp import asyncio -from dataclasses import dataclass -from datetime import datetime, timedelta, timezone import logging +from datetime import datetime, timedelta, timezone from types import TracebackType from typing import Any, AsyncIterator, Generator, Generic, Optional, Type, TypeVar + +import aiohttp +from dataclasses import dataclass from typing_extensions import AsyncContextManager, Awaitable from ya_market import ApiClient, ApiException, RequestorApi, models # type: ignore diff --git a/yapapi/rest/payment.py b/yapapi/rest/payment.py index 86bb31c40..16311efb8 100644 --- a/yapapi/rest/payment.py +++ b/yapapi/rest/payment.py @@ -1,12 +1,13 @@ import asyncio -from dataclasses import dataclass +import logging from datetime import datetime, timedelta, timezone from decimal import Decimal -import logging from typing import AsyncIterator, Iterable, List, Optional, Union, cast -from ya_payment import Account, ApiClient, RequestorApi +from dataclasses import dataclass + import ya_payment.models as yap +from ya_payment import Account, ApiClient, RequestorApi from .common import SuppressedExceptions, is_intermittent_error, repeat_on_error from .resource import ResourceCtx diff --git a/yapapi/script/__init__.py b/yapapi/script/__init__.py index 4d2355cbe..781fbd5f0 100644 --- a/yapapi/script/__init__.py +++ b/yapapi/script/__init__.py @@ -1,17 +1,7 @@ +import itertools from asyncio import InvalidStateError from datetime import timedelta -import itertools -from typing import ( - TYPE_CHECKING, - Any, - Awaitable, - Callable, - Dict, - Iterator, - List, - Optional, - Type, -) +from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, Iterator, List, Optional, Type import yapapi from yapapi.events import CommandEvent, CommandExecuted, ScriptEventType diff --git a/yapapi/script/capture.py b/yapapi/script/capture.py index a3e44a0ee..aa5329a68 100644 --- a/yapapi/script/capture.py +++ b/yapapi/script/capture.py @@ -1,7 +1,8 @@ -from dataclasses import dataclass import enum from typing import Dict, Optional +from dataclasses import dataclass + class CaptureMode(enum.Enum): HEAD = "head" diff --git a/yapapi/script/command.py b/yapapi/script/command.py index 3d8ae94aa..b0da18298 100644 --- a/yapapi/script/command.py +++ b/yapapi/script/command.py @@ -1,29 +1,14 @@ import abc import asyncio -from functools import partial import json +from functools import partial from os import PathLike from pathlib import Path -from typing import ( - TYPE_CHECKING, - Any, - Awaitable, - Callable, - Dict, - List, - Optional, - Type, - Union, -) +from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, List, Optional, Type, Union from yapapi.events import CommandEventType, DownloadFinished, DownloadStarted from yapapi.script.capture import CaptureContext -from yapapi.storage import ( - DOWNLOAD_BYTES_LIMIT_DEFAULT, - Destination, - Source, - StorageProvider, -) +from yapapi.storage import DOWNLOAD_BYTES_LIMIT_DEFAULT, Destination, Source, StorageProvider if TYPE_CHECKING: from yapapi.script import Script diff --git a/yapapi/services/cluster.py b/yapapi/services/cluster.py index ec0ad2431..82ae250cb 100644 --- a/yapapi/services/cluster.py +++ b/yapapi/services/cluster.py @@ -1,17 +1,8 @@ """Implementation of high-level services API.""" -from datetime import datetime, timedelta, timezone import itertools import sys -from typing import ( - AsyncContextManager, - Dict, - Generator, - Generic, - Iterable, - List, - Optional, - Type, -) +from datetime import datetime, timedelta, timezone +from typing import AsyncContextManager, Dict, Generator, Generic, Iterable, List, Optional, Type if sys.version_info >= (3, 8): from contextlib import AsyncExitStack diff --git a/yapapi/services/service.py b/yapapi/services/service.py index 943397268..2f6ddfbb6 100644 --- a/yapapi/services/service.py +++ b/yapapi/services/service.py @@ -1,7 +1,6 @@ import asyncio -from dataclasses import dataclass, field import logging -import statemachine # type: ignore +import uuid from types import TracebackType from typing import ( TYPE_CHECKING, @@ -16,7 +15,9 @@ TypeVar, Union, ) -import uuid + +import statemachine # type: ignore +from dataclasses import dataclass, field from ya_activity.exceptions import ApiException diff --git a/yapapi/services/service_runner.py b/yapapi/services/service_runner.py index 6a01568e7..da889954c 100644 --- a/yapapi/services/service_runner.py +++ b/yapapi/services/service_runner.py @@ -4,16 +4,8 @@ import logging import sys from types import TracebackType -from typing import ( - TYPE_CHECKING, - AsyncContextManager, - List, - Optional, - Set, - Tuple, - Type, - Union, -) +from typing import TYPE_CHECKING, AsyncContextManager, List, Optional, Set, Tuple, Type, Union + from typing_extensions import Final if TYPE_CHECKING: diff --git a/yapapi/services/service_state.py b/yapapi/services/service_state.py index 238f010ff..707f65338 100644 --- a/yapapi/services/service_state.py +++ b/yapapi/services/service_state.py @@ -1,6 +1,7 @@ -import statemachine # type: ignore from typing import TYPE_CHECKING +import statemachine # type: ignore + if TYPE_CHECKING: from .service import ServiceInstance diff --git a/yapapi/storage/__init__.py b/yapapi/storage/__init__.py index fe466571a..91ae33dec 100644 --- a/yapapi/storage/__init__.py +++ b/yapapi/storage/__init__.py @@ -3,14 +3,15 @@ """ import abc -import aiohttp import asyncio import io import os -from os import PathLike import pathlib +from os import PathLike from typing import AsyncIterator, NamedTuple, Optional, Union +import aiohttp + _BUF_SIZE = 40960 DOWNLOAD_BYTES_LIMIT_DEFAULT = 1 * 1024 * 1024 AsyncReader = Union[asyncio.streams.StreamReader, aiohttp.streams.StreamReader] diff --git a/yapapi/storage/gftp.py b/yapapi/storage/gftp.py index 92117913d..17bdfaf9d 100644 --- a/yapapi/storage/gftp.py +++ b/yapapi/storage/gftp.py @@ -2,19 +2,15 @@ Golem File Transfer Storage Provider """ -from async_exit_stack import AsyncExitStack # type: ignore import asyncio import contextlib -from dataclasses import dataclass import json -import jsonrpc_base # type: ignore import logging import os -from os import PathLike -from pathlib import Path -import semantic_version # type: ignore import sys import tempfile +from os import PathLike +from pathlib import Path from types import TracebackType from typing import ( AsyncContextManager, @@ -28,6 +24,11 @@ Union, cast, ) + +import jsonrpc_base # type: ignore +import semantic_version # type: ignore +from async_exit_stack import AsyncExitStack # type: ignore +from dataclasses import dataclass from typing_extensions import Literal, Protocol, TypedDict from yapapi.storage import Content, Destination, Source, StorageProvider diff --git a/yapapi/storage/webdav.py b/yapapi/storage/webdav.py index 1aecd52d6..05897c164 100644 --- a/yapapi/storage/webdav.py +++ b/yapapi/storage/webdav.py @@ -1,11 +1,12 @@ -import aiohttp -from dataclasses import dataclass -from datetime import datetime import logging +import uuid +from datetime import datetime from os import PathLike from typing import AsyncIterator, List, NamedTuple, Optional from urllib.parse import urljoin, urlparse -import uuid + +import aiohttp +from dataclasses import dataclass from . import Content, Destination, Source, StorageProvider diff --git a/yapapi/strategy/base.py b/yapapi/strategy/base.py index dd32fbce9..d4b08ed87 100644 --- a/yapapi/strategy/base.py +++ b/yapapi/strategy/base.py @@ -1,12 +1,13 @@ """Implementation of strategies for choosing offers from market.""" import abc +import logging from copy import deepcopy -from dataclasses import dataclass from datetime import datetime, timezone from decimal import Decimal -import logging from typing import Dict, Optional + +from dataclasses import dataclass from typing_extensions import Final from yapapi import rest diff --git a/yapapi/strategy/dummy.py b/yapapi/strategy/dummy.py index 1d9968bc1..347a7a862 100644 --- a/yapapi/strategy/dummy.py +++ b/yapapi/strategy/dummy.py @@ -1,10 +1,11 @@ from collections import defaultdict -from dataclasses import dataclass from decimal import Decimal -from deprecated import deprecated # type: ignore from types import MappingProxyType from typing import Dict, Mapping, Optional, Union +from dataclasses import dataclass +from deprecated import deprecated # type: ignore + from yapapi import rest from yapapi.props import Activity, com from yapapi.props.builder import DemandBuilder diff --git a/yapapi/strategy/least_expensive.py b/yapapi/strategy/least_expensive.py index 9e5e8e214..e5ddfec8d 100644 --- a/yapapi/strategy/least_expensive.py +++ b/yapapi/strategy/least_expensive.py @@ -1,10 +1,11 @@ +import logging from collections import defaultdict -from dataclasses import dataclass from decimal import Decimal -import logging from types import MappingProxyType from typing import Dict, Mapping, Union +from dataclasses import dataclass + from yapapi import rest from yapapi.props import com from yapapi.props.builder import DemandBuilder diff --git a/yapapi/utils.py b/yapapi/utils.py index 814949cdd..0db9a7b1c 100644 --- a/yapapi/utils.py +++ b/yapapi/utils.py @@ -1,11 +1,11 @@ """Utility functions and classes used within yapapi.""" import asyncio -from datetime import datetime, timezone, tzinfo import enum import functools import logging -from typing import AsyncContextManager, Callable, Optional import warnings +from datetime import datetime, timezone, tzinfo +from typing import AsyncContextManager, Callable, Optional logger = logging.getLogger(__name__) From fadcdd98f4b64bba97a7ceac3147ab1a1c63769e Mon Sep 17 00:00:00 2001 From: approxit Date: Wed, 18 Jan 2023 11:58:19 +0100 Subject: [PATCH 02/20] most of flake8 errors --- .gitignore | 2 +- examples/blender/__init__.py | 0 examples/custom_runtime/__init__.py | 0 .../README.md | 0 examples/custom_usage_counter/__init__.py | 0 .../custom_usage_counter.py | 0 .../Dockerfile | 0 examples/external_api_request/__init__.py | 0 .../external_api_request.py | 2 +- .../golem_sign.pem | 0 .../manifest.json | 0 .../manifest.json.base64.sha256.sig | Bin .../request.sh | 0 .../sign.sh | 0 .../{hello-world => hello_world}/Dockerfile | 0 examples/hello_world/__init__.py | 0 .../{hello-world => hello_world}/hello.py | 0 .../hello_service.py | 0 .../{http-proxy => http_proxy}/Dockerfile | 0 examples/http_proxy/__init__.py | 0 .../{http-proxy => http_proxy}/http_proxy.py | 2 +- examples/low_level_api/__init__.py | 0 .../list-offers.py | 0 examples/market_strategy/__init__.py | 0 .../market_strategy.py | 2 +- examples/scan/__init__.py | 0 examples/simple_service_poc/__init__.py | 0 .../simple_service.py | 1 - .../simple_service/README.md | 0 .../simple_service/__init__.py | 0 .../simple_service/simple_service.Dockerfile | 0 .../simple_service/simple_service.py | 0 .../simple_service/simulate_observations.py | 0 .../simulate_observations_ctl.py | 0 examples/ssh/__init__.py | 0 examples/ssh/ssh.py | 1 - examples/webapp/__init__.py | 0 examples/webapp/http/__init__.py | 0 examples/webapp_fileupload/__init__.py | 0 .../Dockerfile | 0 .../http_file_upload/__init__.py | 0 .../app.py | 0 .../templates/error.html | 0 .../templates/index.html | 0 examples/yacat/__init__.py | 0 examples/yacat/yacat.py | 2 - pyproject.toml | 10 +- tests/__init__.py | 0 tests/conftest.py | 6 +- tests/contrib/__init__.py | 0 tests/contrib/service/__init__.py | 0 tests/drone/__init__.py | 0 tests/drone/drone.py | 16 +-- tests/engine/__init__.py | 0 tests/engine/test_debit_note_intervals.py | 3 +- tests/engine/test_engine.py | 2 - tests/events/__init__.py | 0 tests/events/test_repr.py | 3 +- tests/executor/__init__.py | 0 tests/executor/test_smartq.py | 2 +- tests/goth_tests/conftest.py | 4 +- .../test_agreement_termination/__init__.py | 0 .../test_agreement_termination/requestor.py | 2 +- .../test_async_task_generation/__init__.py | 0 .../test_concurrent_executors/__init__.py | 0 .../test_concurrent_executors/requestor.py | 2 +- .../test_instance_restart/__init__.py | 0 .../test_multiactivity_agreement/__init__.py | 0 .../test_multiactivity_agreement.py | 2 +- tests/goth_tests/test_recycle_ip/__init__.py | 0 .../test_recycle_ip/test_recycle_ip.py | 2 - .../test_renegotiate_proposal/__init__.py | 0 .../test_renegotiate_proposal/requestor.py | 13 +- tests/goth_tests/test_resubscription.py | 15 +-- .../test_run_custom_usage_counter.py | 6 +- tests/goth_tests/test_run_scan.py | 8 +- tests/goth_tests/test_run_simple_service.py | 2 +- tests/payload/__init__.py | 0 tests/props/__init__.py | 0 tests/props/test_from_properties.py | 11 +- tests/rest/__init__.py | 0 tests/script/__init__.py | 0 tests/services/__init__.py | 0 tests/storage/__init__.py | 0 tests/storage/test_gftp.py | 2 +- tests/strategy/test_base.py | 3 +- .../test_decrease_score_for_unconfirmed.py | 10 +- tests/strategy/test_default_strategies.py | 6 +- tests/strategy/test_provider_filter.py | 9 +- tests/test_async_wrapper.py | 2 +- tests/test_ctx.py | 1 - tests/test_log.py | 4 +- tests/test_network.py | 16 +-- tests/test_payment_platforms.py | 2 - tests/test_yapapi.py | 6 +- yapapi/__init__.py | 11 +- yapapi/agreements_pool.py | 16 +-- yapapi/config.py | 18 ++- yapapi/contrib/service/http_proxy.py | 28 ++-- yapapi/contrib/service/socket_proxy.py | 17 ++- yapapi/contrib/strategy/__init__.py | 2 + yapapi/contrib/strategy/provider_filter.py | 15 +-- yapapi/ctx.py | 11 +- yapapi/engine.py | 48 ++++--- yapapi/events.py | 25 ++-- yapapi/executor/__init__.py | 6 +- yapapi/executor/_smartq.py | 17 +-- yapapi/golem.py | 125 ++++++++++-------- yapapi/invoice_manager.py | 19 +-- yapapi/log.py | 8 +- yapapi/network.py | 44 +++--- yapapi/payload/__init__.py | 4 +- yapapi/payload/package.py | 4 +- yapapi/payload/vm.py | 9 +- yapapi/props/__init__.py | 13 +- yapapi/props/base.py | 20 +-- yapapi/props/builder.py | 10 +- yapapi/props/com.py | 5 +- yapapi/rest/__init__.py | 10 +- yapapi/rest/activity.py | 4 +- yapapi/rest/configuration.py | 13 +- yapapi/rest/market.py | 9 +- yapapi/rest/payment.py | 5 +- yapapi/script/__init__.py | 31 +++-- yapapi/script/command.py | 7 +- yapapi/services/__init__.py | 9 ++ yapapi/services/cluster.py | 18 ++- yapapi/services/service.py | 73 +++++----- yapapi/services/service_runner.py | 20 +-- yapapi/storage/__init__.py | 11 +- yapapi/storage/gftp.py | 19 +-- yapapi/strategy/__init__.py | 17 +++ yapapi/strategy/base.py | 18 ++- yapapi/strategy/decrease_score_unconfirmed.py | 6 +- yapapi/strategy/wrapping_strategy.py | 13 +- 135 files changed, 504 insertions(+), 436 deletions(-) create mode 100644 examples/blender/__init__.py create mode 100644 examples/custom_runtime/__init__.py rename examples/{custom-usage-counter => custom_usage_counter}/README.md (100%) create mode 100644 examples/custom_usage_counter/__init__.py rename examples/{custom-usage-counter => custom_usage_counter}/custom_usage_counter.py (100%) mode change 100755 => 100644 rename examples/{external-api-request => external_api_request}/Dockerfile (100%) create mode 100644 examples/external_api_request/__init__.py rename examples/{external-api-request => external_api_request}/external_api_request.py (97%) rename examples/{external-api-request => external_api_request}/golem_sign.pem (100%) rename examples/{external-api-request => external_api_request}/manifest.json (100%) rename examples/{external-api-request => external_api_request}/manifest.json.base64.sha256.sig (100%) rename examples/{external-api-request => external_api_request}/request.sh (100%) mode change 100755 => 100644 rename examples/{external-api-request => external_api_request}/sign.sh (100%) mode change 100755 => 100644 rename examples/{hello-world => hello_world}/Dockerfile (100%) create mode 100644 examples/hello_world/__init__.py rename examples/{hello-world => hello_world}/hello.py (100%) mode change 100755 => 100644 rename examples/{hello-world => hello_world}/hello_service.py (100%) mode change 100755 => 100644 rename examples/{http-proxy => http_proxy}/Dockerfile (100%) create mode 100644 examples/http_proxy/__init__.py rename examples/{http-proxy => http_proxy}/http_proxy.py (99%) create mode 100644 examples/low_level_api/__init__.py rename examples/{low-level-api => low_level_api}/list-offers.py (100%) mode change 100755 => 100644 create mode 100644 examples/market_strategy/__init__.py rename examples/{market-strategy => market_strategy}/market_strategy.py (98%) mode change 100755 => 100644 create mode 100644 examples/scan/__init__.py create mode 100644 examples/simple_service_poc/__init__.py rename examples/{simple-service-poc => simple_service_poc}/simple_service.py (99%) mode change 100755 => 100644 rename examples/{simple-service-poc => simple_service_poc}/simple_service/README.md (100%) create mode 100644 examples/simple_service_poc/simple_service/__init__.py rename examples/{simple-service-poc => simple_service_poc}/simple_service/simple_service.Dockerfile (100%) rename examples/{simple-service-poc => simple_service_poc}/simple_service/simple_service.py (100%) rename examples/{simple-service-poc => simple_service_poc}/simple_service/simulate_observations.py (100%) rename examples/{simple-service-poc => simple_service_poc}/simple_service/simulate_observations_ctl.py (100%) create mode 100644 examples/ssh/__init__.py create mode 100644 examples/webapp/__init__.py create mode 100644 examples/webapp/http/__init__.py create mode 100644 examples/webapp_fileupload/__init__.py rename examples/webapp_fileupload/{http-file-upload => http_file_upload}/Dockerfile (100%) create mode 100644 examples/webapp_fileupload/http_file_upload/__init__.py rename examples/webapp_fileupload/{http-file-upload => http_file_upload}/app.py (100%) rename examples/webapp_fileupload/{http-file-upload => http_file_upload}/templates/error.html (100%) rename examples/webapp_fileupload/{http-file-upload => http_file_upload}/templates/index.html (100%) create mode 100644 examples/yacat/__init__.py create mode 100644 tests/__init__.py create mode 100644 tests/contrib/__init__.py create mode 100644 tests/contrib/service/__init__.py create mode 100644 tests/drone/__init__.py create mode 100644 tests/engine/__init__.py create mode 100644 tests/events/__init__.py create mode 100644 tests/executor/__init__.py create mode 100644 tests/goth_tests/test_agreement_termination/__init__.py create mode 100644 tests/goth_tests/test_async_task_generation/__init__.py create mode 100644 tests/goth_tests/test_concurrent_executors/__init__.py create mode 100644 tests/goth_tests/test_instance_restart/__init__.py create mode 100644 tests/goth_tests/test_multiactivity_agreement/__init__.py create mode 100644 tests/goth_tests/test_recycle_ip/__init__.py create mode 100644 tests/goth_tests/test_renegotiate_proposal/__init__.py create mode 100644 tests/payload/__init__.py create mode 100644 tests/props/__init__.py create mode 100644 tests/rest/__init__.py create mode 100644 tests/script/__init__.py create mode 100644 tests/services/__init__.py create mode 100644 tests/storage/__init__.py diff --git a/.gitignore b/.gitignore index 60171ce6f..8e2f2c1f0 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ __pycache__/ *.log *.png .coverage -.requirements.txt +requirements.txt tests/goth_tests/assets/ diff --git a/examples/blender/__init__.py b/examples/blender/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/custom_runtime/__init__.py b/examples/custom_runtime/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/custom-usage-counter/README.md b/examples/custom_usage_counter/README.md similarity index 100% rename from examples/custom-usage-counter/README.md rename to examples/custom_usage_counter/README.md diff --git a/examples/custom_usage_counter/__init__.py b/examples/custom_usage_counter/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/custom-usage-counter/custom_usage_counter.py b/examples/custom_usage_counter/custom_usage_counter.py old mode 100755 new mode 100644 similarity index 100% rename from examples/custom-usage-counter/custom_usage_counter.py rename to examples/custom_usage_counter/custom_usage_counter.py diff --git a/examples/external-api-request/Dockerfile b/examples/external_api_request/Dockerfile similarity index 100% rename from examples/external-api-request/Dockerfile rename to examples/external_api_request/Dockerfile diff --git a/examples/external_api_request/__init__.py b/examples/external_api_request/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/external-api-request/external_api_request.py b/examples/external_api_request/external_api_request.py similarity index 97% rename from examples/external-api-request/external_api_request.py rename to examples/external_api_request/external_api_request.py index 86b8a8955..c84da7995 100644 --- a/examples/external-api-request/external_api_request.py +++ b/examples/external_api_request/external_api_request.py @@ -81,7 +81,7 @@ async def main(subnet_tag, payment_driver, payment_network): if __name__ == "__main__": parser = build_parser("External API request example") now = datetime.now().strftime("%Y-%m-%d_%H.%M.%S") - parser.set_defaults(log_file=f"external-api-request-yapapi-{now}.log") + parser.set_defaults(log_file=f"external_api_request-yapapi-{now}.log") args = parser.parse_args() run_golem_example( diff --git a/examples/external-api-request/golem_sign.pem b/examples/external_api_request/golem_sign.pem similarity index 100% rename from examples/external-api-request/golem_sign.pem rename to examples/external_api_request/golem_sign.pem diff --git a/examples/external-api-request/manifest.json b/examples/external_api_request/manifest.json similarity index 100% rename from examples/external-api-request/manifest.json rename to examples/external_api_request/manifest.json diff --git a/examples/external-api-request/manifest.json.base64.sha256.sig b/examples/external_api_request/manifest.json.base64.sha256.sig similarity index 100% rename from examples/external-api-request/manifest.json.base64.sha256.sig rename to examples/external_api_request/manifest.json.base64.sha256.sig diff --git a/examples/external-api-request/request.sh b/examples/external_api_request/request.sh old mode 100755 new mode 100644 similarity index 100% rename from examples/external-api-request/request.sh rename to examples/external_api_request/request.sh diff --git a/examples/external-api-request/sign.sh b/examples/external_api_request/sign.sh old mode 100755 new mode 100644 similarity index 100% rename from examples/external-api-request/sign.sh rename to examples/external_api_request/sign.sh diff --git a/examples/hello-world/Dockerfile b/examples/hello_world/Dockerfile similarity index 100% rename from examples/hello-world/Dockerfile rename to examples/hello_world/Dockerfile diff --git a/examples/hello_world/__init__.py b/examples/hello_world/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/hello-world/hello.py b/examples/hello_world/hello.py old mode 100755 new mode 100644 similarity index 100% rename from examples/hello-world/hello.py rename to examples/hello_world/hello.py diff --git a/examples/hello-world/hello_service.py b/examples/hello_world/hello_service.py old mode 100755 new mode 100644 similarity index 100% rename from examples/hello-world/hello_service.py rename to examples/hello_world/hello_service.py diff --git a/examples/http-proxy/Dockerfile b/examples/http_proxy/Dockerfile similarity index 100% rename from examples/http-proxy/Dockerfile rename to examples/http_proxy/Dockerfile diff --git a/examples/http_proxy/__init__.py b/examples/http_proxy/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/http-proxy/http_proxy.py b/examples/http_proxy/http_proxy.py similarity index 99% rename from examples/http-proxy/http_proxy.py rename to examples/http_proxy/http_proxy.py index 63a8e909a..d81bf0d2d 100644 --- a/examples/http-proxy/http_proxy.py +++ b/examples/http_proxy/http_proxy.py @@ -177,7 +177,7 @@ def still_starting(): ), ) now = datetime.now().strftime("%Y-%m-%d_%H.%M.%S") - parser.set_defaults(log_file=f"http-proxy-{now}.log") + parser.set_defaults(log_file=f"http_proxy-{now}.log") args = parser.parse_args() run_golem_example( diff --git a/examples/low_level_api/__init__.py b/examples/low_level_api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/low-level-api/list-offers.py b/examples/low_level_api/list-offers.py old mode 100755 new mode 100644 similarity index 100% rename from examples/low-level-api/list-offers.py rename to examples/low_level_api/list-offers.py diff --git a/examples/market_strategy/__init__.py b/examples/market_strategy/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/market-strategy/market_strategy.py b/examples/market_strategy/market_strategy.py old mode 100755 new mode 100644 similarity index 98% rename from examples/market-strategy/market_strategy.py rename to examples/market_strategy/market_strategy.py index ed904c8ae..3338ea778 --- a/examples/market-strategy/market_strategy.py +++ b/examples/market_strategy/market_strategy.py @@ -97,7 +97,7 @@ async def worker(ctx: WorkContext, tasks): if __name__ == "__main__": parser = build_parser("Select fastest provider using a simple reputation-based market strategy") - parser.set_defaults(log_file="market-strategy-example.log") + parser.set_defaults(log_file="market_strategy-example.log") args = parser.parse_args() run_golem_example( diff --git a/examples/scan/__init__.py b/examples/scan/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/simple_service_poc/__init__.py b/examples/simple_service_poc/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/simple-service-poc/simple_service.py b/examples/simple_service_poc/simple_service.py old mode 100755 new mode 100644 similarity index 99% rename from examples/simple-service-poc/simple_service.py rename to examples/simple_service_poc/simple_service.py index a92e7704f..41c372ceb --- a/examples/simple-service-poc/simple_service.py +++ b/examples/simple_service_poc/simple_service.py @@ -20,7 +20,6 @@ TEXT_COLOR_CYAN, TEXT_COLOR_DEFAULT, TEXT_COLOR_MAGENTA, - TEXT_COLOR_RED, TEXT_COLOR_YELLOW, build_parser, format_usage, diff --git a/examples/simple-service-poc/simple_service/README.md b/examples/simple_service_poc/simple_service/README.md similarity index 100% rename from examples/simple-service-poc/simple_service/README.md rename to examples/simple_service_poc/simple_service/README.md diff --git a/examples/simple_service_poc/simple_service/__init__.py b/examples/simple_service_poc/simple_service/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/simple-service-poc/simple_service/simple_service.Dockerfile b/examples/simple_service_poc/simple_service/simple_service.Dockerfile similarity index 100% rename from examples/simple-service-poc/simple_service/simple_service.Dockerfile rename to examples/simple_service_poc/simple_service/simple_service.Dockerfile diff --git a/examples/simple-service-poc/simple_service/simple_service.py b/examples/simple_service_poc/simple_service/simple_service.py similarity index 100% rename from examples/simple-service-poc/simple_service/simple_service.py rename to examples/simple_service_poc/simple_service/simple_service.py diff --git a/examples/simple-service-poc/simple_service/simulate_observations.py b/examples/simple_service_poc/simple_service/simulate_observations.py similarity index 100% rename from examples/simple-service-poc/simple_service/simulate_observations.py rename to examples/simple_service_poc/simple_service/simulate_observations.py diff --git a/examples/simple-service-poc/simple_service/simulate_observations_ctl.py b/examples/simple_service_poc/simple_service/simulate_observations_ctl.py similarity index 100% rename from examples/simple-service-poc/simple_service/simulate_observations_ctl.py rename to examples/simple_service_poc/simple_service/simulate_observations_ctl.py diff --git a/examples/ssh/__init__.py b/examples/ssh/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/ssh/ssh.py b/examples/ssh/ssh.py index f8ee85e11..4562b15fe 100755 --- a/examples/ssh/ssh.py +++ b/examples/ssh/ssh.py @@ -22,7 +22,6 @@ TEXT_COLOR_CYAN, TEXT_COLOR_DEFAULT, TEXT_COLOR_RED, - TEXT_COLOR_YELLOW, build_parser, print_env_info, run_golem_example, diff --git a/examples/webapp/__init__.py b/examples/webapp/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/webapp/http/__init__.py b/examples/webapp/http/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/webapp_fileupload/__init__.py b/examples/webapp_fileupload/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/webapp_fileupload/http-file-upload/Dockerfile b/examples/webapp_fileupload/http_file_upload/Dockerfile similarity index 100% rename from examples/webapp_fileupload/http-file-upload/Dockerfile rename to examples/webapp_fileupload/http_file_upload/Dockerfile diff --git a/examples/webapp_fileupload/http_file_upload/__init__.py b/examples/webapp_fileupload/http_file_upload/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/webapp_fileupload/http-file-upload/app.py b/examples/webapp_fileupload/http_file_upload/app.py similarity index 100% rename from examples/webapp_fileupload/http-file-upload/app.py rename to examples/webapp_fileupload/http_file_upload/app.py diff --git a/examples/webapp_fileupload/http-file-upload/templates/error.html b/examples/webapp_fileupload/http_file_upload/templates/error.html similarity index 100% rename from examples/webapp_fileupload/http-file-upload/templates/error.html rename to examples/webapp_fileupload/http_file_upload/templates/error.html diff --git a/examples/webapp_fileupload/http-file-upload/templates/index.html b/examples/webapp_fileupload/http_file_upload/templates/index.html similarity index 100% rename from examples/webapp_fileupload/http-file-upload/templates/index.html rename to examples/webapp_fileupload/http_file_upload/templates/index.html diff --git a/examples/yacat/__init__.py b/examples/yacat/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/yacat/yacat.py b/examples/yacat/yacat.py index ca226ee30..4a7d82fa0 100755 --- a/examples/yacat/yacat.py +++ b/examples/yacat/yacat.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 import argparse -import asyncio import math import sys from datetime import datetime, timedelta @@ -21,7 +20,6 @@ TEXT_COLOR_DEFAULT, TEXT_COLOR_GREEN, TEXT_COLOR_RED, - TEXT_COLOR_YELLOW, build_parser, print_env_info, run_golem_example, diff --git a/pyproject.toml b/pyproject.toml index aac03fe1e..afea570f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,6 +85,7 @@ autoflake = "^1" flake8 = "^5" flake8-docstrings = "^1.6" Flake8-pyproject = "^1.2.2" +pyproject-autoflake = "^1.0.2" [tool.poe.tasks] checks = {sequence = ["checks_codestyle", "checks_typing", "checks_license"], help = "Run all available code checks"} @@ -92,12 +93,12 @@ checks_codestyle = {sequence = ["_checks_codestyle_flake8", "_checks_codestyle_i _checks_codestyle_flake8 = "flake8 yapapi tests" _checks_codestyle_isort = "isort --check-only --diff ." _checks_codestyle_black = "black --check --diff ." -checks_typing = {cmd = "mypy --install-types --non-interactive --ignore-missing-imports --check-untyped-defs --warn-unused-ignores --show-error-codes .", help = "Run only code typing checks" } +checks_typing = {cmd = "mypy --ignore-missing-imports --check-untyped-defs --warn-unused-ignores --show-error-codes .", help = "Run only code typing checks" } checks_license = {sequence = ["_checks_license_export", "_checks_license_verify"], help = "Run only license compatibility checks"} _checks_license_export = "poetry export -f requirements.txt -o .requirements.txt" _checks_license_verify = "liccheck -r .requirements.txt" format = {sequence = ["_format_autoflake", "_format_isort", "_format_black"], help = "Run code auto formatting"} -_format_autoflake = "autoflake ." +_format_autoflake = "pautoflake ." _format_isort = "isort ." _format_black = "black ." tests = {sequence = ["tests_unit", "tests_integration"], help = "Run all available tests"} @@ -155,6 +156,11 @@ max-line-length = 100 extend-ignore = [ "E203", # See https://github.com/PyCQA/pycodestyle/issues/373 "E231", # black ignores this rule when formatting + "D100", # TODO: To be fixed by #1087 + "D101", # TODO: To be fixed by #1087 + "D102", # TODO: To be fixed by #1087 + "D103", # TODO: To be fixed by #1087 + "D106", # TODO: To be fixed by #1087 "D105", # No docs for magic method "D104", # No docs for public package "D107", # No docs for __init__ diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/conftest.py b/tests/conftest.py index 74463035b..03cf5850c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,9 +2,6 @@ import pytest -from yapapi.config import ApiConfig -from yapapi.golem import Golem - def pytest_addoption(parser): @@ -18,7 +15,8 @@ def dummy_yagna_engine(monkeypatch): So also e.g. `async with Golem(..., APP_KEY='FAKE_APP_KEY')`, or Golem.start(). But first check if monkeypatches done here don't interefere with - the thing you want to test ofc.""" + the thing you want to test ofc. + """ from yapapi.engine import _Engine from yapapi.storage.gftp import GftpProvider diff --git a/tests/contrib/__init__.py b/tests/contrib/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/contrib/service/__init__.py b/tests/contrib/service/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/drone/__init__.py b/tests/drone/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/drone/drone.py b/tests/drone/drone.py index 7d38364cb..e03dc84a8 100755 --- a/tests/drone/drone.py +++ b/tests/drone/drone.py @@ -8,7 +8,7 @@ examples_dir = pathlib.Path(__file__).resolve().parents[2] / "examples" sys.path.append(str(examples_dir)) -from utils import ( +from utils import ( # noqa: E402 TEXT_COLOR_CYAN, TEXT_COLOR_DEFAULT, TEXT_COLOR_RED, @@ -16,12 +16,12 @@ build_parser, ) -from yapapi import Golem, NoPaymentAccountError, Task, WorkContext -from yapapi import __version__ as yapapi_version -from yapapi import windows_event_loop_fix -from yapapi.log import enable_default_logger -from yapapi.payload import vm -from yapapi.rest.activity import BatchTimeoutError +from yapapi import windows_event_loop_fix # noqa: E402 +from yapapi import Golem, NoPaymentAccountError, Task, WorkContext # noqa: E402 +from yapapi import __version__ as yapapi_version # noqa: E402 +from yapapi.log import enable_default_logger # noqa: E402 +from yapapi.payload import vm # noqa: E402 +from yapapi.rest.activity import BatchTimeoutError # noqa: E402 async def main(subnet_tag, payment_driver=None, payment_network=None): @@ -39,7 +39,7 @@ async def worker(ctx: WorkContext, tasks): script.run("/usr/bin/stress-ng", "--cpu", "1", "--timeout", "1") script.run("/golem/task.sh", "-o", "1024", "-t", "5") script.run("/golem/task.sh", "-f", "/golem/output/output.txt,1048576") - script.download_file(f"/golem/output/output.txt", output_file) + script.download_file("/golem/output/output.txt", output_file) script.run("/golem/task.sh", "-e", "1024", "-t", "5") try: diff --git a/tests/engine/__init__.py b/tests/engine/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/engine/test_debit_note_intervals.py b/tests/engine/test_debit_note_intervals.py index f03ae2c8a..9409a694c 100644 --- a/tests/engine/test_debit_note_intervals.py +++ b/tests/engine/test_debit_note_intervals.py @@ -180,7 +180,8 @@ def test_verify_debit_note_intervals( ({}, {}, lambda: None, 0, 0, False, False), # payable debit note received even though mid-agrement payment have not been negotiated ({}, {"_base__payment_due_date": True}, datetime.now, 0, 0, False, True), - # more than one debit note received before the agreed-upon interval elapsed, we only allow one + # more than one debit note received before the agreed-upon interval elapsed, + # we only allow one ( {"details___ref__demand__properties": {PROP_DEBIT_NOTE_INTERVAL_SEC: 100}}, {}, diff --git a/tests/engine/test_engine.py b/tests/engine/test_engine.py index 5964659f3..357ca483d 100644 --- a/tests/engine/test_engine.py +++ b/tests/engine/test_engine.py @@ -6,8 +6,6 @@ import yapapi.engine import yapapi.rest from tests.factories.golem import GolemFactory -from yapapi import Golem -from yapapi.config import ApiConfig from yapapi.engine import Job diff --git a/tests/events/__init__.py b/tests/events/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/events/test_repr.py b/tests/events/test_repr.py index e915090d2..cac50391a 100644 --- a/tests/events/test_repr.py +++ b/tests/events/test_repr.py @@ -42,7 +42,8 @@ ServiceFinished( job="a-job", agreement="an-agr", activity="an-act", service="a-service" ), - "ServiceFinished(job='a-job', agreement='an-agr', activity='an-act', service='a-service')", + "ServiceFinished(job='a-job', agreement='an-agr', activity='an-act'," + " service='a-service')", ), ( ScriptSent(job="a-job", agreement="an-agr", activity="an-act", script="a-script"), diff --git a/tests/executor/__init__.py b/tests/executor/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/executor/test_smartq.py b/tests/executor/test_smartq.py index a6314ec6d..bb42ec48a 100644 --- a/tests/executor/test_smartq.py +++ b/tests/executor/test_smartq.py @@ -206,7 +206,7 @@ async def worker(): @pytest.mark.asyncio async def test_smartq_finish(): - """check that `consumer.finish()` stops the queue from issuing new items for this consumer""" + """Check that `consumer.finish()` stops the queue from issuing new items for this consumer.""" consumed = [] diff --git a/tests/goth_tests/conftest.py b/tests/goth_tests/conftest.py index ca1e07365..ee50b6642 100644 --- a/tests/goth_tests/conftest.py +++ b/tests/goth_tests/conftest.py @@ -15,7 +15,7 @@ # The same problem occurs when `flaky` is used instead of `pytest-rerunfailures`. # Here we have a patch that is quite ugly, but hopefully harmless. class LoopThatIsNeverClosed(asyncio.AbstractEventLoop): - """Just a loop, but if you try to use it after it was closed you use a fresh loop""" + """Just a loop, but if you try to use it after it was closed you use a fresh loop.""" def __init__(self): self._loop = None @@ -30,7 +30,7 @@ def __getattribute__(self, name): @pytest.fixture def event_loop(): - """This overrides `pytest.asyncio` fixture""" + """Override `pytest.asyncio` fixture with never ending event loop.""" loop = LoopThatIsNeverClosed() yield loop loop.close() diff --git a/tests/goth_tests/test_agreement_termination/__init__.py b/tests/goth_tests/test_agreement_termination/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/goth_tests/test_agreement_termination/requestor.py b/tests/goth_tests/test_agreement_termination/requestor.py index 065b4db48..d98a5b26d 100755 --- a/tests/goth_tests/test_agreement_termination/requestor.py +++ b/tests/goth_tests/test_agreement_termination/requestor.py @@ -20,7 +20,7 @@ async def main(): first_worker = True async def worker(ctx: WorkContext, tasks): - """A worker function for `Golem.execute_tasks()`. + """Execute `Golem.execute_tasks()` as a worker. The first call to this function will produce a worker that sends an invalid `run` command to the provider. diff --git a/tests/goth_tests/test_async_task_generation/__init__.py b/tests/goth_tests/test_async_task_generation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/goth_tests/test_concurrent_executors/__init__.py b/tests/goth_tests/test_concurrent_executors/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/goth_tests/test_concurrent_executors/requestor.py b/tests/goth_tests/test_concurrent_executors/requestor.py index e7c275145..375eb7fcf 100755 --- a/tests/goth_tests/test_concurrent_executors/requestor.py +++ b/tests/goth_tests/test_concurrent_executors/requestor.py @@ -25,7 +25,7 @@ async def main(): first_task = True async def duplicator(work_ctx, tasks): - """A worker that executes `echo {task.data} {task.data}` command on provider. + """Execute `echo {task.data} {task.data}` command on provider as a worker. The command's output is the result of the whole task. diff --git a/tests/goth_tests/test_instance_restart/__init__.py b/tests/goth_tests/test_instance_restart/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/goth_tests/test_multiactivity_agreement/__init__.py b/tests/goth_tests/test_multiactivity_agreement/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/goth_tests/test_multiactivity_agreement/test_multiactivity_agreement.py b/tests/goth_tests/test_multiactivity_agreement/test_multiactivity_agreement.py index 197ce423f..948674731 100644 --- a/tests/goth_tests/test_multiactivity_agreement/test_multiactivity_agreement.py +++ b/tests/goth_tests/test_multiactivity_agreement/test_multiactivity_agreement.py @@ -38,7 +38,7 @@ async def assert_multiple_workers_run(agr_id, events): if m: worker_agr_id = m.group(1) assert worker_agr_id == agr_id, "Worker run for another agreement" - assert not "exception" in line, "Worker finished with error" + assert "exception" not in line, "Worker finished with error" workers_finished += 1 elif re.match("JobFinished", line): break diff --git a/tests/goth_tests/test_recycle_ip/__init__.py b/tests/goth_tests/test_recycle_ip/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/goth_tests/test_recycle_ip/test_recycle_ip.py b/tests/goth_tests/test_recycle_ip/test_recycle_ip.py index b85842f92..23cca5745 100644 --- a/tests/goth_tests/test_recycle_ip/test_recycle_ip.py +++ b/tests/goth_tests/test_recycle_ip/test_recycle_ip.py @@ -4,8 +4,6 @@ import os import re import signal -import sys -import time from pathlib import Path from typing import List diff --git a/tests/goth_tests/test_renegotiate_proposal/__init__.py b/tests/goth_tests/test_renegotiate_proposal/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/goth_tests/test_renegotiate_proposal/requestor.py b/tests/goth_tests/test_renegotiate_proposal/requestor.py index 33bfb2009..3be3922c5 100755 --- a/tests/goth_tests/test_renegotiate_proposal/requestor.py +++ b/tests/goth_tests/test_renegotiate_proposal/requestor.py @@ -5,9 +5,6 @@ from typing_extensions import Final -import ya_market - -from examples import utils from yapapi import props as yp from yapapi.config import ApiConfig from yapapi.log import enable_default_logger @@ -16,7 +13,8 @@ from yapapi.rest.market import OfferProposal DEBIT_NOTE_ACCEPTANCE_TIMEOUT_PROP: Final[str] = "golem.com.payment.debit-notes.accept-timeout?" -# TODO: Investigate number of offers per provider (https://github.com/golemfactory/yapapi/issues/754) +# TODO: Investigate number of offers per provider +# (https://github.com/golemfactory/yapapi/issues/754) PROPOSALS_LIMIT: Final[int] = 6 @@ -28,7 +26,7 @@ async def _respond(proposal: OfferProposal, dbuild) -> str: async def renegotiate_offers(conf: Configuration, subnet_tag: str): - """Rejects every proposal & then renegotiates it""" + """Reject every proposal & then renegotiates it.""" async with conf.market() as client: market_api = Market(client) dbuild = DemandBuilder() @@ -52,7 +50,7 @@ async def renegotiate_offers(conf: Configuration, subnet_tag: str): print(f"[{node_name}] prev_proposal_id: {prev_proposal_id}") if not event.is_draft: if proposals > PROPOSALS_LIMIT: - print(f"[node_name] Skipping additional proposal") + print("[node_name] Skipping additional proposal") break await _respond(event, dbuild) proposals += 1 @@ -61,7 +59,8 @@ async def renegotiate_offers(conf: Configuration, subnet_tag: str): continue print( - f"[{node_name}] Offer: {proposal_id} from {event.issuer} is_draft: {event.is_draft}" + f"[{node_name}] Offer: {proposal_id} from {event.issuer}" + f" is_draft: {event.is_draft}" ) if prev_proposal_id not in rejected_proposals: await event.reject() diff --git a/tests/goth_tests/test_resubscription.py b/tests/goth_tests/test_resubscription.py index dc2ec59be..9f773e0b0 100644 --- a/tests/goth_tests/test_resubscription.py +++ b/tests/goth_tests/test_resubscription.py @@ -1,4 +1,4 @@ -"""Test if subscription expiration is handled correctly by Golem""" +"""Test if subscription expiration is handled correctly by Golem.""" import logging import os import time @@ -33,19 +33,14 @@ class RequestorApi(ya_market.api.requestor_api.RequestorApi): - """A replacement for market API that simulates early subscription expiration. - - A call to `collect_offers(sub_id)` will raise `ApiException` indicating - subscription expiration when at least `SUBSCRIPTION_EXPIRATION_TIME` - elapsed after the given subscription has been created. - """ + """A replacement for market API that simulates early subscription expiration.""" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.subscriptions: Dict[str, float] = {} def subscribe_demand(self, demand, **kwargs): - """Override `RequestorApi.subscribe_demand()` to register subscription create time.""" + """Override parent call with register subscription create time.""" id_coro = super().subscribe_demand(demand, **kwargs) async def coro(): @@ -56,7 +51,7 @@ async def coro(): return coro() def collect_offers(self, subscription_id, **kwargs): - """Override `RequestorApi.collect_offers()`. + """Override parent call with additional error handling. Raise `ApiException(404)` if at least `SUBSCRIPTION_EXPIRATION_TIME` elapsed since the subscription identified by `subscription_id` has been created. @@ -93,7 +88,7 @@ async def unsubscribe_demand(sub_id: str) -> None: async def assert_demand_resubscribed(events: "EventStream[Event]"): - """A temporal assertion that the requestor will have to satisfy.""" + """Assert a temporal condition that the requestor will have to satisfy.""" subscription_ids: Set[str] = set() diff --git a/tests/goth_tests/test_run_custom_usage_counter.py b/tests/goth_tests/test_run_custom_usage_counter.py index e88c5414d..7a20e3bb7 100644 --- a/tests/goth_tests/test_run_custom_usage_counter.py +++ b/tests/goth_tests/test_run_custom_usage_counter.py @@ -31,7 +31,7 @@ async def assert_counter_not_decremented(output_lines: EventStream[str]): value = float(m.group(1)) logger.info(f"Custom usage counter value: {value}") if value < last_value: - raise AssertionError(f"Current custom usage counter was decremented.") + raise AssertionError("Current custom usage counter was decremented.") last_value = value @@ -75,7 +75,7 @@ async def test_run_custom_usage_counter( # This is the default configuration with 2 wasm/VM providers goth_config = load_yaml(goth_config_path, config_overrides) - requestor_path = project_dir / "examples" / "custom-usage-counter" / "custom_usage_counter.py" + requestor_path = project_dir / "examples" / "custom_usage_counter" / "custom_usage_counter.py" runner = Runner( base_log_dir=log_dir, @@ -98,4 +98,4 @@ async def test_run_custom_usage_counter( cmd_monitor.add_assertion(assert_counter_not_decremented) await cmd_monitor.wait_for_pattern(".*All jobs have finished", timeout=300) - logger.info(f"Requestor script finished") + logger.info("Requestor script finished") diff --git a/tests/goth_tests/test_run_scan.py b/tests/goth_tests/test_run_scan.py index b33ed2984..1808da378 100644 --- a/tests/goth_tests/test_run_scan.py +++ b/tests/goth_tests/test_run_scan.py @@ -62,13 +62,13 @@ async def test_run_scan( output = await cmd_monitor.wait_for_pattern( ".*Task finished by provider", timeout=120 ) - matches = re.match(".*by provider 'provider-(\d)', task data: (\d)", output) + matches = re.match(r".*by provider 'provider-(\d)', task data: (\d)", output) providers.add(matches.group(1)) tasks.add(matches.group(2)) assert providers == {"1", "2"} assert tasks == {"0", "1"} - logger.info(f"Scanner tasks completed for the two providers in the network.") + logger.info("Scanner tasks completed for the two providers in the network.") # ensure no more tasks are executed by the two providers logger.info("Waiting to see if another task gets started...") @@ -79,7 +79,7 @@ async def test_run_scan( ] assert len(tasks_finished) == 2 - logger.info(f"As expected, no more tasks started. Issuing a break...") + logger.info("As expected, no more tasks started. Issuing a break...") proc: asyncio.subprocess.Process = await process_monitor.get_process() proc.send_signal(signal.SIGINT) @@ -87,4 +87,4 @@ async def test_run_scan( logger.info("SIGINT sent...") await cmd_monitor.wait_for_pattern(".*All jobs have finished", timeout=20) - logger.info(f"Requestor script finished.") + logger.info("Requestor script finished.") diff --git a/tests/goth_tests/test_run_simple_service.py b/tests/goth_tests/test_run_simple_service.py index 03e9902b7..2a9390fc6 100644 --- a/tests/goth_tests/test_run_simple_service.py +++ b/tests/goth_tests/test_run_simple_service.py @@ -33,7 +33,7 @@ async def test_run_simple_service( # This is the default configuration with 2 wasm/VM providers goth_config = load_yaml(goth_config_path, config_overrides) - requestor_path = project_dir / "examples" / "simple-service-poc" / "simple_service.py" + requestor_path = project_dir / "examples" / "simple_service_poc" / "simple_service.py" runner = Runner( base_log_dir=log_dir, diff --git a/tests/payload/__init__.py b/tests/payload/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/props/__init__.py b/tests/props/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/props/test_from_properties.py b/tests/props/test_from_properties.py index c853be491..95927fec1 100644 --- a/tests/props/test_from_properties.py +++ b/tests/props/test_from_properties.py @@ -1,4 +1,4 @@ -"""Unit tests for `yapapi.props.base.Model.from_properties` method""" +"""Unit tests for `yapapi.props.base.Model.from_properties` method.""" import pytest @@ -52,7 +52,8 @@ "golem.com.usage.vector": ["golem.usage.cpu_sec", "golem.usage.duration_sec"], "golem.com.scheme": "payu", }, - "Invalid properties: expecting the number of linear_coeffs to correspond to usage_vector + 1 (fixed price)", + "Invalid properties: expecting the number of linear_coeffs to correspond to" + " usage_vector + 1 (fixed price)", ), ( { @@ -61,7 +62,8 @@ "golem.com.usage.vector": ["golem.usage.cpu_sec"], "golem.com.scheme": "payu", }, - "Invalid properties: expecting the number of linear_coeffs to correspond to usage_vector + 1 (fixed price)", + "Invalid properties: expecting the number of linear_coeffs to correspond to" + " usage_vector + 1 (fixed price)", ), ( { @@ -79,7 +81,8 @@ "golem.com.usage.vector": "not a vector", "golem.com.scheme": "payu", }, - "Invalid properties: expecting the number of linear_coeffs to correspond to usage_vector + 1 (fixed price)", + "Invalid properties: expecting the number of linear_coeffs to correspond to" + " usage_vector + 1 (fixed price)", ), ], ) diff --git a/tests/rest/__init__.py b/tests/rest/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/script/__init__.py b/tests/script/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/services/__init__.py b/tests/services/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/storage/__init__.py b/tests/storage/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/storage/test_gftp.py b/tests/storage/test_gftp.py index 38bfe05f9..36a031299 100644 --- a/tests/storage/test_gftp.py +++ b/tests/storage/test_gftp.py @@ -113,7 +113,7 @@ async def test_gftp_provider(test_dir, temp_dir, mock_service, monkeypatch): byte_uploads = 0 async def worker(id: int, provider: gftp.GftpProvider): - """A test worker.""" + """Test worker.""" nonlocal num_batches, temp_dir, file_uploads, byte_uploads diff --git a/tests/strategy/test_base.py b/tests/strategy/test_base.py index 71bccec14..7eaca586d 100644 --- a/tests/strategy/test_base.py +++ b/tests/strategy/test_base.py @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta, timezone +from datetime import datetime, timedelta import pytest @@ -13,7 +13,6 @@ PropValueRange, ) from yapapi.strategy.base import ( - DEFAULT_DEBIT_NOTE_ACCEPTANCE_TIMEOUT_SEC, DEFAULT_DEBIT_NOTE_INTERVAL_SEC, DEFAULT_PAYMENT_TIMEOUT_SEC, DEFAULT_PROPERTY_VALUE_RANGES, diff --git a/tests/strategy/test_decrease_score_for_unconfirmed.py b/tests/strategy/test_decrease_score_for_unconfirmed.py index 7c967db85..e868522ec 100644 --- a/tests/strategy/test_decrease_score_for_unconfirmed.py +++ b/tests/strategy/test_decrease_score_for_unconfirmed.py @@ -34,7 +34,7 @@ async def test_6(): @pytest.mark.asyncio @pytest.mark.parametrize("events_def, decreased_providers", sample_data) async def test_decrease_score_for(events_def, decreased_providers): - """Test if DecreaseScoreForUnconfirmedAgreement works as expected""" + """Test if DecreaseScoreForUnconfirmedAgreement works as expected.""" strategy = DecreaseScoreForUnconfirmedAgreement(Always6(), 0.5) for event_cls, event_provider_id in events_def: @@ -48,16 +48,14 @@ async def test_decrease_score_for(events_def, decreased_providers): def empty_event_consumer(event): - """To silience the default logger - it doesn't work with mocked events""" - pass + """To silience the default logger - it doesn't work with mocked events.""" @pytest.mark.asyncio @pytest.mark.parametrize("events_def, decreased_providers", sample_data) async def test_full_DSFUA_workflow(dummy_yagna_engine, events_def, decreased_providers): - """Test if DecreaseScoreForUnconfirmedAgreement is correctly initialized as a default strategy - - that is - if events emitted by the engine reach the event consumer of the default strategy""" + """Test if DecreaseScoreForUnconfirmedAgreement is correctly initialized as a default strategy \ + that is - if events emitted by the engine reach the event consumer of the default strategy.""" golem = Golem(budget=1, event_consumer=empty_event_consumer, app_key="NOT_A_REAL_APPKEY") async with golem: diff --git a/tests/strategy/test_default_strategies.py b/tests/strategy/test_default_strategies.py index dde511754..15d6fd6d2 100644 --- a/tests/strategy/test_default_strategies.py +++ b/tests/strategy/test_default_strategies.py @@ -4,10 +4,8 @@ import pytest -import yapapi.rest.configuration from tests.factories.golem import GolemFactory from tests.factories.rest.market import OfferProposalFactory -from yapapi import Golem from yapapi.props.com import Counter from yapapi.strategy import ( SCORE_NEUTRAL, @@ -79,12 +77,12 @@ async def _test_strategy(strategy, cpu_price_cap, time_price_cap, fixed_price_ca (None, time_price - epsilon, time_price, time_price + epsilon), (None, fixed_price - epsilon, fixed_price, fixed_price + epsilon), ): - if cpu_price_cap == time_price_cap == fixed_price_cap == None: + if cpu_price_cap is time_price_cap is fixed_price_cap is None: strategies = ( LeastExpensiveLinearPayuMS(), LeastExpensiveLinearPayuMS(max_price_for={}), ) - elif cpu_price_cap == time_price_cap == None: + elif cpu_price_cap is time_price_cap is None: strategies = ( LeastExpensiveLinearPayuMS(max_fixed_price=fixed_price_cap), LeastExpensiveLinearPayuMS(max_fixed_price=fixed_price_cap, max_price_for={}), diff --git a/tests/strategy/test_provider_filter.py b/tests/strategy/test_provider_filter.py index 6fad19067..64a0dbc43 100644 --- a/tests/strategy/test_provider_filter.py +++ b/tests/strategy/test_provider_filter.py @@ -14,7 +14,8 @@ ((), (1,), (2,), (1, 2)), ) async def test_restricted_providers(bad_providers): - """Test if the strategy restricts correct providers""" + """Test if the strategy restricts correct providers.""" + strategy = ProviderFilter(Always6(), lambda provider_id: provider_id not in bad_providers) for provider_id in (1, 2, 3): @@ -25,7 +26,8 @@ async def test_restricted_providers(bad_providers): @pytest.mark.asyncio async def test_dynamic_change(): - """Test if changes in the is_allowed function are reflected in scores""" + """Test if changes in the is_allowed function are reflected in scores.""" + something_happened = False def is_allowed(provider_id): @@ -46,7 +48,8 @@ def is_allowed(provider_id): @pytest.mark.asyncio async def test_default_strategy_update(dummy_yagna_engine): - """Test if default strategy extended with ProviderFilter works as expected""" + """Test if default strategy extended with ProviderFilter works as expected.""" + golem = Golem(budget=1, app_key="NOT_A_REAL_APPKEY") golem.strategy = ProviderFilter(golem.strategy, lambda provider_id: provider_id == 2) diff --git a/tests/test_async_wrapper.py b/tests/test_async_wrapper.py index 9d74b9224..be33c112d 100644 --- a/tests/test_async_wrapper.py +++ b/tests/test_async_wrapper.py @@ -64,7 +64,7 @@ def func(interrupt): raise KeyboardInterrupt() async def main(): - """This coroutine mimics how an AsyncWrapper is used in an Executor.""" + """Mimic how an AsyncWrapper is used in an Executor.""" async with AsyncWrapper(func) as wrapper: try: diff --git a/tests/test_ctx.py b/tests/test_ctx.py index dad07a09e..1b1fbdbe5 100644 --- a/tests/test_ctx.py +++ b/tests/test_ctx.py @@ -3,7 +3,6 @@ from functools import partial from unittest import mock -import factory import pytest from yapapi.ctx import WorkContext diff --git a/tests/test_log.py b/tests/test_log.py index 924827bb6..2d1e99c5f 100644 --- a/tests/test_log.py +++ b/tests/test_log.py @@ -40,7 +40,7 @@ def test_log_event_emit_traceback(): try: raise Exception("Hello!") - except: + except Exception: log_event(JobFinished(exc_info=sys.exc_info(), job="42")) @@ -49,7 +49,7 @@ def test_log_event_repr_emit_traceback(): try: raise Exception("Hello!") - except: + except Exception: log_event_repr(JobFinished(exc_info=sys.exc_info(), job="42")) diff --git a/tests/test_network.py b/tests/test_network.py index 41271e6c1..9f35fe9d2 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -107,9 +107,9 @@ async def test_add_node_pool_depleted(self): @pytest.mark.asyncio async def test_id_when_initialized(self): - network = Network(mock.Mock(), f"192.168.0.0/24", "0xdeadbeef") - with pytest.raises(TransitionNotAllowed, match=".*Can't get_id when in initialized.*") as e: - im_gonna_fail = network.network_id + network = Network(mock.Mock(), "192.168.0.0/24", "0xdeadbeef") + with pytest.raises(TransitionNotAllowed, match=".*Can't get_id when in initialized.*"): + getattr(network, "network_id") @pytest.mark.asyncio async def test_id_when_removed(self): @@ -118,8 +118,8 @@ async def test_id_when_removed(self): await network.remove() - with pytest.raises(TransitionNotAllowed, match=".*Can't get_id when in removed.*") as e: - im_gonna_fail = network.network_id + with pytest.raises(TransitionNotAllowed, match=".*Can't get_id when in removed.*"): + getattr(network, "network_id") @pytest.mark.asyncio async def test_remove(self): @@ -131,8 +131,8 @@ async def test_remove(self): @pytest.mark.asyncio async def test_remove_when_initialized(self): - network = Network(mock.Mock(), f"192.168.0.0/24", "0xdeadbeef") - with pytest.raises(TransitionNotAllowed, match=".*Can't stop when in initialized.*") as e: + network = Network(mock.Mock(), "192.168.0.0/24", "0xdeadbeef") + with pytest.raises(TransitionNotAllowed, match=".*Can't stop when in initialized.*"): await network.remove() @pytest.mark.asyncio @@ -141,7 +141,7 @@ async def test_remove_when_removed(self): await network.remove() - with pytest.raises(TransitionNotAllowed, match=".*Can't stop when in removed.*") as e: + with pytest.raises(TransitionNotAllowed, match=".*Can't stop when in removed.*"): await network.remove() @pytest.mark.asyncio diff --git a/tests/test_payment_platforms.py b/tests/test_payment_platforms.py index 452a6f667..71a46917a 100644 --- a/tests/test_payment_platforms.py +++ b/tests/test_payment_platforms.py @@ -7,9 +7,7 @@ from tests.factories.golem import GolemFactory from yapapi import NoPaymentAccountError -from yapapi.config import ApiConfig from yapapi.engine import DEFAULT_DRIVER, DEFAULT_NETWORK -from yapapi.golem import Golem from yapapi.rest.payment import Account, Payment diff --git a/tests/test_yapapi.py b/tests/test_yapapi.py index fa4a0cd46..8ff4056fe 100644 --- a/tests/test_yapapi.py +++ b/tests/test_yapapi.py @@ -19,6 +19,6 @@ async def _asyncio_test(): yapapi.windows_event_loop_fix() - l = asyncio.get_event_loop() - t = l.create_task(_asyncio_test()) - l.run_until_complete(t) + loop = asyncio.get_event_loop() + task = loop.create_task(_asyncio_test()) + loop.run_until_complete(task) diff --git a/yapapi/__init__.py b/yapapi/__init__.py index 80fedd6db..3155544bf 100644 --- a/yapapi/__init__.py +++ b/yapapi/__init__.py @@ -13,9 +13,7 @@ def get_version() -> str: - """ - :return: the version of the yapapi library package - """ + """Return the version of the yapapi library package.""" pyproject_path = Path(__file__).parents[1] / "pyproject.toml" if pyproject_path.exists(): with open(pyproject_path) as f: @@ -27,8 +25,7 @@ def get_version() -> str: def windows_event_loop_fix(): - """ - Set up asyncio to use ProactorEventLoop implementation for new event loops on Windows. + """Set up asyncio to use ProactorEventLoop implementation for new event loops on Windows. This work-around is only needed for Python 3.6 and 3.7. With Python 3.8, `ProactorEventLoop` is already the default on Windows. @@ -43,7 +40,7 @@ class _WindowsEventPolicy(asyncio.events.BaseDefaultEventLoopPolicy): __version__: str = get_version() -__all__ = [ +__all__ = ( "Executor", "props", "rest", @@ -54,4 +51,4 @@ class _WindowsEventPolicy(asyncio.events.BaseDefaultEventLoopPolicy): "ExecOptions", "Golem", "NoPaymentAccountError", -] +) diff --git a/yapapi/agreements_pool.py b/yapapi/agreements_pool.py index 3a9200311..f3962975a 100644 --- a/yapapi/agreements_pool.py +++ b/yapapi/agreements_pool.py @@ -16,7 +16,7 @@ class _BufferedProposal(NamedTuple): - """Providers' proposal with additional local metadata""" + """Providers' proposal with additional local metadata.""" ts: datetime.datetime score: float @@ -25,7 +25,7 @@ class _BufferedProposal(NamedTuple): @dataclass class BufferedAgreement: - """Confirmed agreement with additional local metadata""" + """Confirmed agreement with additional local metadata.""" agreement: Agreement agreement_details: AgreementDetails @@ -36,7 +36,7 @@ class BufferedAgreement: class AgreementsPool: - """Manages proposals and agreements pool""" + """Manages proposals and agreements pool.""" def __init__( self, @@ -51,7 +51,7 @@ def __init__( self.confirmed = 0 async def cycle(self): - """Performs cyclic tasks. + """Perform cyclic tasks. Should be called regularly. """ @@ -67,7 +67,7 @@ async def cycle(self): ) async def add_proposal(self, score: float, proposal: OfferProposal) -> None: - """Adds providers' proposal to the pool of available proposals""" + """Add providers' proposal to the pool of available proposals.""" async with self._lock: self._offer_buffer[proposal.issuer] = _BufferedProposal( datetime.datetime.now(), score, proposal @@ -94,7 +94,7 @@ async def _set_worker(self, agreement_id: str, task: asyncio.Task) -> None: buffered_agreement.worker_task = task async def _get_agreement(self) -> Optional[Agreement]: - """Returns an Agreement + """Return an Agreement. Firstly it tries to reuse agreement from a pool of available agreements (no active worker_task). If that fails it tries to convert offer into agreement. @@ -156,7 +156,7 @@ async def _get_agreement(self) -> Optional[Agreement]: return agreement async def release_agreement(self, agreement_id: str, allow_reuse: bool = True) -> None: - """Marks agreement as unused. + """Mark agreement as unused. If the agreement supports multiple activities and `allow_reuse` is set then it will be returned to the pool and will be available for reuse. @@ -226,7 +226,7 @@ async def terminate_all(self, reason: dict) -> None: await self._terminate_agreement(agreement_id, reason) async def on_agreement_terminated(self, agr_id: str, reason: dict) -> None: - """Reacts to agreement termination event + """React to agreement termination event. Should be called when AgreementTerminated event is received. """ diff --git a/yapapi/config.py b/yapapi/config.py index 8e757d871..ef8fc124a 100644 --- a/yapapi/config.py +++ b/yapapi/config.py @@ -16,13 +16,15 @@ def __str__(self): @dataclass class ApiConfig: - """ - Yagna low level API configuration + """Yagna low level API configuration. + Attributes: app_key: Yagna application key. - If not provided, the default is to get the value from `YAGNA_APPKEY` environment variable. + If not provided, the default is to get the value from `YAGNA_APPKEY` environment + variable. If no value will be found MissingConfiguration error will be thrown - api_url: base URL or all REST API URLs. Example value: http://127.0.10.10:7500 (no trailing slash). + api_url: base URL or all REST API URLs. Example value: http://127.0.10.10:7500 + (no trailing slash). Uses YAGNA_API_URL environment variable market_url: If not provided `api_url` will be used to construct it. Uses YAGNA_MARKET_URL environment variable @@ -34,8 +36,12 @@ class ApiConfig: Uses YAGNA_ACTIVITY_URL environment variable """ - app_key: str = field(default_factory=partial(os.getenv, "YAGNA_APPKEY")) # type: ignore[assignment] - api_url: str = field(default_factory=partial(os.getenv, "YAGNA_API_URL", "http://127.0.0.1:7465")) # type: ignore[assignment] + app_key: str = field( + default_factory=partial(os.getenv, "YAGNA_APPKEY"), + ) # type: ignore[assignment] + api_url: str = field( + default_factory=partial(os.getenv, "YAGNA_API_URL", "http://127.0.0.1:7465"), + ) # type: ignore[assignment] market_url: Optional[str] = field(default_factory=partial(os.getenv, "YAGNA_MARKET_URL")) payment_url: Optional[str] = field(default_factory=partial(os.getenv, "YAGNA_PAYMENT_URL")) net_url: Optional[str] = field(default_factory=partial(os.getenv, "YAGNA_NET_URL")) diff --git a/yapapi/contrib/service/http_proxy.py b/yapapi/contrib/service/http_proxy.py index 7b5489b5b..7a4401b5d 100644 --- a/yapapi/contrib/service/http_proxy.py +++ b/yapapi/contrib/service/http_proxy.py @@ -1,12 +1,10 @@ -""" -Local HTTP Proxy -^^^^^^^^^^^^^^^^ +"""Local HTTP Proxy. A local HTTP proxy that enables easy connections to any VPN-enabled, HTTP-based services launched on Golem providers using yapapi's Services API. For usage in a complete requestor agent app, see the -`http-proxy `_ and +`http_proxy `_ and `webapp `_ examples in the yapapi repository. """ @@ -110,8 +108,7 @@ def __init__( remote_host: Optional[str] = None, response_timeout: float = DEFAULT_TIMEOUT, ): - """ - Initialize the HTTP proxy service + """Initialize the HTTP proxy service. :param remote_port: the port on which the service on the provider's end listens :param remote_host: optional hostname to be used in the headers passed to the remote HTTP @@ -124,9 +121,8 @@ def __init__( self._remote_response_timeout = response_timeout async def handle_request(self, request: web.Request) -> web.Response: - """ - handle a single request coming from a :class:`~LocalHttpProxy` server - by passing it to the HTTP service on the provider's end through the VPN + """Handle a single request coming from a :class:`~LocalHttpProxy` server by passing it to \ + the HTTP service on the provider's end through the VPN. :param request: an `aiohttp.web.Request` :return: an `aiohttp.web.Response` @@ -191,7 +187,7 @@ async def handle_request(self, request: web.Request) -> web.Response: class LocalHttpProxy: - """Runs a local `aiohttp` server and processes requests through instances of + """Runs a local `aiohttp` server and processes requests through instances of \ :class:`~HttpProxyService`. Using `yapapi`'s Network API (:meth:`~yapapi.Golem.create_network`), execution units on the @@ -231,8 +227,7 @@ def __init__( port: int, max_request_size: int = DEFAULT_MAX_REQUEST_SIZE, ): - """ - Initialize the local HTTP proxy + """Initialize the local HTTP proxy. :param cluster: a :class:`~yapapi.services.Cluster` of one or more VPN-connected :class:`~HttpProxyService` instances. @@ -263,11 +258,12 @@ async def _request_handler(self, request: web.Request) -> web.Response: return await instance.handle_request(request) async def run(self): + """Run a local HTTP server. + + Will be listening on the specified port and passing subsequent requests to the + :meth:`~HttpProxyService.handle_request` of the specified cluster in a round-robin fashion. """ - run a local HTTP server, listening on the specified port and passing subsequent requests to - the :meth:`~HttpProxyService.handle_request` of the specified cluster in a round-robin - fashion - """ + runner = web.ServerRunner( _Server(self._request_handler, client_max_size=self._max_request_size) ) # type: ignore diff --git a/yapapi/contrib/service/socket_proxy.py b/yapapi/contrib/service/socket_proxy.py index bf4051f07..73b8324a7 100644 --- a/yapapi/contrib/service/socket_proxy.py +++ b/yapapi/contrib/service/socket_proxy.py @@ -1,6 +1,4 @@ -""" -TCP socket proxy -^^^^^^^^^^^^^^^^ +"""TCP socket proxy. A local proxy that facilitates connections to any VPN-enabled TCP services launched on Golem providers using yapapi's Services API. @@ -13,7 +11,7 @@ import asyncio import itertools import logging -from typing import Dict, Iterator, List, Optional +from typing import Dict, Iterator, List import aiohttp from typing_extensions import Final @@ -35,13 +33,14 @@ class SocketProxyService(Service, abc.ABC): - """ - Base class for services connected to the :class:`~SocketProxy`. + """Base class for services connected to the :class:`~SocketProxy`. + Implements the interface required by the `SocketProxy`. """ remote_ports: List[int] - """List of remote ports to open the proxies for when using `SocketProxy.`:meth:`~SocketProxy.run`.""" + """List of remote ports to open the proxies for when using \ + `SocketProxy.`:meth:`~SocketProxy.run`.""" class ProxyConnection: @@ -200,14 +199,14 @@ def __repr__(self): @property def app_key(self): - """The application key used to authorize access to `yagna`'s REST API.""" + """Return the application key used to authorize access to `yagna`'s REST API.""" return ( self.service.cluster.service_runner._job.engine._api_config.app_key # type: ignore[union-attr] # noqa ) @property def instance_ws(self): - """The websocket URI for the specific remote port of the Service.""" + """Return the websocket URI for the specific remote port of the Service.""" return self.service.network_node.get_websocket_uri(self.remote_port) # type: ignore[union-attr] # noqa async def handler( diff --git a/yapapi/contrib/strategy/__init__.py b/yapapi/contrib/strategy/__init__.py index 53d16f09e..f62df0828 100644 --- a/yapapi/contrib/strategy/__init__.py +++ b/yapapi/contrib/strategy/__init__.py @@ -1 +1,3 @@ from .provider_filter import ProviderFilter + +__all__ = ("ProviderFilter",) diff --git a/yapapi/contrib/strategy/provider_filter.py b/yapapi/contrib/strategy/provider_filter.py index 9466aed22..d140d5f3b 100644 --- a/yapapi/contrib/strategy/provider_filter.py +++ b/yapapi/contrib/strategy/provider_filter.py @@ -1,6 +1,4 @@ -""" -Provider Filter -^^^^^^^^^^^^^^^ +"""Provider Filter. Market strategy wrapper that enables easy exclusion of offers from certain providers using a simple boolean condition, while preserving correct scoring of the remaining offers by the @@ -19,12 +17,13 @@ class ProviderFilter(WrappingMarketStrategy): - """ProviderFilter - extend a market strategy with a layer that excludes offers from certain issuers + """ProviderFilter - extend a market strategy with a layer that excludes offers from certain \ + issuers. :param base_strategy: a market strategy that will be used to score offers from allowed providers - :param is_allowed: a callable that accepts provider_id as an argument and returns either a boolean, - or a boolean-returning awaitable, determining if offers from this provider should be considered - (that is: scored by the `base_strategy`) + :param is_allowed: a callable that accepts provider_id as an argument and returns either a + boolean, or a boolean-returning awaitable, determining if offers from this provider should + be considered (that is: scored by the `base_strategy`) Example 1. Block selected providers:: @@ -64,7 +63,7 @@ def denying_event_consumer(event: events.Event): # NOTE: this will currently work only for **new** offers from the provider, because old offers are already # scored, this should improve in https://github.com/golemfactory/yapapi/issues/820 - """ + """ # noqa: 501 def __init__(self, base_strategy: BaseMarketStrategy, is_allowed: IsAllowedType): super().__init__(base_strategy) diff --git a/yapapi/ctx.py b/yapapi/ctx.py index 7b0b6b88d..3decb0781 100644 --- a/yapapi/ctx.py +++ b/yapapi/ctx.py @@ -1,7 +1,7 @@ import enum import logging from datetime import datetime, timedelta -from typing import Any, Awaitable, Callable, Dict, List, Optional, Type +from typing import Dict, Optional, Type from dataclasses import dataclass, field @@ -13,12 +13,12 @@ from ya_activity.models import ActivityState as yaa_ActivityState from ya_activity.models import ActivityUsage as yaa_ActivityUsage -from yapapi.events import ActivityEventType, CommandExecuted +from yapapi.events import ActivityEventType from yapapi.props.com import ComLinear from yapapi.rest.activity import Activity from yapapi.rest.market import Agreement, AgreementDetails from yapapi.script import Script -from yapapi.storage import DOWNLOAD_BYTES_LIMIT_DEFAULT, StorageProvider +from yapapi.storage import StorageProvider from yapapi.utils import get_local_timezone logger = logging.getLogger(__name__) @@ -72,7 +72,7 @@ def __repr__(self): @property def id(self) -> str: - """Unique identifier for this work context.""" + """Return unique identifier for this work context.""" return self._activity.id @property @@ -109,7 +109,8 @@ def _agreement_details(self) -> AgreementDetails: def new_script( self, timeout: Optional[timedelta] = None, wait_for_results: bool = True ) -> Script: - """Create an instance of :class:`~yapapi.script.Script` attached to this :class:`WorkContext` instance. + """Create an instance of :class:`~yapapi.script.Script` attached to this \ + :class:`WorkContext` instance. This is equivalent to calling `Script(work_context)`. This method is intended to provide a direct link between the two object instances. diff --git a/yapapi/engine.py b/yapapi/engine.py index a13a0fa6d..984a6c9c2 100644 --- a/yapapi/engine.py +++ b/yapapi/engine.py @@ -114,7 +114,8 @@ def __init__( :param budget: maximum budget for payments :param strategy: market strategy used to select providers from the market (e.g. LeastExpensiveLinearPayuMS or DummyMS) - :param event_consumer: callable that will be directly executed on every Event this Engine creates. + :param event_consumer: callable that will be directly executed on every Event this Engine + creates. NOTE: it is expected to be fast or async - if not, it will block the _Engine. :param subnet_tag: use only providers in the subnet with the subnet_tag name. Uses `YAGNA_SUBNET` environment variable, defaults to `None` @@ -382,7 +383,7 @@ def _get_allocation( if allocation.payment_address == item.payer_addr and allocation.payment_platform == item.payment_platform ) - except: + except Exception: raise ValueError(f"No allocation for {item.payment_platform} {item.payer_addr}.") async def _process_invoices(self) -> None: @@ -416,15 +417,17 @@ def _check_for_termination_reason( ) -> Optional[Dict[str, str]]: freq_descr = f"{num_notes} notes/{duration}s" logger.debug( - f"{'Payable Debit notes' if payable else 'Debit notes'} for activity {activity_id}: {freq_descr}" + f"{'Payable Debit notes' if payable else 'Debit notes'} for activity" + f" {activity_id}: {freq_descr}" ) if duration + DEBIT_NOTE_INTERVAL_GRACE_PERIOD < (num_notes - 1) * interval: payable_str = "payable " if payable else "" reason = { - "message": f"Too many {payable_str}debit notes: {freq_descr} (activity: {activity_id})", - "golem.requestor.code": "TooManyPayableDebitNotes" - if payable - else "TooManyDebitNotes", + "message": f"Too many {payable_str}debit notes: {freq_descr} " + f"(activity: {activity_id})", + "golem.requestor.code": ( + "TooManyPayableDebitNotes" if payable else "TooManyDebitNotes", + ), } logger.error( f"Too many {payable_str}debit notes received. %s, activity: %s", @@ -587,7 +590,9 @@ async def _process_debit_note(self, debit_note_id: str) -> None: raise except Exception: job.emit( - events.PaymentFailed, agreement=agreement, exc_info=sys.exc_info() # type: ignore + events.PaymentFailed, + agreement=agreement, + exc_info=sys.exc_info(), # type: ignore ) def accept_debit_notes_for_agreement(self, job_id: str, agreement_id: str) -> None: @@ -635,10 +640,10 @@ async def start_worker( loop = asyncio.get_event_loop() async def worker_task(agreement: Agreement): - """A coroutine run by every worker task. + """Run `run_worker` in a `WorkContext` for activity from given `Agreement`. - It creates an Activity for a given Agreement, then creates a WorkContext for this Activity - and then executes `run_worker` with this WorkContext. + It creates an Activity for a given Agreement, then creates a WorkContext for this + Activity and then executes `run_worker` with this WorkContext. """ if on_agreement_ready: on_agreement_ready(agreement) @@ -652,7 +657,9 @@ async def worker_task(agreement: Agreement): try: activity = await self.create_activity(agreement.id) except Exception: - job.emit(events.ActivityCreateFailed, agreement=agreement, exc_info=sys.exc_info()) # type: ignore + job.emit( + events.ActivityCreateFailed, agreement=agreement, exc_info=sys.exc_info() + ) # type: ignore raise work_context = WorkContext(activity, agreement, self.storage_manager, emitter=job.emit) @@ -735,15 +742,20 @@ async def get_batch_results() -> List[events.CommandEvent]: script = await batch_generator.asend(future_results) def recycle_offer(self, offer: OfferProposal) -> None: - """This offer was already processed, but something happened and we should treat it as a fresh one. + """Mark given offer as a fresh one, regardless of its previous processing. + + This offer was already processed, but something happened and we should treat it as a + fresh one. Currently this "something" is always "we couldn't confirm the agreement with the provider", but someday this might be useful also in other scenarios. Purpose: - * we want to rescore the offer (score might change because of the event that caused recycling) - * if this is a draft offer (and in the current usecase it always is), we want to return to the - negotiations and get a new draft - e.g. because this draft already has an Agreement + * we want to rescore the offer (score might change because of the event that caused + recycling) + * if this is a draft offer (and in the current usecase it always is), we want to return + to the negotiations and get a new draft - e.g. because this draft already has an + Agreement We don't care which Job initiated recycling - it should be recycled by all unfinished Jobs. """ @@ -933,7 +945,9 @@ async def handler(proposal_): raise except Exception: with contextlib.suppress(Exception): - self.emit(events.ProposalFailed, proposal=proposal_, exc_info=sys.exc_info()) # type: ignore + self.emit( + events.ProposalFailed, proposal=proposal_, exc_info=sys.exc_info() + ) # type: ignore finally: semaphore.release() diff --git a/yapapi/events.py b/yapapi/events.py index e85d55c12..61cb9462a 100644 --- a/yapapi/events.py +++ b/yapapi/events.py @@ -17,9 +17,9 @@ * Additional event-specific information, e.g. the reason of the agreement termination for the :class:`AgreementTerminated` event, described along the particular event classes. -Events should be consumed in a strict `read_only` mode: event objects are shared between all event consumers, -and their attributes are used internally by the Golem engine, so any modification may have unexpected -side effects. +Events should be consumed in a strict `read_only` mode: event objects are shared between all event +consumers, and their attributes are used internally by the Golem engine, so any modification may +have unexpected side effects. Attributes shared by various events @@ -72,8 +72,10 @@ * Only leaf events are ever emitted, other events (named :class:`*Event`) are abstract classes * Every abstract class has one more `yapapi` object attached then the parent, e.g. - * :class:`JobEvent` is an :class:`Event` that happened in the context of a particular :attr:`job` - * :class:`AgreementEvent` is a :class:`JobEvent` that happened in the context of a particular :attr:`agreement` + * :class:`JobEvent` is an :class:`Event` that happened in the context of a particular + :attr:`job` + * :class:`AgreementEvent` is a :class:`JobEvent` that happened in the context of a particular + :attr:`agreement` :: @@ -224,7 +226,8 @@ class Event(abc.ABC): """Event creation time""" def __str__(self) -> str: - """Mimics Python's default `repr` format, but excludes the fields `exc_info` and `timestamp` from it. + """Mimics Python's default `repr` format, but excludes the fields `exc_info` and \ + `timestamp` from it. If `exc_info` is not `None`, its underlying exception is included in the result string under the key `exception`. @@ -250,7 +253,7 @@ def __repr__(self) -> str: @property def exception(self) -> Optional[BaseException]: - """Exception associated with this event or `None`""" + """Exception associated with this event or `None`.""" if self.exc_info: return self.exc_info[1] return None @@ -390,7 +393,7 @@ class ProposalReceived(ProposalEvent): @attr.s(auto_attribs=True, repr=False) class ProposalRejected(ProposalEvent): - """We decided to reject provider's proposal because of a :attr:`reason`""" + """We decided to reject provider's proposal because of a :attr:`reason`.""" reason: Optional[str] = None @@ -409,7 +412,7 @@ class ProposalFailed(ProposalEvent): @attr.s(auto_attribs=True, repr=False) class NoProposalsConfirmed(JobEvent): - """We didn't confirm any proposal for a period of :attr:`timeout`""" + """We didn't confirm any proposal for a period of :attr:`timeout`.""" timeout: timedelta @@ -551,8 +554,8 @@ def path(self) -> str: class ShutdownFinished(Event): - """Golem completed the shutdown sequence and is no longer operative""" + """Golem completed the shutdown sequence and is no longer operative.""" class ExecutionInterrupted(Event): - """Golem was stopped by an unhandled exception in code not managed by yapapi""" + """Golem was stopped by an unhandled exception in code not managed by yapapi.""" diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index ef5d00c3b..c9063f495 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -162,7 +162,7 @@ async def _submit( done_queue: asyncio.Queue[Task[D, R]] = asyncio.Queue() def on_task_done(task: Task[D, R], status: TaskStatus) -> None: - """Callback run when `task` is accepted or rejected.""" + """Execute callback when `task` is accepted or rejected.""" if status == TaskStatus.ACCEPTED: done_queue.put_nowait(task) @@ -231,7 +231,9 @@ async def task_generator() -> AsyncGenerator[Task[D, R], None]: pass work_context.emit(events.WorkerFinished) except Exception as e: - work_context.emit(events.WorkerFinished, exc_info=sys.exc_info()) # type: ignore + work_context.emit( + events.WorkerFinished, exc_info=sys.exc_info() + ) # type: ignore await task_gen.athrow(type(e), e) raise finally: diff --git a/yapapi/executor/_smartq.py b/yapapi/executor/_smartq.py index 63a11da66..68fe0ab0d 100644 --- a/yapapi/executor/_smartq.py +++ b/yapapi/executor/_smartq.py @@ -1,7 +1,4 @@ -""" YAPAPI internal module. This is not a part of the public API. It can change at any time. - - -""" +"""YAPAPI internal module. This is not a part of the public API. It can change at any time.""" import asyncio import logging @@ -48,7 +45,8 @@ def data(self) -> Item: class SmartQueue(Generic[Item]): def __init__(self, items: AsyncIterator[Item]): - """ + """Initialize instance. + :param items: the items to be iterated over """ @@ -182,10 +180,8 @@ class Consumer( AsyncIterable[Handle[Item]], ContextManager["Consumer[Item]"], ): - """ - Provides an interface to asynchronously iterate over items in the given queue - while cooperating with other consumers attached to this queue. - """ + """Provides an interface to asynchronously iterate over items in the given queue while \ + cooperating with other consumers attached to this queue.""" def __init__(self, queue: SmartQueue[Item]): self._queue = queue @@ -206,7 +202,8 @@ def __exit__( @property def current_item(self) -> Optional[Item]: - """The most-recent queue item that has been fetched to be processed by this consumer.""" + """Return the most-recent queue item that has been fetched to be processed by this \ + consumer.""" return self._fetched.data if self._fetched else None def finish(self): diff --git a/yapapi/golem.py b/yapapi/golem.py index 717a397fb..5c070d973 100644 --- a/yapapi/golem.py +++ b/yapapi/golem.py @@ -62,16 +62,17 @@ class _EngineKwargs(TypedDict): class Golem: - """The main entrypoint of Golem\\'s high-level API. + """The main entrypoint of Golem's high-level API. Its principal role is providing an interface to run the requestor's payload using one of two modes of operation - executing tasks and running services. - The first one, available through :func:`execute_tasks`, instructs :class:`Golem` to take a sequence of - tasks that the user wishes to compute on Golem and distributes those among the providers. + The first one, available through :func:`execute_tasks`, instructs :class:`Golem` to take a + sequence of tasks that the user wishes to compute on Golem and distributes those among the + providers. - The second one, invoked with :func:`run_service`, makes :class:`Golem` spawn a certain number of instances - of a service based on a single service specification (a specialized implementation + The second one, invoked with :func:`run_service`, makes :class:`Golem` spawn a certain number of + instances of a service based on a single service specification (a specialized implementation inheriting from :class:`~yapapi.services.Service`). While the two modes are not necessarily completely disjoint - in that we can create a @@ -88,10 +89,11 @@ class Golem: certain, discrete phases of a lifetime of each service instance - startup, running and shutdown. - Internally, :class:`Golem`'s job includes running the engine which takes care of first finding the - providers interested in the jobs the requestors want to execute, then negotiating agreements + Internally, :class:`Golem`'s job includes running the engine which takes care of first finding + the providers interested in the jobs the requestors want to execute, then negotiating agreements with them and facilitating the execution of those jobs and lastly, processing payments. For this - reason, it's usually good to have just one instance of :class:`Golem` operative at any given time. + reason, it's usually good to have just one instance of :class:`Golem` operative at any given + time. """ def __init__( @@ -112,7 +114,8 @@ def __init__( :param budget: maximum budget for payments :param strategy: market strategy used to select providers from the market - (e.g. :class:`yapapi.strategy.LeastExpensiveLinearPayuMS` or :class:`yapapi.strategy.DummyMS`) + (e.g. :class:`yapapi.strategy.LeastExpensiveLinearPayuMS` or + :class:`yapapi.strategy.DummyMS`) :param subnet_tag: use only providers in the subnet with the subnet_tag name. Uses `YAGNA_SUBNET` environment variable, defaults to `None` :param payment_driver: name of the payment driver to use. Uses `YAGNA_PAYMENT_DRIVER` @@ -121,14 +124,14 @@ def __init__( :param payment_network: name of the network to use. Uses `YAGNA_PAYMENT_NETWORK` environment variable, defaults to `rinkeby`. Only payment platforms with the specified network will be used - :param event_consumer: a callable that processes events related to the - computation; by default it is a function that logs all events + :param event_consumer: a callable that processes events related to the computation; by + default it is a function that logs all events :param stream_output: stream computation output from providers - :param api_config: configuration of yagna low level api - including but not limited to `YAGNA_APPKEY`, `YAGNA_API_URL` variables + :param api_config: configuration of yagna low level api including but not limited to + `YAGNA_APPKEY`, `YAGNA_API_URL` variables See `:class:`yapapi.config.ApiConfig`` docs for more details - :param app_key: optional Yagna application key. If not provided, the default is to - get the value from `YAGNA_APPKEY` environment variable + :param app_key: optional Yagna application key. If not provided, the default is to get the + value from `YAGNA_APPKEY` environment variable """ self._event_dispatcher = AsyncEventDispatcher() @@ -164,12 +167,13 @@ def add_event_consumer( event_consumer: Callable[[events.Event], None], event_classes_or_names: Iterable[Union[Type[events.Event], str]] = (events.Event,), ): - """Initialize another `event_consumer`, working just like the `event_consumer` passed to :func:`Golem.__init__` + """Initialize another `event_consumer`, working just like the `event_consumer` passed to \ + :func:`Golem.__init__`. :param event_consumer: A callable that will be executed on every event. - :param event_classes_or_names: An iterable defining classes of events that should be passed to - this `event_consumer`. Both classes and class names are accepted (in the latter case classes must be - available in the `yapapi.events` namespace). + :param event_classes_or_names: An iterable defining classes of events that should be passed + to this `event_consumer`. Both classes and class names are accepted (in the latter case + classes must be available in the `yapapi.events` namespace). If this argument is omitted, all events inheriting from :class:`yapapi.events.Event` (i.e. all currently implemented events) will be passed to the `event_consumer`. @@ -219,7 +223,7 @@ def payment_network(self) -> str: @property def strategy(self) -> "BaseMarketStrategy": - """Return the instance of `BaseMarketStrategy` used by the engine""" + """Return the instance of `BaseMarketStrategy` used by the engine.""" return self._engine.strategy @strategy.setter @@ -242,7 +246,7 @@ def subnet_tag(self) -> Optional[str]: @property def operative(self) -> bool: - """Return True if Golem started and didn't stop""" + """Return True if Golem started and didn't stop.""" engine_init_finished = hasattr(self, "_engine") # to avoid special cases in __init__ return engine_init_finished and self._engine.started @@ -264,8 +268,10 @@ async def start(self) -> None: await golem.stop() A repeated call to :func:`Golem.start()`: - * If Golem is already starting, or started and wasn't stopped - will be ignored (and harmless) - * If Golem was stopped - will initialize a new engine that knows nothing about the previous operations + * If Golem is already starting, or started and wasn't stopped - will be ignored + (and harmless) + * If Golem was stopped - will initialize a new engine that knows nothing about the + previous operations """ try: async with self._engine_state_lock: @@ -275,14 +281,15 @@ async def start(self) -> None: self._event_dispatcher.start() await self._engine.start() - except: + except Exception: await self._stop_with_exc_info(*sys.exc_info()) raise async def stop(self) -> None: """Stop the Golem engine after it was started in non-contextmanager mode. - Details: :func:`Golem.start()`""" + Details: :func:`Golem.start()` + """ await self._stop_with_exc_info(None, None, None) async def __aenter__(self) -> "Golem": @@ -327,16 +334,19 @@ async def execute_tasks( :param worker: an async generator that takes a :class:`WorkContext` object and a sequence of tasks, and generates as sequence of scripts to be executed on providers in order to compute given tasks - :param data: an iterable or an async generator of :class:`Task` objects to be computed on providers + :param data: an iterable or an async generator of :class:`Task` objects to be computed on + providers :param payload: specification of the payload that needs to be deployed on providers (for example, a VM runtime package) in order to compute the tasks, passed to the created :class:`Executor` instance - :param max_workers: maximum number of concurrent workers, passed to the :class:`Executor` instance + :param max_workers: maximum number of concurrent workers, passed to the :class:`Executor` + instance :param timeout: timeout for computing all tasks, passed to the :class:`Executor` instance :param job_id: an optional string to identify the job created by this method. Passed as the value of the `id` parameter to :class:`yapapi.engine.Job`. - :param implicit_init: True -> :func:`~yapapi.script.Script.deploy()` and :func:`~yapapi.script.Script.start()` - will be called internally by the :class:`Executor`. False -> those calls must be in the `worker` function + :param implicit_init: True -> :func:`~yapapi.script.Script.deploy()` and + :func:`~yapapi.script.Script.start()` will be called internally by the + :class:`Executor`. False -> those calls must be in the `worker` function :return: an async iterator that yields completed `Task` objects @@ -357,7 +367,7 @@ async def worker(context: WorkContext, tasks: AsyncIterable[Task]): async for completed in golem.execute_tasks(worker, [Task(data=None)], payload=package): print(completed.result.stdout) - """ + """ # noqa: 501 kwargs: Dict[str, Any] = {"payload": payload, "implicit_init": implicit_init} if max_workers: @@ -379,31 +389,36 @@ async def run_service( network: Optional[Network] = None, network_addresses: Optional[List[str]] = None, ) -> Cluster[ServiceType]: - """Run a number of instances of a service represented by a given :class:`~yapapi.services.Service` subclass. + """Run a number of instances of a service represented by a given \ + :class:`~yapapi.services.Service` subclass. - :param service_class: a subclass of :class:`~yapapi.services.Service` that represents the service to be run + :param service_class: a subclass of :class:`~yapapi.services.Service` that represents the + service to be run :param num_instances: optional number of service instances to run. Defaults to a single - instance, unless `instance_params` is given, in which case, the :class:`~yapapi.services.Cluster` will be - created with as many instances as there are elements in the `instance_params` iterable. - if `num_instances` is set to < 1, the :class:`~yapapi.services.Cluster` will still be created but no - instances will be spawned within it. - :param instance_params: optional list of dictionaries of keyword arguments that will be passed - to consecutive, spawned instances. The number of elements in the iterable determines the - number of instances spawned, unless `num_instances` is given, in which case the latter takes - precedence. - In other words, if both `num_instances` and `instance_params` are provided, - the :class:`~yapapi.services.Cluster` will be created with the number of instances determined by - `num_instances` and if there are too few elements in the `instance_params` iterable, it will results in - an error. - :param payload: optional runtime definition for the service; if not provided, the - payload specified by the :func:`~yapapi.services.Service.get_payload()` method of `service_class` is used + instance, unless `instance_params` is given, in which case, the + :class:`~yapapi.services.Cluster` will be created with as many instances as there are + elements in the `instance_params` iterable. if `num_instances` is set to < 1, the + :class:`~yapapi.services.Cluster` will still be created but no instances will be spawned + within it. + :param instance_params: optional list of dictionaries of keyword arguments that will be + passed to consecutive, spawned instances. The number of elements in the iterable + determines the number of instances spawned, unless `num_instances` is given, in which + case the latter takes precedence. + In other words, if both `num_instances` and `instance_params` are provided, the + :class:`~yapapi.services.Cluster` will be created with the number of instances + determined by `num_instances` and if there are too few elements in the `instance_params` + iterable, it will results in an error. + :param payload: optional runtime definition for the service; if not provided, the payload + specified by the :func:`~yapapi.services.Service.get_payload()` method of + `service_class` is used :param expiration: optional expiration datetime for the service :param network: optional :class:`~yapapi.network.Network`, representing a VPN to attach this :class:`~yapapi.services.Cluster`'s instances to - :param network_addresses: optional list of addresses to assign to consecutive spawned instances. - If there are too few addresses given in the `network_addresses` iterable to satisfy - all spawned instances, the rest (or all when the list is empty or not provided at all) - of the addresses will be assigned automatically. + :param network_addresses: optional list of addresses to assign to consecutive spawned + instances. + If there are too few addresses given in the `network_addresses` iterable to satisfy all + spawned instances, the rest (or all when the list is empty or not provided at all) of + the addresses will be assigned automatically. Requires the `network` argument to be provided at the same time. example usage:: @@ -458,7 +473,7 @@ async def main(): print(f"Instance {num} is {instance.state.value} on {instance.provider_name}") await asyncio.sleep(REFRESH_INTERVAL_SEC) - """ + """ # noqa: E501 payload = payload or await service_class.get_payload() if not payload: @@ -490,13 +505,13 @@ async def create_network( mask: Optional[str] = None, gateway: Optional[str] = None, ) -> Network: - """ - Create a VPN inside Golem network. + """Create a VPN inside Golem network. Requires yagna >= 0.8 :param ip: the IP address of the network. May contain netmask, e.g. "192.168.0.0/24" - :param owner_ip: the desired IP address of the requestor node within the newly-created Network + :param owner_ip: the desired IP address of the requestor node within the newly-created + Network :param mask: Optional netmask (only if not provided within the `ip` argument) :param gateway: Optional gateway address for the network """ @@ -516,7 +531,7 @@ def _default_event_consumer() -> Callable[[events.Event], None]: return log_summary(log_event_repr) def _initialize_default_strategy(self) -> DecreaseScoreForUnconfirmedAgreement: - """Create a default strategy and register it's event consumer""" + """Create a default strategy and register it's event consumer.""" base_strategy = LeastExpensiveLinearPayuMS( max_fixed_price=Decimal("1.0"), max_price_for={com.Counter.CPU: Decimal("0.2"), com.Counter.TIME: Decimal("0.1")}, diff --git a/yapapi/invoice_manager.py b/yapapi/invoice_manager.py index 8b635b5da..9fe2324df 100644 --- a/yapapi/invoice_manager.py +++ b/yapapi/invoice_manager.py @@ -1,3 +1,4 @@ +import logging import sys from asyncio import CancelledError from decimal import Decimal @@ -12,6 +13,8 @@ from yapapi.rest.market import Agreement from yapapi.rest.payment import Allocation, Invoice +logger = logging.getLogger(__name__) + @dataclass class AgreementData: @@ -22,11 +25,6 @@ class AgreementData: paid: bool = False -import logging - -logger = logging.getLogger(__name__) - - class InvoiceManager: def __init__(self): self._agreement_data: Dict[str, AgreementData] = {} @@ -42,12 +40,15 @@ def has_payable_unpaid_agreements(self) -> bool: return any(ad.payable and not ad.paid for ad in self._agreement_data.values()) def add_agreement(self, job: "Job", agreement: "Agreement") -> None: - """Inform the InvoiceManager about a new agreement (so that we can use the agreement_id in the future)""" + """Inform the InvoiceManager about a new agreement (so that we can use the agreement_id in \ + the future).""" + ad = self._agreement_data.get(agreement.id) if ad: - # Currently possible if we're having more than one activity for a single agreement - # (We could make some effort to ensure this method is called only once, when the agreement is created, - # but it will make the code more complex as we'll have to reach here from the AgreeementsPool) + # Currently possible if we're having more than one activity for a single agreement + # (We could make some effort to ensure this method is called only once, when the + # agreement is created, but it will make the code more complex as we'll have to reach + # here from the AgreeementsPool) assert ad.job is job and ad.agreement is agreement else: self._agreement_data[agreement.id] = AgreementData(agreement, job) diff --git a/yapapi/log.py b/yapapi/log.py index b225d91cd..ab94a7d20 100644 --- a/yapapi/log.py +++ b/yapapi/log.py @@ -48,7 +48,7 @@ import time from asyncio import CancelledError, get_event_loop from collections import Counter, defaultdict -from datetime import datetime, timedelta, timezone +from datetime import datetime, timedelta from decimal import Decimal from typing import Any, Callable, Dict, List, Optional, Set @@ -443,12 +443,12 @@ def _handle(self, event: events.Event): else: msg = ( f"{offers_collected} {'offer has' if offers_collected == 1 else 'offers have'} " - f"been collected from the market, but no provider has responded for " + "been collected from the market, but no provider has responded for " f"{self.time_waiting_for_proposals.seconds}s." ) msg += ( - f" Make sure you're using the latest released versions of yagna and yapapi," - f" and the correct subnet. " + " Make sure you're using the latest released versions of yagna and yapapi, and the" + " correct subnet. " ) self.logger.warning(msg) diff --git a/yapapi/network.py b/yapapi/network.py index 58ead8638..e829d544a 100644 --- a/yapapi/network.py +++ b/yapapi/network.py @@ -19,9 +19,7 @@ @dataclass class Node: - """ - Describes a node in a VPN, mapping a Golem node id to an IP address. - """ + """Describes a node in a VPN, mapping a Golem node id to an IP address.""" network: "Network" """The :class:`Network` (the specific VPN) this node is part of.""" @@ -33,11 +31,9 @@ class Node: """IP address of this node in this particular VPN.""" def get_deploy_args(self) -> Dict: - """ - Generate a dictionary of arguments that are required for the appropriate - `Deploy` command of an exescript in order to pass the network configuration to the runtime - on the provider's end. - """ + """Generate a dictionary of arguments that are required for the appropriate `Deploy` \ + command of an exescript in order to pass the network configuration to the runtime on the \ + provider's end.""" deploy_args = { "net": [ { @@ -85,9 +81,7 @@ class NetworkState(StateMachine): class Network: - """ - Describes a VPN created between the requestor and the provider nodes within Golem Network. - """ + """Describes a VPN created between the requestor and the provider nodes within Golem Network.""" @classmethod async def create( @@ -104,7 +98,8 @@ async def create( :param net_api: the mid-level binding used directly to perform calls to the REST API. :param ip: the IP address of the network. May contain netmask, e.g. "192.168.0.0/24" :param owner_id: the node ID of the owner of this VPN (the requestor) - :param owner_ip: the desired IP address of the requestor node within the newly-created network + :param owner_ip: the desired IP address of the requestor node within the newly-created + network :param mask: Optional netmask (only if not provided within the `ip` argument) :param gateway: Optional gateway address for the network """ @@ -133,11 +128,13 @@ def __init__( mask: Optional[str] = None, gateway: Optional[str] = None, ): - """ + """Initialize instance. + :param net_api: the mid-level binding used directly to perform calls to the REST API. :param ip: the IP address of the network. May contain netmask, e.g. "192.168.0.0/24" :param owner_id: the node ID of the owner of this VPN (the requestor) - :param owner_ip: the desired IP address of the requestor node within the newly-created network + :param owner_ip: the desired IP address of the requestor node within the newly-created + network :param mask: Optional netmask (only if not provided within the `ip` argument) :param gateway: Optional gateway address for the network """ @@ -175,39 +172,39 @@ async def __aexit__(self, *exc_info) -> None: @property def owner_ip(self) -> str: - """The IP address of the requestor node within the network.""" + """Return the IP address of the requestor node within the network.""" return str(self._owner_ip) @property def state(self) -> State: - """Current state in this network's lifecycle.""" + """Return current state in this network's lifecycle.""" return self._state_machine.current_state @property def network_address(self) -> str: - """The network address of this network, without a netmask.""" + """Return the network address of this network, without a netmask.""" return str(self._ip_network.network_address) @property def netmask(self) -> str: - """The netmask of this network.""" + """Return the netmask of this network.""" return str(self._ip_network.netmask) @property def gateway(self) -> Optional[str]: - """The gateway address within this network, if provided.""" + """Return the gateway address within this network, if provided.""" if self._gateway: return str(self._gateway) return None @property def nodes_dict(self) -> Dict[str, str]: - """Mapping between the IP addresses and Node IDs of the nodes within this network.""" + """Return mapping between the IP addresses and Node IDs of the nodes within this network.""" return {str(v.ip): k for k, v in self._nodes.items()} @property def network_id(self) -> str: - """The automatically-generated, unique ID of this VPN.""" + """Return the automatically-generated, unique ID of this VPN.""" self._state_machine.get_id() assert self._network_id return self._network_id @@ -216,7 +213,8 @@ def _ensure_ip_in_network(self, ip: str): """Ensure the given IP address belongs to the network address range within this VPN.""" if ip_address(ip) not in self._ip_network: raise NetworkError( - f"The given IP ('{ip}') address must belong to the network ('{self._ip_network.with_netmask}')." + f"The given IP ('{ip}') address must belong to the network " + f"('{self._ip_network.with_netmask}')." ) def _ensure_ip_unique(self, ip: str): @@ -319,4 +317,4 @@ def _next_address(self) -> IpAddress: class NetworkError(Exception): - """Exception raised by :class:`Network` when an operation is not possible""" + """Exception raised by :class:`Network` when an operation is not possible.""" diff --git a/yapapi/payload/__init__.py b/yapapi/payload/__init__.py index 3a5e2f0c5..cc5e8bbf5 100644 --- a/yapapi/payload/__init__.py +++ b/yapapi/payload/__init__.py @@ -4,7 +4,7 @@ class Payload(AutodecoratingModel, abc.ABC): - """Base class for descriptions of the payload required by the requestor. + r"""Base class for descriptions of the payload required by the requestor. example usage:: @@ -40,4 +40,4 @@ async def main(): output:: {'properties': {'golem.srv.app.myprop': 'othervalue'}, 'constraints': ['(&(golem.runtime.name=my-runtime)\n\t(golem.inf.mem.gib>=32)\n\t(golem.inf.storage.gib>=1024))']} - """ + """ # noqa: E501 diff --git a/yapapi/payload/package.py b/yapapi/payload/package.py index 81c0882b9..039b35aba 100644 --- a/yapapi/payload/package.py +++ b/yapapi/payload/package.py @@ -9,12 +9,10 @@ class PackageException(Exception): """Exception raised on any problems related to the package repository.""" - pass - @dataclass # type: ignore # mypy issue #5374 class Package(Payload): - """Description of a task package (e.g. a VM image) deployed on the provider nodes""" + """Description of a task package (e.g. a VM image) deployed on the provider nodes.""" @abc.abstractmethod async def resolve_url(self) -> str: diff --git a/yapapi/payload/vm.py b/yapapi/payload/vm.py index 3899aa026..1867bba41 100644 --- a/yapapi/payload/vm.py +++ b/yapapi/payload/vm.py @@ -116,10 +116,12 @@ async def manifest( """ Build a reference to application payload. - :param manifest: base64 encoded Computation Payload Manifest https://handbook.golem.network/requestor-tutorials/vm-runtime/computation-payload-manifest + :param manifest: base64 encoded Computation Payload Manifest + https://handbook.golem.network/requestor-tutorials/vm-runtime/computation-payload-manifest :param manifest_sig: an optional signature of base64 encoded Computation Payload Manifest :param manifest_sig_algorithm: an optional signature algorithm, e.g. "sha256" - :param manifest_cert: an optional base64 encoded public certificate (DER or PEM) matching key used to generate signature + :param manifest_cert: an optional base64 encoded public certificate (DER or PEM) matching key + used to generate signature :param min_mem_gib: minimal memory required to execute application code :param min_storage_gib: minimal disk storage to execute tasks :param min_cpu_threads: minimal available logical CPU cores @@ -200,7 +202,8 @@ async def repo( image_hash="d646d7b93083d817846c2ae5c62c72ca0507782385a2e29291a3d376", ) - example usage with an explicit GVMI image URL (useful to host images outside the Golem repository):: + example usage with an explicit GVMI image URL (useful to host images outside the Golem + repository):: package = await vm.repo( # we still need to provide the image's hash because diff --git a/yapapi/props/__init__.py b/yapapi/props/__init__.py index dbe7ed521..d439fd0a6 100644 --- a/yapapi/props/__init__.py +++ b/yapapi/props/__init__.py @@ -23,7 +23,7 @@ class NodeInfo(Model): @dataclass() class Activity(Model): - """Activity-related Properties""" + """Activity-related Properties.""" cost_cap: Optional[Decimal] = field(default=None, metadata={"key": "golem.activity.cost_cap"}) """Sets a Hard cap on total cost of the Activity (regardless of the usage vector or @@ -63,3 +63,14 @@ class Activity(Model): ActivityKeys = Activity.property_keys() + +__all__ = ( + "InvalidPropertiesError", + "Model", + "constraint", + "prop", + "NodeInfo", + "NodeInfoKeys", + "Activity", + "ActivityKeys", +) diff --git a/yapapi/props/base.py b/yapapi/props/base.py index 25cca942d..d992d0c05 100644 --- a/yapapi/props/base.py +++ b/yapapi/props/base.py @@ -115,9 +115,7 @@ def property_fields(cls) -> typing.List[Field]: @classmethod def constraint_fields(cls) -> typing.List[Field]: - """ - Return a list of constraint fields of a Model. - """ + """Return a list of constraint fields of a Model.""" return [ f for f in fields(cls) @@ -163,8 +161,7 @@ def from_properties(cls: Type[ME], props: Props) -> ME: @classmethod def property_keys(cls): - """ - :return: a mapping between the model's field names and the property keys + """Return a mapping between the model's field names and the property keys. example: ```python @@ -209,10 +206,10 @@ class ModelFieldType(enum.Enum): def constraint( key: str, operator: ConstraintOperator = "=", default=MISSING, default_factory=MISSING ): - """ - Return a constraint-type dataclass field for a Model. + """Return a constraint-type dataclass field for a Model. - :param key: the key of the property on which the constraint is made - e.g. "golem.srv.comp.task_package" + :param key: the key of the property on which the constraint is made - e.g. + "golem.srv.comp.task_package" :param operator: constraint's operator, one of: "=", ">=", "<=" :param default: the default value for the constraint @@ -230,7 +227,8 @@ def constraint( ['(baz<=100)'] ``` """ - return field( # type: ignore # the default / default_factory exception is resolved by the `field` function + # the default / default_factory exception is resolved by the `field` function + return field( # type: ignore default=default, default_factory=default_factory, metadata={ @@ -311,7 +309,9 @@ def join_str_constraints(constraints: List[str], operator: ConstraintGroupOperat example: ```python >>> from dataclasses import dataclass - >>> from yapapi.props.base import Model, constraint, constraint_model_serialize, join_str_constraints + >>> from yapapi.props.base import ( + >>> Model, constraint, constraint_model_serialize, join_str_constraints + >>> ) >>> >>> @dataclass ... class Foo(Model): diff --git a/yapapi/props/builder.py b/yapapi/props/builder.py index a24f2081a..b8fe0a189 100644 --- a/yapapi/props/builder.py +++ b/yapapi/props/builder.py @@ -40,7 +40,6 @@ class DemandBuilder: def __init__(self): self._properties: dict = {} self._constraints: List[str] = [] - pass def __repr__(self): return repr({"properties": self._properties, "constraints": self._constraints}) @@ -90,16 +89,17 @@ async def decorate(self, *decorators: "DemandDecorator"): class DemandDecorator(abc.ABC): - """An interface that specifies classes that can add properties and constraints through a DemandBuilder""" + """An interface that specifies classes that can add properties and constraints through a \ + DemandBuilder.""" @abc.abstractmethod async def decorate_demand(self, demand: DemandBuilder) -> None: - """Add appropriate properties and constraints to a Demand""" + """Add appropriate properties and constraints to a Demand.""" class AutodecoratingModel(Model, DemandDecorator): - """ - Base class, implementing the DemandDecorator interface to automatically decorate a demand using the model's properties and constraints. + """Base class, implementing the DemandDecorator interface to automatically decorate a demand \ + using the model's properties and constraints. example: ```python diff --git a/yapapi/props/com.py b/yapapi/props/com.py index 93671aeb0..2be3177ba 100644 --- a/yapapi/props/com.py +++ b/yapapi/props/com.py @@ -5,7 +5,7 @@ from dataclasses import dataclass, field -from .base import Model, Props, as_list +from .base import Model, Props SCHEME: str = "golem.com.scheme" PRICE_MODEL: str = "golem.com.pricing.model" @@ -39,7 +39,8 @@ class Com(Model): @abc.abstractmethod def calculate_cost(self, usage: List) -> float: - """Calculate the cost by applying the provided usage vector to the underlying pricing model.""" + """Calculate the cost by applying the provided usage vector to the underlying pricing \ + model.""" @abc.abstractmethod def usage_as_dict(self, usage: List) -> Dict: diff --git a/yapapi/rest/__init__.py b/yapapi/rest/__init__.py index 597f06604..daac7e0cb 100644 --- a/yapapi/rest/__init__.py +++ b/yapapi/rest/__init__.py @@ -11,4 +11,12 @@ from .net import Net from .payment import Payment -__all__ = ("Configuration", "Market", "Payment", "Activity", "Net") +__all__ = ( + "activity", + "market", + "Activity", + "Configuration", + "Market", + "Net", + "Payment", +) diff --git a/yapapi/rest/activity.py b/yapapi/rest/activity.py index 00c73785b..89685278f 100644 --- a/yapapi/rest/activity.py +++ b/yapapi/rest/activity.py @@ -42,7 +42,7 @@ async def new_activity(self, agreement_id: str, stream_events: bool = False) -> class Activity(AsyncContextManager["Activity"]): - """Mid-level wrapper for REST's Activity endpoint""" + """Mid-level wrapper for REST's Activity endpoint.""" def __init__( self, @@ -328,7 +328,7 @@ async def __aiter__(self) -> AsyncIterator[CommandEventData]: def _message_event_to_event_data(msg_event: MessageEvent) -> CommandEventData: - """Convert a `MessageEvent` to a matching events.Event subclass and it's kwargs""" + """Convert a `MessageEvent` to a matching events.Event subclass and it's kwargs.""" if msg_event.type != "runtime": raise RuntimeError(f"Unsupported event: {msg_event.type}") diff --git a/yapapi/rest/configuration.py b/yapapi/rest/configuration.py index d6d7ee6b0..9b8dec1ba 100644 --- a/yapapi/rest/configuration.py +++ b/yapapi/rest/configuration.py @@ -1,8 +1,5 @@ -import os from typing import Optional -from typing_extensions import Final - import ya_activity # type: ignore import ya_market # type: ignore import ya_net # type: ignore @@ -45,27 +42,27 @@ def app_key(self) -> str: @property def market_url(self) -> str: - """The URL of the Market REST API""" + """Return URL of the Market REST API.""" return self.__market_url @property def payment_url(self) -> str: - """The URL of the Payment REST API""" + """Return URL of the Payment REST API.""" return self.__payment_url @property def activity_url(self) -> str: - """The URL of the Activity REST API""" + """Return URL of the Activity REST API.""" return self.__activity_url @property def net_url(self) -> str: - """The URL of the Activity REST API""" + """Return URL of the Activity REST API.""" return self.__net_url @property def root_url(self) -> str: - """The root URL of the REST API""" + """Return root URL of the REST API.""" return self.__url def market(self) -> ya_market.ApiClient: diff --git a/yapapi/rest/market.py b/yapapi/rest/market.py index 43b6b31b8..d421c3e88 100644 --- a/yapapi/rest/market.py +++ b/yapapi/rest/market.py @@ -218,9 +218,7 @@ async def __aexit__(self, exc_type, exc_value, traceback): @property def details(self) -> models.Demand: - """ - :return: the Demand for which the Subscription has been registered. - """ + """Return the Demand for which the Subscription has been registered.""" assert self._details is not None, "expected details on list object" return self._details @@ -293,9 +291,8 @@ def __init__(self, api_client: ApiClient): self._api: RequestorApi = RequestorApi(api_client) def subscribe(self, props: dict, constraints: str) -> AsyncResource[Subscription]: - """ - Create a subscription for a demand specified by the supplied properties and constraints. - """ + """Create a subscription for a demand specified by the supplied properties and \ + constraints.""" request = models.DemandOfferBase(properties=props, constraints=constraints) async def create() -> Subscription: diff --git a/yapapi/rest/payment.py b/yapapi/rest/payment.py index 16311efb8..ec55e484e 100644 --- a/yapapi/rest/payment.py +++ b/yapapi/rest/payment.py @@ -128,7 +128,7 @@ def new_allocation( expires: Optional[datetime] = None, make_deposit: bool = False, ) -> ResourceCtx[Allocation]: - """Creates new allocation. + """Create new allocation. - `amount`: Allocation amount. - `expires`: expiration timestamp. set to default 10 years as a work-around. @@ -157,10 +157,9 @@ def new_allocation( ) async def allocations(self) -> AsyncIterator[Allocation]: - """Lists all active allocations. + """Iterate over all active allocations. Example: - Listing all active allocations from yapapi import rest diff --git a/yapapi/script/__init__.py b/yapapi/script/__init__.py index 781fbd5f0..56f57a6a4 100644 --- a/yapapi/script/__init__.py +++ b/yapapi/script/__init__.py @@ -33,10 +33,10 @@ class Script: """Represents a series of commands to be executed on a provider node. - New commands are added to the script either through its :func:`add` method or by calling one of the - convenience methods provided (for example: :func:`run` or :func:`upload_json`). - Adding a new command *does not* result in it being immediately executed. Once ready, a :class:`Script` - instance is meant to be yielded from a worker function (work generator pattern). + New commands are added to the script either through its :func:`add` method or by calling one of + the convenience methods provided (for example: :func:`run` or :func:`upload_json`). + Adding a new command *does not* result in it being immediately executed. Once ready, a + :class:`Script` instance is meant to be yielded from a worker function (work generator pattern). Commands will be run in the order in which they were added to the script. """ @@ -46,14 +46,14 @@ def __init__( timeout: Optional[timedelta] = None, wait_for_results: bool = True, ): - """Initialize a :class:`Script` + """Initialize a :class:`Script`. - :param context: A :class:`yapapi.WorkContext` that will be used to evaluate the script (i.e. to send - commands to the provider) + :param context: A :class:`yapapi.WorkContext` that will be used to evaluate the script (i.e. + to send commands to the provider) :param timeout: Time after which this script's execution should be forcefully interrupted. The default value is `None` which means there's no timeout set. - :param wait_for_results: Whether this script's execution should block until its results are available. - The default value is `True`. + :param wait_for_results: Whether this script's execution should block until its results are + available. The default value is `True`. """ self.timeout = timeout self.wait_for_results = wait_for_results @@ -73,7 +73,7 @@ def emit(self, event_class: Type[ScriptEventType], **kwargs) -> ScriptEventType: def process_batch_event( self, event_class: Type[CommandEvent], event_kwargs: Dict[str, Any] ) -> CommandEvent: - """Event emiting and special events.CommandExecuted logic""" + """Event emitting and special events.CommandExecuted logic.""" command = self._commands[event_kwargs["cmd_idx"]] del event_kwargs["cmd_idx"] event = command.emit(event_class, **event_kwargs) @@ -87,7 +87,10 @@ def process_batch_event( @property def results(self) -> List[CommandExecuted]: - """List of all results of the script commands. Available only after the script execution finished.""" + """Return list of all results of the script commands. + + Available only after the script execution finished. + """ try: return [command._result.result() for command in self._commands] except InvalidStateError: @@ -110,17 +113,17 @@ def _evaluate(self) -> List[BatchCommand]: return batch async def _after(self): - """Hook which is executed after the script has been run on the provider.""" + """Execute "after" hooks after the script has been run on the provider.""" for cmd in self._commands: await cmd.after() async def _before(self): - """Hook which is executed before the script is evaluated and sent to the provider.""" + """Execute "before" hooks before the script is evaluated and sent to the provider.""" for cmd in self._commands: await cmd.before() def add(self, cmd: Command) -> Awaitable[CommandExecuted]: - """Add a :class:`yapapi.script.command.Command` to the :class:`Script`""" + """Add a :class:`yapapi.script.command.Command` to the :class:`Script`.""" self._commands.append(cmd) cmd._set_script(self) return cmd._result diff --git a/yapapi/script/command.py b/yapapi/script/command.py index b0da18298..e8f9a51dc 100644 --- a/yapapi/script/command.py +++ b/yapapi/script/command.py @@ -23,12 +23,11 @@ def evaluate(self) -> BatchCommand: """Evaluate and serialize this command.""" async def after(self) -> None: - """A hook to be executed on requestor's end after the script has finished.""" - pass + """Execute a hook to be executed on requestor's end after the script has finished.""" async def before(self) -> None: - """A hook to be executed on requestor's end before the script is sent to the provider.""" - pass + """Execute a hook to be executed on requestor's end before the script is sent to the \ + provider.""" @staticmethod def _make_batch_command(cmd_name: str, **kwargs) -> BatchCommand: diff --git a/yapapi/services/__init__.py b/yapapi/services/__init__.py index f4ae4fd06..a6dd40b73 100644 --- a/yapapi/services/__init__.py +++ b/yapapi/services/__init__.py @@ -2,3 +2,12 @@ from .service import Service, ServiceInstance, ServiceType from .service_runner import ServiceRunner from .service_state import ServiceState + +__all__ = ( + "Cluster", + "Service", + "ServiceInstance", + "ServiceType", + "ServiceRunner", + "ServiceState", +) diff --git a/yapapi/services/cluster.py b/yapapi/services/cluster.py index 82ae250cb..fceef6a33 100644 --- a/yapapi/services/cluster.py +++ b/yapapi/services/cluster.py @@ -35,7 +35,8 @@ def __init__( """Initialize this Cluster. :param engine: an engine for running service instance - :param service_class: a subclass of :class:`~yapapi.services.Service` that represents the service to be run + :param service_class: a subclass of :class:`~yapapi.services.Service` that represents the + service to be run :param payload: definition of service runtime for this Cluster :param expiration: a date before which all agreements related to running services in this Cluster should be terminated @@ -59,17 +60,18 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): await self._terminate(exc_type, exc_val, exc_tb) async def terminate(self): - """Signal the whole :class:`Cluster` and the underlying :class:`~yapapi.service.ServiceRunner` to stop.""" + """Signal the whole :class:`Cluster` and the underlying \ + :class:`~yapapi.service.ServiceRunner` to stop.""" await self._terminate(None, None, None) def stop(self): - """Stop all services in this :class:`Cluster`""" + """Stop all services in this :class:`Cluster`.""" for instance in self.instances: self.stop_instance(instance) async def _terminate(self, exc_type, exc_val, exc_tb): - # NOTE: this might be called more then once (e.g. by `terminate()` followed by `__aexit__`), - # but it's harmless, so we don't care + # NOTE: this might be called more then once (e.g. by `terminate()` followed by `__aexit__`), + # but it's harmless, so we don't care self.stop() await self._stack.__aexit__(exc_type, exc_val, exc_tb) @@ -79,7 +81,8 @@ def id(self) -> str: @property def expiration(self) -> datetime: - """Return the expiration datetime for agreements related to services in this :class:`Cluster`.""" + """Return the expiration datetime for agreements related to services in this \ + :class:`Cluster`.""" return self.service_runner._job.expiration_time @property @@ -94,7 +97,8 @@ def service_class(self) -> Type[ServiceType]: @property def network(self) -> Optional[Network]: - """Return the :class:`~yapapi.network.Network` record associated with the VPN used by this :class:`Cluster`.""" + """Return the :class:`~yapapi.network.Network` record associated with the VPN used by this \ + :class:`Cluster`.""" return self._network def __repr__(self): diff --git a/yapapi/services/service.py b/yapapi/services/service.py index 2f6ddfbb6..1c5c36d62 100644 --- a/yapapi/services/service.py +++ b/yapapi/services/service.py @@ -108,7 +108,7 @@ def provider_id(self) -> Optional[str]: @property def network(self) -> Optional[Network]: - """Return the :class:`~yapapi.network.Network` to which this instance belongs (if any)""" + """Return the :class:`~yapapi.network.Network` to which this instance belongs (if any).""" return self.network_node.network if self.network_node else None @property @@ -194,7 +194,6 @@ async def get_payload() -> Optional[Payload]: If :func:`get_payload` is not implemented, the payload will need to be provided in the :func:`~yapapi.Golem.run_service` call. """ - pass def get_deploy_args(self) -> Dict: """Return the dictionary of kwargs needed to construct the `Deploy` exescript command.""" @@ -210,16 +209,16 @@ async def start(self) -> AsyncGenerator[Script, Awaitable[List[events.CommandEve Should perform the minimum set of operations after which the instance of a service can be treated as "started", or, in other words, ready to receive service requests. It's up to the - developer of the specific :class:`Service` class to decide what exact operations constitute a - service startup. In the most common scenario :func:`~yapapi.script.Script.deploy()` and + developer of the specific :class:`Service` class to decide what exact operations constitute + a service startup. In the most common scenario :func:`~yapapi.script.Script.deploy()` and :func:`~yapapi.script.Script.start()` are required, check the `Default implementation` section for more details. As a handler implementing the `work generator pattern `_, - it's expected to be a generator that yields :class:`~yapapi.script.Script` (generated using the - service's instance of the :class:`~yapapi.WorkContext` - :attr:`self._ctx`) that are then dispatched - to the activity by the engine. + it's expected to be a generator that yields :class:`~yapapi.script.Script` (generated using + the service's instance of the :class:`~yapapi.WorkContext` - :attr:`self._ctx`) that are + then dispatched to the activity by the engine. Results of those batches can then be retrieved by awaiting the values captured from yield statements. @@ -252,19 +251,20 @@ async def start(self): The default implementation assumes that, in order to accept commands, the runtime needs to be first deployed using the :func:`~yapapi.script.Script.deploy` command, which is analogous - to creation of a container corresponding with the desired payload, and then started using the - :func:`~yapapi.script.Script.start` command, - actually launching the process that runs the aforementioned container. + to creation of a container corresponding with the desired payload, and then started using + the :func:`~yapapi.script.Script.start` command, actually launching the process that runs + the aforementioned container. Additionally, it also assumes that the exe-unit doesn't need any additional parameters - in its :func:`~yapapi.script.Script.start` call (e.g. for the VM runtime, all the required parameters are - already passed as part of the agreement between the requestor and the provider), and parameters - passed to :func:`~yapapi.script.Script.deploy` are returned by :func:`Service.get_deploy_args()` method. + in its :func:`~yapapi.script.Script.start` call (e.g. for the VM runtime, all the required + parameters are already passed as part of the agreement between the requestor and the + provider), and parameters passed to :func:`~yapapi.script.Script.deploy` are returned by + :func:`Service.get_deploy_args()` method. Therefore, this default implementation performs the minimum required for a VM payload to start responding to `run` commands. If your service requires any additional operations - - you'll need to override this method (possibly first yielding from the parent - `super().start()` - generator) - to add appropriate preparatory steps. + you'll need to override this method (possibly first yielding from the parent + - `super().start()` - generator) to add appropriate preparatory steps. In case of runtimes other than VM, `deploy` and/or `start` might be optional or altogether disallowed, or they may take some parameters. It is up to the author of the @@ -287,9 +287,9 @@ async def run(self) -> AsyncGenerator[Script, Awaitable[List[events.CommandEvent As a handler implementing the `work generator pattern `_, - it's expected to be a generator that yields :class:`~yapapi.script.Script` (generated using the - service's instance of the :class:`~yapapi.WorkContext` - :attr:`self._ctx`) that are then dispatched - to the activity by the engine. + it's expected to be a generator that yields :class:`~yapapi.script.Script` (generated using + the service's instance of the :class:`~yapapi.WorkContext` - :attr:`self._ctx`) that are + then dispatched to the activity by the engine. Results of those batches can then be retrieved by awaiting the values captured from `yield` statements. @@ -312,10 +312,10 @@ async def run(self): **Default implementation** Because the nature of the operations required during the "running" state depends directly - on the specifics of a given :class:`Service` and because it's entirely plausible for a service - not to require any direct interaction with the exe-unit (runtime) from the requestor's end - after the service has been started, the default is to just wait indefinitely without - producing any batches. + on the specifics of a given :class:`Service` and because it's entirely plausible for a + service not to require any direct interaction with the exe-unit (runtime) from the + requestor's end after the service has been started, the default is to just wait indefinitely + without producing any batches. """ assert self._ctx is not None @@ -332,9 +332,9 @@ async def shutdown(self) -> AsyncGenerator[Script, Awaitable[List[events.Command As a handler implementing the `work generator pattern `_, - it's expected to be a generator that yields :class:`~yapapi.script.Script` (generated using the - service's instance of the :class:`~yapapi.WorkContext` - :attr:`self._ctx`) that are then dispatched - to the activity by the engine. + it's expected to be a generator that yields :class:`~yapapi.script.Script` (generated using + the service's instance of the :class:`~yapapi.WorkContext` - :attr:`self._ctx`) that are + then dispatched to the activity by the engine. Results of those batches can then be retrieved by awaiting the values captured from yield statements. @@ -370,22 +370,25 @@ async def shutdown(self): async def reset(self) -> None: """Reset the service to the initial state. The default implementation does nothing here. - This method is called internally when the service is restarted in :class:`yapapi.services.ServiceRunner`, - so it is not necessary for services that are never restarted (note that :func:`~yapapi.Golem.run_service()` by - default restarts services that didn't start properly based on :func:`Service.restart_condition()`). + This method is called internally when the service is restarted in + :class:`yapapi.services.ServiceRunner`, so it is not necessary for services that are never + restarted (note that :func:`~yapapi.Golem.run_service()` by default restarts services that + didn't start properly based on :func:`Service.restart_condition()`). - Handlers of a restarted service are called more then once - all of the cleanup necessary between calls - should be implemented here. E.g. if we initialize a counter in :func:`Service.__init__` and increment it - in :func:`Service.start()`, we might want to reset it here to the initial value. + Handlers of a restarted service are called more then once - all of the cleanup necessary + between calls should be implemented here. E.g. if we initialize a counter in + :func:`Service.__init__` and increment it in :func:`Service.start()`, we might want to reset + it here to the initial value. """ - pass @property def restart_condition(self) -> bool: - """The condition, based on which :class:`yapapi.services.ServiceRunner` decides if it should restart this instance. + """Return the condition, based on which :class:`yapapi.services.ServiceRunner` decides if \ + it should restart this instance. - If `restart_condition` returns `True`, the service is restarted. In such case, before putting the service back into the - `pending` state, the `ServiceRunner` calls this service's `reset` method. + If `restart_condition` returns `True`, the service is restarted. In such case, before + putting the service back into the `pending` state, the `ServiceRunner` calls this service's + `reset` method. """ return ( self.exc_info != (None, None, None) and not self.__service_instance.started_successfully diff --git a/yapapi/services/service_runner.py b/yapapi/services/service_runner.py index da889954c..461572c3c 100644 --- a/yapapi/services/service_runner.py +++ b/yapapi/services/service_runner.py @@ -35,11 +35,10 @@ class ServiceRunnerError(Exception): """An error while running a Service.""" - pass - class ControlSignal(enum.Enum): - """Control signal, used to request an instance's state change from the controlling SeviceRunner.""" + """Control signal, used to request an instance's state change from the controlling \ + SeviceRunner.""" stop = "stop" @@ -177,7 +176,8 @@ def _get_handler(instance: ServiceInstance): service_cls_name = type(instance.service).__name__ handler_name = handler.__name__ raise TypeError( - f"Service handler: `{service_cls_name}.{handler_name}` must be an asynchronous generator." + f"Service handler: `{service_cls_name}.{handler_name}` must be an asynchronous" + " generator." ) @staticmethod @@ -444,15 +444,17 @@ def on_agreement_ready(agreement_ready: Agreement) -> None: return async def _ensure_payload_matches(self, service: Service): - # Possible improvement: maybe we should accept services with lower demands then our payload? - # E.g. if service expects 2GB and we have 4GB in our payload, then this seems fine. - # (Not sure how much effort this requires) + # Possible improvement: maybe we should accept services with lower demands then our + # payload? E.g. if service expects 2GB and we have 4GB in our payload, then this seems + # fine. (Not sure how much effort this requires) service_payload = await service.get_payload() our_payload = self._job.payload if service_payload is not None and service_payload != our_payload: logger.error( - "Payload mismatch: service with {service_payload} was added to runner with {our_payload}" + f"Payload mismatch: service with {service_payload} was added to runner" + f" with {our_payload}" ) raise ValueError( - f"Only payload accepted by this service runner is {our_payload}, got {service_payload}" + f"Only payload accepted by this service runner is {our_payload}," + f" got {service_payload}" ) diff --git a/yapapi/storage/__init__.py b/yapapi/storage/__init__.py index 91ae33dec..99fdeccc2 100644 --- a/yapapi/storage/__init__.py +++ b/yapapi/storage/__init__.py @@ -1,6 +1,4 @@ -""" -Storage models. -""" +"""Storage models.""" import abc import asyncio @@ -101,15 +99,14 @@ async def read_file(): async def release_source(self, source: Source) -> None: """Release a source returned by `upload_file` or `upload_bytes`. - The default implementation is to do nothing.""" - pass + The default implementation is to do nothing. + """ class OutputStorageProvider(abc.ABC): @abc.abstractmethod async def new_destination(self, destination_file: Optional[PathLike] = None) -> Destination: - """ - Creates slot for receiving file. + """Create slot for receiving file. Parameters ---------- diff --git a/yapapi/storage/gftp.py b/yapapi/storage/gftp.py index 17bdfaf9d..fc844f779 100644 --- a/yapapi/storage/gftp.py +++ b/yapapi/storage/gftp.py @@ -1,6 +1,4 @@ -""" -Golem File Transfer Storage Provider -""" +"""Golem File Transfer Storage Provider.""" import asyncio import contextlib @@ -54,38 +52,33 @@ class GftpDriver(Protocol): """Golem FTP service API.""" async def version(self) -> str: - """Gets driver version.""" - pass + """Get driver version.""" async def publish(self, *, files: List[str]) -> List[PubLink]: - """Exposes local file as GFTP url. + """Expose local file as GFTP url. `files` : local files to be exposed """ - pass async def close(self, *, urls: List[str]) -> List[CommandStatus]: - """Stops exposing GFTP urls created by [publish(files=[..])](#publish).""" - pass + """Stop exposing GFTP urls created by [publish(files=[..])](#publish).""" async def receive(self, *, output_file: str) -> PubLink: - """Creates GFTP url for receiving file. + """Create GFTP url for receiving file. : `output_file` - """ - pass async def upload(self, *, file: str, url: str): pass async def shutdown(self) -> CommandStatus: - """Stops GFTP service. + """Stop GFTP service. After shutdown all generated urls will be unavailable. """ - pass def service(debug=False) -> AsyncContextManager[GftpDriver]: diff --git a/yapapi/strategy/__init__.py b/yapapi/strategy/__init__.py index 2ea76fdf3..d0127247c 100644 --- a/yapapi/strategy/__init__.py +++ b/yapapi/strategy/__init__.py @@ -14,3 +14,20 @@ from .dummy import DummyMS from .least_expensive import LeastExpensiveLinearPayuMS from .wrapping_strategy import WrappingMarketStrategy + +__all__ = ( + "DEBIT_NOTE_INTERVAL_GRACE_PERIOD", + "PROP_DEBIT_NOTE_ACCEPTANCE_TIMEOUT", + "PROP_DEBIT_NOTE_INTERVAL_SEC", + "PROP_PAYMENT_TIMEOUT_SEC", + "SCORE_NEUTRAL", + "SCORE_REJECTED", + "SCORE_TRUSTED", + "BaseMarketStrategy", + "MarketStrategy", + "PropValueRange", + "DecreaseScoreForUnconfirmedAgreement", + "DummyMS", + "LeastExpensiveLinearPayuMS", + "WrappingMarketStrategy", +) diff --git a/yapapi/strategy/base.py b/yapapi/strategy/base.py index d4b08ed87..e2073182c 100644 --- a/yapapi/strategy/base.py +++ b/yapapi/strategy/base.py @@ -111,16 +111,20 @@ async def invoice_accepted_amount(self, invoice: rest.payment.Invoice) -> Decima """Return the amount we accept to pay for the invoice. Current Golem Engine implementation accepts the invoice if returned amount is not lower than - `invoice.amount` and ignores the invoice otherwise. This will change in the future.""" + `invoice.amount` and ignores the invoice otherwise. This will change in the future. + """ + raise NotImplementedError() @abc.abstractmethod async def debit_note_accepted_amount(self, debit_note: rest.payment.DebitNote) -> Decimal: """Return the amount we accept to pay for the debit note. - Current Golem Engine implementation accepts the debit note if returned amount is not lower than - `debit_note.total_amount_due` and ignores the debit note otherwise. - This will change in the future.""" + Current Golem Engine implementation accepts the debit note if returned amount is not lower + than `debit_note.total_amount_due` and ignores the debit note otherwise. + This will change in the future. + """ + raise NotImplementedError() @@ -134,7 +138,7 @@ class MarketStrategy(BaseMarketStrategy, abc.ABC): @property def acceptable_prop_value_ranges(self) -> Dict[str, PropValueRange]: - """The range of acceptable property values for negotiable properties.""" + """Return the range of acceptable property values for negotiable properties.""" if not hasattr(self, "__acceptable_prop_value_ranges"): # initialize with the overrides self.__acceptable_prop_value_ranges = getattr( @@ -210,8 +214,8 @@ async def respond_to_provider_offer( if prop_value not in acceptable_range: our_value = acceptable_range.clamp(prop_value) logger.debug( - f"Negotiated property %s = %s outside of our accepted range: %s. " - f"Proposing our own value instead: %s", + "Negotiated property %s = %s outside of our accepted range: %s. " + "Proposing our own value instead: %s", prop_key, prop_value, acceptable_range, diff --git a/yapapi/strategy/decrease_score_unconfirmed.py b/yapapi/strategy/decrease_score_unconfirmed.py index e41835185..3d5dd28fd 100644 --- a/yapapi/strategy/decrease_score_unconfirmed.py +++ b/yapapi/strategy/decrease_score_unconfirmed.py @@ -19,7 +19,8 @@ class DecreaseScoreForUnconfirmedAgreement(WrappingMarketStrategy): factor: float def __init__(self, base_strategy, factor): - """ + """Initialize instance. + :param base_strategy: the base strategy around which this strategy is wrapped :param factor: the factor by which the score of an offer for a provider which failed to confirm the last agreement proposed to them will be multiplied @@ -32,7 +33,8 @@ def __init__(self, base_strategy, factor): def on_event(self, event: events.Event) -> None: """Modify the internal `_rejecting_providers` list on `AgreementConfirmed/Rejected`. - This method needs to be added as an event consumer in :func:`yapapi.Golem.add_event_consumer`. + This method needs to be added as an event consumer in + :func:`yapapi.Golem.add_event_consumer`. """ if isinstance(event, events.AgreementConfirmed): self._rejecting_providers.discard(event.provider_id) diff --git a/yapapi/strategy/wrapping_strategy.py b/yapapi/strategy/wrapping_strategy.py index e0ff49818..31bd4b45a 100644 --- a/yapapi/strategy/wrapping_strategy.py +++ b/yapapi/strategy/wrapping_strategy.py @@ -8,22 +8,23 @@ class WrappingMarketStrategy(BaseMarketStrategy, abc.ABC): - """ - Helper abstract class which allows other/user defined strategies to wrap some other strategies, - without overriding the attributes (e.g. defaults) defined on the derived-from strategy. + """Helper abstract class which allows other/user defined strategies to wrap some other \ + strategies, without overriding the attributes (e.g. defaults) defined on the derived-from \ + strategy. WrappingMarketStrategy classes are unusable on their own and always have to wrap some base strategy. - By default all attributes and method calls are forwarded to the `base_strategy`. + By default, all attributes and method calls are forwarded to the `base_strategy`. """ base_strategy: BaseMarketStrategy """base strategy wrapped by this wrapper.""" def __init__(self, base_strategy: BaseMarketStrategy): - """ - :param base_strategy: the base strategy around which this strategy is wrapped + """Initialize instance. + + :param base_strategy: the base strategy around which this strategy is wrapped. """ self.base_strategy = base_strategy From 34588bd8e4679e7341c7e98c4aedbb772cc759a8 Mon Sep 17 00:00:00 2001 From: approxit Date: Wed, 18 Jan 2023 14:18:26 +0100 Subject: [PATCH 03/20] mypy partially done --- docs/sphinx/conf.py | 3 ++- pyproject.toml | 8 +++++++- yapapi/__init__.py | 2 +- yapapi/contrib/service/http_proxy.py | 2 +- yapapi/contrib/service/socket_proxy.py | 2 +- yapapi/ctx.py | 4 ++-- yapapi/engine.py | 16 +++++++--------- yapapi/events.py | 2 +- yapapi/executor/__init__.py | 4 +--- yapapi/golem.py | 2 +- yapapi/invoice_manager.py | 2 +- yapapi/network.py | 2 +- yapapi/payload/vm.py | 2 +- yapapi/props/base.py | 5 ++--- yapapi/props/inf.py | 2 +- yapapi/rest/activity.py | 4 ++-- yapapi/rest/configuration.py | 8 ++++---- yapapi/rest/market.py | 2 +- yapapi/services/cluster.py | 4 ++-- yapapi/services/service.py | 2 +- yapapi/services/service_state.py | 2 +- yapapi/storage/gftp.py | 6 +++--- yapapi/strategy/dummy.py | 2 +- 23 files changed, 45 insertions(+), 43 deletions(-) diff --git a/docs/sphinx/conf.py b/docs/sphinx/conf.py index 9844430ad..ecb7b112d 100644 --- a/docs/sphinx/conf.py +++ b/docs/sphinx/conf.py @@ -12,6 +12,7 @@ # import os import sys +from typing import List sys.path.insert(0, os.path.abspath("../../")) @@ -56,7 +57,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = [] +html_static_path: List[str] = [] # This removes the `yapapi.log` docstrings. # There are two reasons: diff --git a/pyproject.toml b/pyproject.toml index afea570f5..6288b62eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,7 +93,7 @@ checks_codestyle = {sequence = ["_checks_codestyle_flake8", "_checks_codestyle_i _checks_codestyle_flake8 = "flake8 yapapi tests" _checks_codestyle_isort = "isort --check-only --diff ." _checks_codestyle_black = "black --check --diff ." -checks_typing = {cmd = "mypy --ignore-missing-imports --check-untyped-defs --warn-unused-ignores --show-error-codes .", help = "Run only code typing checks" } +checks_typing = {cmd = "mypy --install-types --non-interactive --ignore-missing-imports --check-untyped-defs --warn-unused-ignores --show-error-codes .", help = "Run only code typing checks" } checks_license = {sequence = ["_checks_license_export", "_checks_license_verify"], help = "Run only license compatibility checks"} _checks_license_export = "poetry export -f requirements.txt -o .requirements.txt" _checks_license_verify = "liccheck -r .requirements.txt" @@ -167,6 +167,12 @@ extend-ignore = [ "D202", # We prefer whitelines after docstrings ] +[tool.mypy] +exclude = [ + "^examples/", # TODO: To be fixed by #1088 + "^tests/", # TODO: To be fixed by #1088 +] + [tool.isort] profile = "black" py_version = 36 diff --git a/yapapi/__init__.py b/yapapi/__init__.py index 3155544bf..feac6b86b 100644 --- a/yapapi/__init__.py +++ b/yapapi/__init__.py @@ -36,7 +36,7 @@ def windows_event_loop_fix(): class _WindowsEventPolicy(asyncio.events.BaseDefaultEventLoopPolicy): _loop_factory = asyncio.windows_events.ProactorEventLoop - asyncio.set_event_loop_policy(_WindowsEventPolicy()) # type: ignore + asyncio.set_event_loop_policy(_WindowsEventPolicy()) __version__: str = get_version() diff --git a/yapapi/contrib/service/http_proxy.py b/yapapi/contrib/service/http_proxy.py index 7a4401b5d..05792dbbd 100644 --- a/yapapi/contrib/service/http_proxy.py +++ b/yapapi/contrib/service/http_proxy.py @@ -266,7 +266,7 @@ async def run(self): runner = web.ServerRunner( _Server(self._request_handler, client_max_size=self._max_request_size) - ) # type: ignore + ) await runner.setup() site = web.TCPSite(runner, port=self._port) await site.start() diff --git a/yapapi/contrib/service/socket_proxy.py b/yapapi/contrib/service/socket_proxy.py index 73b8324a7..14a5c3ace 100644 --- a/yapapi/contrib/service/socket_proxy.py +++ b/yapapi/contrib/service/socket_proxy.py @@ -223,7 +223,7 @@ async def run(self): Preferred usage is through `:meth:`~SocketProxy.run_server`. """ server = await asyncio.start_server(self.handler, self.local_address, self.local_port) - addrs = ", ".join(str(sock.getsockname()) for sock in server.sockets) # type: ignore # noqa + addrs = ", ".join(str(sock.getsockname()) for sock in server.sockets) logger.info("Listening on: %s, forwarding to: %s", addrs, self.instance_ws) try: diff --git a/yapapi/ctx.py b/yapapi/ctx.py index 3decb0781..449996dd6 100644 --- a/yapapi/ctx.py +++ b/yapapi/ctx.py @@ -1,7 +1,7 @@ import enum import logging from datetime import datetime, timedelta -from typing import Dict, Optional, Type +from typing import Any, Dict, Optional, Type from dataclasses import dataclass, field @@ -195,7 +195,7 @@ def _build( return cls(mode=mode, fmt=cap_fmt, limit=limit) def to_dict(self) -> Dict: - inner = dict() + inner: Dict[str, Any] = dict() if self.limit: inner[self.mode.value] = self.limit diff --git a/yapapi/engine.py b/yapapi/engine.py index 984a6c9c2..07f7ca6e1 100644 --- a/yapapi/engine.py +++ b/yapapi/engine.py @@ -32,7 +32,7 @@ if sys.version_info >= (3, 7): from contextlib import AsyncExitStack else: - from async_exit_stack import AsyncExitStack # type: ignore + from async_exit_stack import AsyncExitStack from yapapi import events, props, rest from yapapi.agreements_pool import AgreementsPool @@ -222,7 +222,7 @@ async def stop(self, *exc_info) -> Optional[bool]: This *must* be called at the end of the work, by the Engine user. """ if exc_info[0] is not None: - self.emit(events.ExecutionInterrupted, exc_info=exc_info) # type: ignore + self.emit(events.ExecutionInterrupted, exc_info=exc_info) return await self._stack.__aexit__(None, None, None) async def start(self): @@ -426,7 +426,7 @@ def _check_for_termination_reason( "message": f"Too many {payable_str}debit notes: {freq_descr} " f"(activity: {activity_id})", "golem.requestor.code": ( - "TooManyPayableDebitNotes" if payable else "TooManyDebitNotes", + "TooManyPayableDebitNotes" if payable else "TooManyDebitNotes" ), } logger.error( @@ -592,7 +592,7 @@ async def _process_debit_note(self, debit_note_id: str) -> None: job.emit( events.PaymentFailed, agreement=agreement, - exc_info=sys.exc_info(), # type: ignore + exc_info=sys.exc_info(), ) def accept_debit_notes_for_agreement(self, job_id: str, agreement_id: str) -> None: @@ -607,7 +607,7 @@ def add_job(self, job: "Job"): def finalize_job(self, job: "Job"): """Mark a job as finished.""" job.finished.set() - job.emit(events.JobFinished, exc_info=job._exc_info) # type: ignore + job.emit(events.JobFinished, exc_info=job._exc_info) def register_generator(self, generator: AsyncGenerator) -> None: """Register a generator with this engine.""" @@ -657,9 +657,7 @@ async def worker_task(agreement: Agreement): try: activity = await self.create_activity(agreement.id) except Exception: - job.emit( - events.ActivityCreateFailed, agreement=agreement, exc_info=sys.exc_info() - ) # type: ignore + job.emit(events.ActivityCreateFailed, agreement=agreement, exc_info=sys.exc_info()) raise work_context = WorkContext(activity, agreement, self.storage_manager, emitter=job.emit) @@ -947,7 +945,7 @@ async def handler(proposal_): with contextlib.suppress(Exception): self.emit( events.ProposalFailed, proposal=proposal_, exc_info=sys.exc_info() - ) # type: ignore + ) finally: semaphore.release() diff --git a/yapapi/events.py b/yapapi/events.py index 61cb9462a..6e3b46910 100644 --- a/yapapi/events.py +++ b/yapapi/events.py @@ -232,7 +232,7 @@ def __str__(self) -> str: If `exc_info` is not `None`, its underlying exception is included in the result string under the key `exception`. """ - fields: Tuple[attr.Attribute] = attr.fields(self.__class__) # type: ignore + fields: Tuple[attr.Attribute] = attr.fields(self.__class__) field_reprs: List[str] = [] for field in fields: diff --git a/yapapi/executor/__init__.py b/yapapi/executor/__init__.py index c9063f495..d36f75323 100644 --- a/yapapi/executor/__init__.py +++ b/yapapi/executor/__init__.py @@ -231,9 +231,7 @@ async def task_generator() -> AsyncGenerator[Task[D, R], None]: pass work_context.emit(events.WorkerFinished) except Exception as e: - work_context.emit( - events.WorkerFinished, exc_info=sys.exc_info() - ) # type: ignore + work_context.emit(events.WorkerFinished, exc_info=sys.exc_info()) await task_gen.athrow(type(e), e) raise finally: diff --git a/yapapi/golem.py b/yapapi/golem.py index 5c070d973..657854456 100644 --- a/yapapi/golem.py +++ b/yapapi/golem.py @@ -204,7 +204,7 @@ def _parse_event_cls_or_name( return event_cls_or_name else: try: - return getattr(events, event_cls_or_name) # type: ignore + return getattr(events, event_cls_or_name) except AttributeError: raise ValueError( "Second argument must be either an event class, or a name of " diff --git a/yapapi/invoice_manager.py b/yapapi/invoice_manager.py index 9fe2324df..4a75b55c0 100644 --- a/yapapi/invoice_manager.py +++ b/yapapi/invoice_manager.py @@ -114,7 +114,7 @@ async def attempt_payment( ad.job.emit( events.PaymentFailed, agreement=ad.agreement, - exc_info=sys.exc_info(), # type: ignore + exc_info=sys.exc_info(), ) return False else: diff --git a/yapapi/network.py b/yapapi/network.py index e829d544a..36116b89d 100644 --- a/yapapi/network.py +++ b/yapapi/network.py @@ -5,7 +5,7 @@ from urllib.parse import urlparse from dataclasses import dataclass -from statemachine import State, StateMachine # type: ignore +from statemachine import State, StateMachine from ya_net.exceptions import ApiException diff --git a/yapapi/payload/vm.py b/yapapi/payload/vm.py index 1867bba41..97f50dd30 100644 --- a/yapapi/payload/vm.py +++ b/yapapi/payload/vm.py @@ -12,7 +12,7 @@ else: from typing_extensions import Literal -from srvresolver.srv_resolver import SRVRecord, SRVResolver # type: ignore +from srvresolver.srv_resolver import SRVRecord, SRVResolver from yapapi.payload.package import ( Package, diff --git a/yapapi/props/base.py b/yapapi/props/base.py index d992d0c05..5c69b3bff 100644 --- a/yapapi/props/base.py +++ b/yapapi/props/base.py @@ -46,16 +46,15 @@ class _PyField: def encode(self, value: str): def get_type_origin(t): - # type: ignore if hasattr(typing, "get_origin"): - return typing.get_origin(t) # type: ignore + return typing.get_origin(t) else: return getattr(t, "__origin__", None) def get_type_args(t): # >= py3.8 if hasattr(typing, "get_args"): - return typing.get_args(t) # type: ignore + return typing.get_args(t) else: return getattr(t, "__args__") diff --git a/yapapi/props/inf.py b/yapapi/props/inf.py index fdfd56848..aea9b60b6 100644 --- a/yapapi/props/inf.py +++ b/yapapi/props/inf.py @@ -4,7 +4,7 @@ from typing import List, Optional from dataclasses import dataclass -from deprecated import deprecated # type: ignore +from deprecated import deprecated from .base import Model, prop diff --git a/yapapi/rest/activity.py b/yapapi/rest/activity.py index 89685278f..b5a99319f 100644 --- a/yapapi/rest/activity.py +++ b/yapapi/rest/activity.py @@ -6,7 +6,7 @@ from typing import Any, AsyncIterator, Dict, List, Optional, Tuple, Type from aiohttp import ClientPayloadError -from aiohttp_sse_client.client import MessageEvent # type: ignore +from aiohttp_sse_client.client import MessageEvent from dataclasses import dataclass from typing_extensions import AsyncContextManager, AsyncIterable @@ -286,7 +286,7 @@ class StreamingBatch(Batch): """A `Batch` implementation that uses event streaming to return command status.""" async def __aiter__(self) -> AsyncIterator[CommandEventData]: - from aiohttp_sse_client import client as sse_client # type: ignore + from aiohttp_sse_client import client as sse_client api_client = self._activity._api.api_client host = api_client.configuration.host diff --git a/yapapi/rest/configuration.py b/yapapi/rest/configuration.py index 9b8dec1ba..db60a179b 100644 --- a/yapapi/rest/configuration.py +++ b/yapapi/rest/configuration.py @@ -1,11 +1,11 @@ from typing import Optional -import ya_activity # type: ignore -import ya_market # type: ignore -import ya_net # type: ignore +import ya_activity +import ya_market +import ya_net import ya_payment -from yapapi.config import ApiConfig # type: ignore +from yapapi.config import ApiConfig class Configuration(object): diff --git a/yapapi/rest/market.py b/yapapi/rest/market.py index d421c3e88..fed2db511 100644 --- a/yapapi/rest/market.py +++ b/yapapi/rest/market.py @@ -8,7 +8,7 @@ from dataclasses import dataclass from typing_extensions import AsyncContextManager, Awaitable -from ya_market import ApiClient, ApiException, RequestorApi, models # type: ignore +from ya_market import ApiClient, ApiException, RequestorApi, models from ..props import Model, NodeInfo from .common import SuppressedExceptions, is_intermittent_error diff --git a/yapapi/services/cluster.py b/yapapi/services/cluster.py index fceef6a33..d15d9eed2 100644 --- a/yapapi/services/cluster.py +++ b/yapapi/services/cluster.py @@ -9,7 +9,7 @@ from typing import Final else: from typing_extensions import Final - from async_exit_stack import AsyncExitStack # type: ignore + from async_exit_stack import AsyncExitStack from yapapi.engine import Job, _Engine from yapapi.network import Network @@ -152,7 +152,7 @@ def spawn_instances( if network_addresses is not None and len(network_addresses) > ix: network_address = network_addresses[ix] - service = self.service_class(**single_instance_params) # type: ignore + service = self.service_class(**single_instance_params) self.service_runner.add_instance(service, self.network, network_address) service._set_cluster(self) diff --git a/yapapi/services/service.py b/yapapi/services/service.py index 1c5c36d62..756711d16 100644 --- a/yapapi/services/service.py +++ b/yapapi/services/service.py @@ -16,7 +16,7 @@ Union, ) -import statemachine # type: ignore +import statemachine from dataclasses import dataclass, field from ya_activity.exceptions import ApiException diff --git a/yapapi/services/service_state.py b/yapapi/services/service_state.py index 707f65338..4e3cb2138 100644 --- a/yapapi/services/service_state.py +++ b/yapapi/services/service_state.py @@ -1,6 +1,6 @@ from typing import TYPE_CHECKING -import statemachine # type: ignore +import statemachine if TYPE_CHECKING: from .service import ServiceInstance diff --git a/yapapi/storage/gftp.py b/yapapi/storage/gftp.py index fc844f779..44b41a4e1 100644 --- a/yapapi/storage/gftp.py +++ b/yapapi/storage/gftp.py @@ -23,9 +23,9 @@ cast, ) -import jsonrpc_base # type: ignore -import semantic_version # type: ignore -from async_exit_stack import AsyncExitStack # type: ignore +import jsonrpc_base +import semantic_version +from async_exit_stack import AsyncExitStack from dataclasses import dataclass from typing_extensions import Literal, Protocol, TypedDict diff --git a/yapapi/strategy/dummy.py b/yapapi/strategy/dummy.py index 347a7a862..3b25c1661 100644 --- a/yapapi/strategy/dummy.py +++ b/yapapi/strategy/dummy.py @@ -4,7 +4,7 @@ from typing import Dict, Mapping, Optional, Union from dataclasses import dataclass -from deprecated import deprecated # type: ignore +from deprecated import deprecated from yapapi import rest from yapapi.props import Activity, com From 4e7ea6a1980fa9cce7bbae1e6c94e5bcbdfee916 Mon Sep 17 00:00:00 2001 From: approxit Date: Wed, 18 Jan 2023 14:29:01 +0100 Subject: [PATCH 04/20] fixed workflows --- .github/workflows/build_publish.yml | 8 ++++---- .github/workflows/goth-nightly.yml | 4 ++-- .github/workflows/goth.yml | 4 ++-- .github/workflows/tests.yml | 8 ++++---- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build_publish.yml b/.github/workflows/build_publish.yml index ff0391881..ba232cb75 100644 --- a/.github/workflows/build_publish.yml +++ b/.github/workflows/build_publish.yml @@ -18,10 +18,10 @@ jobs: with: poetry-version: 1.2.2 - run: poetry install - - run: poetry run poe test - - run: poetry run poe typecheck - - run: poetry run poe codestyle - - run: poetry run poe liccheck + - run: poetry run poe tests_unit + - run: poetry run poe checks_codestyle + - run: poetry run poe checks_typing + - run: poetry run poe checks_license build: needs: [test] diff --git a/.github/workflows/goth-nightly.yml b/.github/workflows/goth-nightly.yml index 72b5a58d3..61528e4a3 100644 --- a/.github/workflows/goth-nightly.yml +++ b/.github/workflows/goth-nightly.yml @@ -86,8 +86,8 @@ jobs: env: GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - poetry run poe goth-assets - poetry run poe goth-tests + poetry run poe tests_integration_assets + poetry run poe tests_integration - name: Upload test logs uses: actions/upload-artifact@v2 diff --git a/.github/workflows/goth.yml b/.github/workflows/goth.yml index 64d0842ae..eaf77cd7b 100644 --- a/.github/workflows/goth.yml +++ b/.github/workflows/goth.yml @@ -57,8 +57,8 @@ jobs: env: GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - poetry run poe goth-assets - poetry run poe goth-tests + poetry run poe tests_integration_assets + poetry run poe tests_integration - name: Upload test logs uses: actions/upload-artifact@v2 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4cb31de3d..30a845e17 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -50,11 +50,11 @@ jobs: - run: poetry install -E docs if: ${{ steps.extended-checks-sphinx.outputs.ENABLE }} - - run: poetry run poe test - - run: poetry run poe typecheck - - run: poetry run poe codestyle + - run: poetry run poe tests_unit + - run: poetry run poe checks_codestyle + - run: poetry run poe checks_typing if: ${{ steps.extended-checks.outputs.ENABLE }} - - run: poetry run poe liccheck + - run: poetry run poe checks_license if: ${{ steps.extended-checks.outputs.ENABLE }} - run: poetry run poe sphinx -W if: ${{ steps.extended-checks-sphinx.outputs.ENABLE }} From 188a3306481b44365a3a349c4f634775a33f20a2 Mon Sep 17 00:00:00 2001 From: approxit Date: Wed, 18 Jan 2023 14:52:58 +0100 Subject: [PATCH 05/20] fixed ut --- pyproject.toml | 2 +- tests/test_agreements_pool.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6288b62eb..04e1d9c1f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,7 +85,7 @@ autoflake = "^1" flake8 = "^5" flake8-docstrings = "^1.6" Flake8-pyproject = "^1.2.2" -pyproject-autoflake = "^1.0.2" +pyproject-autoflake = "^1.0.2" # TODO: Remove this after upgrading to Python 3.8+ [tool.poe.tasks] checks = {sequence = ["checks_codestyle", "checks_typing", "checks_license"], help = "Run all available code checks"} diff --git a/tests/test_agreements_pool.py b/tests/test_agreements_pool.py index 3e35aaa89..c966f9919 100644 --- a/tests/test_agreements_pool.py +++ b/tests/test_agreements_pool.py @@ -4,7 +4,7 @@ from unittest import mock import pytest -from factories.agreements_pool import BufferedAgreement, BufferedAgreementFactory +from tests.factories.agreements_pool import BufferedAgreement, BufferedAgreementFactory from yapapi import agreements_pool from yapapi.events import AgreementTerminated From 2f84fe70089c0f6c6020dc862d9075f3d97eaf0d Mon Sep 17 00:00:00 2001 From: approxit Date: Wed, 18 Jan 2023 16:20:28 +0100 Subject: [PATCH 06/20] fixed ut again --- tests/test_agreements_pool.py | 2 +- yapapi/script/capture.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_agreements_pool.py b/tests/test_agreements_pool.py index c966f9919..1d752c441 100644 --- a/tests/test_agreements_pool.py +++ b/tests/test_agreements_pool.py @@ -4,8 +4,8 @@ from unittest import mock import pytest -from tests.factories.agreements_pool import BufferedAgreement, BufferedAgreementFactory +from tests.factories.agreements_pool import BufferedAgreement, BufferedAgreementFactory from yapapi import agreements_pool from yapapi.events import AgreementTerminated diff --git a/yapapi/script/capture.py b/yapapi/script/capture.py index aa5329a68..6c7d05e79 100644 --- a/yapapi/script/capture.py +++ b/yapapi/script/capture.py @@ -1,5 +1,5 @@ import enum -from typing import Dict, Optional +from typing import Any, Dict, Optional from dataclasses import dataclass @@ -47,7 +47,7 @@ def _build( return cls(mode=mode, fmt=cap_fmt, limit=limit) def to_dict(self) -> Dict: - inner = dict() + inner: Dict[str, Any] = dict() if self.limit: inner[self.mode.value] = self.limit From f9e3428c6ca366a07c02cd1cc0d2a807adaa1332 Mon Sep 17 00:00:00 2001 From: approxit Date: Wed, 18 Jan 2023 16:39:00 +0100 Subject: [PATCH 07/20] fixed sphinx --- yapapi/golem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yapapi/golem.py b/yapapi/golem.py index 657854456..6a231c606 100644 --- a/yapapi/golem.py +++ b/yapapi/golem.py @@ -399,7 +399,7 @@ async def run_service( :class:`~yapapi.services.Cluster` will be created with as many instances as there are elements in the `instance_params` iterable. if `num_instances` is set to < 1, the :class:`~yapapi.services.Cluster` will still be created but no instances will be spawned - within it. + within it. :param instance_params: optional list of dictionaries of keyword arguments that will be passed to consecutive, spawned instances. The number of elements in the iterable determines the number of instances spawned, unless `num_instances` is given, in which From 264d51441745ef24d74d799d4038f9cc8137d78d Mon Sep 17 00:00:00 2001 From: approxit Date: Thu, 19 Jan 2023 13:41:04 +0100 Subject: [PATCH 08/20] fixed mypy --- yapapi/config.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/yapapi/config.py b/yapapi/config.py index ef8fc124a..7c8a7953c 100644 --- a/yapapi/config.py +++ b/yapapi/config.py @@ -36,12 +36,13 @@ class ApiConfig: Uses YAGNA_ACTIVITY_URL environment variable """ - app_key: str = field( + app_key: str = field( # type: ignore[assignment] default_factory=partial(os.getenv, "YAGNA_APPKEY"), - ) # type: ignore[assignment] - api_url: str = field( + ) + + api_url: str = field( # type: ignore[assignment] default_factory=partial(os.getenv, "YAGNA_API_URL", "http://127.0.0.1:7465"), - ) # type: ignore[assignment] + ) market_url: Optional[str] = field(default_factory=partial(os.getenv, "YAGNA_MARKET_URL")) payment_url: Optional[str] = field(default_factory=partial(os.getenv, "YAGNA_PAYMENT_URL")) net_url: Optional[str] = field(default_factory=partial(os.getenv, "YAGNA_NET_URL")) From f2f931c80bec60a3e9d48e02e224d127442eddb6 Mon Sep 17 00:00:00 2001 From: approxit Date: Thu, 19 Jan 2023 14:10:49 +0100 Subject: [PATCH 09/20] fixed mypy and goth --- pyproject.toml | 12 +++++++++++- tests/goth_tests/test_recycle_ip/test_recycle_ip.py | 2 +- yapapi/props/base.py | 4 ++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 04e1d9c1f..51bfcaee9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,7 +93,7 @@ checks_codestyle = {sequence = ["_checks_codestyle_flake8", "_checks_codestyle_i _checks_codestyle_flake8 = "flake8 yapapi tests" _checks_codestyle_isort = "isort --check-only --diff ." _checks_codestyle_black = "black --check --diff ." -checks_typing = {cmd = "mypy --install-types --non-interactive --ignore-missing-imports --check-untyped-defs --warn-unused-ignores --show-error-codes .", help = "Run only code typing checks" } +checks_typing = {cmd = "mypy .", help = "Run only code typing checks" } checks_license = {sequence = ["_checks_license_export", "_checks_license_verify"], help = "Run only license compatibility checks"} _checks_license_export = "poetry export -f requirements.txt -o .requirements.txt" _checks_license_verify = "liccheck -r .requirements.txt" @@ -172,6 +172,16 @@ exclude = [ "^examples/", # TODO: To be fixed by #1088 "^tests/", # TODO: To be fixed by #1088 ] +install_types = true +non_interactive = true +ignore_missing_imports = true +check_untyped_defs = true +show_error_codes = true +warn_unused_ignores = true + +[[tool.mypy.overrides]] # TODO: Remove this after upgrading to Python 3.8+ +module = "yapapi.props.base" +warn_unused_ignores = false [tool.isort] profile = "black" diff --git a/tests/goth_tests/test_recycle_ip/test_recycle_ip.py b/tests/goth_tests/test_recycle_ip/test_recycle_ip.py index 23cca5745..8413c5884 100644 --- a/tests/goth_tests/test_recycle_ip/test_recycle_ip.py +++ b/tests/goth_tests/test_recycle_ip/test_recycle_ip.py @@ -15,7 +15,7 @@ from goth.runner.log import configure_logging from goth.runner.probe import RequestorProbe -from goth_tests.assertions import assert_all_invoices_accepted, assert_no_errors # isort:skip +from tests.goth_tests.assertions import assert_all_invoices_accepted, assert_no_errors # isort:skip logger = logging.getLogger("goth.test.recycle_ips") diff --git a/yapapi/props/base.py b/yapapi/props/base.py index 5c69b3bff..5451c80d0 100644 --- a/yapapi/props/base.py +++ b/yapapi/props/base.py @@ -47,14 +47,14 @@ class _PyField: def encode(self, value: str): def get_type_origin(t): if hasattr(typing, "get_origin"): - return typing.get_origin(t) + return typing.get_origin(t) # type: ignore[attr-defined] else: return getattr(t, "__origin__", None) def get_type_args(t): # >= py3.8 if hasattr(typing, "get_args"): - return typing.get_args(t) + return typing.get_args(t) # type: ignore[attr-defined] else: return getattr(t, "__args__") From e815b6c21358110345fa8ce6ead0759f164edd78 Mon Sep 17 00:00:00 2001 From: approxit Date: Fri, 20 Jan 2023 14:53:11 +0100 Subject: [PATCH 10/20] updated README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e7d0120a9..bd521e072 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ All of the project's dependencies will be installed to that virtual environment. Declarations of project tasks can be found in `pyproject.toml`. ```bash -poetry run poe test +poetry run poe tests_unit ``` ### Running `goth` integration tests @@ -96,7 +96,7 @@ poetry install -E integration-tests Finally, generate goth's default assets: ```bash -poetry run poe goth-assets +poetry run poe tests_integration_assets ``` #### Running the tests @@ -104,7 +104,7 @@ poetry run poe goth-assets Once you have the environment set up, to run all the integration tests, use: ```bash -poetry run poe goth-tests +poetry run poe tests_integration ``` ### Contributing @@ -112,7 +112,7 @@ poetry run poe goth-tests It is recommended to run unit tests and static code analysis before committing changes. ```bash -poetry run poe check +poetry run poe checks ``` You can clean up the artifacts created during the test runs with: From 7e4026c968ebbf7f4b3d1f5934d519a5425c96ce Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Fri, 27 Jan 2023 11:53:08 +0100 Subject: [PATCH 11/20] add log file path to the `power_outage` test --- tests/goth_tests/test_power_outage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/goth_tests/test_power_outage.py b/tests/goth_tests/test_power_outage.py index 11429a7d6..ae64a27c6 100644 --- a/tests/goth_tests/test_power_outage.py +++ b/tests/goth_tests/test_power_outage.py @@ -64,7 +64,7 @@ async def test_power_outage( provider_1 = runner.get_probes(probe_type=ProviderProbe)[0] async with requestor.run_command_on_host( - f"{blender_path} --subnet-tag goth --min-cpu-threads 1", + f"{blender_path} --subnet-tag goth --min-cpu-threads 1 --log-file {log_dir / 'test_power_outage.log'}", env=os.environ, ) as (_cmd_task, cmd_monitor, _process_monitor): From f599243d88281e648c454122db3df2805b470f64 Mon Sep 17 00:00:00 2001 From: approxit Date: Tue, 31 Jan 2023 14:22:18 +0100 Subject: [PATCH 12/20] format fix --- tests/goth_tests/test_power_outage.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/goth_tests/test_power_outage.py b/tests/goth_tests/test_power_outage.py index ae64a27c6..72994283d 100644 --- a/tests/goth_tests/test_power_outage.py +++ b/tests/goth_tests/test_power_outage.py @@ -64,7 +64,9 @@ async def test_power_outage( provider_1 = runner.get_probes(probe_type=ProviderProbe)[0] async with requestor.run_command_on_host( - f"{blender_path} --subnet-tag goth --min-cpu-threads 1 --log-file {log_dir / 'test_power_outage.log'}", + "{} --subnet-tag goth --min-cpu-threads 1 --log-file {}".format( + blender_path, log_dir / "test_power_outage.log" + ), env=os.environ, ) as (_cmd_task, cmd_monitor, _process_monitor): From fdc137d528c83bed09949fe66997be393f108087 Mon Sep 17 00:00:00 2001 From: approxit Date: Mon, 13 Feb 2023 16:27:56 +0100 Subject: [PATCH 13/20] reverted examples renames --- examples/__init__.py | 0 examples/blender/__init__.py | 0 examples/blender/blender.py | 3 ++- examples/blender/start_stop_blender.py | 12 ++++------- .../README.md | 0 .../custom_usage_counter.py | 0 examples/custom_runtime/__init__.py | 0 examples/custom_usage_counter/__init__.py | 0 .../Dockerfile | 0 .../external_api_request.py | 4 ++-- .../golem_sign.pem | 0 .../manifest.json | 0 .../manifest.json.base64.sha256.sig | Bin .../request.sh | 0 .../sign.sh | 0 examples/external_api_request/__init__.py | 0 .../{hello_world => hello-world}/Dockerfile | 0 .../{hello_world => hello-world}/hello.py | 0 .../hello_service.py | 0 examples/hello_world/__init__.py | 0 .../{http_proxy => http-proxy}/Dockerfile | 0 .../{http_proxy => http-proxy}/http_proxy.py | 9 ++++---- examples/http_proxy/__init__.py | 0 .../list-offers.py | 0 examples/low_level_api/__init__.py | 0 .../market_strategy.py | 11 ++++++---- examples/market_strategy/__init__.py | 0 examples/scan/__init__.py | 0 .../simple_service.py | 8 +++---- .../simple_service/README.md | 0 .../simple_service/simple_service.Dockerfile | 0 .../simple_service/simple_service.py | 6 +++--- .../simple_service/simulate_observations.py | 3 +-- .../simulate_observations_ctl.py | 3 +-- examples/simple_service_poc/__init__.py | 0 .../simple_service/__init__.py | 0 examples/ssh/__init__.py | 0 examples/webapp/__init__.py | 0 examples/webapp/http/__init__.py | 0 examples/webapp/webapp.py | 20 ++++++++++-------- examples/webapp_fileupload/__init__.py | 0 .../Dockerfile | 0 .../app.py | 0 .../templates/error.html | 0 .../templates/index.html | 0 .../http_file_upload/__init__.py | 0 .../webapp_fileupload/webapp_fileupload.py | 20 ++++++++++-------- examples/yacat/__init__.py | 0 examples/yacat/yacat.py | 10 +++------ pyproject.toml | 5 ++++- 50 files changed, 56 insertions(+), 58 deletions(-) delete mode 100644 examples/__init__.py delete mode 100644 examples/blender/__init__.py rename examples/{custom_usage_counter => custom-usage-counter}/README.md (100%) rename examples/{custom_usage_counter => custom-usage-counter}/custom_usage_counter.py (100%) delete mode 100644 examples/custom_runtime/__init__.py delete mode 100644 examples/custom_usage_counter/__init__.py rename examples/{external_api_request => external-api-request}/Dockerfile (100%) rename examples/{external_api_request => external-api-request}/external_api_request.py (96%) rename examples/{external_api_request => external-api-request}/golem_sign.pem (100%) rename examples/{external_api_request => external-api-request}/manifest.json (100%) rename examples/{external_api_request => external-api-request}/manifest.json.base64.sha256.sig (100%) rename examples/{external_api_request => external-api-request}/request.sh (100%) rename examples/{external_api_request => external-api-request}/sign.sh (100%) delete mode 100644 examples/external_api_request/__init__.py rename examples/{hello_world => hello-world}/Dockerfile (100%) rename examples/{hello_world => hello-world}/hello.py (100%) rename examples/{hello_world => hello-world}/hello_service.py (100%) delete mode 100644 examples/hello_world/__init__.py rename examples/{http_proxy => http-proxy}/Dockerfile (100%) rename examples/{http_proxy => http-proxy}/http_proxy.py (97%) delete mode 100644 examples/http_proxy/__init__.py rename examples/{low_level_api => low-level-api}/list-offers.py (100%) delete mode 100644 examples/low_level_api/__init__.py rename examples/{market_strategy => market-strategy}/market_strategy.py (91%) delete mode 100644 examples/market_strategy/__init__.py delete mode 100644 examples/scan/__init__.py rename examples/{simple_service_poc => simple-service-poc}/simple_service.py (97%) rename examples/{simple_service_poc => simple-service-poc}/simple_service/README.md (100%) rename examples/{simple_service_poc => simple-service-poc}/simple_service/simple_service.Dockerfile (100%) rename examples/{simple_service_poc => simple-service-poc}/simple_service/simple_service.py (94%) rename examples/{simple_service_poc => simple-service-poc}/simple_service/simulate_observations.py (87%) rename examples/{simple_service_poc => simple-service-poc}/simple_service/simulate_observations_ctl.py (91%) delete mode 100644 examples/simple_service_poc/__init__.py delete mode 100644 examples/simple_service_poc/simple_service/__init__.py delete mode 100644 examples/ssh/__init__.py delete mode 100644 examples/webapp/__init__.py delete mode 100644 examples/webapp/http/__init__.py delete mode 100644 examples/webapp_fileupload/__init__.py rename examples/webapp_fileupload/{http_file_upload => http-file-upload}/Dockerfile (100%) rename examples/webapp_fileupload/{http_file_upload => http-file-upload}/app.py (100%) rename examples/webapp_fileupload/{http_file_upload => http-file-upload}/templates/error.html (100%) rename examples/webapp_fileupload/{http_file_upload => http-file-upload}/templates/index.html (100%) delete mode 100644 examples/webapp_fileupload/http_file_upload/__init__.py delete mode 100644 examples/yacat/__init__.py diff --git a/examples/__init__.py b/examples/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/blender/__init__.py b/examples/blender/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/blender/blender.py b/examples/blender/blender.py index 74a3ab1f7..b445f9c52 100755 --- a/examples/blender/blender.py +++ b/examples/blender/blender.py @@ -31,7 +31,8 @@ async def main( min_mem_gib=0.5, # only run on provider nodes that have more than 2gb of storage space available min_storage_gib=2.0, - # only run on provider nodes which a certain number of CPU threads (logical CPU cores) available + # only run on provider nodes which a certain number of CPU threads (logical CPU cores) + # available min_cpu_threads=min_cpu_threads, ) diff --git a/examples/blender/start_stop_blender.py b/examples/blender/start_stop_blender.py index 8fd588a0b..9e46a2219 100755 --- a/examples/blender/start_stop_blender.py +++ b/examples/blender/start_stop_blender.py @@ -2,23 +2,19 @@ """Other version of the blender example. -Instead of using Golem in the default way - as a context manager - we directly call `Golem.start()` and `Golem.stop()`. -This way of using Golem might be more convenient for some specific use cases (although doesn't change a lot -in the blender example). +Instead of using Golem in the default way - as a context manager - we directly call `Golem.start()` +and `Golem.stop()`. This way of using Golem might be more convenient for some specific use cases +(although doesn't change a lot in the blender example). """ import pathlib -import sys from datetime import datetime, timedelta from yapapi import Golem, Task, WorkContext from yapapi.payload import vm from yapapi.rest.activity import BatchTimeoutError -examples_dir = pathlib.Path(__file__).resolve().parent.parent -sys.path.append(str(examples_dir)) - -from utils import ( +from ..utils import ( TEXT_COLOR_CYAN, TEXT_COLOR_DEFAULT, TEXT_COLOR_MAGENTA, diff --git a/examples/custom_usage_counter/README.md b/examples/custom-usage-counter/README.md similarity index 100% rename from examples/custom_usage_counter/README.md rename to examples/custom-usage-counter/README.md diff --git a/examples/custom_usage_counter/custom_usage_counter.py b/examples/custom-usage-counter/custom_usage_counter.py similarity index 100% rename from examples/custom_usage_counter/custom_usage_counter.py rename to examples/custom-usage-counter/custom_usage_counter.py diff --git a/examples/custom_runtime/__init__.py b/examples/custom_runtime/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/custom_usage_counter/__init__.py b/examples/custom_usage_counter/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/external_api_request/Dockerfile b/examples/external-api-request/Dockerfile similarity index 100% rename from examples/external_api_request/Dockerfile rename to examples/external-api-request/Dockerfile diff --git a/examples/external_api_request/external_api_request.py b/examples/external-api-request/external_api_request.py similarity index 96% rename from examples/external_api_request/external_api_request.py rename to examples/external-api-request/external_api_request.py index c84da7995..df9a3f62f 100644 --- a/examples/external_api_request/external_api_request.py +++ b/examples/external-api-request/external_api_request.py @@ -46,7 +46,7 @@ async def run(self): future_result = script.run( "/bin/sh", "-c", - f"GOLEM_PRICE=`curl -X 'GET' \ + "GOLEM_PRICE=`curl -X 'GET' \ 'https://api.coingecko.com/api/v3/simple/price?ids=golem&vs_currencies=usd' \ -H 'accept: application/json' | jq .golem.usd`; \ echo ---; \ @@ -81,7 +81,7 @@ async def main(subnet_tag, payment_driver, payment_network): if __name__ == "__main__": parser = build_parser("External API request example") now = datetime.now().strftime("%Y-%m-%d_%H.%M.%S") - parser.set_defaults(log_file=f"external_api_request-yapapi-{now}.log") + parser.set_defaults(log_file=f"external-api-request-yapapi-{now}.log") args = parser.parse_args() run_golem_example( diff --git a/examples/external_api_request/golem_sign.pem b/examples/external-api-request/golem_sign.pem similarity index 100% rename from examples/external_api_request/golem_sign.pem rename to examples/external-api-request/golem_sign.pem diff --git a/examples/external_api_request/manifest.json b/examples/external-api-request/manifest.json similarity index 100% rename from examples/external_api_request/manifest.json rename to examples/external-api-request/manifest.json diff --git a/examples/external_api_request/manifest.json.base64.sha256.sig b/examples/external-api-request/manifest.json.base64.sha256.sig similarity index 100% rename from examples/external_api_request/manifest.json.base64.sha256.sig rename to examples/external-api-request/manifest.json.base64.sha256.sig diff --git a/examples/external_api_request/request.sh b/examples/external-api-request/request.sh similarity index 100% rename from examples/external_api_request/request.sh rename to examples/external-api-request/request.sh diff --git a/examples/external_api_request/sign.sh b/examples/external-api-request/sign.sh similarity index 100% rename from examples/external_api_request/sign.sh rename to examples/external-api-request/sign.sh diff --git a/examples/external_api_request/__init__.py b/examples/external_api_request/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/hello_world/Dockerfile b/examples/hello-world/Dockerfile similarity index 100% rename from examples/hello_world/Dockerfile rename to examples/hello-world/Dockerfile diff --git a/examples/hello_world/hello.py b/examples/hello-world/hello.py similarity index 100% rename from examples/hello_world/hello.py rename to examples/hello-world/hello.py diff --git a/examples/hello_world/hello_service.py b/examples/hello-world/hello_service.py similarity index 100% rename from examples/hello_world/hello_service.py rename to examples/hello-world/hello_service.py diff --git a/examples/hello_world/__init__.py b/examples/hello_world/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/http_proxy/Dockerfile b/examples/http-proxy/Dockerfile similarity index 100% rename from examples/http_proxy/Dockerfile rename to examples/http-proxy/Dockerfile diff --git a/examples/http_proxy/http_proxy.py b/examples/http-proxy/http_proxy.py similarity index 97% rename from examples/http_proxy/http_proxy.py rename to examples/http-proxy/http_proxy.py index d81bf0d2d..ad66224b5 100644 --- a/examples/http_proxy/http_proxy.py +++ b/examples/http-proxy/http_proxy.py @@ -1,7 +1,5 @@ #!/usr/bin/env python3 -""" -a simple http proxy example -""" +"""A simple http proxy example.""" import asyncio import pathlib import shlex @@ -123,7 +121,8 @@ def still_starting(): await proxy.run() print( - f"{TEXT_COLOR_CYAN}Local HTTP server listening on:\nhttp://localhost:{port}{TEXT_COLOR_DEFAULT}" + f"{TEXT_COLOR_CYAN}Local HTTP server listening on:\n" + f"http://localhost:{port}{TEXT_COLOR_DEFAULT}" ) # wait until Ctrl-C @@ -177,7 +176,7 @@ def still_starting(): ), ) now = datetime.now().strftime("%Y-%m-%d_%H.%M.%S") - parser.set_defaults(log_file=f"http_proxy-{now}.log") + parser.set_defaults(log_file=f"http-proxy-{now}.log") args = parser.parse_args() run_golem_example( diff --git a/examples/http_proxy/__init__.py b/examples/http_proxy/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/low_level_api/list-offers.py b/examples/low-level-api/list-offers.py similarity index 100% rename from examples/low_level_api/list-offers.py rename to examples/low-level-api/list-offers.py diff --git a/examples/low_level_api/__init__.py b/examples/low_level_api/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/market_strategy/market_strategy.py b/examples/market-strategy/market_strategy.py similarity index 91% rename from examples/market_strategy/market_strategy.py rename to examples/market-strategy/market_strategy.py index 3338ea778..ab07874e8 100644 --- a/examples/market_strategy/market_strategy.py +++ b/examples/market-strategy/market_strategy.py @@ -21,11 +21,13 @@ class FastestProviderStrategy(MarketStrategy): - """Strategy that ignores all offer parameters (including pricing) and just selects the fastest provider. + """Strategy that ignores all offer parameters (including pricing) and just selects the fastest \ + provider. Decision algorithm: * we always try any new provider - * if there are no new providers, we select the one with shortest average execution time in past runs + * if there are no new providers, we select the one with shortest average execution time in past + runs """ def __init__(self): @@ -45,7 +47,8 @@ async def score_offer(self, offer): # Non-draft offers are not shown to limit the number of lines printed if previous_runs: print( - f"Scored known provider: {provider_id}: {score} ({len(previous_runs)} runs, avg time {avg_time})" + f"Scored known provider: {provider_id}: " + f"{score} ({len(previous_runs)} runs, avg time {avg_time})" ) else: print(f"Found new provider: {provider_id}, default score {SCORE_TRUSTED}") @@ -97,7 +100,7 @@ async def worker(ctx: WorkContext, tasks): if __name__ == "__main__": parser = build_parser("Select fastest provider using a simple reputation-based market strategy") - parser.set_defaults(log_file="market_strategy-example.log") + parser.set_defaults(log_file="market-strategy-example.log") args = parser.parse_args() run_golem_example( diff --git a/examples/market_strategy/__init__.py b/examples/market_strategy/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/scan/__init__.py b/examples/scan/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/simple_service_poc/simple_service.py b/examples/simple-service-poc/simple_service.py similarity index 97% rename from examples/simple_service_poc/simple_service.py rename to examples/simple-service-poc/simple_service.py index 41c372ceb..a7898f8c8 100644 --- a/examples/simple_service_poc/simple_service.py +++ b/examples/simple-service-poc/simple_service.py @@ -1,7 +1,5 @@ #!/usr/bin/env python3 -""" -the requestor agent controlling and interacting with the "simple service" -""" +"""The requestor agent controlling and interacting with the "simple service".""" import asyncio import pathlib import random @@ -54,7 +52,7 @@ async def get_payload(): ) async def start(self): - """handler responsible for starting the service.""" + """Handle starting the service.""" # perform the initialization of the Service async for script in super().start(): @@ -147,7 +145,7 @@ async def main( def print_instances(): print( - f"instances: " + "instances: " + str( [ f"{s.name}: {s.state.value}" diff --git a/examples/simple_service_poc/simple_service/README.md b/examples/simple-service-poc/simple_service/README.md similarity index 100% rename from examples/simple_service_poc/simple_service/README.md rename to examples/simple-service-poc/simple_service/README.md diff --git a/examples/simple_service_poc/simple_service/simple_service.Dockerfile b/examples/simple-service-poc/simple_service/simple_service.Dockerfile similarity index 100% rename from examples/simple_service_poc/simple_service/simple_service.Dockerfile rename to examples/simple-service-poc/simple_service/simple_service.Dockerfile diff --git a/examples/simple_service_poc/simple_service/simple_service.py b/examples/simple-service-poc/simple_service/simple_service.py similarity index 94% rename from examples/simple_service_poc/simple_service/simple_service.py rename to examples/simple-service-poc/simple_service/simple_service.py index 2025e588f..cf4d5d2b0 100644 --- a/examples/simple_service_poc/simple_service/simple_service.py +++ b/examples/simple-service-poc/simple_service/simple_service.py @@ -1,7 +1,7 @@ #!/usr/local/bin/python -""" -a very basic "stub" that exposes a few commands of an imagined, very simple CLI-based -service that is able to accumulate some linear, time-based values and present it stats +"""A very basic "stub" that exposes a few commands of an imagined, very simple CLI-based service. + +Service is able to accumulate some linear, time-based values and present it stats (characteristics of the statistical distribution of the data collected so far) or provide distribution and time-series plots of the collected data. diff --git a/examples/simple_service_poc/simple_service/simulate_observations.py b/examples/simple-service-poc/simple_service/simulate_observations.py similarity index 87% rename from examples/simple_service_poc/simple_service/simulate_observations.py rename to examples/simple-service-poc/simple_service/simulate_observations.py index c2f04fba7..1345ef569 100644 --- a/examples/simple_service_poc/simple_service/simulate_observations.py +++ b/examples/simple-service-poc/simple_service/simulate_observations.py @@ -1,6 +1,5 @@ #!/usr/local/bin/python -""" -the "hello world" service here just adds randomized numbers with normal distribution +"""The "hello world" service here just adds randomized numbers with normal distribution. in a real-world example, this could be e.g. a thermometer connected to the provider's machine providing its inputs into the database or some other piece of information diff --git a/examples/simple_service_poc/simple_service/simulate_observations_ctl.py b/examples/simple-service-poc/simple_service/simulate_observations_ctl.py similarity index 91% rename from examples/simple_service_poc/simple_service/simulate_observations_ctl.py rename to examples/simple-service-poc/simple_service/simulate_observations_ctl.py index 851ac4ccb..20f21293e 100644 --- a/examples/simple_service_poc/simple_service/simulate_observations_ctl.py +++ b/examples/simple-service-poc/simple_service/simulate_observations_ctl.py @@ -1,6 +1,5 @@ #!/usr/local/bin/python -""" -a helper, control script that starts and stops our example `simulate_observations` service +"""a helper, control script that starts and stops our example `simulate_observations` service. [ part of the VM image that's deployed by the runtime on the Provider's end. ] """ diff --git a/examples/simple_service_poc/__init__.py b/examples/simple_service_poc/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/simple_service_poc/simple_service/__init__.py b/examples/simple_service_poc/simple_service/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/ssh/__init__.py b/examples/ssh/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/webapp/__init__.py b/examples/webapp/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/webapp/http/__init__.py b/examples/webapp/http/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/webapp/webapp.py b/examples/webapp/webapp.py index 091937983..2c22b1676 100755 --- a/examples/webapp/webapp.py +++ b/examples/webapp/webapp.py @@ -1,7 +1,5 @@ #!/usr/bin/env python3 import asyncio -import pathlib -import sys from datetime import datetime, timedelta from yapapi import Golem @@ -9,10 +7,7 @@ from yapapi.payload import vm from yapapi.services import Service, ServiceState -examples_dir = pathlib.Path(__file__).resolve().parent.parent -sys.path.append(str(examples_dir)) - -from utils import ( +from ..utils import ( TEXT_COLOR_CYAN, TEXT_COLOR_DEFAULT, build_parser, @@ -50,12 +45,18 @@ async def start(self): script.run( "/bin/bash", "-c", - f"cd /webapp && python app.py --db-address {self._db_address} --db-port {self._db_port} initdb", + f"cd /webapp && python app.py " + f"--db-address {self._db_address} " + f"--db-port {self._db_port}" + f" initdb", ) script.run( "/bin/bash", "-c", - f"cd /webapp && python app.py --db-address {self._db_address} --db-port {self._db_port} run > /webapp/out 2> /webapp/err &", + f"cd /webapp && python app.py " + f"--db-address {self._db_address} " + f"--db-port {self._db_port} " + f"run > /webapp/out 2> /webapp/err &", ) yield script @@ -144,7 +145,8 @@ def raise_exception_if_still_starting(cluster): await proxy.run() print( - f"{TEXT_COLOR_CYAN}Local HTTP server listening on:\nhttp://localhost:{port}{TEXT_COLOR_DEFAULT}" + f"{TEXT_COLOR_CYAN}Local HTTP server listening on:\n" + f"http://localhost:{port}{TEXT_COLOR_DEFAULT}" ) # wait until Ctrl-C diff --git a/examples/webapp_fileupload/__init__.py b/examples/webapp_fileupload/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/webapp_fileupload/http_file_upload/Dockerfile b/examples/webapp_fileupload/http-file-upload/Dockerfile similarity index 100% rename from examples/webapp_fileupload/http_file_upload/Dockerfile rename to examples/webapp_fileupload/http-file-upload/Dockerfile diff --git a/examples/webapp_fileupload/http_file_upload/app.py b/examples/webapp_fileupload/http-file-upload/app.py similarity index 100% rename from examples/webapp_fileupload/http_file_upload/app.py rename to examples/webapp_fileupload/http-file-upload/app.py diff --git a/examples/webapp_fileupload/http_file_upload/templates/error.html b/examples/webapp_fileupload/http-file-upload/templates/error.html similarity index 100% rename from examples/webapp_fileupload/http_file_upload/templates/error.html rename to examples/webapp_fileupload/http-file-upload/templates/error.html diff --git a/examples/webapp_fileupload/http_file_upload/templates/index.html b/examples/webapp_fileupload/http-file-upload/templates/index.html similarity index 100% rename from examples/webapp_fileupload/http_file_upload/templates/index.html rename to examples/webapp_fileupload/http-file-upload/templates/index.html diff --git a/examples/webapp_fileupload/http_file_upload/__init__.py b/examples/webapp_fileupload/http_file_upload/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/webapp_fileupload/webapp_fileupload.py b/examples/webapp_fileupload/webapp_fileupload.py index 2462f3458..f6c2fb3ad 100755 --- a/examples/webapp_fileupload/webapp_fileupload.py +++ b/examples/webapp_fileupload/webapp_fileupload.py @@ -1,7 +1,5 @@ #!/usr/bin/env python3 import asyncio -import pathlib -import sys from datetime import datetime, timedelta from yapapi import Golem @@ -9,10 +7,7 @@ from yapapi.payload import vm from yapapi.services import Service, ServiceState -examples_dir = pathlib.Path(__file__).resolve().parent.parent -sys.path.append(str(examples_dir)) - -from utils import ( +from ..utils import ( TEXT_COLOR_CYAN, TEXT_COLOR_DEFAULT, build_parser, @@ -51,12 +46,18 @@ async def start(self): script.run( "/bin/bash", "-c", - f"cd /webapp && python app.py --db-address {self._db_address} --db-port {self._db_port} initdb", + f"cd /webapp && python app.py " + f"--db-address {self._db_address} " + f"--db-port {self._db_port} " + f"initdb", ) script.run( "/bin/bash", "-c", - f"cd /webapp && python app.py --db-address {self._db_address} --db-port {self._db_port} run > /logs/out 2> /logs/err &", + f"cd /webapp && python app.py " + f"--db-address {self._db_address} " + f"--db-port {self._db_port} " + f"run > /logs/out 2> /logs/err &", ) yield script @@ -145,7 +146,8 @@ def raise_exception_if_still_starting(cluster): await proxy.run() print( - f"{TEXT_COLOR_CYAN}Local HTTP server listening on:\nhttp://localhost:{port}{TEXT_COLOR_DEFAULT}" + f"{TEXT_COLOR_CYAN}Local HTTP server listening on:\n" + f"http://localhost:{port}{TEXT_COLOR_DEFAULT}" ) # wait until Ctrl-C diff --git a/examples/yacat/__init__.py b/examples/yacat/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/yacat/yacat.py b/examples/yacat/yacat.py index 4a7d82fa0..37f9311a7 100755 --- a/examples/yacat/yacat.py +++ b/examples/yacat/yacat.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import argparse import math -import sys from datetime import datetime, timedelta from pathlib import Path from tempfile import gettempdir @@ -12,10 +11,7 @@ from yapapi.payload import vm from yapapi.rest.activity import CommandExecutionError -examples_dir = Path(__file__).resolve().parent.parent -sys.path.append(str(examples_dir)) - -from utils import ( +from ..utils import ( TEXT_COLOR_CYAN, TEXT_COLOR_DEFAULT, TEXT_COLOR_GREEN, @@ -112,7 +108,7 @@ async def perform_mask_attack(ctx: WorkContext, tasks: AsyncIterable[Task]): worker_output_path = f"/golem/output/{output_name}" script = ctx.new_script(timeout=MASK_ATTACK_TIMEOUT) - script.run(f"/bin/sh", "-c", _make_attack_command(skip, limit, worker_output_path)) + script.run("/bin/sh", "-c", _make_attack_command(skip, limit, worker_output_path)) try: output_file = Path(gettempdir()) / output_name script.download_file(worker_output_path, str(output_file)) @@ -137,7 +133,7 @@ def _make_attack_command(skip: int, limit: int, output_path: str) -> str: def _parse_result(potfile_line: str) -> Optional[str]: - """Helper function which parses a single .potfile line and returns the password part. + """Parse a single .potfile line and returns the password part. Hashcat uses its .potfile format to report results. In this format, each line consists of the hash and its matching word, separated with a colon (e.g. `asdf1234:password`). diff --git a/pyproject.toml b/pyproject.toml index 51bfcaee9..4596bec83 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -90,7 +90,7 @@ pyproject-autoflake = "^1.0.2" # TODO: Remove this after upgrading to Python 3.8 [tool.poe.tasks] checks = {sequence = ["checks_codestyle", "checks_typing", "checks_license"], help = "Run all available code checks"} checks_codestyle = {sequence = ["_checks_codestyle_flake8", "_checks_codestyle_isort", "_checks_codestyle_black"], help = "Run only code style checks"} -_checks_codestyle_flake8 = "flake8 yapapi tests" +_checks_codestyle_flake8 = "flake8 yapapi tests examples/*" _checks_codestyle_isort = "isort --check-only --diff ." _checks_codestyle_black = "black --check --diff ." checks_typing = {cmd = "mypy .", help = "Run only code typing checks" } @@ -166,6 +166,9 @@ extend-ignore = [ "D107", # No docs for __init__ "D202", # We prefer whitelines after docstrings ] +per-file-ignores = [ + "examples/*:E402", +] [tool.mypy] exclude = [ From 2e1b4d3f9ab5789160f51b26ef04494a43aae619 Mon Sep 17 00:00:00 2001 From: approxit Date: Wed, 15 Feb 2023 12:19:02 +0100 Subject: [PATCH 14/20] fixed utils import in examples --- examples/blender/start_stop_blender.py | 6 +++++- examples/webapp/webapp.py | 7 ++++++- examples/webapp_fileupload/webapp_fileupload.py | 7 ++++++- examples/yacat/yacat.py | 8 +++++++- 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/examples/blender/start_stop_blender.py b/examples/blender/start_stop_blender.py index 9e46a2219..b0dee3bf8 100755 --- a/examples/blender/start_stop_blender.py +++ b/examples/blender/start_stop_blender.py @@ -8,13 +8,17 @@ """ import pathlib +import sys from datetime import datetime, timedelta from yapapi import Golem, Task, WorkContext from yapapi.payload import vm from yapapi.rest.activity import BatchTimeoutError -from ..utils import ( +examples_dir = pathlib.Path(__file__).resolve().parent.parent +sys.path.append(str(examples_dir)) + +from utils import ( TEXT_COLOR_CYAN, TEXT_COLOR_DEFAULT, TEXT_COLOR_MAGENTA, diff --git a/examples/webapp/webapp.py b/examples/webapp/webapp.py index 2c22b1676..b3c41d429 100755 --- a/examples/webapp/webapp.py +++ b/examples/webapp/webapp.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 import asyncio +import pathlib +import sys from datetime import datetime, timedelta from yapapi import Golem @@ -7,7 +9,10 @@ from yapapi.payload import vm from yapapi.services import Service, ServiceState -from ..utils import ( +examples_dir = pathlib.Path(__file__).resolve().parent.parent +sys.path.append(str(examples_dir)) + +from utils import ( TEXT_COLOR_CYAN, TEXT_COLOR_DEFAULT, build_parser, diff --git a/examples/webapp_fileupload/webapp_fileupload.py b/examples/webapp_fileupload/webapp_fileupload.py index f6c2fb3ad..ad61d1146 100755 --- a/examples/webapp_fileupload/webapp_fileupload.py +++ b/examples/webapp_fileupload/webapp_fileupload.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 import asyncio +import pathlib +import sys from datetime import datetime, timedelta from yapapi import Golem @@ -7,7 +9,10 @@ from yapapi.payload import vm from yapapi.services import Service, ServiceState -from ..utils import ( +examples_dir = pathlib.Path(__file__).resolve().parent.parent +sys.path.append(str(examples_dir)) + +from utils import ( TEXT_COLOR_CYAN, TEXT_COLOR_DEFAULT, build_parser, diff --git a/examples/yacat/yacat.py b/examples/yacat/yacat.py index 37f9311a7..1e8c2ec5a 100755 --- a/examples/yacat/yacat.py +++ b/examples/yacat/yacat.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 import argparse import math +import pathlib +import sys from datetime import datetime, timedelta from pathlib import Path from tempfile import gettempdir @@ -11,7 +13,11 @@ from yapapi.payload import vm from yapapi.rest.activity import CommandExecutionError -from ..utils import ( +examples_dir = pathlib.Path(__file__).resolve().parent.parent +sys.path.append(str(examples_dir)) + + +from utils import ( TEXT_COLOR_CYAN, TEXT_COLOR_DEFAULT, TEXT_COLOR_GREEN, From 187f99182ae8e006e89624ec52bb1d1fa8fec865 Mon Sep 17 00:00:00 2001 From: approxit Date: Wed, 15 Feb 2023 11:51:19 +0100 Subject: [PATCH 15/20] example fix --- tests/goth_tests/test_run_custom_usage_counter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/goth_tests/test_run_custom_usage_counter.py b/tests/goth_tests/test_run_custom_usage_counter.py index 7a20e3bb7..1a57f10c2 100644 --- a/tests/goth_tests/test_run_custom_usage_counter.py +++ b/tests/goth_tests/test_run_custom_usage_counter.py @@ -75,7 +75,7 @@ async def test_run_custom_usage_counter( # This is the default configuration with 2 wasm/VM providers goth_config = load_yaml(goth_config_path, config_overrides) - requestor_path = project_dir / "examples" / "custom_usage_counter" / "custom_usage_counter.py" + requestor_path = project_dir / "examples" / "custom-usage-counter" / "custom_usage_counter.py" runner = Runner( base_log_dir=log_dir, From 61c8c8e5c513eb0ab08b06b121d4c5cbe6630420 Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Wed, 22 Feb 2023 12:18:58 +0100 Subject: [PATCH 16/20] add logfile, re-add +x chmod --- examples/custom-usage-counter/custom_usage_counter.py | 2 ++ 1 file changed, 2 insertions(+) mode change 100644 => 100755 examples/custom-usage-counter/custom_usage_counter.py diff --git a/examples/custom-usage-counter/custom_usage_counter.py b/examples/custom-usage-counter/custom_usage_counter.py old mode 100644 new mode 100755 index 7593d1050..49a7224c6 --- a/examples/custom-usage-counter/custom_usage_counter.py +++ b/examples/custom-usage-counter/custom_usage_counter.py @@ -102,6 +102,8 @@ def print_instances(): type=int, help="How long should the the service run (in seconds, default: %(default)s)", ) + now = datetime.now().strftime("%Y-%m-%d_%H.%M.%S") + parser.set_defaults(log_file=f"custom-counters-yapapi-{now}.log") args = parser.parse_args() enable_default_logger( log_file=args.log_file, From 8db87da30f0132c45e64883712c6ac5cc4eee7af Mon Sep 17 00:00:00 2001 From: approxit Date: Mon, 27 Feb 2023 13:54:41 +0100 Subject: [PATCH 17/20] chmod +x --- examples/external-api-request/request.sh | 0 examples/external-api-request/sign.sh | 0 examples/hello-world/hello.py | 0 examples/hello-world/hello_service.py | 0 examples/low-level-api/list-offers.py | 0 examples/market-strategy/market_strategy.py | 0 examples/simple-service-poc/simple_service.py | 0 7 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 examples/external-api-request/request.sh mode change 100644 => 100755 examples/external-api-request/sign.sh mode change 100644 => 100755 examples/hello-world/hello.py mode change 100644 => 100755 examples/hello-world/hello_service.py mode change 100644 => 100755 examples/low-level-api/list-offers.py mode change 100644 => 100755 examples/market-strategy/market_strategy.py mode change 100644 => 100755 examples/simple-service-poc/simple_service.py diff --git a/examples/external-api-request/request.sh b/examples/external-api-request/request.sh old mode 100644 new mode 100755 diff --git a/examples/external-api-request/sign.sh b/examples/external-api-request/sign.sh old mode 100644 new mode 100755 diff --git a/examples/hello-world/hello.py b/examples/hello-world/hello.py old mode 100644 new mode 100755 diff --git a/examples/hello-world/hello_service.py b/examples/hello-world/hello_service.py old mode 100644 new mode 100755 diff --git a/examples/low-level-api/list-offers.py b/examples/low-level-api/list-offers.py old mode 100644 new mode 100755 diff --git a/examples/market-strategy/market_strategy.py b/examples/market-strategy/market_strategy.py old mode 100644 new mode 100755 diff --git a/examples/simple-service-poc/simple_service.py b/examples/simple-service-poc/simple_service.py old mode 100644 new mode 100755 From d5b6b0dfb23f10416bfd3c432a429e89dac06435 Mon Sep 17 00:00:00 2001 From: approxit Date: Fri, 3 Mar 2023 15:47:52 +0100 Subject: [PATCH 18/20] commented out bad goth test --- .../{test_run_simple_service.py => _test_run_simple_service.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/goth_tests/{test_run_simple_service.py => _test_run_simple_service.py} (100%) diff --git a/tests/goth_tests/test_run_simple_service.py b/tests/goth_tests/_test_run_simple_service.py similarity index 100% rename from tests/goth_tests/test_run_simple_service.py rename to tests/goth_tests/_test_run_simple_service.py From 9733d36c7c93e345a4f3e7d341d482a1907475a0 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Fri, 3 Mar 2023 15:52:48 +0100 Subject: [PATCH 19/20] Fix test_run_simple_service wrong module name --- .../{_test_run_simple_service.py => test_run_simple_service.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename tests/goth_tests/{_test_run_simple_service.py => test_run_simple_service.py} (97%) diff --git a/tests/goth_tests/_test_run_simple_service.py b/tests/goth_tests/test_run_simple_service.py similarity index 97% rename from tests/goth_tests/_test_run_simple_service.py rename to tests/goth_tests/test_run_simple_service.py index 2a9390fc6..03e9902b7 100644 --- a/tests/goth_tests/_test_run_simple_service.py +++ b/tests/goth_tests/test_run_simple_service.py @@ -33,7 +33,7 @@ async def test_run_simple_service( # This is the default configuration with 2 wasm/VM providers goth_config = load_yaml(goth_config_path, config_overrides) - requestor_path = project_dir / "examples" / "simple_service_poc" / "simple_service.py" + requestor_path = project_dir / "examples" / "simple-service-poc" / "simple_service.py" runner = Runner( base_log_dir=log_dir, From 4b68e012947fa57f6bef4e9420d579e9e2f631b9 Mon Sep 17 00:00:00 2001 From: approxit Date: Tue, 7 Mar 2023 11:54:32 +0100 Subject: [PATCH 20/20] Apply suggestions from code review Co-authored-by: shadeofblue --- examples/simple-service-poc/simple_service/simple_service.py | 2 +- yapapi/contrib/service/http_proxy.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/simple-service-poc/simple_service/simple_service.py b/examples/simple-service-poc/simple_service/simple_service.py index cf4d5d2b0..84f981b51 100644 --- a/examples/simple-service-poc/simple_service/simple_service.py +++ b/examples/simple-service-poc/simple_service/simple_service.py @@ -1,7 +1,7 @@ #!/usr/local/bin/python """A very basic "stub" that exposes a few commands of an imagined, very simple CLI-based service. -Service is able to accumulate some linear, time-based values and present it stats +The Service is able to accumulate some linear, time-based values and present it stats (characteristics of the statistical distribution of the data collected so far) or provide distribution and time-series plots of the collected data. diff --git a/yapapi/contrib/service/http_proxy.py b/yapapi/contrib/service/http_proxy.py index 05792dbbd..714936a2d 100644 --- a/yapapi/contrib/service/http_proxy.py +++ b/yapapi/contrib/service/http_proxy.py @@ -4,7 +4,7 @@ Golem providers using yapapi's Services API. For usage in a complete requestor agent app, see the -`http_proxy `_ and +`http-proxy `_ and `webapp `_ examples in the yapapi repository. """