0.10 to master

.gitignore

@@ -22,3 +22,5 @@ poetry.lock
 
 # local config files
 .pre-commit-config.yaml
+
+/build

README.md

@@ -139,7 +139,7 @@ It's possible to set various elements of `yagna` configuration through environme
 - `YAGNA_PAYMENT_NETWORK`, Ethereum network name for `yagna` to use, e.g. `rinkeby`
 - `YAGNA_PAYMENT_DRIVER`, payment driver name for `yagna` to use, e.g. `erc20`
 - `YAGNA_PAYMENT_URL`, URL to `yagna` payment API, e.g. `http://localhost:7500/payment-api/v1`
-- `YAGNA_SUBNET`, name of the `yagna` sub network to be used, e.g. `devnet-beta`
+- `YAGNA_SUBNET`, name of the `yagna` sub network to be used, e.g. `public`
 - `YAPAPI_USE_GFTP_CLOSE`, if set to a _truthy_ value (e.g. "1", "Y", "True", "on") then `yapapi`
   will ask `gftp` to close files when there's no need to publish them any longer. This may greatly
   reduce the number of files kept open while `yapapi` is running but requires `yagna`

docs/sphinx/api.rst

@@ -27,7 +27,7 @@ Service
 -------
 
 .. autoclass:: yapapi.services.Service
-    :members: id, provider_name, state, is_available, start, run, shutdown, reset, send_message, send_message_nowait, receive_message, receive_message_nowait, get_payload, network, network_node
+    :members: id, provider_name, state, is_available, start, run, shutdown, reset, is_activity_responsive, restart_condition, send_message, send_message_nowait, receive_message, receive_message_nowait, get_payload, network, network_node
 
 Cluster
 -------
@@ -75,11 +75,11 @@ Package
 .. autoclass:: yapapi.payload.package.Package
 
 
-vm.repo
+vm
 -------
 
 .. automodule:: yapapi.payload.vm
-    :members: repo
+    :members: repo, manifest
 
 
 Execution control
@@ -151,14 +151,22 @@ Yapapi Contrib
 
 .. automodule:: yapapi.contrib
 
-.. automodule:: yapapi.contrib.strategy.provider_filter
-
-.. autoclass:: yapapi.contrib.strategy.ProviderFilter
-
 .. automodule:: yapapi.contrib.service.http_proxy
 
 .. autoclass:: yapapi.contrib.service.http_proxy.LocalHttpProxy
     :members: __init__, run, stop
 
 .. autoclass:: yapapi.contrib.service.http_proxy.HttpProxyService
     :members: __init__, handle_request
+
+.. automodule:: yapapi.contrib.service.socket_proxy
+
+.. autoclass:: yapapi.contrib.service.socket_proxy.SocketProxy
+    :members: __init__, run, run_server, stop
+
+.. autoclass:: yapapi.contrib.service.socket_proxy.SocketProxyService
+    :members: remote_ports
+
+.. automodule:: yapapi.contrib.strategy.provider_filter
+
+.. autoclass:: yapapi.contrib.strategy.ProviderFilter

examples/external-api-request/external_api_request.py

@@ -22,13 +22,13 @@ async def get_payload():
         manifest = open("manifest.json", "rb").read()
         manifest = base64.b64encode(manifest).decode("utf-8")
 
-        manifest_sig = open("manifest.json.base64.sign.sha256", "rb").read()
+        manifest_sig = open("manifest.json.base64.sha256.sig", "rb").read()
         manifest_sig = base64.b64encode(manifest_sig).decode("utf-8")
 
         manifest_sig_algorithm = "sha256"
 
-        # both DER and PEM formats are supported
-        manifest_cert = open("requestor.cert.der", "rb").read()
+        # DER, PEM and PEM chain formats are supported
+        manifest_cert = open("golem_sign.pem", "rb").read()
         manifest_cert = base64.b64encode(manifest_cert).decode("utf-8")
 
         return await vm.manifest(

examples/external-api-request/golem_sign.pem

@@ -0,0 +1,13 @@
+-----BEGIN CERTIFICATE-----
+MIIB/zCCAYWgAwIBAgIUL+slWOaCypC0+VZM96VMW3lN3RkwCgYIKoZIzj0EAwIw
+ODEWMBQGA1UECgwNR29sZW0gRmFjdG9yeTEeMBwGA1UEAwwVR29sZW0gRmFjdG9y
+eSBST09UIENBMB4XDTIyMTEyMzExNDA1M1oXDTI1MTEyMjExNDA1M1owQDEWMBQG
+A1UECgwNR29sZW0gRmFjdG9yeTEmMCQGA1UEAwwdR29sZW0gRmFjdG9yeSBJTlRF
+Uk1FRElBVEUgQ0EwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAASSFvpQd9Q0hE33
+VcdWXx57BNYfxL82IechAhcOFZMfuxlUIh3scIrvVu7IKK/5WZEeMw3SlU7zQrHf
+xDQhW7FZo2UwYzAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/BAgwBgEB/wIBATA9
+BgNVHR8BAf8EMzAxMC+gLaArhilodHRwczovL2NhLmdvbGVtLm5ldHdvcmsvY3Js
+L0dPTEVNLUNBLmNybDAKBggqhkjOPQQDAgNoADBlAjAMz6iyHqudwY0WwfOGSEWu
+QtJ+/x0EztMzHVKIc1sFrNFhKTm1rNozlX1k5dGaMZ0CMQDrCcCBs9XiVCjRIjsG
+UqSjwyXJprSAQn4g3kFNOLCHrzOo/5IS9MZUH0lnekbaa1o=
+-----END CERTIFICATE-----

examples/external-api-request/sign.sh

@@ -7,4 +7,4 @@ fi
 
 PRIVATE_KEY=$1
 base64 manifest.json --wrap=0 > manifest.json.base64
-openssl dgst -sha256 -sign $PRIVATE_KEY -out manifest.json.base64.sign.sha256 manifest.json.base64
+openssl dgst -sha256 -sign $PRIVATE_KEY -out manifest.json.base64.sha256.sig manifest.json.base64

examples/hello-world/hello.py

@@ -24,7 +24,7 @@ async def main():
 
     tasks = [Task(data=None)]
 
-    async with Golem(budget=1.0, subnet_tag="devnet-beta") as golem:
+    async with Golem(budget=1.0, subnet_tag="public") as golem:
         async for completed in golem.execute_tasks(worker, tasks, payload=package):
             print(completed.result.stdout)
 

examples/hello-world/hello_service.py

@@ -48,7 +48,7 @@ async def run(self):
 
 
 async def main():
-    async with Golem(budget=1.0, subnet_tag="devnet-beta") as golem:
+    async with Golem(budget=1.0, subnet_tag="public") as golem:
         cluster = await golem.run_service(DateService, num_instances=1)
         start_time = datetime.now()
 

examples/utils/__init__.py

@@ -35,7 +35,7 @@ def build_parser(description: str) -> argparse.ArgumentParser:
     parser.add_argument(
         "--payment-network", "--network", help="Payment network name, for example `rinkeby`"
     )
-    parser.add_argument("--subnet-tag", help="Subnet name, for example `devnet-beta`")
+    parser.add_argument("--subnet-tag", help="Subnet name, for example `public`")
     parser.add_argument(
         "--log-file",
         default=str(default_log_path),

examples/webapp/webapp.py

@@ -45,7 +45,7 @@ async def start(self):
         async for script in super().start():
             yield script
 
-        script = self._ctx.new_script(timeout=timedelta(seconds=10))
+        script = self._ctx.new_script(timeout=timedelta(seconds=20))
 
         script.run(
             "/bin/bash",

examples/webapp_fileupload/webapp_fileupload.py

@@ -60,14 +60,6 @@ async def start(self):
         )
         yield script
 
-    async def run(self):
-        while True:
-            await asyncio.sleep(5)
-            script = self._ctx.new_script()
-            script.download_file("/logs/out", "webapp_out.txt")
-            script.download_file("/logs/err", "webapp_err.txt")
-            yield script
-
 
 class DbService(Service):
     @staticmethod
@@ -191,7 +183,7 @@ def raise_exception_if_still_starting(cluster):
         help="The local port to listen on",
     )
     now = datetime.now().strftime("%Y-%m-%d_%H.%M.%S")
-    parser.set_defaults(log_file=f"webapp-yapapi-{now}.log")
+    parser.set_defaults(log_file=f"webapp-fileupload-yapapi-{now}.log")
     args = parser.parse_args()
 
     run_golem_example(

pyproject.toml

@@ -114,7 +114,7 @@ ya-market = "0.1.0"
 [tool.poe.tasks]
 test = "pytest --cov=yapapi --ignore tests/goth_tests"
 goth-assets = "python -m goth create-assets tests/goth_tests/assets"
-goth-tests = "pytest -svx tests/goth_tests --config-override docker-compose.build-environment.use-prerelease=true --config-path tests/goth_tests/assets/goth-config-testing.yml --ssh-verify-connection --reruns 3 --only-rerun AssertionError --only-rerun TimeoutError --only-rerun goth.runner.exceptions.TemporalAssertionError --only-rerun urllib.error.URLError --only-rerun goth.runner.exceptions.CommandError"
+goth-tests = "pytest -svx tests/goth_tests  --config-override docker-compose.build-environment.use-prerelease=true --config-path tests/goth_tests/assets/goth-config-testing.yml --ssh-verify-connection --reruns 3 --only-rerun AssertionError --only-rerun TimeoutError --only-rerun goth.runner.exceptions.TemporalAssertionError --only-rerun urllib.error.URLError --only-rerun goth.runner.exceptions.CommandError"
 typecheck = "mypy --check-untyped-defs --no-implicit-optional ."
 _codestyle_isort = "isort --check-only --diff ."
 _codestyle_black = "black --check --diff ."

tests/factories/props/__init__.py

@@ -8,4 +8,4 @@ class Meta:
         model = NodeInfo
 
     name = factory.Faker("pystr")
-    subnet_tag = "devnet-beta"
+    subnet_tag = "public"

tests/goth_tests/test_recycle_ip/ssh_recycle_ip.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+import asyncio
+from datetime import timedelta
+import random
+import string
+
+from yapapi import Golem
+from yapapi.contrib.service.socket_proxy import SocketProxy, SocketProxyService
+from yapapi.payload import vm
+
+first_time = True
+
+
+class SshService(SocketProxyService):
+    remote_port = 22
+
+    def __init__(self, proxy: SocketProxy):
+        super().__init__()
+        self.proxy = proxy
+
+    @staticmethod
+    async def get_payload():
+        return await vm.repo(
+            image_hash="1e06505997e8bd1b9e1a00bd10d255fc6a390905e4d6840a22a79902",  # ssh example
+            capabilities=[vm.VM_CAPS_VPN],
+        )
+
+    async def start(self):
+        global first_time
+
+        async for script in super().start():
+            yield script
+
+        password = "".join(random.choice(string.ascii_letters + string.digits) for _ in range(8))
+
+        script = self._ctx.new_script(timeout=timedelta(seconds=10))
+
+        if first_time:
+            first_time = False
+            raise Exception("intentional failure on the first run")
+
+        script.run("/bin/bash", "-c", "syslogd")
+        script.run("/bin/bash", "-c", "ssh-keygen -A")
+        script.run("/bin/bash", "-c", f'echo -e "{password}\n{password}" | passwd')
+        script.run("/bin/bash", "-c", "/usr/sbin/sshd")
+        yield script
+
+        server = await self.proxy.run_server(self, self.remote_port)
+
+        print(
+            f"connect with:\n"
+            f"ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no "
+            f"-p {server.local_port} root@{server.local_address}"
+        )
+        print(f"password: {password}")
+
+
+async def main():
+    async with Golem(
+        budget=1.0,
+        subnet_tag="goth",
+    ) as golem:
+        network = await golem.create_network("192.168.0.1/24")
+        proxy = SocketProxy(ports=[2222])
+
+        async with network:
+            cluster = await golem.run_service(
+                SshService,
+                network=network,
+                instance_params=[{"proxy": proxy}],
+                network_addresses=["192.168.0.2"],
+            )
+            instances = cluster.instances
+
+            while True:
+                print(instances)
+                print(".....", network._nodes)
+                try:
+                    await asyncio.sleep(5)
+                except (KeyboardInterrupt, asyncio.CancelledError):
+                    break
+
+            await proxy.stop()
+            cluster.stop()
+
+            cnt = 0
+            while cnt < 3 and any(s.is_available for s in instances):
+                print(instances)
+                await asyncio.sleep(5)
+                cnt += 1
+
+
+if __name__ == "__main__":
+    try:
+        loop = asyncio.get_event_loop()
+        task = loop.create_task(main())
+        loop.run_until_complete(task)
+    except KeyboardInterrupt:
+        pass

tests/goth_tests/test_recycle_ip/test_recycle_ip.py

@@ -0,0 +1,117 @@
+"""An integration test scenario that runs the SSH example requestor app."""
+import asyncio
+import logging
+import os
+from pathlib import Path
+import pexpect
+import pytest
+import re
+import signal
+import sys
+import time
+from typing import List
+
+from goth.configuration import Override, load_yaml
+from goth.runner import Runner
+from goth.runner.log import configure_logging
+from goth.runner.probe import RequestorProbe
+
+from goth_tests.assertions import assert_all_invoices_accepted, assert_no_errors  # isort:skip
+
+logger = logging.getLogger("goth.test.recycle_ips")
+
+SUBNET_TAG = "goth"
+
+
+@pytest.mark.asyncio
+async def test_recycle_ip(
+    log_dir: Path,
+    project_dir: Path,
+    goth_config_path: Path,
+    config_overrides: List[Override],
+    ssh_verify_connection: bool,
+) -> None:
+
+    if ssh_verify_connection:
+        ssh_check = pexpect.spawn("/usr/bin/which ssh")
+        exit_code = ssh_check.wait()
+        if exit_code != 0:
+            raise ProcessLookupError(
+                "ssh binary not found, please install it or check your PATH. "
+                "You may also skip the connection check by omitting `--ssh-verify-connection`."
+            )
+
+    configure_logging(log_dir)
+
+    # This is the default configuration with 2 wasm/VM providers
+    goth_config = load_yaml(goth_config_path, config_overrides)
+
+    # disable the mitm proxy used to capture the requestor agent -> daemon API calls
+    # because it doesn't support websockets which are neeeded to enable VPN connections
+    requestor = [c for c in goth_config.containers if c.name == "requestor"][0]
+    requestor.use_proxy = False
+
+    requestor_path = str(Path(__file__).parent / "ssh_recycle_ip.py")
+
+    runner = Runner(
+        base_log_dir=log_dir,
+        compose_config=goth_config.compose_config,
+    )
+
+    async with runner(goth_config.containers):
+
+        requestor = runner.get_probes(probe_type=RequestorProbe)[0]
+
+        async with requestor.run_command_on_host(
+            f"{requestor_path} --subnet-tag {SUBNET_TAG}",
+            env=os.environ,
+        ) as (_cmd_task, cmd_monitor, process_monitor):
+            cmd_monitor.add_assertion(assert_no_errors)
+            cmd_monitor.add_assertion(assert_all_invoices_accepted)
+
+            ssh_connections = []
+
+            # A longer timeout to account for downloading a VM image
+            ssh_string = await cmd_monitor.wait_for_pattern("ssh -o", timeout=120)
+            matches = re.match("ssh -o .* -p (\\d+) (root@.*)", ssh_string)
+            port = matches.group(1)
+            address = matches.group(2)
+            password = re.sub("password: ", "", await cmd_monitor.wait_for_pattern("password:"))
+
+            ssh_connections.append((port, address, password))
+
+            await cmd_monitor.wait_for_pattern(".*SshService running on provider", timeout=10)
+            logger.info("SSH service instances started")
+
+            if not ssh_verify_connection:
+                logger.warning(
+                    "Skipping SSH connection check. Use `--ssh-verify-connection` to perform it."
+                )
+            else:
+                for port, address, password in ssh_connections:
+                    args = [
+                        "ssh",
+                        "-o",
+                        "UserKnownHostsFile=/dev/null",
+                        "-o",
+                        "StrictHostKeyChecking=no",
+                        "-p",
+                        port,
+                        address,
+                        "uname -v",
+                    ]
+
+                    logger.debug("running ssh with: %s", args)
+
+                    ssh = pexpect.spawn(" ".join(args))
+                    ssh.expect("[pP]assword:", timeout=5)
+                    ssh.sendline(password)
+                    ssh.expect("#1-Alpine SMP", timeout=5)
+                    ssh.expect(pexpect.EOF, timeout=5)
+                    logger.info("Connection to port %s confirmed.", port)
+
+                logger.info("SSH connections confirmed.")
+
+            proc: asyncio.subprocess.Process = await process_monitor.get_process()
+            proc.send_signal(signal.SIGINT)
+            logger.info("Sent SIGINT...")