REST api support

.github/workflows/test.yml

@@ -16,11 +16,20 @@ on:
         required: false
         type: string
         default: "wakuorg/go-waku:latest"
+      protocol:
+        description: "Protocol used to comunicate inside the network"
+        required: true
+        type: choice
+        default: "REST"
+        options:
+        - "REST"
+        - "RPC"
 
 env:
   FORCE_COLOR: "1"
   NODE_1: ${{ inputs.node1 || 'wakuorg/nwaku:deploy-wakuv2-test' }}
   NODE_2: ${{ inputs.node2 || 'wakuorg/go-waku:latest' }}
+  PROTOCOL: ${{ inputs.protocol || 'REST' }}
 
 jobs:
 

README.md

@@ -1,18 +1,25 @@
 # waku-interop-tests
 
-Waku interop testing between various implementation of the [Waku v2 protocol](https://rfc.vac.dev/spec/10/).
+Waku e2e and interop framework used to test various implementation of the [Waku v2 protocol](https://rfc.vac.dev/spec/10/).
 
-## Setup
+## Setup and contribute
 
 ```shell
 git clone git@github.com:waku-org/waku-interop-tests.git
 cd waku-interop-tests
 python -m venv .venv
 source .venv/bin/activate
 pip install -r requirements.txt
+pre-commit install
+(optional) Overwrite default vars from src/env_vars.py via cli env vars or by adding a .env file
 pytest
 ```
 
+## CI
+
+- Test runs via github actions
+- [Allure Test Reports](https://waku-org.github.io/waku-interop-tests/3/) are published via github pages
+
 ## License
 
 Licensed and distributed under either of

pytest.ini

@@ -1,5 +1,5 @@
 [pytest]
-addopts = --instafail --tb=short --color=auto
+addopts = -s --instafail --tb=short --color=auto
 log_level = DEBUG
 log_cli = True
 log_file = log/test.log

requirements.txt

@@ -1,11 +1,13 @@
 allure-pytest
 black
 docker
+marshmallow-dataclass
 pre-commit
 pyright
 pytest
 pytest-instafail
 pytest-xdist
 pytest-rerunfailures
+python-dotenv
 requests
 tenacity

src/data_classes.py

@@ -1,24 +1,24 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
+from marshmallow_dataclass import class_schema
 from typing import Optional
 
 
-@dataclass
-class KeyPair:
-    privateKey: str
-    publicKey: str
-
-
 @dataclass
 class MessageRpcQuery:
-    payload: str  # Hex encoded data string without `0x` prefix.
-    contentTopic: Optional[str] = None
-    timestamp: Optional[int] = None  # Unix epoch time in nanoseconds as a 64-bit integer value.
+    payload: str
+    contentTopic: str
+    timestamp: Optional[int] = None
 
 
 @dataclass
 class MessageRpcResponse:
     payload: str
-    contentTopic: Optional[str] = None
-    version: Optional[int] = None
-    timestamp: Optional[int] = None  # Unix epoch time in nanoseconds as a 64-bit integer value.
-    ephemeral: Optional[bool] = None
+    contentTopic: str
+    version: Optional[int]
+    timestamp: int
+    ephemeral: Optional[bool]
+    rateLimitProof: Optional[dict] = field(default_factory=dict)
+    rate_limit_proof: Optional[dict] = field(default_factory=dict)
+
+
+message_rpc_response_schema = class_schema(MessageRpcResponse)()

src/env_vars.py

@@ -1,21 +1,25 @@
 import os
-import logging
+from dotenv import load_dotenv
 
-logger = logging.getLogger(__name__)
+load_dotenv()  # This will load environment variables from a .env file if it exists
 
 
-def get_env_var(var_name, default):
+def get_env_var(var_name, default=None):
     env_var = os.getenv(var_name, default)
-    logger.debug(f"{var_name}: {env_var}")
+    if env_var is not None:
+        print(f"{var_name}: {env_var}")
+    else:
+        print(f"{var_name} is not set; using default value: {default}")
     return env_var
 
 
 # Configuration constants. Need to be upercase to appear in reports
-NODE_1 = get_env_var("NODE_1", "wakuorg/nwaku:deploy-wakuv2-test")
+NODE_1 = get_env_var("NODE_1", "wakuorg/nwaku:latest")
 NODE_2 = get_env_var("NODE_2", "wakuorg/go-waku:latest")
 LOG_DIR = get_env_var("LOG_DIR", "./log")
 NETWORK_NAME = get_env_var("NETWORK_NAME", "waku")
 SUBNET = get_env_var("SUBNET", "172.18.0.0/16")
 IP_RANGE = get_env_var("IP_RANGE", "172.18.0.0/24")
 GATEWAY = get_env_var("GATEWAY", "172.18.0.1")
 DEFAULT_PUBSUBTOPIC = get_env_var("DEFAULT_PUBSUBTOPIC", "/waku/2/default-waku/proto")
+PROTOCOL = get_env_var("PROTOCOL", "REST")

src/node/api_clients/base_client.py

@@ -0,0 +1,44 @@
+import logging
+import requests
+from tenacity import retry, stop_after_delay, wait_fixed
+from abc import ABC, abstractmethod
+
+logger = logging.getLogger(__name__)
+
+
+class BaseClient(ABC):
+    # The retry decorator is applied to handle transient errors gracefully. This is particularly
+    # useful when running tests in parallel, where occasional network-related errors such as
+    # connection drops, timeouts, or temporary unavailability of a service can occur. Retrying
+    # ensures that such intermittent issues don't cause the tests to fail outright.
+    @retry(stop=stop_after_delay(2), wait=wait_fixed(0.1), reraise=True)
+    def make_request(self, method, url, headers=None, data=None):
+        logger.debug("%s call: %s with payload: %s", method.upper(), url, data)
+        response = requests.request(method.upper(), url, headers=headers, data=data)
+        try:
+            response.raise_for_status()
+        except requests.HTTPError as http_err:
+            logger.error("HTTP error occurred: %s", http_err)
+            raise
+        except Exception as err:
+            logger.error("An error occurred: %s", err)
+            raise
+        else:
+            logger.info("Response status code: %s", response.status_code)
+        return response
+
+    @abstractmethod
+    def info(self):
+        pass
+
+    @abstractmethod
+    def set_subscriptions(self, pubsub_topics):
+        pass
+
+    @abstractmethod
+    def send_message(self, message, pubsub_topic):
+        pass
+
+    @abstractmethod
+    def get_messages(self, pubsub_topic):
+        pass

src/node/api_clients/rest.py

@@ -0,0 +1,31 @@
+import logging
+import json
+from dataclasses import asdict
+from urllib.parse import quote
+from src.node.api_clients.base_client import BaseClient
+
+logger = logging.getLogger(__name__)
+
+
+class REST(BaseClient):
+    def __init__(self, rest_port):
+        self._rest_port = rest_port
+
+    def rest_call(self, method, endpoint, payload=None):
+        url = f"http://127.0.0.1:{self._rest_port}/{endpoint}"
+        headers = {"Content-Type": "application/json"}
+        return self.make_request(method, url, headers=headers, data=payload)
+
+    def info(self):
+        info_response = self.rest_call("get", "debug/v1/info")
+        return info_response.json()
+
+    def set_subscriptions(self, pubsub_topics):
+        return self.rest_call("post", "relay/v1/subscriptions", json.dumps(pubsub_topics))
+
+    def send_message(self, message, pubsub_topic):
+        return self.rest_call("post", f"relay/v1/messages/{quote(pubsub_topic, safe='')}", json.dumps(asdict(message)))
+
+    def get_messages(self, pubsub_topic):
+        get_messages_response = self.rest_call("get", f"relay/v1/messages/{quote(pubsub_topic, safe='')}")
+        return get_messages_response.json()

src/node/api_clients/rpc.py

@@ -0,0 +1,35 @@
+import logging
+import json
+from dataclasses import asdict
+from src.node.api_clients.base_client import BaseClient
+
+logger = logging.getLogger(__name__)
+
+
+class RPC(BaseClient):
+    def __init__(self, rpc_port, image_name):
+        self._image_name = image_name
+        self._rpc_port = rpc_port
+
+    def rpc_call(self, endpoint, params=[]):
+        url = f"http://127.0.0.1:{self._rpc_port}"
+        headers = {"Content-Type": "application/json"}
+        payload = {"jsonrpc": "2.0", "method": endpoint, "params": params, "id": 1}
+        return self.make_request("post", url, headers=headers, data=json.dumps(payload))
+
+    def info(self):
+        info_response = self.rpc_call("get_waku_v2_debug_v1_info", [])
+        return info_response.json()["result"]
+
+    def set_subscriptions(self, pubsub_topics):
+        if "nwaku" in self._image_name:
+            return self.rpc_call("post_waku_v2_relay_v1_subscriptions", [pubsub_topics])
+        else:
+            return self.rpc_call("post_waku_v2_relay_v1_subscription", [pubsub_topics])
+
+    def send_message(self, message, pubsub_topic):
+        return self.rpc_call("post_waku_v2_relay_v1_message", [pubsub_topic, asdict(message)])
+
+    def get_messages(self, pubsub_topic):
+        get_messages_response = self.rpc_call("get_waku_v2_relay_v1_messages", [pubsub_topic])
+        return get_messages_response.json()["result"]

src/node/docker_mananger.py

@@ -14,34 +14,39 @@ class DockerManager:
     def __init__(self, image):
         self._image = image
         self._client = docker.from_env()
-        logger.debug(f"Docker client initialized with image {self._image}")
+        logger.debug("Docker client initialized with image %s", self._image)
 
     def create_network(self, network_name=NETWORK_NAME):
-        logger.debug(f"Attempting to create or retrieve network '{network_name}'.")
+        logger.debug("Attempting to create or retrieve network %s", network_name)
         networks = self._client.networks.list(names=[network_name])
         if networks:
-            logger.debug(f"Network '{network_name}' already exists.")
+            logger.debug("Network %s already exists", network_name)
             return networks[0]
 
         network = self._client.networks.create(
             network_name,
             driver="bridge",
             ipam=IPAMConfig(driver="default", pool_configs=[IPAMPool(subnet=SUBNET, iprange=IP_RANGE, gateway=GATEWAY)]),
         )
-        logger.debug(f"Network '{network_name}' created.")
+        logger.debug("Network %s created", network_name)
         return network
 
     def start_container(self, image_name, ports, args, log_path, container_ip):
-        command = [f"--{key}={value}" for key, value in args.items()]
+        cli_args = []
+        for key, value in args.items():
+            if isinstance(value, list):  # Check if value is a list
+                cli_args.extend([f"--{key}={item}" for item in value])  # Add a command for each item in the list
+            else:
+                cli_args.append(f"--{key}={value}")  # Add a single command
         port_bindings = {f"{port}/tcp": ("", port) for port in ports}
-        logger.debug(f"Starting container with image '{image_name}'.")
-
-        container = self._client.containers.run(image_name, command=command, ports=port_bindings, detach=True, auto_remove=True)
+        logger.debug("Starting container with image %s", image_name)
+        logger.debug("Using args %s", cli_args)
+        container = self._client.containers.run(image_name, command=cli_args, ports=port_bindings, detach=True, remove=True, auto_remove=True)
 
         network = self._client.networks.get(NETWORK_NAME)
         network.connect(container, ipv4_address=container_ip)
 
-        logger.debug(f"Container started with ID {container.short_id}. Setting up logs at '{log_path}'.")
+        logger.debug("Container started with ID %s. Setting up logs at %s", container.short_id, log_path)
         log_thread = threading.Thread(target=self._log_container_output, args=(container, log_path))
         log_thread.daemon = True
         log_thread.start()
@@ -54,26 +59,26 @@ def _log_container_output(self, container, log_path):
             for chunk in container.logs(stream=True):
                 log_file.write(chunk)
 
-    def generate_ports(self, base_port=None, count=4):
+    def generate_ports(self, base_port=None, count=5):
         if base_port is None:
             base_port = random.randint(1024, 65535 - count)
         ports = [base_port + i for i in range(count)]
-        logger.debug(f"Generated ports: {ports}")
+        logger.debug("Generated ports %s", ports)
         return ports
 
     @staticmethod
     def generate_random_ext_ip():
         base_ip_fragments = ["172", "18"]
         ext_ip = ".".join(base_ip_fragments + [str(random.randint(0, 255)) for _ in range(2)])
-        logger.debug(f"Generated random external IP: {ext_ip}")
+        logger.debug("Generated random external IP %s", ext_ip)
         return ext_ip
 
     def is_container_running(self, container):
         try:
             refreshed_container = self._client.containers.get(container.id)
             return refreshed_container.status == "running"
         except NotFound:
-            logger.error(f"Container with ID {container.id} not found.")
+            logger.error("Container with ID %s not found", container.id)
             return False
 
     @property