fix: python sample wit dependencies and modules references (#7)

* feat: python sample with wit dependencies
---------
Co-authored-by: Cedric <cleroux@logcraft.io>

.gitignore

@@ -1,2 +1,4 @@
 .vscode/
-target/
+target/
+*.pyc
+*.pyo

samples/.gitignore

@@ -1 +1,10 @@
-*.wasm
+# Ignore matching files
+poetry.lock
+*.wasm
+# Ignore any matching folders
+__pycache__/
+.venv/
+.env/
+# ignore generated bindings
+plugins/
+schemas/

samples/python/Makefile

@@ -0,0 +1,18 @@
+PLUGIN_FILE=my-plugin.wasm
+PLUGIN_FOLDER=myplugin
+
+schemas:
+	poetry run python3 myplugin/helpers/schemas.py
+
+bindings:
+	poetry run componentize-py --wit-path wit --world logcraft:lgc/plugins@0.1.0 bindings ${PLUGIN_FOLDER}
+
+build:
+	make clear
+	make schemas
+	poetry run componentize-py --wit-path wit --world logcraft:lgc/plugins@0.1.0 componentize -p ${PLUGIN_FOLDER} main -o ${PLUGIN_FILE}
+
+clear:
+	rm -f ${PLUGIN_FILE}
+	rm -rf ${PLUGIN_FOLDER}/plugins
+	rm -rf ${PLUGIN_FOLDER}/schemas

samples/python/README.md

@@ -1,14 +1,14 @@
 # Python Plugin Sample
 
-This is an example plugin using python.
+This is an example plugin using Python.
 
 ---
 **Documentation**: <a href="https://docs.logcraft.io" target="_blank">https://docs.logcraft.io</a>
 ---
 
 ## Poetry
 
-This example uses python poetry to manage dependencies but feel free to use your prefered package manager.
+This example uses Python poetry to manage dependencies but feel free to use your preferred package manager.
 
 ## Build
 
@@ -17,29 +17,33 @@ Building a plugin is a 2 steps process:
 1. Build the bindings for your IDE. This is optional but advised for development.
 
 ```bash
-poetry run componentize-py --wit-path wit --world plugins bindings myplugin
-
+poetry run componentize-py --wit-path wit --world logcraft:lgc/plugins@0.1.0 bindings myplugin
 ```
 
-2. Build the plugin. This step automatically build the bindings regardless if you did it in the previous step or not. The resulting wasm file is the LogCraft CLI plugin.
+2. Build the plugin. This step automatically builds the bindings regardless if you did it in the previous step or not. The resulting wasm file is the LogCraft CLI plugin.
 
 ```bash
-poetry run componentize-py --wit-path wit --world plugin componentize -p myplugin main -o my-plugin.wasm
+poetry run componentize-py --wit-path wit --world logcraft:lgc/plugins@0.1.0 componentize -p myplugin main -o my-plugin.wasm
 ```
 
+Or, with the provided Makefile:
 
-## Important
+```bash
+% make bindings
+% make build
+% make clear
+```
 
-As of june 2024, `componentize-py` uses cpython runtime without `zlib`. This is an issue that has consequences: we cannot use python `requests` library and probably others.
+## Important
+As of June 2024, `componentize-py` uses cpython runtime without `zlib`. This is an issue that has consequences: we cannot use python `requests` library and probably others.
 
 This is a known and identified problem that will be fixed in the future by the `componentize-py` team.
 
-To workaround this issue, we created a http library based on `sink`, this is available in this sample python application (`myplugin.client`).
-
+To workaround this issue, we created an HTTP library based on `sink`, which is available in this sample Python application (`myplugin.client`).
 
 ## WIT
 
-The wit files provides from:
+The wit files are provided from:
 
-1. `wit/world.wit` and `wit/plugin.wit` are LogCraft specific configuration files. These files define input/outputs of plugins.
+1. `wit/world.wit` and `wit/plugin.`wit` are LogCraft-specific configuration files. These files define the inputs/outputs of plugins.
 2. `wit/deps/` come from [wasi-http](https://github.com/WebAssembly/wasi-http/tree/main/wit/deps)

samples/python/myplugin/helpers/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) 2023 LogCraft, SAS.
+# SPDX-License-Identifier: MPL-2.0

samples/python/myplugin/helpers/client/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) 2023 LogCraft, SAS.
+# SPDX-License-Identifier: MPL-2.0

samples/python/myplugin/client/pool_loop.py → samples/python/myplugin/helpers/client/pool_loop.py

@@ -10,13 +10,13 @@
 import asyncio
 import socket
 import subprocess
+from typing import Optional, cast
 
 from plugins.types import Ok, Err
-from plugins.imports import types, streams, poll, outgoing_handler
+from plugins.imports import types, streams, poll
 from plugins.imports.types import IncomingBody, OutgoingBody, OutgoingRequest, IncomingResponse
 from plugins.imports.streams import StreamErrorClosed, InputStream
 from plugins.imports.poll import Pollable
-from typing import Optional, cast
 
 # Maximum number of bytes to read at a time
 READ_SIZE: int = 16 * 1024

samples/python/myplugin/client/req.py → samples/python/myplugin/helpers/client/req.py

@@ -4,13 +4,15 @@
 import traceback
 import poll_loop
 from poll_loop import PollLoop, Sink, Stream
+
 from plugins.types import Ok, Err
 from plugins.imports.types import (
     IncomingResponse, Method, Method_Get, Method_Head, Method_Post, Method_Put, Method_Delete, Method_Connect, Method_Options,
     Method_Trace, Method_Patch, Method_Other, IncomingRequest, IncomingBody, ResponseOutparam, OutgoingResponse,
     Fields, Scheme, Scheme_Http, Scheme_Https, Scheme_Other, OutgoingRequest, OutgoingBody
 )
 from plugins.imports.streams import StreamError_Closed
+
 from dataclasses import dataclass
 from collections.abc import MutableMapping
 from typing import Optional

samples/python/myplugin/helpers/schemas.py

@@ -0,0 +1,32 @@
+# Copyright (c) 2023 LogCraft, SAS.
+# SPDX-License-Identifier: MPL-2.0
+import os
+
+def generate():
+    """
+    Convert .k files from the 'package' directory to .py files
+    """
+    root = os.path.join(os.path.dirname(__file__), '../')
+    root = os.path.abspath(root)
+
+    # schemas directory
+    schemas_dir = os.path.join(root, 'schemas')
+    if not os.path.exists(schemas_dir):
+        os.makedirs(schemas_dir)
+
+    # convert .k files to .py files
+    for filename in ["settings.k", "rule.k"]:
+        filepath = os.path.join(root, 'package', filename)
+        filepath = os.path.abspath(filepath)
+
+        with open(filepath, 'r') as f:
+            content = f.read()
+
+        variable = filename.replace('.k', '')
+        f_out = os.path.join(schemas_dir, filename.replace('.k', '.py'))
+        with open(f_out, 'w') as f:
+            f.write("%s = '''%s'''" % (variable, content))
+
+
+if __name__ == "__main__":
+    generate()

samples/python/myplugin/main.py

@@ -1,68 +1,187 @@
 # Copyright (c) 2023 LogCraft, SAS.
 # SPDX-License-Identifier: MPL-2.0
-
+import json
 from typing import Optional
 
 # bindings generated by `componentize-py`
-from plugins import Plugins
-from plugins.exports.plugin import Metadata
 from plugins.types import Err, Ok, Some, Result
+from plugins.exports import Plugin
+from plugins.exports.plugin import Metadata
 
+# bindings generated by `helpers.schemas.generate()`
+from schemas.settings import settings
+from schemas.rule import rule
 
+# NOTE:
 # As of June 2024, the `requests` library is not supported due to missing dependencies
-# in the cpython runtime used by componentize-py (ssl support, zlib).
+# in the CPython runtime used by componentize-py (ssl support, zlib).
 #
 # A fix is planned for the future, so in the mean time, we use our own http library
 # derivated from sink
 # https://github.com/bytecodealliance/componentize-py/issues/96
+from helpers.client.req import Request, send
+
+class Plugin(Plugin):
+    """
+    The following functions are called by `lgc` and they receive these parameters, or a
+    combinaison of them:
+    
+    `name`:
+        a string representing the name of the detection, this is what we have in the detection
+        file.
+
+    `config`: 
+        a JSON string representing the service configuration. This is an dict representing the
+        service configuration. The available keys are defined in the `package/settings.k` file.
+        
+        For example, in this sample code we are using a copy of the settings.k from the Splunk
+        plugin, so we should expect parameters such as 'endpoint', 'authorization', etc.
 
-from client.req import Request, Response, send
+    `params`:
+        similar to `config`, the `params` parameter represents the detection rule (i.e. the yaml
+        file). The available keys are defined in the `package/rule.k` file.
 
+    NOTE:
+        This code sample is minimalistic and does not implement the actual logic to create, read,
+        update or delete a detection. It only shows how to parse the JSON strings received from
+        the CLI and how to return the results. In a real plugin, consider implementing classes to
+        represent the settings.k and rule.k files.
+    """
 
-class Plugin(Plugins):
     # func() -> metadata;
     def load(self) -> Metadata:
         """
-        The `load()` function is called when the plugin is installed using `lgc plugins install`.
+        The `load()` function is called when the plugin is installed using
+        `lgc plugins install /path/to/my-plugin.wasm`.
 
-        It should return a `Metadata` object containing the plugin's name, version, author, and description.
-        Make sure the name respect kebab-case (lowercase and separated by dashes). The provided information
-        will be displayed in the lgc.yaml file.
+        It should return a `Metadata` object containing the plugin's name, version, author, and
+        description. Make sure the name respect kebab-case (lowercase and separated by dashes).
+        
+        This information will be registered/displayed in the lgc.yaml file.
         """
         return Metadata("my-plugin", "0.1.0", "LogCraft", "This is a famous plugin")
 
     # func() -> string;
     def settings(self) -> str:
-        return  "OK"
+        """
+        The `settings()` function is called by several `lgc` capabilities such as
+        `lgc validate` or `lgc services configure my-plugin`.
+        """
+        return settings
 
     # func() -> string;
     def schema(self) -> str:
-        return  "OK"
+        """
+        The `schema()` function is called by `lgc plugins schema my-plugin`.
+        """
+        return rule
 
     # func(config: string, name: string, params: string) -> result<option<string>, string>;
     def create(self, config: str, name: str, params: str) -> Result[Optional[str], str]:
-        return Ok(Some("create()"))
+        # decode the json string received from the CLI
+        try:
+            config = json.loads(config)  # service configuration (settings.k)
+            params = json.loads(params)  # detection rule (rule.k)
+        except Exception as e:
+            raise Err(str(e))
+
+        # Depending on the settings.k, we need to assemble an url that will be used to
+        # create the detection. Here we are using the `endpoint` key from the settings.k
+        # to create the url.
+        url = f"{config['endpoint']}/some/service/remote/path/{name}"
+        resp = send(Request("POST", url, {}, None))
+
+        if resp.status == 201:
+            # Rule content not needed
+            return ""
+        raise Err(str(resp.status))
 
     # func(config: string, name: string, params: string) -> result<option<string>, string>;
     def read(self, config: str, name: str, params: str) -> Optional[str]:
-       return Ok(Some("read()"))
+        # decode the json string received from the CLI
+        try:
+            config = json.loads(config)  # service configuration (settings.k)
+            params = json.loads(params)  # detection rule (rule.k)
+        except Exception as e:
+            raise Err(str(e))
+
+        # Depending on the settings.k, we need to assemble an url that will be used to
+        # retrieve a detection. Here we are using the `endpoint` key from the settings.k
+        # to create the url.
+        url = f"{config['endpoint']}/some/service/remote/path/{name}"
+        resp = send(Request("GET", url, {}, None))
+
+        if resp.status == 200:
+            # return a json/dict object as a string (representing the rule.k)
+            # ex: return json.dumps({"rule": "my-rule"})
+            return str(resp.body)
+        # If 404, detection does not exist and will be created
+        elif resp.status == 404:
+            return None
+        # For any other HTTP code return an error
+        else:
+            raise Err(f"Error: HTTP/{resp.status}")
 
     # func(config: string, name: string, params: string) -> result<option<string>, string>;
     def update(self, config: str, name: str, params: str) -> Optional[str]:
-        return Ok(Some("update()"))
+        # decode the json string received from the CLI
+        try:
+            config = json.loads(config)  # service configuration (settings.k)
+            params = json.loads(params)  # detection rule (rule.k)
+        except Exception as e:
+            raise Err(str(e))
+
+        # Depending on the settings.k, we need to assemble an url that will be used to
+        # retrieve a detection. Here we are using the `endpoint` key from the settings.k
+        # to create the url.
+        url = f"{config['endpoint']}/some/service/remote/path/{name}"
+        resp = send(Request("PUT", url, {}, None))
+
+        if resp.status == 200:
+            # Rule content not needed
+            return ""
+        raise Err(str(resp.status))
 
     # func(config: string, name: string, params: string) -> result<option<string>, string>;
     def delete(self, config: str, name: str, params: str) -> Optional[str]:
-        return Ok(Some("delete()"))
+        # decode the json string received from the CLI
+        try:
+            config = json.loads(config)  # service configuration (settings.k)
+            params = json.loads(params)  # detection rule (rule.k)
+        except Exception as e:
+            raise Err(str(e))
+
+        # Depending on the settings.k, we need to assemble an url that will be used to
+        # retrieve a detection. Here we are using the `endpoint` key from the settings.k
+        # to create the url.
+        url = f"{config['endpoint']}/some/service/remote/path/{name}"
+        resp = send(Request("DELETE", url, {}, None))
+
+        if resp.status == 201:
+            # Rule content not needed
+            return ""
+        elif resp.status == 404:
+            # Detection doesn't exists
+            return None
+        raise Err(str(resp.status))
 
     # ping: func(config: string) -> result<bool, string>;
-    def ping(self, config: str) -> int: 
+    def ping(self, config: str) -> int:
+        """
+        `lgc services ping` will call this function to check if the service is up and running.
+
+        This is a sample implementation of the `ping` function that sends a GET request
+        to `https://google.fr` and returns the status code, or an error if the request
+        fails.
+        """
         try:
-            resp = send(Request("GET", "https://google.fr", {}, None))
+            # Service configuration
+            config = json.loads(config)
+            # Make GET request with service provided endpoint
+            resp = send(Request("GET", config["endpoint"], {}, None))
         except Exception as e:
             raise Err(str(e))
 
-        if resp.status_code >= 400:
-            raise Err(str(resp.status_code))
-
-        return Ok(resp.status_code)
+        if resp.status >= 400:
+            raise Err(str(resp.status))
+        return resp.status