Add pytest plugin Log10 managed evaluation

.github/workflows/ruff.yml

@@ -8,10 +8,10 @@ jobs:
 
       - uses: chartboost/ruff-action@v1
         with:
-          src: log10/ examples/ tests/
+          src: src/ examples/ tests/
           args: check --no-fix
 
       - uses: chartboost/ruff-action@v1
         with:
-          src: log10/ examples/ tests/
+          src: src/ examples/ tests/
           args: format --diff

pyproject.toml

@@ -16,7 +16,7 @@ classifiers = [
     "Operating System :: OS Independent",
 ]
 packages = [
-    { include = "log10" },
+    { include = "log10", from = "src" },
 ]
 
 [tool.poetry.scripts]
@@ -55,6 +55,8 @@ click = {version = "^8.1.7", optional = true}
 rich = {version = "^13.7.1", optional = true}
 tabulate = {version = "^0.9.0", optional = true}
 pandas = {version = ">=2", optional = true}
+# pytest = {version = ">=8.0.0", optional = true}
+# pytest-metadata = {version = ">=1.0.0", optional = true}
 
 [tool.poetry.extras]
 autofeedback_icl = ["magentic"]
@@ -67,6 +69,11 @@ mosaicml = ["mosaicml-cli"]
 google-generativeai = ["google-generativeai"]
 lamini = ["lamini"]
 cli = ["click", "rich", "tabulate", "pandas"]
+pytest = ["pytest_log10_managed_evaluation"]
+
+[tool.poetry.dependencies.pytest_log10_managed_evaluation]
+path = "src/pytest_log10_managed_evaluation"
+optional = true
 
 [tool.ruff]
 # Never enforce `E501` (line length violations).

log10/langchain.py → src/log10/langchain.py

@@ -1,16 +1,11 @@
+import logging
 import time
 import uuid
-from uuid import UUID
-
-from langchain.schema import AIMessage, BaseMessage, HumanMessage, SystemMessage
-
-
-"""Callback Handler that prints to std out."""
-import logging
 from typing import Any, Dict, List, Optional, Union
+from uuid import UUID
 
 from langchain.callbacks.base import BaseCallbackHandler
-from langchain.schema import AgentAction, AgentFinish, LLMResult
+from langchain.schema import AgentAction, AgentFinish, AIMessage, BaseMessage, HumanMessage, LLMResult, SystemMessage
 
 from log10.llm import LLM, Kind, Message
 

src/pytest_log10_managed_evaluation/README.md

@@ -0,0 +1,76 @@
+# Pytest Log10 Managed Evaluation
+
+A pytest plugin for managing evaluation in Log10 platform.
+
+[TODO]: add link to the evaluation doc
+
+## Installation
+After [configuring the Log10 environment variables](https://docs.log10.io/observability/advanced/logging#configuration),
+```bash
+pip install 'log10-io[pytest]'
+```
+
+## Options
+
+| Option | Type | Description |
+|--------|------|-------------|
+| `--eval-session-name` | Command-line | Set name for the evaluation session |
+| `--local` | Command-line | Run pytest locally without showing the session in Log10. A JSON report file will be saved locally. |
+| `--pretty-print` | Command-line | Pretty-print the JSON report with indentation. |
+| `eval_session_name` | INI | Set name for the evaluation session. Set in `pytest.ini` or `setup.cfg`. |
+
+## Usage
+
+Once installed, the plugin is automatically enabled when you run pytest. Simply execute your tests as you normally would, e.g.:
+```bash
+pytest tests
+```
+
+This will run your tests and upload the results to the Log10 platform for managed evaluation.
+
+### Running Tests Locally
+
+If you prefer to run tests locally without uploading results to Log10, use the `--local` option:
+
+```bash
+pytest tests --local
+```
+
+When using the `--local` option, a JSON report will be generated and saved in the `.pytest_log10_eval_reports` folder in your project directory. This is useful for:
+
+- Debugging and local development
+- Reviewing test results before uploading to Log10
+- Running tests in environments without Log10 access
+
+### Customizing the Evaluation Session
+This helps in organizing and identifying specific test runs in the Log10 platform.
+To assign a custom name to your evaluation session, use the `--eval-session-name` option:
+
+```bash
+pytest tests --eval-session-name <your-test-session-name>
+```
+
+Use `eval_session_name` in INI file
+
+You can set a default evaluation session name in your pytest configuration file. This will be used if the `--eval-session-name` command-line option is not provided.
+
+Example in `pytest.ini`:
+
+```ini
+[pytest]
+eval_session_name = <your-test-session-name>
+```
+
+### Pretty-Printing JSON Reports
+
+For improved readability of local JSON reports, use the `--pretty-print` option:
+
+```bash
+pytest tests --local --pretty-print
+```
+
+This will format the JSON report with proper indentation, making it easier to read and analyze.
+
+
+## Acknowledgments
+This project is based on [pytest-json-report](https://github.com/numirias/pytest-json-report), licensed under the MIT License. We've modified and extended it to integrate with the Log10 platform and add additional features.

src/pytest_log10_managed_evaluation/pyproject.toml

@@ -0,0 +1,42 @@
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry]
+name = "pytest_log10_managed_evaluation"
+description = "pytest plugin for log10 managed evaluation"
+
+version = "0.13.0" # Match this with log10-io version
+authors = ["log10 team"]
+license = "MIT"
+classifiers = [
+    "Framework :: Pytest",
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Topic :: Software Development :: Testing",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: Implementation :: CPython",
+    "Programming Language :: Python :: Implementation :: PyPy",
+    "License :: OSI Approved :: MIT License",
+]
+
+packages = [
+    { include = "pytest_log10_managed_evaluation", from = "." }
+]
+
+[tool.poetry.dependencies]
+python = ">=3.10,<4.0"
+pytest = ">=6.2.0"
+pytest-metadata = ">=1.0.0"
+
+[project.urls]
+"Homepage" = "https://github.com/log10-io/log10"
+"Bug Tracker" = "https://github.com/log10-io/log10/issues"
+
+[tool.poetry.plugins.pytest11]
+log10_managed_evaluation = "pytest_log10_managed_evaluation.plugin"