list app evaluations / continue from last evaluation / update app eva…

agenta-backend/agenta_backend/models/api/app_evaluation_model.py

@@ -1,11 +1,13 @@
-from pydantic import BaseModel
-from typing import Optional, List
+from pydantic import BaseModel, Field
+from typing import Optional, List, Dict
 from datetime import datetime
 
 
 class ComparisonTable(BaseModel):
     id: str
     variants: Optional[List[str]]
+    app_name: str
+    dataset: Dict[str, str] = Field(...)
     created_at: datetime
     updated_at: datetime
 
@@ -30,7 +32,12 @@ class EvaluationRow(BaseModel):
 
 class EvaluationRowUpdate(BaseModel):
     vote: str
+    outputs: List[EvaluationRowOutput]
 
 
 class NewComparisonTable(BaseModel):
+    app_name: str
     variants: List[str]
+    inputs: List[str]
+    dataset: Dict[str, str] = Field(...)
+    status: str = Field(...)

agenta-backend/agenta_backend/routers/app_evaluation_router.py

@@ -1,6 +1,6 @@
 from fastapi import HTTPException, APIRouter, Body
 from agenta_backend.models.api.app_evaluation_model import ComparisonTable, EvaluationRow, EvaluationRowUpdate, NewComparisonTable
-from agenta_backend.services.db_mongo import comparison_tables, evaluation_rows
+from agenta_backend.services.db_mongo import comparison_tables, evaluation_rows, datasets
 from datetime import datetime
 from bson import ObjectId
 from typing import List
@@ -18,16 +18,49 @@ async def create_comparison_table(newComparisonTableData: NewComparisonTable = B
     Returns:
         _description_
     """
-    comparison_table = dict()
+    comparison_table = newComparisonTableData.dict()
     comparison_table["created_at"] = comparison_table["updated_at"] = datetime.utcnow()
-    comparison_table["variants"] = newComparisonTableData.variants
-    result = await comparison_tables.insert_one(comparison_table)
-    if result.acknowledged:
-        comparison_table["id"] = str(result.inserted_id)
+
+    newComparisonTable = await comparison_tables.insert_one(comparison_table)
+
+    if newComparisonTable.acknowledged:
+        datasetId = comparison_table["dataset"]["_id"]
+        dataset = await datasets.find_one({"_id": ObjectId(datasetId)})
+        csvdata = dataset["csvdata"]
+        for datum in csvdata:
+            evaluation_row = {
+                "comparison_table_id": str(newComparisonTable.inserted_id),
+                "inputs": [{'input_name': name, 'input_value': datum[name]} for name in comparison_table["inputs"]],
+                "outputs": [],
+                "vote": "",
+                "created_at": datetime.utcnow(),
+                "updated_at": datetime.utcnow()
+            }
+            await evaluation_rows.insert_one(evaluation_row)
+
+        comparison_table["id"] = str(newComparisonTable.inserted_id)
         return comparison_table
     else:
         raise HTTPException(status_code=500, detail="Failed to create evaluation_row")
 
+@router.get("/{comparison_table_id}/evaluation_rows", response_model=List[EvaluationRow])
+async def fetch_evaluation_rows(comparison_table_id: str):
+    """Creates an empty evaluation row
+
+    Arguments:
+        evaluation_row -- _description_
+
+    Raises:
+        HTTPException: _description_
+
+    Returns:
+        _description_
+    """
+    cursor = evaluation_rows.find({"comparison_table_id": comparison_table_id})
+    items = await cursor.to_list(length=100)    # limit length to 100 for the example
+    for item in items:
+        item['id'] = str(item['_id'])
+    return items
 
 @router.post("/{comparison_table_id}/evaluation_row", response_model=EvaluationRow)
 async def create_evaluation_row(evaluation_row: EvaluationRow):
@@ -72,7 +105,10 @@ async def update_evaluation_row(evaluation_row_id: str, evaluation_row: Evaluati
     evaluation_row_dict["updated_at"] = datetime.utcnow()
     result = await evaluation_rows.update_one(
         {'_id': ObjectId(evaluation_row_id)},
-        {'$set': {'vote': evaluation_row_dict["vote"]}}
+        {'$set': {
+            'vote': evaluation_row_dict["vote"],
+            'outputs': evaluation_row_dict["outputs"]
+        }}
     )
     if result.acknowledged:
         return evaluation_row_dict
@@ -81,8 +117,8 @@ async def update_evaluation_row(evaluation_row_id: str, evaluation_row: Evaluati
 
 
 @router.get("/", response_model=List[ComparisonTable])
-async def get_comparison_table_id():
-    """lists the ids of all comparison tables
+async def fetch_list_comparison_tables():
+    """lists of all comparison tables
 
     Returns:
         _description_
@@ -94,6 +130,21 @@ async def get_comparison_table_id():
     return items
 
 
+@router.get("/{comparison_table_id}", response_model=ComparisonTable)
+async def fetch_comparison_table(comparison_table_id: str):
+    """Fetch one comparison table
+
+    Returns:
+        _description_
+    """
+    comparison_table = await comparison_tables.find_one({"_id" : ObjectId(comparison_table_id)})
+    if comparison_table:
+        comparison_table["id"] = str(comparison_table["_id"])
+        return comparison_table
+    else:
+        raise HTTPException(status_code=404, detail=f"dataset with id {comparison_table_id} not found")
+
+
 @router.get("/{comparison_table_id}/votes_data")
 async def fetch_results(comparison_table_id: str):
     """Fetch all the results for one the comparison table