From 7657dda02daa5766b273ba49bddda780e27dddfc Mon Sep 17 00:00:00 2001 From: Juan Pablo Vega Date: Fri, 15 Nov 2024 18:12:21 +0100 Subject: [PATCH 01/18] update testset_router to allow users to fetch test sets without app_id --- .../agenta_backend/routers/testset_router.py | 67 ++++++++++++------- 1 file changed, 42 insertions(+), 25 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/testset_router.py b/agenta-backend/agenta_backend/routers/testset_router.py index 05268749cf..141660498c 100644 --- a/agenta-backend/agenta_backend/routers/testset_router.py +++ b/agenta-backend/agenta_backend/routers/testset_router.py @@ -315,7 +315,6 @@ async def update_testset( @router.get("/", operation_id="get_testsets") async def get_testsets( - app_id: str, request: Request, ) -> List[TestSetOutputResponse]: """ @@ -328,34 +327,52 @@ async def get_testsets( - `HTTPException` with status code 404 if no testsets are found. """ - app = await db_manager.fetch_app_by_id(app_id=app_id) - if isCloudEE(): - has_permission = await check_action_access( - user_uid=request.state.user_id, - project_id=str(app.project_id), - permission=Permission.VIEW_TESTSET, + try: + if isCloudEE(): + has_permission = await check_action_access( + user_uid=request.state.user_id, + project_id=request.state.project_id, + permission=Permission.VIEW_TESTSET, + ) + + logger.debug( + "User has Permission to view Testsets: %s", + has_permission, + ) + + if not has_permission: + error_msg = ( + "You do not have permission to perform this action. " + + "Please contact your organization admin." + ) + logger.error(error_msg) + + return JSONResponse( + status_code=403, + content={"detail": error_msg}, + ) + + testsets = await db_manager.fetch_testsets_by_project_id( + project_id=request.state.project_id, ) - logger.debug(f"User has Permission to view Testsets: {has_permission}") - if not has_permission: - error_msg = f"You do not have permission to perform this action. Please contact your organization admin." - logger.error(error_msg) - return JSONResponse( - {"detail": error_msg}, - status_code=403, + + return [ + TestSetOutputResponse( + _id=str(testset.id), # type: ignore + name=testset.name, + created_at=str(testset.created_at), + updated_at=str(testset.updated_at), ) + for testset in testsets + ] - testsets = await db_manager.fetch_testsets_by_project_id( - project_id=str(app.project_id) - ) - return [ - TestSetOutputResponse( - _id=str(testset.id), # type: ignore - name=testset.name, - created_at=str(testset.created_at), - updated_at=str(testset.updated_at), + except Exception as e: + logger.exception(f"An error occurred: {str(e)}") + + raise HTTPException( + status_code=500, + detail=str(e), ) - for testset in testsets - ] @router.get("/{testset_id}/", operation_id="get_single_testset") From fa076a64d8d40f03c5e402a4aaa8abd7738e3828 Mon Sep 17 00:00:00 2001 From: Juan Pablo Vega Date: Fri, 15 Nov 2024 18:24:47 +0100 Subject: [PATCH 02/18] remove app_id when fetching testsets in web UI --- .../HumanEvaluationModal/HumanEvaluationModal.tsx | 2 +- .../Playground/AddToTestSetDrawer/AddToTestSetDrawer.tsx | 2 +- agenta-web/src/components/Playground/LoadTestsModal.tsx | 2 +- .../evaluations/NewEvaluation/NewEvaluationModal.tsx | 2 +- .../autoEvaluation/EvaluatorsModal/EvaluatorsModal.tsx | 2 +- .../pages/testset/modals/CreateTestsetFromScratch.tsx | 2 +- .../src/components/pages/testset/modals/UploadTestset.tsx | 2 +- agenta-web/src/pages/apps/testsets/index.tsx | 2 +- agenta-web/src/services/testsets/api/index.ts | 8 ++++---- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/agenta-web/src/components/HumanEvaluationModal/HumanEvaluationModal.tsx b/agenta-web/src/components/HumanEvaluationModal/HumanEvaluationModal.tsx index 084bf22fbc..d088970d10 100644 --- a/agenta-web/src/components/HumanEvaluationModal/HumanEvaluationModal.tsx +++ b/agenta-web/src/components/HumanEvaluationModal/HumanEvaluationModal.tsx @@ -150,7 +150,7 @@ const HumanEvaluationModal = ({ const appId = router.query.app_id?.toString() || "" - const {testsets, isTestsetsLoadingError} = useLoadTestsetsList(appId) + const {testsets, isTestsetsLoadingError} = useLoadTestsetsList() const [variantsInputs, setVariantsInputs] = useState>({}) diff --git a/agenta-web/src/components/Playground/AddToTestSetDrawer/AddToTestSetDrawer.tsx b/agenta-web/src/components/Playground/AddToTestSetDrawer/AddToTestSetDrawer.tsx index 1d835e28dd..6dbd00f86d 100644 --- a/agenta-web/src/components/Playground/AddToTestSetDrawer/AddToTestSetDrawer.tsx +++ b/agenta-web/src/components/Playground/AddToTestSetDrawer/AddToTestSetDrawer.tsx @@ -101,7 +101,7 @@ const AddToTestSetDrawer: React.FC = ({params, isChatVariant, ...props}) const dirty = useRef(false) const router = useRouter() const appId = router.query.app_id as string - const {testsets, mutate, isTestsetsLoading, isTestsetsLoadingError} = useLoadTestsetsList(appId) + const {testsets, mutate, isTestsetsLoading, isTestsetsLoadingError} = useLoadTestsetsList() const storedValue = localStorage.getItem(`selectedTestset_${appId}`)?.replace(/"/g, "") const [selectedTestset, setSelectedTestset] = useLocalStorage( `selectedTestset_${appId}`, diff --git a/agenta-web/src/components/Playground/LoadTestsModal.tsx b/agenta-web/src/components/Playground/LoadTestsModal.tsx index e3fcedf621..9cbb589349 100644 --- a/agenta-web/src/components/Playground/LoadTestsModal.tsx +++ b/agenta-web/src/components/Playground/LoadTestsModal.tsx @@ -30,7 +30,7 @@ const LoadTestsModal: React.FC = (props) => { const appId = router.query.app_id as string - const {testsets, isTestsetsLoading, isTestsetsLoadingError} = useLoadTestsetsList(appId) + const {testsets, isTestsetsLoading, isTestsetsLoadingError} = useLoadTestsetsList() const options = testsets?.map((item: Record) => ({ label: item.name, diff --git a/agenta-web/src/components/pages/evaluations/NewEvaluation/NewEvaluationModal.tsx b/agenta-web/src/components/pages/evaluations/NewEvaluation/NewEvaluationModal.tsx index 3ed4d5b8b5..6f2733a58f 100644 --- a/agenta-web/src/components/pages/evaluations/NewEvaluation/NewEvaluationModal.tsx +++ b/agenta-web/src/components/pages/evaluations/NewEvaluation/NewEvaluationModal.tsx @@ -83,7 +83,7 @@ const NewEvaluationModal: React.FC = ({onSuccess, ...props}) => { useEffect(() => { setFetching(true) form.resetFields() - Promise.all([fetchTestsets(appId), fetchVariants(appId)]) + Promise.all([fetchTestsets(), fetchVariants(appId)]) .then(([testSets, variants]) => { setTestSets(testSets) setVariants(variants) diff --git a/agenta-web/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/EvaluatorsModal.tsx b/agenta-web/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/EvaluatorsModal.tsx index a9e70174ed..461edfe199 100644 --- a/agenta-web/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/EvaluatorsModal.tsx +++ b/agenta-web/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/EvaluatorsModal.tsx @@ -67,7 +67,7 @@ const EvaluatorsModal = ({...props}: EvaluatorsModalProps) => { fetchAllEvaluators(), fetchAllEvaluatorConfigs(appId), fetchVariants(appId), - fetchTestsets(appId), + fetchTestsets(), ]).then(([evaluators, configs, variants, testsets]) => { setEvaluators(evaluators) setEvaluatorConfigs(configs) diff --git a/agenta-web/src/components/pages/testset/modals/CreateTestsetFromScratch.tsx b/agenta-web/src/components/pages/testset/modals/CreateTestsetFromScratch.tsx index 7fefb9e377..6e9b6ae2f7 100644 --- a/agenta-web/src/components/pages/testset/modals/CreateTestsetFromScratch.tsx +++ b/agenta-web/src/components/pages/testset/modals/CreateTestsetFromScratch.tsx @@ -51,7 +51,7 @@ const CreateTestsetFromScratch: React.FC = ({ mode === "rename" ? (editTestsetValues?.name as string) : "", ) const [isLoading, setIsLoading] = useState(false) - const {mutate} = useLoadTestsetsList(appId) + const {mutate} = useLoadTestsetsList() const generateInitialRowData = async (): Promise => { const backendVariants = await fetchVariants(appId) diff --git a/agenta-web/src/components/pages/testset/modals/UploadTestset.tsx b/agenta-web/src/components/pages/testset/modals/UploadTestset.tsx index da1dc8c834..7bf64d557e 100644 --- a/agenta-web/src/components/pages/testset/modals/UploadTestset.tsx +++ b/agenta-web/src/components/pages/testset/modals/UploadTestset.tsx @@ -61,7 +61,7 @@ const UploadTestset: React.FC = ({setCurrent, onCancel, appId}) => { const [testsetName, setTestsetName] = useState("") const [uploadLoading, setUploadLoading] = useState(false) const [fileProgress, setFileProgress] = useState({} as UploadFile) - const {mutate} = useLoadTestsetsList(appId) + const {mutate} = useLoadTestsetsList() const onFinish = async (values: any) => { const {file} = values diff --git a/agenta-web/src/pages/apps/testsets/index.tsx b/agenta-web/src/pages/apps/testsets/index.tsx index 7a6cde0221..41ce28d7bf 100644 --- a/agenta-web/src/pages/apps/testsets/index.tsx +++ b/agenta-web/src/pages/apps/testsets/index.tsx @@ -58,7 +58,7 @@ const Testset = () => { const {apps, isLoading: isAppsLoading} = useAppsData() const appId = apps[0]?.app_id const [selectedRowKeys, setSelectedRowKeys] = useState([]) - const {testsets, isTestsetsLoading, mutate} = useLoadTestsetsList(appId) + const {testsets, isTestsetsLoading, mutate} = useLoadTestsetsList() const [isCreateTestsetModalOpen, setIsCreateTestsetModalOpen] = useState(false) const [searchTerm, setSearchTerm] = useState("") const [testsetCreationMode, setTestsetCreationMode] = useState("create") diff --git a/agenta-web/src/services/testsets/api/index.ts b/agenta-web/src/services/testsets/api/index.ts index e634588e2a..87a8e92def 100644 --- a/agenta-web/src/services/testsets/api/index.ts +++ b/agenta-web/src/services/testsets/api/index.ts @@ -10,9 +10,9 @@ import {axiosFetcher} from "@/services/api" // - update: PUT data to server // - delete: DELETE data from server -export const useLoadTestsetsList = (appId: string) => { +export const useLoadTestsetsList = () => { const {data, error, mutate, isLoading} = useSWR( - () => (appId ? `${getAgentaApiUrl()}/api/testsets/?app_id=${appId}` : null), + `${getAgentaApiUrl()}/api/testsets`, axiosFetcher, {revalidateOnFocus: false, shouldRetryOnError: false}, ) @@ -25,8 +25,8 @@ export const useLoadTestsetsList = (appId: string) => { } } -export const fetchTestsets = async (appId: string) => { - const response = await axios.get(`${getAgentaApiUrl()}/api/testsets/?app_id=${appId}`) +export const fetchTestsets = async () => { + const response = await axios.get(`${getAgentaApiUrl()}/api/testsets`) return response.data } From 4b2ed7f531b525000b373d01b5af83cf2ae2bb9c Mon Sep 17 00:00:00 2001 From: Juan Pablo Vega Date: Fri, 15 Nov 2024 19:21:28 +0100 Subject: [PATCH 03/18] remove app_id in create/fetch testsets --- .../agenta_backend/routers/testset_router.py | 29 +++--- .../agenta_backend/services/db_manager.py | 34 +------ .../test_variant_testset_router.py | 65 +++++------- .../agenta/client/backend/testsets/client.py | 48 ++------- .../testsets/create_with_upload/curl.ts | 3 +- .../testsets/create_with_upload/python.ts | 5 +- .../testsets/create_with_upload/typescript.ts | 4 +- .../HumanEvaluationModal.tsx | 2 +- .../AddToTestSetDrawer/AddToTestSetDrawer.tsx | 7 +- .../components/Playground/LoadTestsModal.tsx | 2 - .../SaveTestsetModal/SaveTestsetModal.tsx | 4 +- .../testset/modals/CreateTestsetFromApi.tsx | 8 +- .../modals/CreateTestsetFromScratch.tsx | 2 +- .../pages/testset/modals/UploadTestset.tsx | 1 - agenta-web/src/services/testsets/api/index.ts | 4 +- cookbook/evaluations_with_sdk.ipynb | 71 +++++++++----- docs/docs/evaluation/02-create-test-sets.mdx | 2 +- docs/docs/evaluation/05-sdk-evaluation.mdx | 2 +- .../docs/reference/api/create-testset.api.mdx | 7 +- docs/docs/reference/openapi.json | 98 +++++++------------ docs/docs/tutorials/sdk/evaluate-with-SDK.mdx | 2 +- response.txt | 2 +- 22 files changed, 156 insertions(+), 246 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/testset_router.py b/agenta-backend/agenta_backend/routers/testset_router.py index 141660498c..c7d3311af1 100644 --- a/agenta-backend/agenta_backend/routers/testset_router.py +++ b/agenta-backend/agenta_backend/routers/testset_router.py @@ -53,7 +53,6 @@ async def upload_file( upload_type: str = Form(None), file: UploadFile = File(...), testset_name: Optional[str] = File(None), - app_id: str = Form(None), ): """ Uploads a CSV or JSON file and saves its data to MongoDB. @@ -67,11 +66,10 @@ async def upload_file( dict: The result of the upload process. """ - app = await db_manager.fetch_app_by_id(app_id=app_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=str(app.project_id), + project_id=request.state.project_id, permission=Permission.CREATE_TESTSET, ) logger.debug(f"User has Permission to upload Testset: {has_permission}") @@ -114,7 +112,8 @@ async def upload_file( try: testset = await db_manager.create_testset( - app=app, project_id=str(app.project_id), testset_data=document + project_id=request.state.project_id, + testset_data=document, ) return TestSetSimpleResponse( id=str(testset.id), @@ -132,7 +131,6 @@ async def import_testset( request: Request, endpoint: str = Form(None), testset_name: str = Form(None), - app_id: str = Form(None), ): """ Import JSON testset data from an endpoint and save it to MongoDB. @@ -145,11 +143,10 @@ async def import_testset( dict: The result of the import process. """ - app = await db_manager.fetch_app_by_id(app_id=app_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=str(app.project_id), + project_id=request.state.project_id, permission=Permission.CREATE_TESTSET, ) logger.debug(f"User has Permission to import Testset: {has_permission}") @@ -180,7 +177,8 @@ async def import_testset( document["csvdata"].append(row) testset = await db_manager.create_testset( - app=app, project_id=str(app.project_id), testset_data=document + project_id=request.state.project_id, + testset_data=document, ) return TestSetSimpleResponse( id=str(testset.id), @@ -204,30 +202,30 @@ async def import_testset( @router.post( - "/{app_id}/", response_model=TestSetSimpleResponse, operation_id="create_testset" + "/{app_id}", + response_model=TestSetSimpleResponse, + operation_id="deprecating_create_testset", ) +@router.post("/", response_model=TestSetSimpleResponse, operation_id="create_testset") async def create_testset( - app_id: str, csvdata: NewTestset, request: Request, ): """ - Create a testset with given name and app_name, save the testset to MongoDB. + Create a testset with given name, save the testset to MongoDB. Args: name (str): name of the test set. - app_name (str): name of the application. testset (Dict[str, str]): test set data. Returns: str: The id of the test set created. """ - app = await db_manager.fetch_app_by_id(app_id=app_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=str(app.project_id), + project_id=request.state.project_id, permission=Permission.CREATE_TESTSET, ) logger.debug(f"User has Permission to create Testset: {has_permission}") @@ -245,7 +243,8 @@ async def create_testset( "csvdata": csvdata.csvdata, } testset_instance = await db_manager.create_testset( - app=app, project_id=str(app.project_id), testset_data=testset_data + project_id=request.state.project_id, + testset_data=testset_data, ) if testset_instance is not None: return TestSetSimpleResponse( diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 0baf67510d..1c6773d51e 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -1843,38 +1843,6 @@ async def remove_testsets(testset_ids: List[str]): await session.commit() -async def remove_app_testsets(app_id: str): - """Returns a list of testsets owned by an app. - - Args: - app_id (str): The name of the app - - Returns: - int: The number of testsets deleted - """ - - # Find testsets owned by the app - deleted_count: int = 0 - - async with engine.session() as session: - result = await session.execute( - select(TestSetDB).filter_by(app_id=uuid.UUID(app_id)) - ) - testsets = result.scalars().all() - - if len(testsets) == 0: - logger.info(f"No testsets found for app {app_id}") - return 0 - - for testset in testsets: - await session.delete(testset) - deleted_count += 1 - logger.info(f"{deleted_count} testset(s) deleted for app {app_id}") - - await session.commit() - return deleted_count - - async def remove_base_from_db(base_id: str): """ Remove a base from the database. @@ -2037,7 +2005,7 @@ async def fetch_testset_by_id(testset_id: str) -> Optional[TestSetDB]: return testset -async def create_testset(app: AppDB, project_id: str, testset_data: Dict[str, Any]): +async def create_testset(project_id: str, testset_data: Dict[str, Any]): """ Creates a testset. diff --git a/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_testset_router.py b/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_testset_router.py index f8b9e73dc7..850bfa7c06 100644 --- a/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_testset_router.py +++ b/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_testset_router.py @@ -29,34 +29,29 @@ @pytest.mark.asyncio async def test_create_testset(): - async with engine.session() as session: - result = await session.execute( - select(AppDB).filter_by(app_name="app_variant_test") - ) - app = result.scalars().first() - - payload = { - "name": "create_testset_main", - "csvdata": [ - { - "country": "Comoros", - "correct_answer": "The capital of Comoros is Moroni", - }, - { - "country": "Kyrgyzstan", - "correct_answer": "The capital of Kyrgyzstan is Bishkek", - }, - { - "country": "Azerbaijan", - "correct_answer": "The capital of Azerbaijan is Baku", - }, - ], - } - response = await test_client.post( - f"{BACKEND_API_HOST}/testsets/{str(app.id)}/", json=payload - ) - assert response.status_code == 200 - assert response.json()["name"] == payload["name"] + payload = { + "name": "create_testset_main", + "csvdata": [ + { + "country": "Comoros", + "correct_answer": "The capital of Comoros is Moroni", + }, + { + "country": "Kyrgyzstan", + "correct_answer": "The capital of Kyrgyzstan is Bishkek", + }, + { + "country": "Azerbaijan", + "correct_answer": "The capital of Azerbaijan is Baku", + }, + ], + } + response = await test_client.post( + f"{BACKEND_API_HOST}/testsets", + json=payload, + ) + assert response.status_code == 200 + assert response.json()["name"] == payload["name"] @pytest.mark.asyncio @@ -101,18 +96,10 @@ async def test_update_testset(): @pytest.mark.asyncio async def test_get_testsets(): - async with engine.session() as session: - result = await session.execute( - select(AppDB).filter_by(app_name="app_variant_test") - ) - app = result.scalars().first() + response = await test_client.get(f"{BACKEND_API_HOST}/testsets") - response = await test_client.get( - f"{BACKEND_API_HOST}/testsets/?app_id={str(app.id)}" - ) - - assert response.status_code == 200 - assert len(response.json()) == 2 + assert response.status_code == 200 + assert len(response.json()) == 2 @pytest.mark.asyncio() diff --git a/agenta-cli/agenta/client/backend/testsets/client.py b/agenta-cli/agenta/client/backend/testsets/client.py index 0ea1f2eae0..fc10205700 100644 --- a/agenta-cli/agenta/client/backend/testsets/client.py +++ b/agenta-cli/agenta/client/backend/testsets/client.py @@ -28,7 +28,6 @@ def upload_file( file: core.File, upload_type: typing.Optional[str] = OMIT, testset_name: typing.Optional[str] = OMIT, - app_id: typing.Optional[str] = OMIT, request_options: typing.Optional[RequestOptions] = None, ) -> TestSetSimpleResponse: """ @@ -51,8 +50,6 @@ def upload_file( testset_name : typing.Optional[str] - app_id : typing.Optional[str] - request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -77,7 +74,6 @@ def upload_file( data={ "upload_type": upload_type, "testset_name": testset_name, - "app_id": app_id, }, files={ "file": file, @@ -173,18 +169,16 @@ def import_testset( def create_testset( self, - app_id: str, *, name: str, csvdata: typing.Sequence[typing.Dict[str, typing.Optional[typing.Any]]], request_options: typing.Optional[RequestOptions] = None, ) -> TestSetSimpleResponse: """ - Create a testset with given name and app_name, save the testset to MongoDB. + Create a testset with given name, save the testset to MongoDB. Args: name (str): name of the test set. - app_name (str): name of the application. testset (Dict[str, str]): test set data. Returns: @@ -192,8 +186,6 @@ def create_testset( Parameters ---------- - app_id : str - name : str csvdata : typing.Sequence[typing.Dict[str, typing.Optional[typing.Any]]] @@ -215,13 +207,12 @@ def create_testset( base_url="https://yourhost.com/path/to/api", ) client.testsets.create_testset( - app_id="app_id", name="name", csvdata=[{"key": "value"}], ) """ _response = self._client_wrapper.httpx_client.request( - f"testsets/{jsonable_encoder(app_id)}", + "testsets", method="POST", json={ "name": name, @@ -405,7 +396,7 @@ def update_testset( raise ApiError(status_code=_response.status_code, body=_response_json) def get_testsets( - self, *, app_id: str, request_options: typing.Optional[RequestOptions] = None + self, *, request_options: typing.Optional[RequestOptions] = None ) -> typing.List[TestSetOutputResponse]: """ Get all testsets. @@ -420,8 +411,6 @@ def get_testsets( Parameters ---------- - app_id : str - request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -438,16 +427,11 @@ def get_testsets( api_key="YOUR_API_KEY", base_url="https://yourhost.com/path/to/api", ) - client.testsets.get_testsets( - app_id="app_id", - ) + client.testsets.get_testsets() """ _response = self._client_wrapper.httpx_client.request( "testsets", method="GET", - params={ - "app_id": app_id, - }, request_options=request_options, ) try: @@ -557,7 +541,6 @@ async def upload_file( file: core.File, upload_type: typing.Optional[str] = OMIT, testset_name: typing.Optional[str] = OMIT, - app_id: typing.Optional[str] = OMIT, request_options: typing.Optional[RequestOptions] = None, ) -> TestSetSimpleResponse: """ @@ -580,8 +563,6 @@ async def upload_file( testset_name : typing.Optional[str] - app_id : typing.Optional[str] - request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -614,7 +595,6 @@ async def main() -> None: data={ "upload_type": upload_type, "testset_name": testset_name, - "app_id": app_id, }, files={ "file": file, @@ -718,18 +698,16 @@ async def main() -> None: async def create_testset( self, - app_id: str, *, name: str, csvdata: typing.Sequence[typing.Dict[str, typing.Optional[typing.Any]]], request_options: typing.Optional[RequestOptions] = None, ) -> TestSetSimpleResponse: """ - Create a testset with given name and app_name, save the testset to MongoDB. + Create a testset with given name, save the testset to MongoDB. Args: name (str): name of the test set. - app_name (str): name of the application. testset (Dict[str, str]): test set data. Returns: @@ -737,8 +715,6 @@ async def create_testset( Parameters ---------- - app_id : str - name : str csvdata : typing.Sequence[typing.Dict[str, typing.Optional[typing.Any]]] @@ -765,7 +741,6 @@ async def create_testset( async def main() -> None: await client.testsets.create_testset( - app_id="app_id", name="name", csvdata=[{"key": "value"}], ) @@ -774,7 +749,7 @@ async def main() -> None: asyncio.run(main()) """ _response = await self._client_wrapper.httpx_client.request( - f"testsets/{jsonable_encoder(app_id)}", + "testsets", method="POST", json={ "name": name, @@ -974,7 +949,7 @@ async def main() -> None: raise ApiError(status_code=_response.status_code, body=_response_json) async def get_testsets( - self, *, app_id: str, request_options: typing.Optional[RequestOptions] = None + self, *, request_options: typing.Optional[RequestOptions] = None ) -> typing.List[TestSetOutputResponse]: """ Get all testsets. @@ -989,8 +964,6 @@ async def get_testsets( Parameters ---------- - app_id : str - request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -1012,9 +985,7 @@ async def get_testsets( async def main() -> None: - await client.testsets.get_testsets( - app_id="app_id", - ) + await client.testsets.get_testsets() asyncio.run(main()) @@ -1022,9 +993,6 @@ async def main() -> None: _response = await self._client_wrapper.httpx_client.request( "testsets", method="GET", - params={ - "app_id": app_id, - }, request_options=request_options, ) try: diff --git a/agenta-web/src/code_snippets/testsets/create_with_upload/curl.ts b/agenta-web/src/code_snippets/testsets/create_with_upload/curl.ts index 6d3cce6844..4070c47273 100644 --- a/agenta-web/src/code_snippets/testsets/create_with_upload/curl.ts +++ b/agenta-web/src/code_snippets/testsets/create_with_upload/curl.ts @@ -1,10 +1,9 @@ import {isDemo} from "@/lib/helpers/utils" -export default function cURLCode(uri: string, appId: string): string { +export default function cURLCode(uri: string): string { return `curl -X POST ${uri} \\ -H 'Content-Type: multipart/form-data' \\ -F 'file=@/path/to/your/file.csv' \\ -F 'testset_name=your_testset_name' \\ --F 'app_id=${appId}' \\ ${!isDemo() ? "" : "-H 'Authorization: your_api_key'"}` } diff --git a/agenta-web/src/code_snippets/testsets/create_with_upload/python.ts b/agenta-web/src/code_snippets/testsets/create_with_upload/python.ts index 3176f462ac..249b367092 100644 --- a/agenta-web/src/code_snippets/testsets/create_with_upload/python.ts +++ b/agenta-web/src/code_snippets/testsets/create_with_upload/python.ts @@ -1,16 +1,15 @@ import {isDemo} from "@/lib/helpers/utils" -export default function pythonCode(uri: string, appId: string): string { +export default function pythonCode(uri: string): string { return `import requests url = '${uri}' file_path = '/path/to/your/file.csv' testset_name = 'your_testset_name' -appId = '${appId}' with open(file_path, 'rb') as file: files = {'file': file} - data = {'testset_name': testset_name, 'app_id': appId} + data = {'testset_name': testset_name} response = requests.post(url, files=files, data=data${ !isDemo() ? "" : ", headers={'Authorization': 'your_api_key'}" }) diff --git a/agenta-web/src/code_snippets/testsets/create_with_upload/typescript.ts b/agenta-web/src/code_snippets/testsets/create_with_upload/typescript.ts index 1816542ffc..d08b2495de 100644 --- a/agenta-web/src/code_snippets/testsets/create_with_upload/typescript.ts +++ b/agenta-web/src/code_snippets/testsets/create_with_upload/typescript.ts @@ -1,7 +1,7 @@ import {isDemo} from "@/lib/helpers/utils" import {js as beautify} from "js-beautify" -export default function tsCode(uri: string, appId: string): string { +export default function tsCode(uri: string): string { const codeString = `import axios from 'axios'; const fs = require('fs'); const FormData = require('form-data'); @@ -9,12 +9,10 @@ export default function tsCode(uri: string, appId: string): string { const url = '${uri}'; const filePath = './cypress/data/countries-genders.csv'; const testsetName = 'tribalafa'; - const appId = '${appId}'; const formData = new FormData(); formData.append('file', fs.createReadStream(filePath)); formData.append('testset_name', testsetName); - formData.append('app_id', appId); const config = { headers: { diff --git a/agenta-web/src/components/HumanEvaluationModal/HumanEvaluationModal.tsx b/agenta-web/src/components/HumanEvaluationModal/HumanEvaluationModal.tsx index d088970d10..b03f4ebd05 100644 --- a/agenta-web/src/components/HumanEvaluationModal/HumanEvaluationModal.tsx +++ b/agenta-web/src/components/HumanEvaluationModal/HumanEvaluationModal.tsx @@ -340,7 +340,7 @@ const HumanEvaluationModal = ({ setError({ message: getErrorMessage(err), btnText: "Go to Test sets", - endpoint: `/apps/${appId}/testsets`, + endpoint: `/apps/testsets`, }) } }) diff --git a/agenta-web/src/components/Playground/AddToTestSetDrawer/AddToTestSetDrawer.tsx b/agenta-web/src/components/Playground/AddToTestSetDrawer/AddToTestSetDrawer.tsx index 6dbd00f86d..2246216901 100644 --- a/agenta-web/src/components/Playground/AddToTestSetDrawer/AddToTestSetDrawer.tsx +++ b/agenta-web/src/components/Playground/AddToTestSetDrawer/AddToTestSetDrawer.tsx @@ -100,11 +100,10 @@ const AddToTestSetDrawer: React.FC = ({params, isChatVariant, ...props}) const [shouldRender, setShouldRender] = useState(false) const dirty = useRef(false) const router = useRouter() - const appId = router.query.app_id as string const {testsets, mutate, isTestsetsLoading, isTestsetsLoadingError} = useLoadTestsetsList() - const storedValue = localStorage.getItem(`selectedTestset_${appId}`)?.replace(/"/g, "") + const storedValue = localStorage.getItem(`selectedTestset`)?.replace(/"/g, "") const [selectedTestset, setSelectedTestset] = useLocalStorage( - `selectedTestset_${appId}`, + `selectedTestset`, "", ) @@ -177,7 +176,7 @@ const AddToTestSetDrawer: React.FC = ({params, isChatVariant, ...props}) }) const promise = isNew - ? createNewTestset(appId, name, newRows) + ? createNewTestset(name, newRows) : updateTestset(selectedTestset!, name, [...csvdata, ...newRows]) promise .then(() => { diff --git a/agenta-web/src/components/Playground/LoadTestsModal.tsx b/agenta-web/src/components/Playground/LoadTestsModal.tsx index 9cbb589349..93d32be7bb 100644 --- a/agenta-web/src/components/Playground/LoadTestsModal.tsx +++ b/agenta-web/src/components/Playground/LoadTestsModal.tsx @@ -28,8 +28,6 @@ const LoadTestsModal: React.FC = (props) => { const [isOpen, setIsOpen] = useState(false) const [selectedSet, setSelectedSet] = useState("") - const appId = router.query.app_id as string - const {testsets, isTestsetsLoading, isTestsetsLoadingError} = useLoadTestsetsList() const options = testsets?.map((item: Record) => ({ diff --git a/agenta-web/src/components/SaveTestsetModal/SaveTestsetModal.tsx b/agenta-web/src/components/SaveTestsetModal/SaveTestsetModal.tsx index 02099ea9b1..61fe420962 100644 --- a/agenta-web/src/components/SaveTestsetModal/SaveTestsetModal.tsx +++ b/agenta-web/src/components/SaveTestsetModal/SaveTestsetModal.tsx @@ -1,5 +1,4 @@ import React, {useEffect, useState} from "react" -import {useAppId} from "@/hooks/useAppId" import {Evaluation, EvaluationScenario} from "@/lib/Types" import {EvaluationFlow} from "@/lib/enums" import {createNewTestset} from "@/services/testsets/api" @@ -21,7 +20,6 @@ const SaveTestsetModal: React.FC = ({ onSuccess, ...props }) => { - const appId = useAppId() const [form] = Form.useForm() const [submitLoading, setSubmitLoading] = useState(false) @@ -46,7 +44,7 @@ const SaveTestsetModal: React.FC = ({ } }) - createNewTestset(appId, values.testset_name, newRows) + createNewTestset(values.testset_name, newRows) .then(() => onSuccess(values.testset_name)) .catch(console.error) .finally(() => { diff --git a/agenta-web/src/components/pages/testset/modals/CreateTestsetFromApi.tsx b/agenta-web/src/components/pages/testset/modals/CreateTestsetFromApi.tsx index cbdb670424..eff78031e9 100644 --- a/agenta-web/src/components/pages/testset/modals/CreateTestsetFromApi.tsx +++ b/agenta-web/src/components/pages/testset/modals/CreateTestsetFromApi.tsx @@ -75,7 +75,7 @@ const CreateTestsetFromApi: React.FC = ({setCurrent, onCancel, appId}) => const [selectedLang, setSelectedLang] = useState("python") const uploadURI = `${getAgentaApiUrl()}/api/testsets/upload` - const jsonURI = `${getAgentaApiUrl()}/api/testsets/${appId}` + const jsonURI = `${getAgentaApiUrl()}/api/testsets` const params = `{ "name": "testset_name",}` @@ -87,9 +87,9 @@ const CreateTestsetFromApi: React.FC = ({setCurrent, onCancel, appId}) => } const csvCodeSnippets: Record = { - python: pythonCodeUpload(uploadURI, appId), - bash: cURLCodeUpload(uploadURI, appId), - typescript: tsCodeUpload(uploadURI, appId), + python: pythonCodeUpload(uploadURI), + bash: cURLCodeUpload(uploadURI), + typescript: tsCodeUpload(uploadURI), } const codeSnippets = uploadType === "csv" ? csvCodeSnippets : jsonCodeSnippets diff --git a/agenta-web/src/components/pages/testset/modals/CreateTestsetFromScratch.tsx b/agenta-web/src/components/pages/testset/modals/CreateTestsetFromScratch.tsx index 6e9b6ae2f7..9e616c2be6 100644 --- a/agenta-web/src/components/pages/testset/modals/CreateTestsetFromScratch.tsx +++ b/agenta-web/src/components/pages/testset/modals/CreateTestsetFromScratch.tsx @@ -67,7 +67,7 @@ const CreateTestsetFromScratch: React.FC = ({ setIsLoading(true) try { const rowData = data || (await generateInitialRowData()) - const response = await createNewTestset(appId, testsetName, rowData) + const response = await createNewTestset(testsetName, rowData) message.success("Test set created successfully") router.push(`/apps/testsets/${response.data.id}`) } catch (error) { diff --git a/agenta-web/src/components/pages/testset/modals/UploadTestset.tsx b/agenta-web/src/components/pages/testset/modals/UploadTestset.tsx index 7bf64d557e..93f590275d 100644 --- a/agenta-web/src/components/pages/testset/modals/UploadTestset.tsx +++ b/agenta-web/src/components/pages/testset/modals/UploadTestset.tsx @@ -83,7 +83,6 @@ const UploadTestset: React.FC = ({setCurrent, onCancel, appId}) => { if (testsetName && testsetName.trim() !== "") { formData.append("testset_name", testsetName) } - formData.append("app_id", appId) try { setUploadLoading(true) diff --git a/agenta-web/src/services/testsets/api/index.ts b/agenta-web/src/services/testsets/api/index.ts index 87a8e92def..d0e9c2112f 100644 --- a/agenta-web/src/services/testsets/api/index.ts +++ b/agenta-web/src/services/testsets/api/index.ts @@ -30,8 +30,8 @@ export const fetchTestsets = async () => { return response.data } -export async function createNewTestset(appId: string, testsetName: string, testsetData: any) { - const response = await axios.post(`${getAgentaApiUrl()}/api/testsets/${appId}/`, { +export async function createNewTestset(testsetName: string, testsetData: any) { + const response = await axios.post(`${getAgentaApiUrl()}/api/testsets`, { name: testsetName, csvdata: testsetData, }) diff --git a/cookbook/evaluations_with_sdk.ipynb b/cookbook/evaluations_with_sdk.ipynb index 2a4b0f42ae..01566778e6 100644 --- a/cookbook/evaluations_with_sdk.ipynb +++ b/cookbook/evaluations_with_sdk.ipynb @@ -56,6 +56,7 @@ "\n", "# You can find the application ID in the URL. For example, in the URL https://cloud.agenta.ai/apps/666dde95962bbaffdb0072b5/playground?variant=app.default, the application ID is `666dde95962bbaffdb0072b5`.\n", "from agenta.client.backend.client import AgentaApi\n", + "\n", "# Let's list the applications\n", "client.apps.list_apps()" ] @@ -66,13 +67,12 @@ "metadata": {}, "outputs": [], "source": [ - "\n", "app_id = \"667d8cfad1812781f7e375d9\"\n", "\n", "# You can create the API key under the settings page. If you are using the OSS version, you should keep this as an empty string\n", "api_key = \"EUqJGOUu.xxxx\"\n", "\n", - "# Host. \n", + "# Host.\n", "host = \"https://cloud.agenta.ai\"\n", "\n", "# Initialize the client\n", @@ -96,21 +96,25 @@ "from agenta.client.backend.types.new_testset import NewTestset\n", "\n", "csvdata = [\n", - " {\"country\": \"france\", \"capital\": \"Paris\"},\n", - " {\"country\": \"Germany\", \"capital\": \"paris\"}\n", - " ]\n", + " {\"country\": \"france\", \"capital\": \"Paris\"},\n", + " {\"country\": \"Germany\", \"capital\": \"paris\"},\n", + "]\n", "\n", - "response = client.testsets.create_testset(app_id=app_id, request=NewTestset(name=\"test set\", csvdata=csvdata))\n", + "response = client.testsets.create_testset(\n", + " request=NewTestset(name=\"test set\", csvdata=csvdata)\n", + ")\n", "test_set_id = response.id\n", "\n", "# let's now update it\n", "\n", "csvdata = [\n", - " {\"country\": \"france\", \"capital\": \"Paris\"},\n", - " {\"country\": \"Germany\", \"capital\": \"Berlin\"}\n", - " ]\n", + " {\"country\": \"france\", \"capital\": \"Paris\"},\n", + " {\"country\": \"Germany\", \"capital\": \"Berlin\"},\n", + "]\n", "\n", - "client.testsets.update_testset(testset_id=test_set_id, request=NewTestset(name=\"test set\", csvdata=csvdata))" + "client.testsets.update_testset(\n", + " testset_id=test_set_id, request=NewTestset(name=\"test set\", csvdata=csvdata)\n", + ")" ] }, { @@ -128,7 +132,12 @@ "source": [ "# Create an evaluator that performs an exact match comparison on the 'capital' column\n", "# You can find the list of evaluator keys and evaluators and their configurations in https://github.com/Agenta-AI/agenta/blob/main/agenta-backend/agenta_backend/resources/evaluators/evaluators.py\n", - "response = client.evaluators.create_new_evaluator_config(app_id=app_id, name=\"capital_evaluator\", evaluator_key=\"auto_exact_match\", settings_values={\"correct_answer_key\": \"capital\"})\n", + "response = client.evaluators.create_new_evaluator_config(\n", + " app_id=app_id,\n", + " name=\"capital_evaluator\",\n", + " evaluator_key=\"auto_exact_match\",\n", + " settings_values={\"correct_answer_key\": \"capital\"},\n", + ")\n", "exact_match_eval_id = response.id\n", "\n", "code_snippet = \"\"\"\n", @@ -146,7 +155,12 @@ " return 0.0\n", "\"\"\"\n", "\n", - "response = client.evaluators.create_new_evaluator_config(app_id=app_id, name=\"capital_letter_evaluator\", evaluator_key=\"auto_custom_code_run\", settings_values={\"code\": code_snippet})\n", + "response = client.evaluators.create_new_evaluator_config(\n", + " app_id=app_id,\n", + " name=\"capital_letter_evaluator\",\n", + " evaluator_key=\"auto_custom_code_run\",\n", + " settings_values={\"code\": code_snippet},\n", + ")\n", "letter_match_eval_id = response.id" ] }, @@ -186,13 +200,19 @@ "source": [ "# Run an evaluation\n", "from agenta.client.backend.types.llm_run_rate_limit import LlmRunRateLimit\n", - "response = client.evaluations.create_evaluation(app_id=app_id, variant_ids=[myvariant_id], testset_id=test_set_id, evaluators_configs=[exact_match_eval_id, letter_match_eval_id],\n", - " rate_limit=LlmRunRateLimit(\n", - " batch_size=10, # number of rows to call in parallel\n", - " max_retries=3, # max number of time to retry a failed llm call\n", - " retry_delay=2, # delay before retrying a failed llm call\n", - " delay_between_batches=5, # delay between batches\n", - " ),)\n", + "\n", + "response = client.evaluations.create_evaluation(\n", + " app_id=app_id,\n", + " variant_ids=[myvariant_id],\n", + " testset_id=test_set_id,\n", + " evaluators_configs=[exact_match_eval_id, letter_match_eval_id],\n", + " rate_limit=LlmRunRateLimit(\n", + " batch_size=10, # number of rows to call in parallel\n", + " max_retries=3, # max number of time to retry a failed llm call\n", + " retry_delay=2, # delay before retrying a failed llm call\n", + " delay_between_batches=5, # delay between batches\n", + " ),\n", + ")\n", "print(response)" ] }, @@ -203,7 +223,7 @@ "outputs": [], "source": [ "# check the status\n", - "client.evaluations.fetch_evaluation_status('667d98fbd1812781f7e3761a')" + "client.evaluations.fetch_evaluation_status(\"667d98fbd1812781f7e3761a\")" ] }, { @@ -213,9 +233,12 @@ "outputs": [], "source": [ "# fetch the overall results\n", - "response = client.evaluations.fetch_evaluation_results('667d98fbd1812781f7e3761a')\n", + "response = client.evaluations.fetch_evaluation_results(\"667d98fbd1812781f7e3761a\")\n", "\n", - "results = [(evaluator[\"evaluator_config\"][\"name\"], evaluator[\"result\"]) for evaluator in response[\"results\"]]\n", + "results = [\n", + " (evaluator[\"evaluator_config\"][\"name\"], evaluator[\"result\"])\n", + " for evaluator in response[\"results\"]\n", + "]\n", "# End of Selection" ] }, @@ -226,7 +249,9 @@ "outputs": [], "source": [ "# fetch the detailed results\n", - "client.evaluations.fetch_evaluation_scenarios(evaluations_ids='667d98fbd1812781f7e3761a')" + "client.evaluations.fetch_evaluation_scenarios(\n", + " evaluations_ids=\"667d98fbd1812781f7e3761a\"\n", + ")" ] } ], diff --git a/docs/docs/evaluation/02-create-test-sets.mdx b/docs/docs/evaluation/02-create-test-sets.mdx index 315d4fd3a3..31a90cdcc4 100644 --- a/docs/docs/evaluation/02-create-test-sets.mdx +++ b/docs/docs/evaluation/02-create-test-sets.mdx @@ -85,7 +85,7 @@ Here's an example of such a call: **HTTP Request:** ``` -POST /testsets/{app_id}/ +POST /testsets ``` diff --git a/docs/docs/evaluation/05-sdk-evaluation.mdx b/docs/docs/evaluation/05-sdk-evaluation.mdx index de8acd5174..c8c88886f9 100644 --- a/docs/docs/evaluation/05-sdk-evaluation.mdx +++ b/docs/docs/evaluation/05-sdk-evaluation.mdx @@ -65,7 +65,7 @@ csvdata = [ {"country": "Germany", "capital": "Berlin"} ] -response = client.testsets.create_testset(app_id=app_id, request=NewTestset(name="test set", csvdata=csvdata)) +response = client.testsets.create_testset(request=NewTestset(name="test set", csvdata=csvdata)) test_set_id = response.id ``` diff --git a/docs/docs/reference/api/create-testset.api.mdx b/docs/docs/reference/api/create-testset.api.mdx index bd7dc5d6ac..0b1212d93f 100644 --- a/docs/docs/reference/api/create-testset.api.mdx +++ b/docs/docs/reference/api/create-testset.api.mdx @@ -1,7 +1,7 @@ --- id: create-testset title: "Create Testset" -description: "Create a testset with given name and app_name, save the testset to MongoDB." +description: "Create a testset with given name, save the testset to MongoDB." sidebar_label: "Create Testset" hide_title: true hide_table_of_contents: true @@ -34,18 +34,17 @@ import TabItem from "@theme/TabItem"; -Create a testset with given name and app_name, save the testset to MongoDB. +Create a testset with given name save the testset to MongoDB. Args: name (str): name of the test set. -app_name (str): name of the application. testset (Dict[str, str]): test set data. Returns: diff --git a/docs/docs/reference/openapi.json b/docs/docs/reference/openapi.json index 300bd62509..cfdb6ce1ce 100644 --- a/docs/docs/reference/openapi.json +++ b/docs/docs/reference/openapi.json @@ -3462,43 +3462,31 @@ } } }, - "/testsets/{app_id}": { - "post": { + "/testsets/{testset_id}": { + "get": { "tags": [ "Testsets" ], - "summary": "Create Testset", - "description": "Create a testset with given name and app_name, save the testset to MongoDB.\n\nArgs:\nname (str): name of the test set.\napp_name (str): name of the application.\ntestset (Dict[str, str]): test set data.\n\nReturns:\nstr: The id of the test set created.", - "operationId": "create_testset", + "summary": "Get Single Testset", + "description": "Fetch a specific testset in a MongoDB collection using its _id.\n\nArgs:\n testset_id (str): The _id of the testset to fetch.\n\nReturns:\n The requested testset if found, else an HTTPException.", + "operationId": "get_single_testset", "parameters": [ { "required": true, "schema": { "type": "string", - "title": "App Id" + "title": "Testset Id" }, - "name": "app_id", + "name": "testset_id", "in": "path" } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/NewTestset" - } - } - }, - "required": true - }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { - "schema": { - "$ref": "#/components/schemas/TestSetSimpleResponse" - } + "schema": {} } } }, @@ -3513,16 +3501,14 @@ } } } - } - }, - "/testsets/{testset_id}": { - "get": { + }, + "put": { "tags": [ "Testsets" ], - "summary": "Get Single Testset", - "description": "Fetch a specific testset in a MongoDB collection using its _id.\n\nArgs:\n testset_id (str): The _id of the testset to fetch.\n\nReturns:\n The requested testset if found, else an HTTPException.", - "operationId": "get_single_testset", + "summary": "Update Testset", + "description": "Update a testset with given id, update the testset in MongoDB.\n\nArgs:\ntestset_id (str): id of the test set to be updated.\ncsvdata (NewTestset): New data to replace the old testset.\n\nReturns:\nstr: The id of the test set updated.", + "operationId": "update_testset", "parameters": [ { "required": true, @@ -3534,6 +3520,16 @@ "in": "path" } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewTestset" + } + } + }, + "required": true + }, "responses": { "200": { "description": "Successful Response", @@ -3554,25 +3550,17 @@ } } } - }, - "put": { + } + }, + "/testsets": { + "post": { "tags": [ "Testsets" ], - "summary": "Update Testset", - "description": "Update a testset with given id, update the testset in MongoDB.\n\nArgs:\ntestset_id (str): id of the test set to be updated.\ncsvdata (NewTestset): New data to replace the old testset.\n\nReturns:\nstr: The id of the test set updated.", - "operationId": "update_testset", - "parameters": [ - { - "required": true, - "schema": { - "type": "string", - "title": "Testset Id" - }, - "name": "testset_id", - "in": "path" - } - ], + "summary": "Create Testset", + "description": "Create a testset with given name, save the testset to MongoDB.\n\nArgs:\nname (str): name of the test set.\ntestset (Dict[str, str]): test set data.\n\nReturns:\nstr: The id of the test set created.", + "operationId": "create_testset", + "parameters": [], "requestBody": { "content": { "application/json": { @@ -3588,7 +3576,9 @@ "description": "Successful Response", "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/TestSetSimpleResponse" + } } } }, @@ -3603,9 +3593,7 @@ } } } - } - }, - "/testsets": { + }, "get": { "tags": [ "Testsets" @@ -3613,17 +3601,7 @@ "summary": "Get Testsets", "description": "Get all testsets.\n\nReturns:\n- A list of testset objects.\n\nRaises:\n- `HTTPException` with status code 404 if no testsets are found.", "operationId": "get_testsets", - "parameters": [ - { - "required": true, - "schema": { - "type": "string", - "title": "App Id" - }, - "name": "app_id", - "in": "query" - } - ], + "parameters": [], "responses": { "200": { "description": "Successful Response", @@ -4405,10 +4383,6 @@ "testset_name": { "type": "string", "title": "Testset Name" - }, - "app_id": { - "type": "string", - "title": "App Id" } }, "type": "object", diff --git a/docs/docs/tutorials/sdk/evaluate-with-SDK.mdx b/docs/docs/tutorials/sdk/evaluate-with-SDK.mdx index a5827f3773..e2b0e918a8 100644 --- a/docs/docs/tutorials/sdk/evaluate-with-SDK.mdx +++ b/docs/docs/tutorials/sdk/evaluate-with-SDK.mdx @@ -67,7 +67,7 @@ csvdata = [ {"country": "Germany", "capital": "paris"} ] -response = client.testsets.create_testset(app_id=app_id, request=NewTestset(name="test set", csvdata=csvdata)) +response = client.testsets.create_testset(request=NewTestset(name="test set", csvdata=csvdata)) test_set_id = response.id # let's now update it diff --git a/response.txt b/response.txt index 6fe67f331e..58592e9a71 100644 --- a/response.txt +++ b/response.txt @@ -1 +1 @@ -{"openapi":"3.0.2","info":{"title":"FastAPI","version":"0.1.0"},"paths":{"/api/profile/":{"get":{"summary":"User Profile","operationId":"user_profile_api_profile__get","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}}}}},"/api/apps/{app_id}/variants/":{"get":{"summary":"List App Variants","description":"Retrieve a list of app variants for a given app ID.\n\nArgs:\n app_id (str): The ID of the app to retrieve variants for.\n stoken_session (SessionContainer, optional): The session container to verify the user's session. Defaults to Depends(verify_session()).\n\nReturns:\n List[AppVariantOutput]: A list of app variants for the given app ID.","operationId":"list_app_variants_api_apps__app_id__variants__get","parameters":[{"required":true,"schema":{"title":"App Id","type":"string"},"name":"app_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response List App Variants Api Apps App Id Variants Get","type":"array","items":{"$ref":"#/components/schemas/AppVariantOutput"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/apps/get_variant_by_env/":{"get":{"summary":"Get Variant By Env","description":"Retrieve the app variant based on the provided app_id and environment.\n\nArgs:\n app_id (str): The ID of the app to retrieve the variant for.\n environment (str): The environment of the app variant to retrieve.\n stoken_session (SessionContainer, optional): The session token container. Defaults to Depends(verify_session()).\n\nRaises:\n HTTPException: If the app variant is not found (status_code=500), or if a ValueError is raised (status_code=400), or if any other exception is raised (status_code=500).\n\nReturns:\n AppVariantOutput: The retrieved app variant.","operationId":"get_variant_by_env_api_apps_get_variant_by_env__get","parameters":[{"required":true,"schema":{"title":"App Id","type":"string"},"name":"app_id","in":"query"},{"required":true,"schema":{"title":"Environment","type":"string"},"name":"environment","in":"query"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AppVariantOutput"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/apps/":{"get":{"summary":"List Apps","description":"Retrieve a list of apps filtered by app_name and org_id.\n\nArgs:\n app_name (Optional[str]): The name of the app to filter by.\n org_id (Optional[str]): The ID of the organization to filter by.\n stoken_session (SessionContainer): The session container.\n\nReturns:\n List[App]: A list of apps filtered by app_name and org_id.\n\nRaises:\n HTTPException: If there was an error retrieving the list of apps.","operationId":"list_apps_api_apps__get","parameters":[{"required":false,"schema":{"title":"App Name","type":"string"},"name":"app_name","in":"query"},{"required":false,"schema":{"title":"Org Id","type":"string"},"name":"org_id","in":"query"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response List Apps Api Apps Get","type":"array","items":{"$ref":"#/components/schemas/App"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}},"post":{"summary":"Create App","description":"Create a new app for a user or organization.\n\nArgs:\n payload (CreateApp): The payload containing the app name and organization ID (optional).\n stoken_session (SessionContainer): The session container containing the user's session token.\n\nReturns:\n CreateAppOutput: The output containing the newly created app's ID and name.\n\nRaises:\n HTTPException: If there is an error creating the app or the user does not have permission to access the app.","operationId":"create_app_api_apps__post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateApp"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateAppOutput"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/apps/{app_id}/variant/from-image/":{"post":{"summary":"Add Variant From Image","description":"Add a new variant to an app based on a Docker image.\n\nArgs:\n app_id (str): The ID of the app to add the variant to.\n payload (AddVariantFromImagePayload): The payload containing information about the variant to add.\n stoken_session (SessionContainer, optional): The session container. Defaults to Depends(verify_session()).\n\nRaises:\n HTTPException: If the feature flag is set to \"demo\" or if the image does not have a tag starting with the registry name (agenta-server) or if the image is not found or if the user does not have access to the app.\n\nReturns:\n dict: The newly added variant.","operationId":"add_variant_from_image_api_apps__app_id__variant_from_image__post","parameters":[{"required":true,"schema":{"title":"App Id","type":"string"},"name":"app_id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/AddVariantFromImagePayload"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/apps/{app_id}/":{"delete":{"summary":"Remove App","description":"Remove app, all its variant, containers and images\n\nArguments:\n app -- App to remove","operationId":"remove_app_api_apps__app_id___delete","parameters":[{"required":true,"schema":{"title":"App Id","type":"string"},"name":"app_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/apps/app_and_variant_from_template/":{"post":{"summary":"Create App And Variant From Template","description":"Create an app and variant from a template.\n\nArgs:\n payload (CreateAppVariant): The payload containing the app and variant information.\n stoken_session (SessionContainer, optional): The session container. Defaults to Depends(verify_session()).\n\nRaises:\n HTTPException: If the user has reached the app limit or if an app with the same name already exists.\n\nReturns:\n AppVariantOutput: The output of the created app variant.","operationId":"create_app_and_variant_from_template_api_apps_app_and_variant_from_template__post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateAppVariant"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AppVariantOutput"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/apps/{app_id}/environments/":{"get":{"summary":"List Environments","description":"Retrieve a list of environments for a given app ID.\n\nArgs:\n app_id (str): The ID of the app to retrieve environments for.\n stoken_session (SessionContainer, optional): The session container. Defaults to Depends(verify_session()).\n\nReturns:\n List[EnvironmentOutput]: A list of environment objects.","operationId":"list_environments_api_apps__app_id__environments__get","parameters":[{"required":true,"schema":{"title":"App Id","type":"string"},"name":"app_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response List Environments Api Apps App Id Environments Get","type":"array","items":{"$ref":"#/components/schemas/EnvironmentOutput"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/variants/from-base/":{"post":{"summary":"Add Variant From Base And Config","description":"Add a new variant based on an existing one.\nSame as POST /config\n\nArgs:\n payload (AddVariantFromBasePayload): Payload containing base variant ID, new variant name, and parameters.\n stoken_session (SessionContainer, optional): Session container. Defaults to result of verify_session().\n\nRaises:\n HTTPException: Raised if the variant could not be added or accessed.\n\nReturns:\n Union[AppVariantOutput, Any]: New variant details or exception.","operationId":"add_variant_from_base_and_config_api_variants_from_base__post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/AddVariantFromBasePayload"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Add Variant From Base And Config Api Variants From Base Post","anyOf":[{"$ref":"#/components/schemas/AppVariantOutput"},{}]}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/variants/{variant_id}/":{"put":{"summary":"Start Variant","description":"Start a variant of an app.\n\nArgs:\n variant_id (str): The ID of the variant to start.\n action (VariantAction): The action to perform on the variant (start).\n env_vars (Optional[DockerEnvVars], optional): The environment variables to inject to the Docker container. Defaults to None.\n stoken_session (SessionContainer, optional): The session container. Defaults to Depends(verify_session()).\n\nReturns:\n URI: The URL of the started variant.\n\nRaises:\n HTTPException: If the app container cannot be started.","operationId":"start_variant_api_variants__variant_id___put","parameters":[{"required":true,"schema":{"title":"Variant Id","type":"string"},"name":"variant_id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Body_start_variant_api_variants__variant_id___put"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/URI"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}},"delete":{"summary":"Remove Variant","description":"Remove a variant from the server.\nIn the case it's the last variant using the image, stop the container and remove the image.\n\nArguments:\n app_variant -- AppVariant to remove\n\nRaises:\n HTTPException: If there is a problem removing the app variant","operationId":"remove_variant_api_variants__variant_id___delete","parameters":[{"required":true,"schema":{"title":"Variant Id","type":"string"},"name":"variant_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/variants/{variant_id}/parameters/":{"put":{"summary":"Update Variant Parameters","description":"Updates the parameters for an app variant.\n\nArgs:\n variant_id (str): The ID of the app variant to update.\n payload (UpdateVariantParameterPayload): The payload containing the updated parameters.\n stoken_session (SessionContainer, optional): The session container. Defaults to Depends(verify_session()).\n\nRaises:\n HTTPException: If there is an error while trying to update the app variant.\n\nReturns:\n JSONResponse: A JSON response containing the updated app variant parameters.","operationId":"update_variant_parameters_api_variants__variant_id__parameters__put","parameters":[{"required":true,"schema":{"title":"Variant Id","type":"string"},"name":"variant_id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateVariantParameterPayload"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/variants/{variant_id}/image/":{"put":{"summary":"Update Variant Image","description":"Updates the image used in an app variant.\n\nArgs:\n variant_id (str): The ID of the app variant to update.\n image (Image): The image information to update.\n\nRaises:\n HTTPException: If an error occurs while trying to update the app variant.\n\nReturns:\n JSONResponse: A JSON response indicating whether the update was successful or not.","operationId":"update_variant_image_api_variants__variant_id__image__put","parameters":[{"required":true,"schema":{"title":"Variant Id","type":"string"},"name":"variant_id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Image"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/":{"get":{"summary":"Fetch List Evaluations","description":"Fetches a list of evaluations, optionally filtered by an app ID.\n\nArgs:\n app_id (Optional[str]): An optional app ID to filter the evaluations.\n\nReturns:\n List[Evaluation]: A list of evaluations.","operationId":"fetch_list_evaluations_api_evaluations__get","parameters":[{"required":true,"schema":{"title":"App Id","type":"string"},"name":"app_id","in":"query"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Fetch List Evaluations Api Evaluations Get","type":"array","items":{"$ref":"#/components/schemas/Evaluation"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}},"post":{"summary":"Create Evaluation","description":"Creates a new comparison table document\nRaises:\n HTTPException: _description_\nReturns:\n _description_","operationId":"create_evaluation_api_evaluations__post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/NewEvaluation"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SimpleEvaluationOutput"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}},"delete":{"summary":"Delete Evaluations","description":"Delete specific comparison tables based on their unique IDs.\n\nArgs:\ndelete_evaluations (List[str]): The unique identifiers of the comparison tables to delete.\n\nReturns:\nA list of the deleted comparison tables' IDs.","operationId":"delete_evaluations_api_evaluations__delete","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/DeleteEvaluation"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Delete Evaluations Api Evaluations Delete","type":"array","items":{"type":"string"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/{evaluation_id}/":{"get":{"summary":"Fetch Evaluation","description":"Fetches a single evaluation based on its ID.\n\nArgs:\n evaluation_id (str): The ID of the evaluation to fetch.\n\nReturns:\n Evaluation: The fetched evaluation.","operationId":"fetch_evaluation_api_evaluations__evaluation_id___get","parameters":[{"required":true,"schema":{"title":"Evaluation Id","type":"string"},"name":"evaluation_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Evaluation"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}},"put":{"summary":"Update Evaluation Router","description":"Updates an evaluation's status.\n\nRaises:\n HTTPException: If the columns in the test set do not match with the inputs in the variant.\n\nReturns:\n None: A 204 No Content status code, indicating that the update was successful.","operationId":"update_evaluation_router_api_evaluations__evaluation_id___put","parameters":[{"required":true,"schema":{"title":"Evaluation Id","type":"string"},"name":"evaluation_id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/EvaluationUpdate"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/{evaluation_id}/evaluation_scenarios/":{"get":{"summary":"Fetch Evaluation Scenarios","description":"Fetches evaluation scenarios for a given evaluation ID.\n\nArguments:\n evaluation_id (str): The ID of the evaluation for which to fetch scenarios.\n\nRaises:\n HTTPException: If the evaluation is not found or access is denied.\n\nReturns:\n List[EvaluationScenario]: A list of evaluation scenarios.","operationId":"fetch_evaluation_scenarios_api_evaluations__evaluation_id__evaluation_scenarios__get","parameters":[{"required":true,"schema":{"title":"Evaluation Id","type":"string"},"name":"evaluation_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Fetch Evaluation Scenarios Api Evaluations Evaluation Id Evaluation Scenarios Get","type":"array","items":{"$ref":"#/components/schemas/EvaluationScenario"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/{evaluation_id}/evaluation_scenario/":{"post":{"summary":"Create Evaluation Scenario","description":"Create a new evaluation scenario for a given evaluation ID.\n\nRaises:\n HTTPException: If evaluation not found or access denied.\n\nReturns:\n None: 204 No Content status code upon success.","operationId":"create_evaluation_scenario_api_evaluations__evaluation_id__evaluation_scenario__post","parameters":[{"required":true,"schema":{"title":"Evaluation Id","type":"string"},"name":"evaluation_id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/EvaluationScenario"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/{evaluation_id}/evaluation_scenario/{evaluation_scenario_id}/{evaluation_type}/":{"put":{"summary":"Update Evaluation Scenario Router","description":"Updates an evaluation scenario's vote or score based on its type.\n\nRaises:\n HTTPException: If update fails or unauthorized.\n\nReturns:\n None: 204 No Content status code upon successful update.","operationId":"update_evaluation_scenario_router_api_evaluations__evaluation_id__evaluation_scenario__evaluation_scenario_id___evaluation_type___put","parameters":[{"required":true,"schema":{"title":"Evaluation Scenario Id","type":"string"},"name":"evaluation_scenario_id","in":"path"},{"required":true,"schema":{"$ref":"#/components/schemas/EvaluationType"},"name":"evaluation_type","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/EvaluationScenarioUpdate"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/evaluation_scenario/ai_critique/":{"post":{"summary":"Evaluate Ai Critique","description":"Evaluate AI critique based on the given payload.\n\nArgs:\n payload (AICritiqueCreate): The payload containing data for AI critique evaluation.\n stoken_session (SessionContainer): The session container verified by `verify_session`.\n\nReturns:\n str: The output of the AI critique evaluation.\n\nRaises:\n HTTPException: If any exception occurs during the evaluation.","operationId":"evaluate_ai_critique_api_evaluations_evaluation_scenario_ai_critique__post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/AICritiqueCreate"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Evaluate Ai Critique Api Evaluations Evaluation Scenario Ai Critique Post","type":"string"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/evaluation_scenario/{evaluation_scenario_id}/score/":{"get":{"summary":"Get Evaluation Scenario Score Router","description":"Fetch the score of a specific evaluation scenario.\n\nArgs:\n evaluation_scenario_id: The ID of the evaluation scenario to fetch.\n stoken_session: Session data, verified by `verify_session`.\n\nReturns:\n Dictionary containing the scenario ID and its score.","operationId":"get_evaluation_scenario_score_router_api_evaluations_evaluation_scenario__evaluation_scenario_id__score__get","parameters":[{"required":true,"schema":{"title":"Evaluation Scenario Id","type":"string"},"name":"evaluation_scenario_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Get Evaluation Scenario Score Router Api Evaluations Evaluation Scenario Evaluation Scenario Id Score Get","type":"object","additionalProperties":{"type":"string"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}},"put":{"summary":"Update Evaluation Scenario Score Router","description":"Updates the score of an evaluation scenario.\n\nRaises:\n HTTPException: Server error if the evaluation update fails.\n\nReturns:\n None: 204 No Content status code upon successful update.","operationId":"update_evaluation_scenario_score_router_api_evaluations_evaluation_scenario__evaluation_scenario_id__score__put","parameters":[{"required":true,"schema":{"title":"Evaluation Scenario Id","type":"string"},"name":"evaluation_scenario_id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/EvaluationScenarioScoreUpdate"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/{evaluation_id}/results/":{"get":{"summary":"Fetch Results","description":"Fetch all the results for one the comparison table\n\nArguments:\n evaluation_id -- _description_\n\nReturns:\n _description_","operationId":"fetch_results_api_evaluations__evaluation_id__results__get","parameters":[{"required":true,"schema":{"title":"Evaluation Id","type":"string"},"name":"evaluation_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/custom_evaluation/":{"post":{"summary":"Create Custom Evaluation","description":"Create evaluation with custom python code.\n\n Args:\n \ncustom_evaluation_payload (CreateCustomEvaluation): the required payload","operationId":"create_custom_evaluation_api_evaluations_custom_evaluation__post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateCustomEvaluation"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/custom_evaluation/{id}":{"put":{"summary":"Update Custom Evaluation","description":"Update a custom code evaluation.\nArgs:\n id (str): the ID of the custom evaluation to update\n updated_data (CreateCustomEvaluation): the payload with updated data\n stoken_session (SessionContainer): session container for authentication","operationId":"update_custom_evaluation_api_evaluations_custom_evaluation__id__put","parameters":[{"required":true,"schema":{"title":"Id","type":"string"},"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateCustomEvaluation"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/custom_evaluation/list/{app_id}/":{"get":{"summary":"List Custom Evaluations","description":"List the custom code evaluations for a given app.\n\nArgs:\n app_id (str): the id of the app\n\nReturns:\n List[CustomEvaluationOutput]: a list of custom evaluation","operationId":"list_custom_evaluations_api_evaluations_custom_evaluation_list__app_id___get","parameters":[{"required":true,"schema":{"title":"App Id","type":"string"},"name":"app_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response List Custom Evaluations Api Evaluations Custom Evaluation List App Id Get","type":"array","items":{"$ref":"#/components/schemas/CustomEvaluationOutput"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/custom_evaluation/{id}/":{"get":{"summary":"Get Custom Evaluation","description":"Get the custom code evaluation detail.\n\nArgs:\n id (str): the id of the custom evaluation\n\nReturns:\n CustomEvaluationDetail: Detail of the custom evaluation","operationId":"get_custom_evaluation_api_evaluations_custom_evaluation__id___get","parameters":[{"required":true,"schema":{"title":"Id","type":"string"},"name":"id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CustomEvaluationDetail"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/custom_evaluation/{app_name}/names/":{"get":{"summary":"Get Custom Evaluation Names","description":"Get the names of custom evaluation for a given app.\n\nArgs:\n app_name (str): the name of the app the evaluation belongs to\n\nReturns:\n List[CustomEvaluationNames]: the list of name of custom evaluations","operationId":"get_custom_evaluation_names_api_evaluations_custom_evaluation__app_name__names__get","parameters":[{"required":true,"schema":{"title":"App Name","type":"string"},"name":"app_name","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Get Custom Evaluation Names Api Evaluations Custom Evaluation App Name Names Get","type":"array","items":{"$ref":"#/components/schemas/CustomEvaluationNames"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/custom_evaluation/execute/{evaluation_id}/":{"post":{"summary":"Execute Custom Evaluation","description":"Execute a custom evaluation code.\n\nArgs:\n evaluation_id (str): the custom evaluation id\n payload (ExecuteCustomEvaluationCode): the required payload\n\nReturns:\n float: the result of the evaluation custom code","operationId":"execute_custom_evaluation_api_evaluations_custom_evaluation_execute__evaluation_id___post","parameters":[{"required":true,"schema":{"title":"Evaluation Id","type":"string"},"name":"evaluation_id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExecuteCustomEvaluationCode"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/webhook_example_fake/":{"post":{"summary":"Webhook Example Fake","description":"Returns a fake score response for example webhook evaluation\n\nReturns:\n _description_","operationId":"webhook_example_fake_api_evaluations_webhook_example_fake__post","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/EvaluationWebhook"}}}}}}},"/api/testsets/upload/":{"post":{"summary":"Upload File","description":"Uploads a CSV or JSON file and saves its data to MongoDB.\n\nArgs:\nupload_type : Either a json or csv file.\n file (UploadFile): The CSV or JSON file to upload.\n testset_name (Optional): the name of the testset if provided.\n\nReturns:\n dict: The result of the upload process.","operationId":"upload_file_api_testsets_upload__post","requestBody":{"content":{"multipart/form-data":{"schema":{"$ref":"#/components/schemas/Body_upload_file_api_testsets_upload__post"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TestSetSimpleResponse"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/testsets/endpoint/":{"post":{"summary":"Import Testset","description":"Import JSON testset data from an endpoint and save it to MongoDB.\n\nArgs:\n endpoint (str): An endpoint URL to import data from.\n testset_name (str): the name of the testset if provided.\n\nReturns:\n dict: The result of the import process.","operationId":"import_testset_api_testsets_endpoint__post","requestBody":{"content":{"application/x-www-form-urlencoded":{"schema":{"$ref":"#/components/schemas/Body_import_testset_api_testsets_endpoint__post"}}}},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TestSetSimpleResponse"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/testsets/{app_id}/":{"post":{"summary":"Create Testset","description":"Create a testset with given name and app_name, save the testset to MongoDB.\n\nArgs:\nname (str): name of the test set.\napp_name (str): name of the application.\ntestset (Dict[str, str]): test set data.\n\nReturns:\nstr: The id of the test set created.","operationId":"create_testset_api_testsets__app_id___post","parameters":[{"required":true,"schema":{"title":"App Id","type":"string"},"name":"app_id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/NewTestset"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/testsets/{testset_id}/":{"get":{"tags":["testsets"],"summary":"Get Testset","description":"Fetch a specific testset in a MongoDB collection using its _id.\n\nArgs:\n testset_id (str): The _id of the testset to fetch.\n\nReturns:\n The requested testset if found, else an HTTPException.","operationId":"get_testset_api_testsets__testset_id___get","parameters":[{"required":true,"schema":{"title":"Testset Id","type":"string"},"name":"testset_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}},"put":{"summary":"Update Testset","description":"Update a testset with given id, update the testset in MongoDB.\n\nArgs:\ntestset_id (str): id of the test set to be updated.\ncsvdata (NewTestset): New data to replace the old testset.\n\nReturns:\nstr: The id of the test set updated.","operationId":"update_testset_api_testsets__testset_id___put","parameters":[{"required":true,"schema":{"title":"Testset Id","type":"string"},"name":"testset_id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/NewTestset"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/testsets/":{"get":{"tags":["testsets"],"summary":"Get Testsets","description":"Get all testsets.\n\nReturns:\n- A list of testset objects.\n\nRaises:\n- `HTTPException` with status code 404 if no testsets are found.","operationId":"get_testsets_api_testsets__get","parameters":[{"required":true,"schema":{"title":"App Id","type":"string"},"name":"app_id","in":"query"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Get Testsets Api Testsets Get","type":"array","items":{"$ref":"#/components/schemas/TestSetOutputResponse"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}},"delete":{"summary":"Delete Testsets","description":"Delete specific testsets based on their unique IDs.\n\nArgs:\ntestset_ids (List[str]): The unique identifiers of the testsets to delete.\n\nReturns:\nA list of the deleted testsets' IDs.","operationId":"delete_testsets_api_testsets__delete","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/DeleteTestsets"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Delete Testsets Api Testsets Delete","type":"array","items":{"type":"string"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/containers/build_image/":{"post":{"summary":"Build Image","description":"Builds a Docker image from a tar file containing the application code.\n\nArgs:\n app_id (str): The ID of the application to build the image for.\n base_name (str): The base name of the image to build.\n tar_file (UploadFile): The tar file containing the application code.\n stoken_session (SessionContainer): The session container for the user making the request.\n\nReturns:\n Image: The Docker image that was built.","operationId":"build_image_api_containers_build_image__post","parameters":[{"required":true,"schema":{"title":"App Id","type":"string"},"name":"app_id","in":"query"},{"required":true,"schema":{"title":"Base Name","type":"string"},"name":"base_name","in":"query"}],"requestBody":{"content":{"multipart/form-data":{"schema":{"$ref":"#/components/schemas/Body_build_image_api_containers_build_image__post"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Image"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/containers/restart_container/":{"post":{"summary":"Restart Docker Container","description":"Restart docker container.\n\nArgs:\n payload (RestartAppContainer) -- the required data (app_name and variant_name)","operationId":"restart_docker_container_api_containers_restart_container__post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/RestartAppContainer"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Restart Docker Container Api Containers Restart Container Post","type":"object"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/containers/templates/":{"get":{"summary":"Container Templates","description":"Returns a list of templates available for creating new containers.\n\nParameters:\nstoken_session (SessionContainer): The session container for the user.\n\nReturns:\n\nUnion[List[Template], str]: A list of templates or an error message.","operationId":"container_templates_api_containers_templates__get","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Container Templates Api Containers Templates Get","anyOf":[{"type":"array","items":{"$ref":"#/components/schemas/Template"}},{"type":"string"}]}}}}}}},"/api/containers/container_url/":{"get":{"summary":"Construct App Container Url","description":"Constructs the URL for an app container based on the provided base_id or variant_id.\n\nArgs:\n base_id (Optional[str]): The ID of the base to use for the app container.\n variant_id (Optional[str]): The ID of the variant to use for the app container.\n stoken_session (SessionContainer): The session container for the user.\n\nReturns:\n URI: The URI for the app container.\n\nRaises:\n HTTPException: If the base or variant cannot be found or the user does not have access.","operationId":"construct_app_container_url_api_containers_container_url__get","parameters":[{"required":false,"schema":{"title":"Base Id","type":"string"},"name":"base_id","in":"query"},{"required":false,"schema":{"title":"Variant Id","type":"string"},"name":"variant_id","in":"query"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/URI"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/environments/deploy/":{"post":{"summary":"Deploy To Environment","description":"Deploys a given variant to an environment\n\nArgs:\n environment_name: Name of the environment to deploy to.\n variant_id: variant id to deploy.\n stoken_session: . Defaults to Depends(verify_session()).\n\nRaises:\n HTTPException: If the deployment fails.","operationId":"deploy_to_environment_api_environments_deploy__post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/DeployToEnvironmentPayload"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/observability/traces/":{"post":{"summary":"Create Trace","operationId":"create_trace_api_observability_traces__post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateTrace"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Create Trace Api Observability Traces Post","type":"string"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/observability/traces/{app_id}/{variant_id}/":{"get":{"summary":"Get Traces","operationId":"get_traces_api_observability_traces__app_id___variant_id___get","parameters":[{"required":true,"schema":{"title":"App Id","type":"string"},"name":"app_id","in":"path"},{"required":true,"schema":{"title":"Variant Id","type":"string"},"name":"variant_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Get Traces Api Observability Traces App Id Variant Id Get","type":"array","items":{"$ref":"#/components/schemas/Trace"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/observability/traces/{trace_id}/":{"get":{"summary":"Get Trace","operationId":"get_trace_api_observability_traces__trace_id___get","parameters":[{"required":true,"schema":{"title":"Trace Id","type":"string"},"name":"trace_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Trace"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}},"put":{"summary":"Update Trace Status","operationId":"update_trace_status_api_observability_traces__trace_id___put","parameters":[{"required":true,"schema":{"title":"Trace Id","type":"string"},"name":"trace_id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateTrace"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Update Trace Status Api Observability Traces Trace Id Put","type":"boolean"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/observability/spans/":{"post":{"summary":"Create Span","operationId":"create_span_api_observability_spans__post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateSpan"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Create Span Api Observability Spans Post","type":"string"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/observability/spans/{trace_id}/":{"get":{"summary":"Get Spans Of Trace","operationId":"get_spans_of_trace_api_observability_spans__trace_id___get","parameters":[{"required":true,"schema":{"title":"Trace Id","type":"string"},"name":"trace_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Get Spans Of Trace Api Observability Spans Trace Id Get","type":"array","items":{"$ref":"#/components/schemas/Span"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/observability/feedbacks/{trace_id}/":{"get":{"summary":"Get Feedbacks","operationId":"get_feedbacks_api_observability_feedbacks__trace_id___get","parameters":[{"required":true,"schema":{"title":"Trace Id","type":"string"},"name":"trace_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Get Feedbacks Api Observability Feedbacks Trace Id Get","type":"array","items":{"$ref":"#/components/schemas/Feedback"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}},"post":{"summary":"Create Feedback","operationId":"create_feedback_api_observability_feedbacks__trace_id___post","parameters":[{"required":true,"schema":{"title":"Trace Id","type":"string"},"name":"trace_id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateFeedback"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Create Feedback Api Observability Feedbacks Trace Id Post","type":"string"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/observability/feedbacks/{trace_id}/{feedback_id}/":{"get":{"summary":"Get Feedback","operationId":"get_feedback_api_observability_feedbacks__trace_id___feedback_id___get","parameters":[{"required":true,"schema":{"title":"Trace Id","type":"string"},"name":"trace_id","in":"path"},{"required":true,"schema":{"title":"Feedback Id","type":"string"},"name":"feedback_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Feedback"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}},"put":{"summary":"Update Feedback","operationId":"update_feedback_api_observability_feedbacks__trace_id___feedback_id___put","parameters":[{"required":true,"schema":{"title":"Trace Id","type":"string"},"name":"trace_id","in":"path"},{"required":true,"schema":{"title":"Feedback Id","type":"string"},"name":"feedback_id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateFeedback"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Feedback"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/organizations/":{"get":{"summary":"List Organizations","description":"Returns a list of organizations associated with the user's session.\n\nArgs:\n stoken_session (SessionContainer): The user's session token.\n\nReturns:\n list[Organization]: A list of organizations associated with the user's session.\n\nRaises:\n HTTPException: If there is an error retrieving the organizations from the database.","operationId":"list_organizations_api_organizations__get","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response List Organizations Api Organizations Get","type":"array","items":{"$ref":"#/components/schemas/Organization"}}}}}}}},"/api/organizations/own/":{"get":{"summary":"Get User Organization","operationId":"get_user_organization_api_organizations_own__get","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}}}}},"/api/bases/":{"get":{"summary":"List Bases","description":"Retrieve a list of bases filtered by app_id and base_name.\n\nArgs:\n request (Request): The incoming request.\n app_id (Optional[str], optional): The ID of the app to filter by. Defaults to None.\n base_name (Optional[str], optional): The name of the base to filter by. Defaults to None.\n\nReturns:\n List[BaseOutput]: A list of BaseOutput objects representing the filtered bases.\n\nRaises:\n HTTPException: If there was an error retrieving the bases.","operationId":"list_bases_api_bases__get","parameters":[{"required":false,"schema":{"title":"App Id","type":"string"},"name":"app_id","in":"query"},{"required":false,"schema":{"title":"Base Name","type":"string"},"name":"base_name","in":"query"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response List Bases Api Bases Get","type":"array","items":{"$ref":"#/components/schemas/BaseOutput"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/configs/":{"get":{"summary":"Get Config","operationId":"get_config_api_configs__get","parameters":[{"required":true,"schema":{"title":"Base Id","type":"string"},"name":"base_id","in":"query"},{"required":false,"schema":{"title":"Config Name","type":"string"},"name":"config_name","in":"query"},{"required":false,"schema":{"title":"Environment Name","type":"string"},"name":"environment_name","in":"query"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GetConfigReponse"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}},"post":{"summary":"Save Config","operationId":"save_config_api_configs__post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/SaveConfigPayload"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}}},"components":{"schemas":{"AICritiqueCreate":{"title":"AICritiqueCreate","required":["correct_answer","inputs","outputs"],"type":"object","properties":{"correct_answer":{"title":"Correct Answer","type":"string"},"llm_app_prompt_template":{"title":"Llm App Prompt Template","type":"string"},"inputs":{"title":"Inputs","type":"array","items":{"$ref":"#/components/schemas/EvaluationScenarioInput"}},"outputs":{"title":"Outputs","type":"array","items":{"$ref":"#/components/schemas/EvaluationScenarioOutput"}},"evaluation_prompt_template":{"title":"Evaluation Prompt Template","type":"string"},"open_ai_key":{"title":"Open Ai Key","type":"string"}}},"AddVariantFromBasePayload":{"title":"AddVariantFromBasePayload","required":["base_id","new_variant_name","new_config_name","parameters"],"type":"object","properties":{"base_id":{"title":"Base Id","type":"string"},"new_variant_name":{"title":"New Variant Name","type":"string"},"new_config_name":{"title":"New Config Name","type":"string"},"parameters":{"title":"Parameters","type":"object"}}},"AddVariantFromImagePayload":{"title":"AddVariantFromImagePayload","required":["variant_name","docker_id","tags"],"type":"object","properties":{"variant_name":{"title":"Variant Name","type":"string"},"docker_id":{"title":"Docker Id","type":"string"},"tags":{"title":"Tags","type":"string"},"base_name":{"title":"Base Name","type":"string"},"config_name":{"title":"Config Name","type":"string"}}},"App":{"title":"App","required":["app_id","app_name"],"type":"object","properties":{"app_id":{"title":"App Id","type":"string"},"app_name":{"title":"App Name","type":"string"}}},"AppVariantOutput":{"title":"AppVariantOutput","required":["app_id","app_name","variant_id","variant_name","organization_id","user_id","base_name","base_id","config_name","config_id"],"type":"object","properties":{"app_id":{"title":"App Id","type":"string"},"app_name":{"title":"App Name","type":"string"},"variant_id":{"title":"Variant Id","type":"string"},"variant_name":{"title":"Variant Name","type":"string"},"parameters":{"title":"Parameters","type":"object"},"previous_variant_name":{"title":"Previous Variant Name","type":"string"},"organization_id":{"title":"Organization Id","type":"string"},"user_id":{"title":"User Id","type":"string"},"base_name":{"title":"Base Name","type":"string"},"base_id":{"title":"Base Id","type":"string"},"config_name":{"title":"Config Name","type":"string"},"config_id":{"title":"Config Id","type":"string"},"uri":{"title":"Uri","type":"string"}}},"BaseOutput":{"title":"BaseOutput","required":["base_id","base_name"],"type":"object","properties":{"base_id":{"title":"Base Id","type":"string"},"base_name":{"title":"Base Name","type":"string"}}},"Body_build_image_api_containers_build_image__post":{"title":"Body_build_image_api_containers_build_image__post","required":["tar_file"],"type":"object","properties":{"tar_file":{"title":"Tar File","type":"string","format":"binary"}}},"Body_import_testset_api_testsets_endpoint__post":{"title":"Body_import_testset_api_testsets_endpoint__post","type":"object","properties":{"endpoint":{"title":"Endpoint","type":"string"},"testset_name":{"title":"Testset Name","type":"string"},"app_id":{"title":"App Id","type":"string"}}},"Body_start_variant_api_variants__variant_id___put":{"title":"Body_start_variant_api_variants__variant_id___put","required":["action"],"type":"object","properties":{"action":{"$ref":"#/components/schemas/VariantAction"},"env_vars":{"$ref":"#/components/schemas/DockerEnvVars"}}},"Body_upload_file_api_testsets_upload__post":{"title":"Body_upload_file_api_testsets_upload__post","required":["file"],"type":"object","properties":{"upload_type":{"title":"Upload Type","type":"string"},"file":{"title":"File","type":"string","format":"binary"},"testset_name":{"title":"Testset Name","type":"string"},"app_id":{"title":"App Id","type":"string"}}},"CreateApp":{"title":"CreateApp","required":["app_name"],"type":"object","properties":{"app_name":{"title":"App Name","type":"string"},"organization_id":{"title":"Organization Id","type":"string"}}},"CreateAppOutput":{"title":"CreateAppOutput","required":["app_id","app_name"],"type":"object","properties":{"app_id":{"title":"App Id","type":"string"},"app_name":{"title":"App Name","type":"string"}}},"CreateAppVariant":{"title":"CreateAppVariant","required":["app_name","template_id","env_vars"],"type":"object","properties":{"app_name":{"title":"App Name","type":"string"},"template_id":{"title":"Template Id","type":"string"},"env_vars":{"title":"Env Vars","type":"object","additionalProperties":{"type":"string"}},"organization_id":{"title":"Organization Id","type":"string"}}},"CreateCustomEvaluation":{"title":"CreateCustomEvaluation","required":["evaluation_name","python_code","app_id"],"type":"object","properties":{"evaluation_name":{"title":"Evaluation Name","type":"string"},"python_code":{"title":"Python Code","type":"string"},"app_id":{"title":"App Id","type":"string"}}},"CreateFeedback":{"title":"CreateFeedback","type":"object","properties":{"feedback":{"title":"Feedback","type":"string"},"score":{"title":"Score","type":"number"},"meta":{"title":"Meta","type":"object"}}},"CreateSpan":{"title":"CreateSpan","required":["event_name","start_time","status","end_time"],"type":"object","properties":{"parent_span_id":{"title":"Parent Span Id","type":"string"},"meta":{"title":"Meta","type":"object"},"event_name":{"title":"Event Name","type":"string"},"event_type":{"title":"Event Type","type":"string"},"start_time":{"title":"Start Time","type":"string","format":"date-time"},"duration":{"title":"Duration","type":"integer"},"status":{"title":"Status","type":"string"},"end_time":{"title":"End Time","type":"string","format":"date-time"},"inputs":{"title":"Inputs","type":"array","items":{"type":"string"}},"outputs":{"title":"Outputs","type":"array","items":{"type":"string"}},"prompt_template":{"title":"Prompt Template","type":"string"},"tokens_input":{"title":"Tokens Input","type":"integer"},"tokens_output":{"title":"Tokens Output","type":"integer"},"token_total":{"title":"Token Total","type":"integer"},"cost":{"title":"Cost","type":"number"},"tags":{"title":"Tags","type":"array","items":{"type":"string"}}}},"CreateTrace":{"title":"CreateTrace","required":["latency","status","start_time","end_time","spans"],"type":"object","properties":{"app_id":{"title":"App Id","type":"string"},"variant_id":{"title":"Variant Id","type":"string"},"cost":{"title":"Cost","type":"number"},"latency":{"title":"Latency","type":"number"},"status":{"title":"Status","type":"string"},"token_consumption":{"title":"Token Consumption","type":"integer"},"tags":{"title":"Tags","type":"array","items":{"type":"string"}},"start_time":{"title":"Start Time","type":"string","format":"date-time"},"end_time":{"title":"End Time","type":"string","format":"date-time"},"spans":{"title":"Spans","type":"array","items":{"type":"string"}}}},"CustomEvaluationDetail":{"title":"CustomEvaluationDetail","required":["id","app_id","evaluation_name","python_code","created_at","updated_at"],"type":"object","properties":{"id":{"title":"Id","type":"string"},"app_id":{"title":"App Id","type":"string"},"evaluation_name":{"title":"Evaluation Name","type":"string"},"python_code":{"title":"Python Code","type":"string"},"created_at":{"title":"Created At","type":"string","format":"date-time"},"updated_at":{"title":"Updated At","type":"string","format":"date-time"}}},"CustomEvaluationNames":{"title":"CustomEvaluationNames","required":["id","evaluation_name"],"type":"object","properties":{"id":{"title":"Id","type":"string"},"evaluation_name":{"title":"Evaluation Name","type":"string"}}},"CustomEvaluationOutput":{"title":"CustomEvaluationOutput","required":["id","app_id","evaluation_name","created_at"],"type":"object","properties":{"id":{"title":"Id","type":"string"},"app_id":{"title":"App Id","type":"string"},"evaluation_name":{"title":"Evaluation Name","type":"string"},"created_at":{"title":"Created At","type":"string","format":"date-time"}}},"DeleteEvaluation":{"title":"DeleteEvaluation","required":["evaluations_ids"],"type":"object","properties":{"evaluations_ids":{"title":"Evaluations Ids","type":"array","items":{"type":"string"}}}},"DeleteTestsets":{"title":"DeleteTestsets","required":["testset_ids"],"type":"object","properties":{"testset_ids":{"title":"Testset Ids","type":"array","items":{"type":"string"}}}},"DeployToEnvironmentPayload":{"title":"DeployToEnvironmentPayload","required":["environment_name","variant_id"],"type":"object","properties":{"environment_name":{"title":"Environment Name","type":"string"},"variant_id":{"title":"Variant Id","type":"string"}}},"DockerEnvVars":{"title":"DockerEnvVars","required":["env_vars"],"type":"object","properties":{"env_vars":{"title":"Env Vars","type":"object","additionalProperties":{"type":"string"}}}},"EnvironmentOutput":{"title":"EnvironmentOutput","required":["name","app_id"],"type":"object","properties":{"name":{"title":"Name","type":"string"},"app_id":{"title":"App Id","type":"string"},"deployed_app_variant_id":{"title":"Deployed App Variant Id","type":"string"},"deployed_variant_name":{"title":"Deployed Variant Name","type":"string"}}},"Evaluation":{"title":"Evaluation","required":["id","app_id","user_id","user_username","evaluation_type","variant_ids","variant_names","testset_id","testset_name","status","created_at","updated_at"],"type":"object","properties":{"id":{"title":"Id","type":"string"},"app_id":{"title":"App Id","type":"string"},"user_id":{"title":"User Id","type":"string"},"user_username":{"title":"User Username","type":"string"},"evaluation_type":{"$ref":"#/components/schemas/EvaluationType"},"evaluation_type_settings":{"$ref":"#/components/schemas/EvaluationTypeSettings"},"variant_ids":{"title":"Variant Ids","type":"array","items":{"type":"string"}},"variant_names":{"title":"Variant Names","type":"array","items":{"type":"string"}},"testset_id":{"title":"Testset Id","type":"string"},"testset_name":{"title":"Testset Name","type":"string"},"status":{"title":"Status","type":"string"},"created_at":{"title":"Created At","type":"string","format":"date-time"},"updated_at":{"title":"Updated At","type":"string","format":"date-time"}}},"EvaluationScenario":{"title":"EvaluationScenario","required":["evaluation_id","inputs","outputs"],"type":"object","properties":{"id":{"title":"Id","type":"string"},"evaluation_id":{"title":"Evaluation Id","type":"string"},"inputs":{"title":"Inputs","type":"array","items":{"$ref":"#/components/schemas/EvaluationScenarioInput"}},"outputs":{"title":"Outputs","type":"array","items":{"$ref":"#/components/schemas/EvaluationScenarioOutput"}},"vote":{"title":"Vote","type":"string"},"score":{"title":"Score","type":"string"},"evaluation":{"title":"Evaluation","type":"string"},"correct_answer":{"title":"Correct Answer","type":"string"},"is_pinned":{"title":"Is Pinned","type":"boolean"},"note":{"title":"Note","type":"string"}}},"EvaluationScenarioInput":{"title":"EvaluationScenarioInput","required":["input_name","input_value"],"type":"object","properties":{"input_name":{"title":"Input Name","type":"string"},"input_value":{"title":"Input Value","type":"string"}}},"EvaluationScenarioOutput":{"title":"EvaluationScenarioOutput","required":["variant_id","variant_output"],"type":"object","properties":{"variant_id":{"title":"Variant Id","type":"string"},"variant_output":{"title":"Variant Output","type":"string"}}},"EvaluationScenarioScoreUpdate":{"title":"EvaluationScenarioScoreUpdate","required":["score"],"type":"object","properties":{"score":{"title":"Score","type":"number"}}},"EvaluationScenarioUpdate":{"title":"EvaluationScenarioUpdate","type":"object","properties":{"vote":{"title":"Vote","type":"string"},"score":{"title":"Score","type":"string"},"correct_answer":{"title":"Correct Answer","type":"string"},"outputs":{"title":"Outputs","type":"array","items":{"$ref":"#/components/schemas/EvaluationScenarioOutput"}},"inputs":{"title":"Inputs","type":"array","items":{"$ref":"#/components/schemas/EvaluationScenarioInput"}},"is_pinned":{"title":"Is Pinned","type":"boolean"},"note":{"title":"Note","type":"string"}}},"EvaluationStatusEnum":{"title":"EvaluationStatusEnum","enum":["EVALUATION_INITIALIZED","EVALUATION_STARTED","COMPARISON_RUN_STARTED","EVALUATION_FINISHED"],"type":"string","description":"An enumeration."},"EvaluationType":{"title":"EvaluationType","enum":["auto_exact_match","auto_similarity_match","auto_regex_test","auto_webhook_test","auto_ai_critique","human_a_b_testing","human_scoring","custom_code_run"],"type":"string","description":"An enumeration."},"EvaluationTypeSettings":{"title":"EvaluationTypeSettings","type":"object","properties":{"similarity_threshold":{"title":"Similarity Threshold","type":"number"},"regex_pattern":{"title":"Regex Pattern","type":"string"},"regex_should_match":{"title":"Regex Should Match","type":"boolean"},"webhook_url":{"title":"Webhook Url","type":"string"},"custom_code_evaluation_id":{"title":"Custom Code Evaluation Id","type":"string"},"llm_app_prompt_template":{"title":"Llm App Prompt Template","type":"string"},"evaluation_prompt_template":{"title":"Evaluation Prompt Template","type":"string"}}},"EvaluationUpdate":{"title":"EvaluationUpdate","type":"object","properties":{"status":{"$ref":"#/components/schemas/EvaluationStatusEnum"},"evaluation_type_settings":{"$ref":"#/components/schemas/EvaluationTypeSettings"}}},"EvaluationWebhook":{"title":"EvaluationWebhook","required":["score"],"type":"object","properties":{"score":{"title":"Score","type":"number"}}},"ExecuteCustomEvaluationCode":{"title":"ExecuteCustomEvaluationCode","required":["inputs","app_id","variant_id","correct_answer","outputs"],"type":"object","properties":{"inputs":{"title":"Inputs","type":"array","items":{"type":"object"}},"app_id":{"title":"App Id","type":"string"},"variant_id":{"title":"Variant Id","type":"string"},"correct_answer":{"title":"Correct Answer","type":"string"},"outputs":{"title":"Outputs","type":"array","items":{"type":"object"}}}},"Feedback":{"title":"Feedback","required":["feedback_id"],"type":"object","properties":{"feedback":{"title":"Feedback","type":"string"},"score":{"title":"Score","type":"number"},"meta":{"title":"Meta","type":"object"},"feedback_id":{"title":"Feedback Id","type":"string"},"created_at":{"title":"Created At","type":"string","format":"date-time"}}},"GetConfigReponse":{"title":"GetConfigReponse","required":["config_id","config_name","current_version","parameters"],"type":"object","properties":{"config_id":{"title":"Config Id","type":"string"},"config_name":{"title":"Config Name","type":"string"},"current_version":{"title":"Current Version","type":"integer"},"parameters":{"title":"Parameters","type":"object"}}},"HTTPValidationError":{"title":"HTTPValidationError","type":"object","properties":{"detail":{"title":"Detail","type":"array","items":{"$ref":"#/components/schemas/ValidationError"}}}},"Image":{"title":"Image","required":["docker_id","tags"],"type":"object","properties":{"docker_id":{"title":"Docker Id","type":"string"},"tags":{"title":"Tags","type":"string"},"organization_id":{"title":"Organization Id","type":"string"}}},"NewEvaluation":{"title":"NewEvaluation","required":["app_id","variant_ids","evaluation_type","inputs","testset_id","status"],"type":"object","properties":{"app_id":{"title":"App Id","type":"string"},"variant_ids":{"title":"Variant Ids","type":"array","items":{"type":"string"}},"evaluation_type":{"$ref":"#/components/schemas/EvaluationType"},"evaluation_type_settings":{"$ref":"#/components/schemas/EvaluationTypeSettings"},"inputs":{"title":"Inputs","type":"array","items":{"type":"string"}},"testset_id":{"title":"Testset Id","type":"string"},"status":{"title":"Status","type":"string"}}},"NewTestset":{"title":"NewTestset","required":["name","csvdata"],"type":"object","properties":{"name":{"title":"Name","type":"string"},"csvdata":{"title":"Csvdata","type":"array","items":{"type":"object","additionalProperties":{"type":"string"}}}}},"Organization":{"title":"Organization","required":["name","owner"],"type":"object","properties":{"id":{"title":"Id","type":"string"},"name":{"title":"Name","type":"string"},"description":{"title":"Description","type":"string"},"type":{"title":"Type","type":"string"},"owner":{"title":"Owner","type":"string"},"members":{"title":"Members","type":"array","items":{"type":"string"}},"invitations":{"title":"Invitations","type":"array","items":{}}}},"RestartAppContainer":{"title":"RestartAppContainer","required":["variant_id"],"type":"object","properties":{"variant_id":{"title":"Variant Id","type":"string"}}},"SaveConfigPayload":{"title":"SaveConfigPayload","required":["base_id","config_name","parameters","overwrite"],"type":"object","properties":{"base_id":{"title":"Base Id","type":"string"},"config_name":{"title":"Config Name","type":"string"},"parameters":{"title":"Parameters","type":"object"},"overwrite":{"title":"Overwrite","type":"boolean"}}},"SimpleEvaluationOutput":{"title":"SimpleEvaluationOutput","required":["id","variant_ids","app_id","status","evaluation_type"],"type":"object","properties":{"id":{"title":"Id","type":"string"},"variant_ids":{"title":"Variant Ids","type":"array","items":{"type":"string"}},"app_id":{"title":"App Id","type":"string"},"status":{"title":"Status","type":"string"},"evaluation_type":{"$ref":"#/components/schemas/EvaluationType"}}},"Span":{"title":"Span","required":["event_name","start_time","status","end_time","span_id"],"type":"object","properties":{"parent_span_id":{"title":"Parent Span Id","type":"string"},"meta":{"title":"Meta","type":"object"},"event_name":{"title":"Event Name","type":"string"},"event_type":{"title":"Event Type","type":"string"},"start_time":{"title":"Start Time","type":"string","format":"date-time"},"duration":{"title":"Duration","type":"integer"},"status":{"title":"Status","type":"string"},"end_time":{"title":"End Time","type":"string","format":"date-time"},"inputs":{"title":"Inputs","type":"array","items":{"type":"string"}},"outputs":{"title":"Outputs","type":"array","items":{"type":"string"}},"prompt_template":{"title":"Prompt Template","type":"string"},"tokens_input":{"title":"Tokens Input","type":"integer"},"tokens_output":{"title":"Tokens Output","type":"integer"},"token_total":{"title":"Token Total","type":"integer"},"cost":{"title":"Cost","type":"number"},"tags":{"title":"Tags","type":"array","items":{"type":"string"}},"span_id":{"title":"Span Id","type":"string"}}},"Template":{"title":"Template","required":["id","image"],"type":"object","properties":{"id":{"title":"Id","type":"string"},"image":{"$ref":"#/components/schemas/TemplateImageInfo"}}},"TemplateImageInfo":{"title":"TemplateImageInfo","required":["name","size","digest","title","description","last_pushed","repo_name"],"type":"object","properties":{"name":{"title":"Name","type":"string"},"size":{"title":"Size","type":"integer"},"digest":{"title":"Digest","type":"string"},"title":{"title":"Title","type":"string"},"description":{"title":"Description","type":"string"},"last_pushed":{"title":"Last Pushed","type":"string","format":"date-time"},"repo_name":{"title":"Repo Name","type":"string"}}},"TestSetOutputResponse":{"title":"TestSetOutputResponse","required":["_id","name","created_at"],"type":"object","properties":{"_id":{"title":" Id","type":"string"},"name":{"title":"Name","type":"string"},"created_at":{"title":"Created At","type":"string","format":"date-time"}}},"TestSetSimpleResponse":{"title":"TestSetSimpleResponse","required":["id","name","created_at"],"type":"object","properties":{"id":{"title":"Id","type":"string"},"name":{"title":"Name","type":"string"},"created_at":{"title":"Created At","type":"string"}}},"Trace":{"title":"Trace","required":["latency","status","start_time","end_time","trace_id","spans"],"type":"object","properties":{"app_id":{"title":"App Id","type":"string"},"variant_id":{"title":"Variant Id","type":"string"},"cost":{"title":"Cost","type":"number"},"latency":{"title":"Latency","type":"number"},"status":{"title":"Status","type":"string"},"token_consumption":{"title":"Token Consumption","type":"integer"},"tags":{"title":"Tags","type":"array","items":{"type":"string"}},"start_time":{"title":"Start Time","type":"string","format":"date-time"},"end_time":{"title":"End Time","type":"string","format":"date-time"},"trace_id":{"title":"Trace Id","type":"string"},"spans":{"title":"Spans","type":"array","items":{"type":"string"}},"feedbacks":{"title":"Feedbacks","type":"array","items":{"$ref":"#/components/schemas/Feedback"}}}},"URI":{"title":"URI","required":["uri"],"type":"object","properties":{"uri":{"title":"Uri","type":"string"}}},"UpdateFeedback":{"title":"UpdateFeedback","required":["feedback"],"type":"object","properties":{"feedback":{"title":"Feedback","type":"string"},"score":{"title":"Score","type":"number"},"meta":{"title":"Meta","type":"object"}}},"UpdateTrace":{"title":"UpdateTrace","required":["status"],"type":"object","properties":{"status":{"title":"Status","type":"string"}}},"UpdateVariantParameterPayload":{"title":"UpdateVariantParameterPayload","required":["parameters"],"type":"object","properties":{"parameters":{"title":"Parameters","type":"object"}}},"ValidationError":{"title":"ValidationError","required":["loc","msg","type"],"type":"object","properties":{"loc":{"title":"Location","type":"array","items":{"anyOf":[{"type":"string"},{"type":"integer"}]}},"msg":{"title":"Message","type":"string"},"type":{"title":"Error Type","type":"string"}}},"VariantAction":{"title":"VariantAction","required":["action"],"type":"object","properties":{"action":{"$ref":"#/components/schemas/VariantActionEnum"}}},"VariantActionEnum":{"title":"VariantActionEnum","enum":["START","STOP"],"type":"string","description":"An enumeration."}}}} \ No newline at end of file +{"openapi":"3.0.2","info":{"title":"FastAPI","version":"0.1.0"},"paths":{"/api/profile/":{"get":{"summary":"User Profile","operationId":"user_profile_api_profile__get","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}}}}},"/api/apps/{app_id}/variants/":{"get":{"summary":"List App Variants","description":"Retrieve a list of app variants for a given app ID.\n\nArgs:\n app_id (str): The ID of the app to retrieve variants for.\n stoken_session (SessionContainer, optional): The session container to verify the user's session. Defaults to Depends(verify_session()).\n\nReturns:\n List[AppVariantOutput]: A list of app variants for the given app ID.","operationId":"list_app_variants_api_apps__app_id__variants__get","parameters":[{"required":true,"schema":{"title":"App Id","type":"string"},"name":"app_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response List App Variants Api Apps App Id Variants Get","type":"array","items":{"$ref":"#/components/schemas/AppVariantOutput"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/apps/get_variant_by_env/":{"get":{"summary":"Get Variant By Env","description":"Retrieve the app variant based on the provided app_id and environment.\n\nArgs:\n app_id (str): The ID of the app to retrieve the variant for.\n environment (str): The environment of the app variant to retrieve.\n stoken_session (SessionContainer, optional): The session token container. Defaults to Depends(verify_session()).\n\nRaises:\n HTTPException: If the app variant is not found (status_code=500), or if a ValueError is raised (status_code=400), or if any other exception is raised (status_code=500).\n\nReturns:\n AppVariantOutput: The retrieved app variant.","operationId":"get_variant_by_env_api_apps_get_variant_by_env__get","parameters":[{"required":true,"schema":{"title":"App Id","type":"string"},"name":"app_id","in":"query"},{"required":true,"schema":{"title":"Environment","type":"string"},"name":"environment","in":"query"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AppVariantOutput"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/apps/":{"get":{"summary":"List Apps","description":"Retrieve a list of apps filtered by app_name and org_id.\n\nArgs:\n app_name (Optional[str]): The name of the app to filter by.\n org_id (Optional[str]): The ID of the organization to filter by.\n stoken_session (SessionContainer): The session container.\n\nReturns:\n List[App]: A list of apps filtered by app_name and org_id.\n\nRaises:\n HTTPException: If there was an error retrieving the list of apps.","operationId":"list_apps_api_apps__get","parameters":[{"required":false,"schema":{"title":"App Name","type":"string"},"name":"app_name","in":"query"},{"required":false,"schema":{"title":"Org Id","type":"string"},"name":"org_id","in":"query"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response List Apps Api Apps Get","type":"array","items":{"$ref":"#/components/schemas/App"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}},"post":{"summary":"Create App","description":"Create a new app for a user or organization.\n\nArgs:\n payload (CreateApp): The payload containing the app name and organization ID (optional).\n stoken_session (SessionContainer): The session container containing the user's session token.\n\nReturns:\n CreateAppOutput: The output containing the newly created app's ID and name.\n\nRaises:\n HTTPException: If there is an error creating the app or the user does not have permission to access the app.","operationId":"create_app_api_apps__post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateApp"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateAppOutput"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/apps/{app_id}/variant/from-image/":{"post":{"summary":"Add Variant From Image","description":"Add a new variant to an app based on a Docker image.\n\nArgs:\n app_id (str): The ID of the app to add the variant to.\n payload (AddVariantFromImagePayload): The payload containing information about the variant to add.\n stoken_session (SessionContainer, optional): The session container. Defaults to Depends(verify_session()).\n\nRaises:\n HTTPException: If the feature flag is set to \"demo\" or if the image does not have a tag starting with the registry name (agenta-server) or if the image is not found or if the user does not have access to the app.\n\nReturns:\n dict: The newly added variant.","operationId":"add_variant_from_image_api_apps__app_id__variant_from_image__post","parameters":[{"required":true,"schema":{"title":"App Id","type":"string"},"name":"app_id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/AddVariantFromImagePayload"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/apps/{app_id}/":{"delete":{"summary":"Remove App","description":"Remove app, all its variant, containers and images\n\nArguments:\n app -- App to remove","operationId":"remove_app_api_apps__app_id___delete","parameters":[{"required":true,"schema":{"title":"App Id","type":"string"},"name":"app_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/apps/app_and_variant_from_template/":{"post":{"summary":"Create App And Variant From Template","description":"Create an app and variant from a template.\n\nArgs:\n payload (CreateAppVariant): The payload containing the app and variant information.\n stoken_session (SessionContainer, optional): The session container. Defaults to Depends(verify_session()).\n\nRaises:\n HTTPException: If the user has reached the app limit or if an app with the same name already exists.\n\nReturns:\n AppVariantOutput: The output of the created app variant.","operationId":"create_app_and_variant_from_template_api_apps_app_and_variant_from_template__post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateAppVariant"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AppVariantOutput"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/apps/{app_id}/environments/":{"get":{"summary":"List Environments","description":"Retrieve a list of environments for a given app ID.\n\nArgs:\n app_id (str): The ID of the app to retrieve environments for.\n stoken_session (SessionContainer, optional): The session container. Defaults to Depends(verify_session()).\n\nReturns:\n List[EnvironmentOutput]: A list of environment objects.","operationId":"list_environments_api_apps__app_id__environments__get","parameters":[{"required":true,"schema":{"title":"App Id","type":"string"},"name":"app_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response List Environments Api Apps App Id Environments Get","type":"array","items":{"$ref":"#/components/schemas/EnvironmentOutput"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/variants/from-base/":{"post":{"summary":"Add Variant From Base And Config","description":"Add a new variant based on an existing one.\nSame as POST /config\n\nArgs:\n payload (AddVariantFromBasePayload): Payload containing base variant ID, new variant name, and parameters.\n stoken_session (SessionContainer, optional): Session container. Defaults to result of verify_session().\n\nRaises:\n HTTPException: Raised if the variant could not be added or accessed.\n\nReturns:\n Union[AppVariantOutput, Any]: New variant details or exception.","operationId":"add_variant_from_base_and_config_api_variants_from_base__post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/AddVariantFromBasePayload"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Add Variant From Base And Config Api Variants From Base Post","anyOf":[{"$ref":"#/components/schemas/AppVariantOutput"},{}]}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/variants/{variant_id}/":{"put":{"summary":"Start Variant","description":"Start a variant of an app.\n\nArgs:\n variant_id (str): The ID of the variant to start.\n action (VariantAction): The action to perform on the variant (start).\n env_vars (Optional[DockerEnvVars], optional): The environment variables to inject to the Docker container. Defaults to None.\n stoken_session (SessionContainer, optional): The session container. Defaults to Depends(verify_session()).\n\nReturns:\n URI: The URL of the started variant.\n\nRaises:\n HTTPException: If the app container cannot be started.","operationId":"start_variant_api_variants__variant_id___put","parameters":[{"required":true,"schema":{"title":"Variant Id","type":"string"},"name":"variant_id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Body_start_variant_api_variants__variant_id___put"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/URI"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}},"delete":{"summary":"Remove Variant","description":"Remove a variant from the server.\nIn the case it's the last variant using the image, stop the container and remove the image.\n\nArguments:\n app_variant -- AppVariant to remove\n\nRaises:\n HTTPException: If there is a problem removing the app variant","operationId":"remove_variant_api_variants__variant_id___delete","parameters":[{"required":true,"schema":{"title":"Variant Id","type":"string"},"name":"variant_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/variants/{variant_id}/parameters/":{"put":{"summary":"Update Variant Parameters","description":"Updates the parameters for an app variant.\n\nArgs:\n variant_id (str): The ID of the app variant to update.\n payload (UpdateVariantParameterPayload): The payload containing the updated parameters.\n stoken_session (SessionContainer, optional): The session container. Defaults to Depends(verify_session()).\n\nRaises:\n HTTPException: If there is an error while trying to update the app variant.\n\nReturns:\n JSONResponse: A JSON response containing the updated app variant parameters.","operationId":"update_variant_parameters_api_variants__variant_id__parameters__put","parameters":[{"required":true,"schema":{"title":"Variant Id","type":"string"},"name":"variant_id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateVariantParameterPayload"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/variants/{variant_id}/image/":{"put":{"summary":"Update Variant Image","description":"Updates the image used in an app variant.\n\nArgs:\n variant_id (str): The ID of the app variant to update.\n image (Image): The image information to update.\n\nRaises:\n HTTPException: If an error occurs while trying to update the app variant.\n\nReturns:\n JSONResponse: A JSON response indicating whether the update was successful or not.","operationId":"update_variant_image_api_variants__variant_id__image__put","parameters":[{"required":true,"schema":{"title":"Variant Id","type":"string"},"name":"variant_id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Image"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/":{"get":{"summary":"Fetch List Evaluations","description":"Fetches a list of evaluations, optionally filtered by an app ID.\n\nArgs:\n app_id (Optional[str]): An optional app ID to filter the evaluations.\n\nReturns:\n List[Evaluation]: A list of evaluations.","operationId":"fetch_list_evaluations_api_evaluations__get","parameters":[{"required":true,"schema":{"title":"App Id","type":"string"},"name":"app_id","in":"query"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Fetch List Evaluations Api Evaluations Get","type":"array","items":{"$ref":"#/components/schemas/Evaluation"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}},"post":{"summary":"Create Evaluation","description":"Creates a new comparison table document\nRaises:\n HTTPException: _description_\nReturns:\n _description_","operationId":"create_evaluation_api_evaluations__post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/NewEvaluation"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SimpleEvaluationOutput"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}},"delete":{"summary":"Delete Evaluations","description":"Delete specific comparison tables based on their unique IDs.\n\nArgs:\ndelete_evaluations (List[str]): The unique identifiers of the comparison tables to delete.\n\nReturns:\nA list of the deleted comparison tables' IDs.","operationId":"delete_evaluations_api_evaluations__delete","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/DeleteEvaluation"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Delete Evaluations Api Evaluations Delete","type":"array","items":{"type":"string"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/{evaluation_id}/":{"get":{"summary":"Fetch Evaluation","description":"Fetches a single evaluation based on its ID.\n\nArgs:\n evaluation_id (str): The ID of the evaluation to fetch.\n\nReturns:\n Evaluation: The fetched evaluation.","operationId":"fetch_evaluation_api_evaluations__evaluation_id___get","parameters":[{"required":true,"schema":{"title":"Evaluation Id","type":"string"},"name":"evaluation_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Evaluation"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}},"put":{"summary":"Update Evaluation Router","description":"Updates an evaluation's status.\n\nRaises:\n HTTPException: If the columns in the test set do not match with the inputs in the variant.\n\nReturns:\n None: A 204 No Content status code, indicating that the update was successful.","operationId":"update_evaluation_router_api_evaluations__evaluation_id___put","parameters":[{"required":true,"schema":{"title":"Evaluation Id","type":"string"},"name":"evaluation_id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/EvaluationUpdate"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/{evaluation_id}/evaluation_scenarios/":{"get":{"summary":"Fetch Evaluation Scenarios","description":"Fetches evaluation scenarios for a given evaluation ID.\n\nArguments:\n evaluation_id (str): The ID of the evaluation for which to fetch scenarios.\n\nRaises:\n HTTPException: If the evaluation is not found or access is denied.\n\nReturns:\n List[EvaluationScenario]: A list of evaluation scenarios.","operationId":"fetch_evaluation_scenarios_api_evaluations__evaluation_id__evaluation_scenarios__get","parameters":[{"required":true,"schema":{"title":"Evaluation Id","type":"string"},"name":"evaluation_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Fetch Evaluation Scenarios Api Evaluations Evaluation Id Evaluation Scenarios Get","type":"array","items":{"$ref":"#/components/schemas/EvaluationScenario"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/{evaluation_id}/evaluation_scenario/":{"post":{"summary":"Create Evaluation Scenario","description":"Create a new evaluation scenario for a given evaluation ID.\n\nRaises:\n HTTPException: If evaluation not found or access denied.\n\nReturns:\n None: 204 No Content status code upon success.","operationId":"create_evaluation_scenario_api_evaluations__evaluation_id__evaluation_scenario__post","parameters":[{"required":true,"schema":{"title":"Evaluation Id","type":"string"},"name":"evaluation_id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/EvaluationScenario"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/{evaluation_id}/evaluation_scenario/{evaluation_scenario_id}/{evaluation_type}/":{"put":{"summary":"Update Evaluation Scenario Router","description":"Updates an evaluation scenario's vote or score based on its type.\n\nRaises:\n HTTPException: If update fails or unauthorized.\n\nReturns:\n None: 204 No Content status code upon successful update.","operationId":"update_evaluation_scenario_router_api_evaluations__evaluation_id__evaluation_scenario__evaluation_scenario_id___evaluation_type___put","parameters":[{"required":true,"schema":{"title":"Evaluation Scenario Id","type":"string"},"name":"evaluation_scenario_id","in":"path"},{"required":true,"schema":{"$ref":"#/components/schemas/EvaluationType"},"name":"evaluation_type","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/EvaluationScenarioUpdate"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/evaluation_scenario/ai_critique/":{"post":{"summary":"Evaluate Ai Critique","description":"Evaluate AI critique based on the given payload.\n\nArgs:\n payload (AICritiqueCreate): The payload containing data for AI critique evaluation.\n stoken_session (SessionContainer): The session container verified by `verify_session`.\n\nReturns:\n str: The output of the AI critique evaluation.\n\nRaises:\n HTTPException: If any exception occurs during the evaluation.","operationId":"evaluate_ai_critique_api_evaluations_evaluation_scenario_ai_critique__post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/AICritiqueCreate"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Evaluate Ai Critique Api Evaluations Evaluation Scenario Ai Critique Post","type":"string"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/evaluation_scenario/{evaluation_scenario_id}/score/":{"get":{"summary":"Get Evaluation Scenario Score Router","description":"Fetch the score of a specific evaluation scenario.\n\nArgs:\n evaluation_scenario_id: The ID of the evaluation scenario to fetch.\n stoken_session: Session data, verified by `verify_session`.\n\nReturns:\n Dictionary containing the scenario ID and its score.","operationId":"get_evaluation_scenario_score_router_api_evaluations_evaluation_scenario__evaluation_scenario_id__score__get","parameters":[{"required":true,"schema":{"title":"Evaluation Scenario Id","type":"string"},"name":"evaluation_scenario_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Get Evaluation Scenario Score Router Api Evaluations Evaluation Scenario Evaluation Scenario Id Score Get","type":"object","additionalProperties":{"type":"string"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}},"put":{"summary":"Update Evaluation Scenario Score Router","description":"Updates the score of an evaluation scenario.\n\nRaises:\n HTTPException: Server error if the evaluation update fails.\n\nReturns:\n None: 204 No Content status code upon successful update.","operationId":"update_evaluation_scenario_score_router_api_evaluations_evaluation_scenario__evaluation_scenario_id__score__put","parameters":[{"required":true,"schema":{"title":"Evaluation Scenario Id","type":"string"},"name":"evaluation_scenario_id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/EvaluationScenarioScoreUpdate"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/{evaluation_id}/results/":{"get":{"summary":"Fetch Results","description":"Fetch all the results for one the comparison table\n\nArguments:\n evaluation_id -- _description_\n\nReturns:\n _description_","operationId":"fetch_results_api_evaluations__evaluation_id__results__get","parameters":[{"required":true,"schema":{"title":"Evaluation Id","type":"string"},"name":"evaluation_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/custom_evaluation/":{"post":{"summary":"Create Custom Evaluation","description":"Create evaluation with custom python code.\n\n Args:\n \ncustom_evaluation_payload (CreateCustomEvaluation): the required payload","operationId":"create_custom_evaluation_api_evaluations_custom_evaluation__post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateCustomEvaluation"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/custom_evaluation/{id}":{"put":{"summary":"Update Custom Evaluation","description":"Update a custom code evaluation.\nArgs:\n id (str): the ID of the custom evaluation to update\n updated_data (CreateCustomEvaluation): the payload with updated data\n stoken_session (SessionContainer): session container for authentication","operationId":"update_custom_evaluation_api_evaluations_custom_evaluation__id__put","parameters":[{"required":true,"schema":{"title":"Id","type":"string"},"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateCustomEvaluation"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/custom_evaluation/list/{app_id}/":{"get":{"summary":"List Custom Evaluations","description":"List the custom code evaluations for a given app.\n\nArgs:\n app_id (str): the id of the app\n\nReturns:\n List[CustomEvaluationOutput]: a list of custom evaluation","operationId":"list_custom_evaluations_api_evaluations_custom_evaluation_list__app_id___get","parameters":[{"required":true,"schema":{"title":"App Id","type":"string"},"name":"app_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response List Custom Evaluations Api Evaluations Custom Evaluation List App Id Get","type":"array","items":{"$ref":"#/components/schemas/CustomEvaluationOutput"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/custom_evaluation/{id}/":{"get":{"summary":"Get Custom Evaluation","description":"Get the custom code evaluation detail.\n\nArgs:\n id (str): the id of the custom evaluation\n\nReturns:\n CustomEvaluationDetail: Detail of the custom evaluation","operationId":"get_custom_evaluation_api_evaluations_custom_evaluation__id___get","parameters":[{"required":true,"schema":{"title":"Id","type":"string"},"name":"id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CustomEvaluationDetail"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/custom_evaluation/{app_name}/names/":{"get":{"summary":"Get Custom Evaluation Names","description":"Get the names of custom evaluation for a given app.\n\nArgs:\n app_name (str): the name of the app the evaluation belongs to\n\nReturns:\n List[CustomEvaluationNames]: the list of name of custom evaluations","operationId":"get_custom_evaluation_names_api_evaluations_custom_evaluation__app_name__names__get","parameters":[{"required":true,"schema":{"title":"App Name","type":"string"},"name":"app_name","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Get Custom Evaluation Names Api Evaluations Custom Evaluation App Name Names Get","type":"array","items":{"$ref":"#/components/schemas/CustomEvaluationNames"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/custom_evaluation/execute/{evaluation_id}/":{"post":{"summary":"Execute Custom Evaluation","description":"Execute a custom evaluation code.\n\nArgs:\n evaluation_id (str): the custom evaluation id\n payload (ExecuteCustomEvaluationCode): the required payload\n\nReturns:\n float: the result of the evaluation custom code","operationId":"execute_custom_evaluation_api_evaluations_custom_evaluation_execute__evaluation_id___post","parameters":[{"required":true,"schema":{"title":"Evaluation Id","type":"string"},"name":"evaluation_id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExecuteCustomEvaluationCode"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/evaluations/webhook_example_fake/":{"post":{"summary":"Webhook Example Fake","description":"Returns a fake score response for example webhook evaluation\n\nReturns:\n _description_","operationId":"webhook_example_fake_api_evaluations_webhook_example_fake__post","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/EvaluationWebhook"}}}}}}},"/api/testsets/upload/":{"post":{"summary":"Upload File","description":"Uploads a CSV or JSON file and saves its data to MongoDB.\n\nArgs:\nupload_type : Either a json or csv file.\n file (UploadFile): The CSV or JSON file to upload.\n testset_name (Optional): the name of the testset if provided.\n\nReturns:\n dict: The result of the upload process.","operationId":"upload_file_api_testsets_upload__post","requestBody":{"content":{"multipart/form-data":{"schema":{"$ref":"#/components/schemas/Body_upload_file_api_testsets_upload__post"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TestSetSimpleResponse"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/testsets/endpoint/":{"post":{"summary":"Import Testset","description":"Import JSON testset data from an endpoint and save it to MongoDB.\n\nArgs:\n endpoint (str): An endpoint URL to import data from.\n testset_name (str): the name of the testset if provided.\n\nReturns:\n dict: The result of the import process.","operationId":"import_testset_api_testsets_endpoint__post","requestBody":{"content":{"application/x-www-form-urlencoded":{"schema":{"$ref":"#/components/schemas/Body_import_testset_api_testsets_endpoint__post"}}}},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TestSetSimpleResponse"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/testsets":{"post":{"summary":"Create Testset","description":"Create a testset with given name, save the testset to MongoDB.\n\nArgs:\nname (str): name of the test set.\ntestset (Dict[str, str]): test set data.\n\nReturns:\nstr: The id of the test set created.","operationId":"create_testset_api_testsets__post","parameters":[],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/NewTestset"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/testsets/{testset_id}/":{"get":{"tags":["testsets"],"summary":"Get Testset","description":"Fetch a specific testset in a MongoDB collection using its _id.\n\nArgs:\n testset_id (str): The _id of the testset to fetch.\n\nReturns:\n The requested testset if found, else an HTTPException.","operationId":"get_testset_api_testsets__testset_id___get","parameters":[{"required":true,"schema":{"title":"Testset Id","type":"string"},"name":"testset_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}},"put":{"summary":"Update Testset","description":"Update a testset with given id, update the testset in MongoDB.\n\nArgs:\ntestset_id (str): id of the test set to be updated.\ncsvdata (NewTestset): New data to replace the old testset.\n\nReturns:\nstr: The id of the test set updated.","operationId":"update_testset_api_testsets__testset_id___put","parameters":[{"required":true,"schema":{"title":"Testset Id","type":"string"},"name":"testset_id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/NewTestset"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/testsets/":{"get":{"tags":["testsets"],"summary":"Get Testsets","description":"Get all testsets.\n\nReturns:\n- A list of testset objects.\n\nRaises:\n- `HTTPException` with status code 404 if no testsets are found.","operationId":"get_testsets_api_testsets__get","parameters":[],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Get Testsets Api Testsets Get","type":"array","items":{"$ref":"#/components/schemas/TestSetOutputResponse"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}},"delete":{"summary":"Delete Testsets","description":"Delete specific testsets based on their unique IDs.\n\nArgs:\ntestset_ids (List[str]): The unique identifiers of the testsets to delete.\n\nReturns:\nA list of the deleted testsets' IDs.","operationId":"delete_testsets_api_testsets__delete","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/DeleteTestsets"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Delete Testsets Api Testsets Delete","type":"array","items":{"type":"string"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/containers/build_image/":{"post":{"summary":"Build Image","description":"Builds a Docker image from a tar file containing the application code.\n\nArgs:\n app_id (str): The ID of the application to build the image for.\n base_name (str): The base name of the image to build.\n tar_file (UploadFile): The tar file containing the application code.\n stoken_session (SessionContainer): The session container for the user making the request.\n\nReturns:\n Image: The Docker image that was built.","operationId":"build_image_api_containers_build_image__post","parameters":[{"required":true,"schema":{"title":"App Id","type":"string"},"name":"app_id","in":"query"},{"required":true,"schema":{"title":"Base Name","type":"string"},"name":"base_name","in":"query"}],"requestBody":{"content":{"multipart/form-data":{"schema":{"$ref":"#/components/schemas/Body_build_image_api_containers_build_image__post"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Image"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/containers/restart_container/":{"post":{"summary":"Restart Docker Container","description":"Restart docker container.\n\nArgs:\n payload (RestartAppContainer) -- the required data (app_name and variant_name)","operationId":"restart_docker_container_api_containers_restart_container__post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/RestartAppContainer"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Restart Docker Container Api Containers Restart Container Post","type":"object"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/containers/templates/":{"get":{"summary":"Container Templates","description":"Returns a list of templates available for creating new containers.\n\nParameters:\nstoken_session (SessionContainer): The session container for the user.\n\nReturns:\n\nUnion[List[Template], str]: A list of templates or an error message.","operationId":"container_templates_api_containers_templates__get","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Container Templates Api Containers Templates Get","anyOf":[{"type":"array","items":{"$ref":"#/components/schemas/Template"}},{"type":"string"}]}}}}}}},"/api/containers/container_url/":{"get":{"summary":"Construct App Container Url","description":"Constructs the URL for an app container based on the provided base_id or variant_id.\n\nArgs:\n base_id (Optional[str]): The ID of the base to use for the app container.\n variant_id (Optional[str]): The ID of the variant to use for the app container.\n stoken_session (SessionContainer): The session container for the user.\n\nReturns:\n URI: The URI for the app container.\n\nRaises:\n HTTPException: If the base or variant cannot be found or the user does not have access.","operationId":"construct_app_container_url_api_containers_container_url__get","parameters":[{"required":false,"schema":{"title":"Base Id","type":"string"},"name":"base_id","in":"query"},{"required":false,"schema":{"title":"Variant Id","type":"string"},"name":"variant_id","in":"query"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/URI"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/environments/deploy/":{"post":{"summary":"Deploy To Environment","description":"Deploys a given variant to an environment\n\nArgs:\n environment_name: Name of the environment to deploy to.\n variant_id: variant id to deploy.\n stoken_session: . Defaults to Depends(verify_session()).\n\nRaises:\n HTTPException: If the deployment fails.","operationId":"deploy_to_environment_api_environments_deploy__post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/DeployToEnvironmentPayload"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/observability/traces/":{"post":{"summary":"Create Trace","operationId":"create_trace_api_observability_traces__post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateTrace"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Create Trace Api Observability Traces Post","type":"string"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/observability/traces/{app_id}/{variant_id}/":{"get":{"summary":"Get Traces","operationId":"get_traces_api_observability_traces__app_id___variant_id___get","parameters":[{"required":true,"schema":{"title":"App Id","type":"string"},"name":"app_id","in":"path"},{"required":true,"schema":{"title":"Variant Id","type":"string"},"name":"variant_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Get Traces Api Observability Traces App Id Variant Id Get","type":"array","items":{"$ref":"#/components/schemas/Trace"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/observability/traces/{trace_id}/":{"get":{"summary":"Get Trace","operationId":"get_trace_api_observability_traces__trace_id___get","parameters":[{"required":true,"schema":{"title":"Trace Id","type":"string"},"name":"trace_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Trace"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}},"put":{"summary":"Update Trace Status","operationId":"update_trace_status_api_observability_traces__trace_id___put","parameters":[{"required":true,"schema":{"title":"Trace Id","type":"string"},"name":"trace_id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateTrace"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Update Trace Status Api Observability Traces Trace Id Put","type":"boolean"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/observability/spans/":{"post":{"summary":"Create Span","operationId":"create_span_api_observability_spans__post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateSpan"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Create Span Api Observability Spans Post","type":"string"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/observability/spans/{trace_id}/":{"get":{"summary":"Get Spans Of Trace","operationId":"get_spans_of_trace_api_observability_spans__trace_id___get","parameters":[{"required":true,"schema":{"title":"Trace Id","type":"string"},"name":"trace_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Get Spans Of Trace Api Observability Spans Trace Id Get","type":"array","items":{"$ref":"#/components/schemas/Span"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/observability/feedbacks/{trace_id}/":{"get":{"summary":"Get Feedbacks","operationId":"get_feedbacks_api_observability_feedbacks__trace_id___get","parameters":[{"required":true,"schema":{"title":"Trace Id","type":"string"},"name":"trace_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Get Feedbacks Api Observability Feedbacks Trace Id Get","type":"array","items":{"$ref":"#/components/schemas/Feedback"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}},"post":{"summary":"Create Feedback","operationId":"create_feedback_api_observability_feedbacks__trace_id___post","parameters":[{"required":true,"schema":{"title":"Trace Id","type":"string"},"name":"trace_id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateFeedback"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response Create Feedback Api Observability Feedbacks Trace Id Post","type":"string"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/observability/feedbacks/{trace_id}/{feedback_id}/":{"get":{"summary":"Get Feedback","operationId":"get_feedback_api_observability_feedbacks__trace_id___feedback_id___get","parameters":[{"required":true,"schema":{"title":"Trace Id","type":"string"},"name":"trace_id","in":"path"},{"required":true,"schema":{"title":"Feedback Id","type":"string"},"name":"feedback_id","in":"path"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Feedback"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}},"put":{"summary":"Update Feedback","operationId":"update_feedback_api_observability_feedbacks__trace_id___feedback_id___put","parameters":[{"required":true,"schema":{"title":"Trace Id","type":"string"},"name":"trace_id","in":"path"},{"required":true,"schema":{"title":"Feedback Id","type":"string"},"name":"feedback_id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateFeedback"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Feedback"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/organizations/":{"get":{"summary":"List Organizations","description":"Returns a list of organizations associated with the user's session.\n\nArgs:\n stoken_session (SessionContainer): The user's session token.\n\nReturns:\n list[Organization]: A list of organizations associated with the user's session.\n\nRaises:\n HTTPException: If there is an error retrieving the organizations from the database.","operationId":"list_organizations_api_organizations__get","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response List Organizations Api Organizations Get","type":"array","items":{"$ref":"#/components/schemas/Organization"}}}}}}}},"/api/organizations/own/":{"get":{"summary":"Get User Organization","operationId":"get_user_organization_api_organizations_own__get","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}}}}},"/api/bases/":{"get":{"summary":"List Bases","description":"Retrieve a list of bases filtered by app_id and base_name.\n\nArgs:\n request (Request): The incoming request.\n app_id (Optional[str], optional): The ID of the app to filter by. Defaults to None.\n base_name (Optional[str], optional): The name of the base to filter by. Defaults to None.\n\nReturns:\n List[BaseOutput]: A list of BaseOutput objects representing the filtered bases.\n\nRaises:\n HTTPException: If there was an error retrieving the bases.","operationId":"list_bases_api_bases__get","parameters":[{"required":false,"schema":{"title":"App Id","type":"string"},"name":"app_id","in":"query"},{"required":false,"schema":{"title":"Base Name","type":"string"},"name":"base_name","in":"query"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"title":"Response List Bases Api Bases Get","type":"array","items":{"$ref":"#/components/schemas/BaseOutput"}}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/configs/":{"get":{"summary":"Get Config","operationId":"get_config_api_configs__get","parameters":[{"required":true,"schema":{"title":"Base Id","type":"string"},"name":"base_id","in":"query"},{"required":false,"schema":{"title":"Config Name","type":"string"},"name":"config_name","in":"query"},{"required":false,"schema":{"title":"Environment Name","type":"string"},"name":"environment_name","in":"query"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GetConfigReponse"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}},"post":{"summary":"Save Config","operationId":"save_config_api_configs__post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/SaveConfigPayload"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}}},"components":{"schemas":{"AICritiqueCreate":{"title":"AICritiqueCreate","required":["correct_answer","inputs","outputs"],"type":"object","properties":{"correct_answer":{"title":"Correct Answer","type":"string"},"llm_app_prompt_template":{"title":"Llm App Prompt Template","type":"string"},"inputs":{"title":"Inputs","type":"array","items":{"$ref":"#/components/schemas/EvaluationScenarioInput"}},"outputs":{"title":"Outputs","type":"array","items":{"$ref":"#/components/schemas/EvaluationScenarioOutput"}},"evaluation_prompt_template":{"title":"Evaluation Prompt Template","type":"string"},"open_ai_key":{"title":"Open Ai Key","type":"string"}}},"AddVariantFromBasePayload":{"title":"AddVariantFromBasePayload","required":["base_id","new_variant_name","new_config_name","parameters"],"type":"object","properties":{"base_id":{"title":"Base Id","type":"string"},"new_variant_name":{"title":"New Variant Name","type":"string"},"new_config_name":{"title":"New Config Name","type":"string"},"parameters":{"title":"Parameters","type":"object"}}},"AddVariantFromImagePayload":{"title":"AddVariantFromImagePayload","required":["variant_name","docker_id","tags"],"type":"object","properties":{"variant_name":{"title":"Variant Name","type":"string"},"docker_id":{"title":"Docker Id","type":"string"},"tags":{"title":"Tags","type":"string"},"base_name":{"title":"Base Name","type":"string"},"config_name":{"title":"Config Name","type":"string"}}},"App":{"title":"App","required":["app_id","app_name"],"type":"object","properties":{"app_id":{"title":"App Id","type":"string"},"app_name":{"title":"App Name","type":"string"}}},"AppVariantOutput":{"title":"AppVariantOutput","required":["app_id","app_name","variant_id","variant_name","organization_id","user_id","base_name","base_id","config_name","config_id"],"type":"object","properties":{"app_id":{"title":"App Id","type":"string"},"app_name":{"title":"App Name","type":"string"},"variant_id":{"title":"Variant Id","type":"string"},"variant_name":{"title":"Variant Name","type":"string"},"parameters":{"title":"Parameters","type":"object"},"previous_variant_name":{"title":"Previous Variant Name","type":"string"},"organization_id":{"title":"Organization Id","type":"string"},"user_id":{"title":"User Id","type":"string"},"base_name":{"title":"Base Name","type":"string"},"base_id":{"title":"Base Id","type":"string"},"config_name":{"title":"Config Name","type":"string"},"config_id":{"title":"Config Id","type":"string"},"uri":{"title":"Uri","type":"string"}}},"BaseOutput":{"title":"BaseOutput","required":["base_id","base_name"],"type":"object","properties":{"base_id":{"title":"Base Id","type":"string"},"base_name":{"title":"Base Name","type":"string"}}},"Body_build_image_api_containers_build_image__post":{"title":"Body_build_image_api_containers_build_image__post","required":["tar_file"],"type":"object","properties":{"tar_file":{"title":"Tar File","type":"string","format":"binary"}}},"Body_import_testset_api_testsets_endpoint__post":{"title":"Body_import_testset_api_testsets_endpoint__post","type":"object","properties":{"endpoint":{"title":"Endpoint","type":"string"},"testset_name":{"title":"Testset Name","type":"string"}}},"Body_start_variant_api_variants__variant_id___put":{"title":"Body_start_variant_api_variants__variant_id___put","required":["action"],"type":"object","properties":{"action":{"$ref":"#/components/schemas/VariantAction"},"env_vars":{"$ref":"#/components/schemas/DockerEnvVars"}}},"Body_upload_file_api_testsets_upload__post":{"title":"Body_upload_file_api_testsets_upload__post","required":["file"],"type":"object","properties":{"upload_type":{"title":"Upload Type","type":"string"},"file":{"title":"File","type":"string","format":"binary"},"testset_name":{"title":"Testset Name","type":"string"}}},"CreateApp":{"title":"CreateApp","required":["app_name"],"type":"object","properties":{"app_name":{"title":"App Name","type":"string"},"organization_id":{"title":"Organization Id","type":"string"}}},"CreateAppOutput":{"title":"CreateAppOutput","required":["app_id","app_name"],"type":"object","properties":{"app_id":{"title":"App Id","type":"string"},"app_name":{"title":"App Name","type":"string"}}},"CreateAppVariant":{"title":"CreateAppVariant","required":["app_name","template_id","env_vars"],"type":"object","properties":{"app_name":{"title":"App Name","type":"string"},"template_id":{"title":"Template Id","type":"string"},"env_vars":{"title":"Env Vars","type":"object","additionalProperties":{"type":"string"}},"organization_id":{"title":"Organization Id","type":"string"}}},"CreateCustomEvaluation":{"title":"CreateCustomEvaluation","required":["evaluation_name","python_code","app_id"],"type":"object","properties":{"evaluation_name":{"title":"Evaluation Name","type":"string"},"python_code":{"title":"Python Code","type":"string"},"app_id":{"title":"App Id","type":"string"}}},"CreateFeedback":{"title":"CreateFeedback","type":"object","properties":{"feedback":{"title":"Feedback","type":"string"},"score":{"title":"Score","type":"number"},"meta":{"title":"Meta","type":"object"}}},"CreateSpan":{"title":"CreateSpan","required":["event_name","start_time","status","end_time"],"type":"object","properties":{"parent_span_id":{"title":"Parent Span Id","type":"string"},"meta":{"title":"Meta","type":"object"},"event_name":{"title":"Event Name","type":"string"},"event_type":{"title":"Event Type","type":"string"},"start_time":{"title":"Start Time","type":"string","format":"date-time"},"duration":{"title":"Duration","type":"integer"},"status":{"title":"Status","type":"string"},"end_time":{"title":"End Time","type":"string","format":"date-time"},"inputs":{"title":"Inputs","type":"array","items":{"type":"string"}},"outputs":{"title":"Outputs","type":"array","items":{"type":"string"}},"prompt_template":{"title":"Prompt Template","type":"string"},"tokens_input":{"title":"Tokens Input","type":"integer"},"tokens_output":{"title":"Tokens Output","type":"integer"},"token_total":{"title":"Token Total","type":"integer"},"cost":{"title":"Cost","type":"number"},"tags":{"title":"Tags","type":"array","items":{"type":"string"}}}},"CreateTrace":{"title":"CreateTrace","required":["latency","status","start_time","end_time","spans"],"type":"object","properties":{"app_id":{"title":"App Id","type":"string"},"variant_id":{"title":"Variant Id","type":"string"},"cost":{"title":"Cost","type":"number"},"latency":{"title":"Latency","type":"number"},"status":{"title":"Status","type":"string"},"token_consumption":{"title":"Token Consumption","type":"integer"},"tags":{"title":"Tags","type":"array","items":{"type":"string"}},"start_time":{"title":"Start Time","type":"string","format":"date-time"},"end_time":{"title":"End Time","type":"string","format":"date-time"},"spans":{"title":"Spans","type":"array","items":{"type":"string"}}}},"CustomEvaluationDetail":{"title":"CustomEvaluationDetail","required":["id","app_id","evaluation_name","python_code","created_at","updated_at"],"type":"object","properties":{"id":{"title":"Id","type":"string"},"app_id":{"title":"App Id","type":"string"},"evaluation_name":{"title":"Evaluation Name","type":"string"},"python_code":{"title":"Python Code","type":"string"},"created_at":{"title":"Created At","type":"string","format":"date-time"},"updated_at":{"title":"Updated At","type":"string","format":"date-time"}}},"CustomEvaluationNames":{"title":"CustomEvaluationNames","required":["id","evaluation_name"],"type":"object","properties":{"id":{"title":"Id","type":"string"},"evaluation_name":{"title":"Evaluation Name","type":"string"}}},"CustomEvaluationOutput":{"title":"CustomEvaluationOutput","required":["id","app_id","evaluation_name","created_at"],"type":"object","properties":{"id":{"title":"Id","type":"string"},"app_id":{"title":"App Id","type":"string"},"evaluation_name":{"title":"Evaluation Name","type":"string"},"created_at":{"title":"Created At","type":"string","format":"date-time"}}},"DeleteEvaluation":{"title":"DeleteEvaluation","required":["evaluations_ids"],"type":"object","properties":{"evaluations_ids":{"title":"Evaluations Ids","type":"array","items":{"type":"string"}}}},"DeleteTestsets":{"title":"DeleteTestsets","required":["testset_ids"],"type":"object","properties":{"testset_ids":{"title":"Testset Ids","type":"array","items":{"type":"string"}}}},"DeployToEnvironmentPayload":{"title":"DeployToEnvironmentPayload","required":["environment_name","variant_id"],"type":"object","properties":{"environment_name":{"title":"Environment Name","type":"string"},"variant_id":{"title":"Variant Id","type":"string"}}},"DockerEnvVars":{"title":"DockerEnvVars","required":["env_vars"],"type":"object","properties":{"env_vars":{"title":"Env Vars","type":"object","additionalProperties":{"type":"string"}}}},"EnvironmentOutput":{"title":"EnvironmentOutput","required":["name","app_id"],"type":"object","properties":{"name":{"title":"Name","type":"string"},"app_id":{"title":"App Id","type":"string"},"deployed_app_variant_id":{"title":"Deployed App Variant Id","type":"string"},"deployed_variant_name":{"title":"Deployed Variant Name","type":"string"}}},"Evaluation":{"title":"Evaluation","required":["id","app_id","user_id","user_username","evaluation_type","variant_ids","variant_names","testset_id","testset_name","status","created_at","updated_at"],"type":"object","properties":{"id":{"title":"Id","type":"string"},"app_id":{"title":"App Id","type":"string"},"user_id":{"title":"User Id","type":"string"},"user_username":{"title":"User Username","type":"string"},"evaluation_type":{"$ref":"#/components/schemas/EvaluationType"},"evaluation_type_settings":{"$ref":"#/components/schemas/EvaluationTypeSettings"},"variant_ids":{"title":"Variant Ids","type":"array","items":{"type":"string"}},"variant_names":{"title":"Variant Names","type":"array","items":{"type":"string"}},"testset_id":{"title":"Testset Id","type":"string"},"testset_name":{"title":"Testset Name","type":"string"},"status":{"title":"Status","type":"string"},"created_at":{"title":"Created At","type":"string","format":"date-time"},"updated_at":{"title":"Updated At","type":"string","format":"date-time"}}},"EvaluationScenario":{"title":"EvaluationScenario","required":["evaluation_id","inputs","outputs"],"type":"object","properties":{"id":{"title":"Id","type":"string"},"evaluation_id":{"title":"Evaluation Id","type":"string"},"inputs":{"title":"Inputs","type":"array","items":{"$ref":"#/components/schemas/EvaluationScenarioInput"}},"outputs":{"title":"Outputs","type":"array","items":{"$ref":"#/components/schemas/EvaluationScenarioOutput"}},"vote":{"title":"Vote","type":"string"},"score":{"title":"Score","type":"string"},"evaluation":{"title":"Evaluation","type":"string"},"correct_answer":{"title":"Correct Answer","type":"string"},"is_pinned":{"title":"Is Pinned","type":"boolean"},"note":{"title":"Note","type":"string"}}},"EvaluationScenarioInput":{"title":"EvaluationScenarioInput","required":["input_name","input_value"],"type":"object","properties":{"input_name":{"title":"Input Name","type":"string"},"input_value":{"title":"Input Value","type":"string"}}},"EvaluationScenarioOutput":{"title":"EvaluationScenarioOutput","required":["variant_id","variant_output"],"type":"object","properties":{"variant_id":{"title":"Variant Id","type":"string"},"variant_output":{"title":"Variant Output","type":"string"}}},"EvaluationScenarioScoreUpdate":{"title":"EvaluationScenarioScoreUpdate","required":["score"],"type":"object","properties":{"score":{"title":"Score","type":"number"}}},"EvaluationScenarioUpdate":{"title":"EvaluationScenarioUpdate","type":"object","properties":{"vote":{"title":"Vote","type":"string"},"score":{"title":"Score","type":"string"},"correct_answer":{"title":"Correct Answer","type":"string"},"outputs":{"title":"Outputs","type":"array","items":{"$ref":"#/components/schemas/EvaluationScenarioOutput"}},"inputs":{"title":"Inputs","type":"array","items":{"$ref":"#/components/schemas/EvaluationScenarioInput"}},"is_pinned":{"title":"Is Pinned","type":"boolean"},"note":{"title":"Note","type":"string"}}},"EvaluationStatusEnum":{"title":"EvaluationStatusEnum","enum":["EVALUATION_INITIALIZED","EVALUATION_STARTED","COMPARISON_RUN_STARTED","EVALUATION_FINISHED"],"type":"string","description":"An enumeration."},"EvaluationType":{"title":"EvaluationType","enum":["auto_exact_match","auto_similarity_match","auto_regex_test","auto_webhook_test","auto_ai_critique","human_a_b_testing","human_scoring","custom_code_run"],"type":"string","description":"An enumeration."},"EvaluationTypeSettings":{"title":"EvaluationTypeSettings","type":"object","properties":{"similarity_threshold":{"title":"Similarity Threshold","type":"number"},"regex_pattern":{"title":"Regex Pattern","type":"string"},"regex_should_match":{"title":"Regex Should Match","type":"boolean"},"webhook_url":{"title":"Webhook Url","type":"string"},"custom_code_evaluation_id":{"title":"Custom Code Evaluation Id","type":"string"},"llm_app_prompt_template":{"title":"Llm App Prompt Template","type":"string"},"evaluation_prompt_template":{"title":"Evaluation Prompt Template","type":"string"}}},"EvaluationUpdate":{"title":"EvaluationUpdate","type":"object","properties":{"status":{"$ref":"#/components/schemas/EvaluationStatusEnum"},"evaluation_type_settings":{"$ref":"#/components/schemas/EvaluationTypeSettings"}}},"EvaluationWebhook":{"title":"EvaluationWebhook","required":["score"],"type":"object","properties":{"score":{"title":"Score","type":"number"}}},"ExecuteCustomEvaluationCode":{"title":"ExecuteCustomEvaluationCode","required":["inputs","app_id","variant_id","correct_answer","outputs"],"type":"object","properties":{"inputs":{"title":"Inputs","type":"array","items":{"type":"object"}},"app_id":{"title":"App Id","type":"string"},"variant_id":{"title":"Variant Id","type":"string"},"correct_answer":{"title":"Correct Answer","type":"string"},"outputs":{"title":"Outputs","type":"array","items":{"type":"object"}}}},"Feedback":{"title":"Feedback","required":["feedback_id"],"type":"object","properties":{"feedback":{"title":"Feedback","type":"string"},"score":{"title":"Score","type":"number"},"meta":{"title":"Meta","type":"object"},"feedback_id":{"title":"Feedback Id","type":"string"},"created_at":{"title":"Created At","type":"string","format":"date-time"}}},"GetConfigReponse":{"title":"GetConfigReponse","required":["config_id","config_name","current_version","parameters"],"type":"object","properties":{"config_id":{"title":"Config Id","type":"string"},"config_name":{"title":"Config Name","type":"string"},"current_version":{"title":"Current Version","type":"integer"},"parameters":{"title":"Parameters","type":"object"}}},"HTTPValidationError":{"title":"HTTPValidationError","type":"object","properties":{"detail":{"title":"Detail","type":"array","items":{"$ref":"#/components/schemas/ValidationError"}}}},"Image":{"title":"Image","required":["docker_id","tags"],"type":"object","properties":{"docker_id":{"title":"Docker Id","type":"string"},"tags":{"title":"Tags","type":"string"},"organization_id":{"title":"Organization Id","type":"string"}}},"NewEvaluation":{"title":"NewEvaluation","required":["app_id","variant_ids","evaluation_type","inputs","testset_id","status"],"type":"object","properties":{"app_id":{"title":"App Id","type":"string"},"variant_ids":{"title":"Variant Ids","type":"array","items":{"type":"string"}},"evaluation_type":{"$ref":"#/components/schemas/EvaluationType"},"evaluation_type_settings":{"$ref":"#/components/schemas/EvaluationTypeSettings"},"inputs":{"title":"Inputs","type":"array","items":{"type":"string"}},"testset_id":{"title":"Testset Id","type":"string"},"status":{"title":"Status","type":"string"}}},"NewTestset":{"title":"NewTestset","required":["name","csvdata"],"type":"object","properties":{"name":{"title":"Name","type":"string"},"csvdata":{"title":"Csvdata","type":"array","items":{"type":"object","additionalProperties":{"type":"string"}}}}},"Organization":{"title":"Organization","required":["name","owner"],"type":"object","properties":{"id":{"title":"Id","type":"string"},"name":{"title":"Name","type":"string"},"description":{"title":"Description","type":"string"},"type":{"title":"Type","type":"string"},"owner":{"title":"Owner","type":"string"},"members":{"title":"Members","type":"array","items":{"type":"string"}},"invitations":{"title":"Invitations","type":"array","items":{}}}},"RestartAppContainer":{"title":"RestartAppContainer","required":["variant_id"],"type":"object","properties":{"variant_id":{"title":"Variant Id","type":"string"}}},"SaveConfigPayload":{"title":"SaveConfigPayload","required":["base_id","config_name","parameters","overwrite"],"type":"object","properties":{"base_id":{"title":"Base Id","type":"string"},"config_name":{"title":"Config Name","type":"string"},"parameters":{"title":"Parameters","type":"object"},"overwrite":{"title":"Overwrite","type":"boolean"}}},"SimpleEvaluationOutput":{"title":"SimpleEvaluationOutput","required":["id","variant_ids","app_id","status","evaluation_type"],"type":"object","properties":{"id":{"title":"Id","type":"string"},"variant_ids":{"title":"Variant Ids","type":"array","items":{"type":"string"}},"app_id":{"title":"App Id","type":"string"},"status":{"title":"Status","type":"string"},"evaluation_type":{"$ref":"#/components/schemas/EvaluationType"}}},"Span":{"title":"Span","required":["event_name","start_time","status","end_time","span_id"],"type":"object","properties":{"parent_span_id":{"title":"Parent Span Id","type":"string"},"meta":{"title":"Meta","type":"object"},"event_name":{"title":"Event Name","type":"string"},"event_type":{"title":"Event Type","type":"string"},"start_time":{"title":"Start Time","type":"string","format":"date-time"},"duration":{"title":"Duration","type":"integer"},"status":{"title":"Status","type":"string"},"end_time":{"title":"End Time","type":"string","format":"date-time"},"inputs":{"title":"Inputs","type":"array","items":{"type":"string"}},"outputs":{"title":"Outputs","type":"array","items":{"type":"string"}},"prompt_template":{"title":"Prompt Template","type":"string"},"tokens_input":{"title":"Tokens Input","type":"integer"},"tokens_output":{"title":"Tokens Output","type":"integer"},"token_total":{"title":"Token Total","type":"integer"},"cost":{"title":"Cost","type":"number"},"tags":{"title":"Tags","type":"array","items":{"type":"string"}},"span_id":{"title":"Span Id","type":"string"}}},"Template":{"title":"Template","required":["id","image"],"type":"object","properties":{"id":{"title":"Id","type":"string"},"image":{"$ref":"#/components/schemas/TemplateImageInfo"}}},"TemplateImageInfo":{"title":"TemplateImageInfo","required":["name","size","digest","title","description","last_pushed","repo_name"],"type":"object","properties":{"name":{"title":"Name","type":"string"},"size":{"title":"Size","type":"integer"},"digest":{"title":"Digest","type":"string"},"title":{"title":"Title","type":"string"},"description":{"title":"Description","type":"string"},"last_pushed":{"title":"Last Pushed","type":"string","format":"date-time"},"repo_name":{"title":"Repo Name","type":"string"}}},"TestSetOutputResponse":{"title":"TestSetOutputResponse","required":["_id","name","created_at"],"type":"object","properties":{"_id":{"title":" Id","type":"string"},"name":{"title":"Name","type":"string"},"created_at":{"title":"Created At","type":"string","format":"date-time"}}},"TestSetSimpleResponse":{"title":"TestSetSimpleResponse","required":["id","name","created_at"],"type":"object","properties":{"id":{"title":"Id","type":"string"},"name":{"title":"Name","type":"string"},"created_at":{"title":"Created At","type":"string"}}},"Trace":{"title":"Trace","required":["latency","status","start_time","end_time","trace_id","spans"],"type":"object","properties":{"app_id":{"title":"App Id","type":"string"},"variant_id":{"title":"Variant Id","type":"string"},"cost":{"title":"Cost","type":"number"},"latency":{"title":"Latency","type":"number"},"status":{"title":"Status","type":"string"},"token_consumption":{"title":"Token Consumption","type":"integer"},"tags":{"title":"Tags","type":"array","items":{"type":"string"}},"start_time":{"title":"Start Time","type":"string","format":"date-time"},"end_time":{"title":"End Time","type":"string","format":"date-time"},"trace_id":{"title":"Trace Id","type":"string"},"spans":{"title":"Spans","type":"array","items":{"type":"string"}},"feedbacks":{"title":"Feedbacks","type":"array","items":{"$ref":"#/components/schemas/Feedback"}}}},"URI":{"title":"URI","required":["uri"],"type":"object","properties":{"uri":{"title":"Uri","type":"string"}}},"UpdateFeedback":{"title":"UpdateFeedback","required":["feedback"],"type":"object","properties":{"feedback":{"title":"Feedback","type":"string"},"score":{"title":"Score","type":"number"},"meta":{"title":"Meta","type":"object"}}},"UpdateTrace":{"title":"UpdateTrace","required":["status"],"type":"object","properties":{"status":{"title":"Status","type":"string"}}},"UpdateVariantParameterPayload":{"title":"UpdateVariantParameterPayload","required":["parameters"],"type":"object","properties":{"parameters":{"title":"Parameters","type":"object"}}},"ValidationError":{"title":"ValidationError","required":["loc","msg","type"],"type":"object","properties":{"loc":{"title":"Location","type":"array","items":{"anyOf":[{"type":"string"},{"type":"integer"}]}},"msg":{"title":"Message","type":"string"},"type":{"title":"Error Type","type":"string"}}},"VariantAction":{"title":"VariantAction","required":["action"],"type":"object","properties":{"action":{"$ref":"#/components/schemas/VariantActionEnum"}}},"VariantActionEnum":{"title":"VariantActionEnum","enum":["START","STOP"],"type":"string","description":"An enumeration."}}}} \ No newline at end of file From bd582ed69151ecdff5e0e66b7c3f30882c8db8bf Mon Sep 17 00:00:00 2001 From: Juan Pablo Vega Date: Fri, 15 Nov 2024 19:32:00 +0100 Subject: [PATCH 04/18] always show test sets --- agenta-web/src/components/Sidebar/config.tsx | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/agenta-web/src/components/Sidebar/config.tsx b/agenta-web/src/components/Sidebar/config.tsx index eaea606695..0d8926ff57 100644 --- a/agenta-web/src/components/Sidebar/config.tsx +++ b/agenta-web/src/components/Sidebar/config.tsx @@ -60,7 +60,6 @@ export const useSidebarConfig = () => { tooltip: "Create new applications or switch between your existing projects.", link: "/apps", icon: , - divider: apps.length === 0 ? true : false, }, { key: "app-testsets-link", @@ -68,15 +67,13 @@ export const useSidebarConfig = () => { tooltip: "Create and manage testsets for evaluation purposes.", link: `/apps/testsets`, icon: , - isHidden: apps.length === 0, }, { key: "app-observability-link", title: "Observability", link: `/observability`, icon: , - divider: true, - isHidden: apps.length === 0, + divider: apps.length === 0 ? true : false, }, { key: `${currentApp?.app_name || ""}_key`, From 99d027a2de023ac44702f0ab7a7c9853c6f56652 Mon Sep 17 00:00:00 2001 From: Juan Pablo Vega Date: Fri, 15 Nov 2024 19:42:50 +0100 Subject: [PATCH 05/18] move testsets to general page, i.e. remove /apps --- agenta-web/cypress/support/commands/evaluations.ts | 2 +- .../components/HumanEvaluationModal/HumanEvaluationModal.tsx | 2 +- .../src/components/HumanEvaluations/AbTestingEvaluation.tsx | 2 +- .../src/components/HumanEvaluations/SingleModelEvaluation.tsx | 2 +- agenta-web/src/components/Sidebar/config.tsx | 2 +- .../pages/evaluations/autoEvaluation/AutoEvaluation.tsx | 2 +- .../pages/evaluations/evaluationCompare/EvaluationCompare.tsx | 2 +- .../evaluations/evaluationScenarios/EvaluationScenarios.tsx | 2 +- .../overview/automaticEvaluation/AutomaticEvalOverview.tsx | 2 +- .../pages/testset/modals/CreateTestsetFromScratch.tsx | 2 +- .../src/pages/{apps => }/testsets/[testset_id]/index.tsx | 0 agenta-web/src/pages/{apps => }/testsets/index.tsx | 4 ++-- 12 files changed, 12 insertions(+), 12 deletions(-) rename agenta-web/src/pages/{apps => }/testsets/[testset_id]/index.tsx (100%) rename agenta-web/src/pages/{apps => }/testsets/index.tsx (98%) diff --git a/agenta-web/cypress/support/commands/evaluations.ts b/agenta-web/cypress/support/commands/evaluations.ts index 796efd2a32..065651fcfc 100644 --- a/agenta-web/cypress/support/commands/evaluations.ts +++ b/agenta-web/cypress/support/commands/evaluations.ts @@ -49,7 +49,7 @@ Cypress.Commands.add("createVariant", () => { Cypress.Commands.add("createVariantsAndTestsets", () => { cy.createVariant() - cy.visit("/apps/testsets") + cy.visit("/testsets") cy.url().should("include", "/testsets") cy.get('[data-cy="create-testset-modal-button"]').click() cy.get(".ant-modal-content").should("exist") diff --git a/agenta-web/src/components/HumanEvaluationModal/HumanEvaluationModal.tsx b/agenta-web/src/components/HumanEvaluationModal/HumanEvaluationModal.tsx index b03f4ebd05..2383cac732 100644 --- a/agenta-web/src/components/HumanEvaluationModal/HumanEvaluationModal.tsx +++ b/agenta-web/src/components/HumanEvaluationModal/HumanEvaluationModal.tsx @@ -340,7 +340,7 @@ const HumanEvaluationModal = ({ setError({ message: getErrorMessage(err), btnText: "Go to Test sets", - endpoint: `/apps/testsets`, + endpoint: `/testsets`, }) } }) diff --git a/agenta-web/src/components/HumanEvaluations/AbTestingEvaluation.tsx b/agenta-web/src/components/HumanEvaluations/AbTestingEvaluation.tsx index 5befb8753c..56b7902dda 100644 --- a/agenta-web/src/components/HumanEvaluations/AbTestingEvaluation.tsx +++ b/agenta-web/src/components/HumanEvaluations/AbTestingEvaluation.tsx @@ -353,7 +353,7 @@ const AbTestingEvaluation = ({viewType}: {viewType: "evaluation" | "overview"}) icon: , onClick: (e) => { e.domEvent.stopPropagation() - router.push(`/apps/testsets/${record.testset._id}`) + router.push(`/testsets/${record.testset._id}`) }, }, {type: "divider"}, diff --git a/agenta-web/src/components/HumanEvaluations/SingleModelEvaluation.tsx b/agenta-web/src/components/HumanEvaluations/SingleModelEvaluation.tsx index 34c217e234..ba6acb2e40 100644 --- a/agenta-web/src/components/HumanEvaluations/SingleModelEvaluation.tsx +++ b/agenta-web/src/components/HumanEvaluations/SingleModelEvaluation.tsx @@ -257,7 +257,7 @@ const SingleModelEvaluation = ({viewType}: {viewType: "evaluation" | "overview"} icon: , onClick: (e) => { e.domEvent.stopPropagation() - router.push(`/apps/testsets/${record.testset._id}`) + router.push(`/testsets/${record.testset._id}`) }, }, {type: "divider"}, diff --git a/agenta-web/src/components/Sidebar/config.tsx b/agenta-web/src/components/Sidebar/config.tsx index 0d8926ff57..efbf7303ff 100644 --- a/agenta-web/src/components/Sidebar/config.tsx +++ b/agenta-web/src/components/Sidebar/config.tsx @@ -65,7 +65,7 @@ export const useSidebarConfig = () => { key: "app-testsets-link", title: "Test Sets", tooltip: "Create and manage testsets for evaluation purposes.", - link: `/apps/testsets`, + link: `/testsets`, icon: , }, { diff --git a/agenta-web/src/components/pages/evaluations/autoEvaluation/AutoEvaluation.tsx b/agenta-web/src/components/pages/evaluations/autoEvaluation/AutoEvaluation.tsx index df1ea198e7..2380da611a 100644 --- a/agenta-web/src/components/pages/evaluations/autoEvaluation/AutoEvaluation.tsx +++ b/agenta-web/src/components/pages/evaluations/autoEvaluation/AutoEvaluation.tsx @@ -430,7 +430,7 @@ const AutoEvaluation = () => { icon: , onClick: (e) => { e.domEvent.stopPropagation() - router.push(`/apps/testsets/${record.testset.id}`) + router.push(`/testsets/${record.testset.id}`) }, }, {type: "divider"}, diff --git a/agenta-web/src/components/pages/evaluations/evaluationCompare/EvaluationCompare.tsx b/agenta-web/src/components/pages/evaluations/evaluationCompare/EvaluationCompare.tsx index d99561780e..85f36dcb32 100644 --- a/agenta-web/src/components/pages/evaluations/evaluationCompare/EvaluationCompare.tsx +++ b/agenta-web/src/components/pages/evaluations/evaluationCompare/EvaluationCompare.tsx @@ -458,7 +458,7 @@ const EvaluationCompareMode: React.FC = () => { Testset: - + {testset?.name || ""} diff --git a/agenta-web/src/components/pages/evaluations/evaluationScenarios/EvaluationScenarios.tsx b/agenta-web/src/components/pages/evaluations/evaluationScenarios/EvaluationScenarios.tsx index f03fde7dc8..5eece5ac39 100644 --- a/agenta-web/src/components/pages/evaluations/evaluationScenarios/EvaluationScenarios.tsx +++ b/agenta-web/src/components/pages/evaluations/evaluationScenarios/EvaluationScenarios.tsx @@ -349,7 +349,7 @@ const EvaluationScenarios: React.FC = () => { Testset: - + {evalaution?.testset.name || ""} diff --git a/agenta-web/src/components/pages/overview/automaticEvaluation/AutomaticEvalOverview.tsx b/agenta-web/src/components/pages/overview/automaticEvaluation/AutomaticEvalOverview.tsx index e65db174d9..3a797cac52 100644 --- a/agenta-web/src/components/pages/overview/automaticEvaluation/AutomaticEvalOverview.tsx +++ b/agenta-web/src/components/pages/overview/automaticEvaluation/AutomaticEvalOverview.tsx @@ -370,7 +370,7 @@ const AutomaticEvalOverview = () => { icon: , onClick: (e) => { e.domEvent.stopPropagation() - router.push(`/apps/testsets/${record.testset.id}`) + router.push(`/testsets/${record.testset.id}`) }, }, {type: "divider"}, diff --git a/agenta-web/src/components/pages/testset/modals/CreateTestsetFromScratch.tsx b/agenta-web/src/components/pages/testset/modals/CreateTestsetFromScratch.tsx index 9e616c2be6..22e2b47da0 100644 --- a/agenta-web/src/components/pages/testset/modals/CreateTestsetFromScratch.tsx +++ b/agenta-web/src/components/pages/testset/modals/CreateTestsetFromScratch.tsx @@ -69,7 +69,7 @@ const CreateTestsetFromScratch: React.FC = ({ const rowData = data || (await generateInitialRowData()) const response = await createNewTestset(testsetName, rowData) message.success("Test set created successfully") - router.push(`/apps/testsets/${response.data.id}`) + router.push(`/testsets/${response.data.id}`) } catch (error) { console.error("Error saving test set:", error) message.error("Failed to create Test set. Please try again!") diff --git a/agenta-web/src/pages/apps/testsets/[testset_id]/index.tsx b/agenta-web/src/pages/testsets/[testset_id]/index.tsx similarity index 100% rename from agenta-web/src/pages/apps/testsets/[testset_id]/index.tsx rename to agenta-web/src/pages/testsets/[testset_id]/index.tsx diff --git a/agenta-web/src/pages/apps/testsets/index.tsx b/agenta-web/src/pages/testsets/index.tsx similarity index 98% rename from agenta-web/src/pages/apps/testsets/index.tsx rename to agenta-web/src/pages/testsets/index.tsx index 41ce28d7bf..b94dc3fb12 100644 --- a/agenta-web/src/pages/apps/testsets/index.tsx +++ b/agenta-web/src/pages/testsets/index.tsx @@ -157,7 +157,7 @@ const Testset = () => { icon: , onClick: (e) => { e.domEvent.stopPropagation() - router.push(`/apps/testsets/${record._id}`) + router.push(`/testsets/${record._id}`) }, }, { @@ -262,7 +262,7 @@ const Testset = () => { pagination={false} onRow={(record) => { return { - onClick: () => router.push(`/apps/testsets/${record._id}`), + onClick: () => router.push(`/testsets/${record._id}`), style: {cursor: "pointer"}, } }} From 490a6f1251af509fe38c2bae67488af2821417fe Mon Sep 17 00:00:00 2001 From: Juan Pablo Vega Date: Fri, 15 Nov 2024 19:55:46 +0100 Subject: [PATCH 06/18] remove app_id from testsets --- .../src/components/TestSetTable/TestsetTable.tsx | 12 +----------- .../pages/testset/modals/CreateTestsetFromApi.tsx | 3 +-- .../testset/modals/CreateTestsetFromScratch.tsx | 13 +------------ .../pages/testset/modals/UploadTestset.tsx | 3 +-- .../src/components/pages/testset/modals/index.tsx | 7 ++----- agenta-web/src/pages/testsets/index.tsx | 9 --------- 6 files changed, 6 insertions(+), 41 deletions(-) diff --git a/agenta-web/src/components/TestSetTable/TestsetTable.tsx b/agenta-web/src/components/TestSetTable/TestsetTable.tsx index 10c4a515df..8edaa31445 100644 --- a/agenta-web/src/components/TestSetTable/TestsetTable.tsx +++ b/agenta-web/src/components/TestSetTable/TestsetTable.tsx @@ -92,9 +92,6 @@ const TestsetTable: React.FC = ({mode}) => { const router = useRouter() const {appTheme} = useAppTheme() - const {apps, isLoading: isAppsLoading} = useAppsData() - - const appId = apps[0]?.app_id const {testset_id} = router.query useBlockNavigation(unSavedChanges, { @@ -115,13 +112,6 @@ const TestsetTable: React.FC = ({mode}) => { } }, [rowData, testsetName, columnDefs, inputValues]) - useUpdateEffect(() => { - if ((apps.length === 0 || !apps) && !isAppsLoading) { - message.warning("To view the test set, you first need to create an app.") - router.push("/apps") - } - }, [isAppsLoading]) - useEffect(() => { async function applyColData(colData: {field: string}[] = []) { const newColDefs = createNewColDefs(colData) @@ -140,7 +130,7 @@ const TestsetTable: React.FC = ({mode}) => { ) }) } - }, [writeMode, testset_id, appId]) + }, [writeMode, testset_id]) const handleExportClick = () => { const csvData = convertToCsv( diff --git a/agenta-web/src/components/pages/testset/modals/CreateTestsetFromApi.tsx b/agenta-web/src/components/pages/testset/modals/CreateTestsetFromApi.tsx index eff78031e9..50e2aa1648 100644 --- a/agenta-web/src/components/pages/testset/modals/CreateTestsetFromApi.tsx +++ b/agenta-web/src/components/pages/testset/modals/CreateTestsetFromApi.tsx @@ -43,7 +43,6 @@ const useStyles = createUseStyles((theme: JSSTheme) => ({ type Props = { setCurrent: React.Dispatch> onCancel: () => void - appId: string } type LanguageCodeBlockProps = { selectedLang: string @@ -68,7 +67,7 @@ const LanguageCodeBlock = ({selectedLang, codeSnippets}: LanguageCodeBlockProps) ) } -const CreateTestsetFromApi: React.FC = ({setCurrent, onCancel, appId}) => { +const CreateTestsetFromApi: React.FC = ({setCurrent, onCancel}) => { const classes = useStyles() const router = useRouter() const [uploadType, setUploadType] = useState<"csv" | "json">("csv") diff --git a/agenta-web/src/components/pages/testset/modals/CreateTestsetFromScratch.tsx b/agenta-web/src/components/pages/testset/modals/CreateTestsetFromScratch.tsx index 22e2b47da0..12e7add235 100644 --- a/agenta-web/src/components/pages/testset/modals/CreateTestsetFromScratch.tsx +++ b/agenta-web/src/components/pages/testset/modals/CreateTestsetFromScratch.tsx @@ -43,7 +43,6 @@ const CreateTestsetFromScratch: React.FC = ({ setEditTestsetValues, setCurrent, onCancel, - appId, }) => { const classes = useStyles() const router = useRouter() @@ -53,20 +52,10 @@ const CreateTestsetFromScratch: React.FC = ({ const [isLoading, setIsLoading] = useState(false) const {mutate} = useLoadTestsetsList() - const generateInitialRowData = async (): Promise => { - const backendVariants = await fetchVariants(appId) - const variant = backendVariants[0] - const inputParams = await getVariantInputParameters(appId, variant) - const fields = [...inputParams.map((param) => param.name), "correct_answer"] - return Array(3) - .fill({}) - .map(() => fields.reduce((acc, field) => ({...acc, [field]: ""}), {})) - } - const handleCreateTestset = async (data?: KeyValuePair[]) => { setIsLoading(true) try { - const rowData = data || (await generateInitialRowData()) + const rowData = data const response = await createNewTestset(testsetName, rowData) message.success("Test set created successfully") router.push(`/testsets/${response.data.id}`) diff --git a/agenta-web/src/components/pages/testset/modals/UploadTestset.tsx b/agenta-web/src/components/pages/testset/modals/UploadTestset.tsx index 93f590275d..2f636b28da 100644 --- a/agenta-web/src/components/pages/testset/modals/UploadTestset.tsx +++ b/agenta-web/src/components/pages/testset/modals/UploadTestset.tsx @@ -49,10 +49,9 @@ const useStyles = createUseStyles((theme: JSSTheme) => ({ type Props = { setCurrent: React.Dispatch> onCancel: () => void - appId: string } -const UploadTestset: React.FC = ({setCurrent, onCancel, appId}) => { +const UploadTestset: React.FC = ({setCurrent, onCancel}) => { const classes = useStyles() const router = useRouter() const [form] = Form.useForm() diff --git a/agenta-web/src/components/pages/testset/modals/index.tsx b/agenta-web/src/components/pages/testset/modals/index.tsx index a59d5b554a..f02fe36ef1 100644 --- a/agenta-web/src/components/pages/testset/modals/index.tsx +++ b/agenta-web/src/components/pages/testset/modals/index.tsx @@ -27,7 +27,6 @@ type Props = { setEditTestsetValues: React.Dispatch> current: number setCurrent: React.Dispatch> - appId: string } & React.ComponentProps const TestsetModal: React.FC = ({ @@ -37,7 +36,6 @@ const TestsetModal: React.FC = ({ setEditTestsetValues, current, setCurrent, - appId, ...props }) => { const classes = useStyles() @@ -63,16 +61,15 @@ const TestsetModal: React.FC = ({ onCancel={onCancel} editTestsetValues={editTestsetValues} setEditTestsetValues={setEditTestsetValues} - appId={appId} /> ), }, { - content: , + content: , }, { content: ( - + ), }, ] diff --git a/agenta-web/src/pages/testsets/index.tsx b/agenta-web/src/pages/testsets/index.tsx index b94dc3fb12..bff833049f 100644 --- a/agenta-web/src/pages/testsets/index.tsx +++ b/agenta-web/src/pages/testsets/index.tsx @@ -56,7 +56,6 @@ const Testset = () => { const classes = useStyles() const router = useRouter() const {apps, isLoading: isAppsLoading} = useAppsData() - const appId = apps[0]?.app_id const [selectedRowKeys, setSelectedRowKeys] = useState([]) const {testsets, isTestsetsLoading, mutate} = useLoadTestsetsList() const [isCreateTestsetModalOpen, setIsCreateTestsetModalOpen] = useState(false) @@ -65,13 +64,6 @@ const Testset = () => { const [editTestsetValues, setEditTestsetValues] = useState(null) const [current, setCurrent] = useState(0) - useUpdateEffect(() => { - if ((apps.length === 0 || !apps) && !isAppsLoading) { - message.warning("To view the test set, you first need to create an app.") - router.push("/apps") - } - }, [isAppsLoading]) - const rowSelection = { onChange: (selectedRowKeys: React.Key[]) => { setSelectedRowKeys(selectedRowKeys) @@ -278,7 +270,6 @@ const Testset = () => { testsetCreationMode={testsetCreationMode} setTestsetCreationMode={setTestsetCreationMode} open={isCreateTestsetModalOpen} - appId={appId} onCancel={() => { setIsCreateTestsetModalOpen(false) }} From dcfa623bc055dc7e15070f11cbe56e65c9966235 Mon Sep 17 00:00:00 2001 From: Juan Pablo Vega Date: Fri, 15 Nov 2024 20:12:45 +0100 Subject: [PATCH 07/18] fix testsets deletion --- .../agenta_backend/routers/evaluation_router.py | 8 ++++---- .../agenta/client/backend/evaluations/client.py | 11 ----------- agenta-web/src/lib/helpers/evaluate.ts | 11 +---------- agenta-web/src/pages/testsets/index.tsx | 2 +- agenta-web/src/services/evaluations/api/index.ts | 3 +-- ...evaluation-ids-evaluations-by-resource-get.api.mdx | 6 ------ docs/docs/reference/openapi.json | 9 --------- 7 files changed, 7 insertions(+), 43 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/evaluation_router.py b/agenta-backend/agenta_backend/routers/evaluation_router.py index d3cd298181..6ef89bc48c 100644 --- a/agenta-backend/agenta_backend/routers/evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/evaluation_router.py @@ -33,7 +33,6 @@ response_model=List[str], ) async def fetch_evaluation_ids( - app_id: str, resource_type: str, request: Request, resource_ids: List[str] = Query(None), @@ -52,11 +51,10 @@ async def fetch_evaluation_ids( List[str]: A list of evaluation ids. """ try: - app = await db_manager.fetch_app_by_id(app_id=app_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=str(app.project_id), + project_id=request.state.project_id, permission=Permission.VIEW_EVALUATION, ) logger.debug( @@ -70,7 +68,9 @@ async def fetch_evaluation_ids( status_code=403, ) evaluations = await db_manager.fetch_evaluations_by_resource( - resource_type, str(app.project_id), resource_ids + resource_type, + request.state.project_id, + resource_ids, ) return list(map(lambda x: str(x.id), evaluations)) except Exception as exc: diff --git a/agenta-cli/agenta/client/backend/evaluations/client.py b/agenta-cli/agenta/client/backend/evaluations/client.py index ce3dd7dcca..a8190dd1f2 100644 --- a/agenta-cli/agenta/client/backend/evaluations/client.py +++ b/agenta-cli/agenta/client/backend/evaluations/client.py @@ -26,7 +26,6 @@ def __init__(self, *, client_wrapper: SyncClientWrapper): def fetch_evaluation_ids( self, *, - app_id: str, resource_type: str, resource_ids: typing.Optional[typing.Union[str, typing.Sequence[str]]] = None, request_options: typing.Optional[RequestOptions] = None, @@ -35,7 +34,6 @@ def fetch_evaluation_ids( Fetches evaluation ids for a given resource type and id. Arguments: - app_id (str): The ID of the app for which to fetch evaluations. resource_type (str): The type of resource for which to fetch evaluations. resource_ids List[ObjectId]: The IDs of resource for which to fetch evaluations. @@ -47,8 +45,6 @@ def fetch_evaluation_ids( Parameters ---------- - app_id : str - resource_type : str resource_ids : typing.Optional[typing.Union[str, typing.Sequence[str]]] @@ -70,7 +66,6 @@ def fetch_evaluation_ids( base_url="https://yourhost.com/path/to/api", ) client.evaluations.fetch_evaluation_ids( - app_id="app_id", resource_type="resource_type", ) """ @@ -78,7 +73,6 @@ def fetch_evaluation_ids( "evaluations/by_resource", method="GET", params={ - "app_id": app_id, "resource_type": resource_type, "resource_ids": resource_ids, }, @@ -714,7 +708,6 @@ def __init__(self, *, client_wrapper: AsyncClientWrapper): async def fetch_evaluation_ids( self, *, - app_id: str, resource_type: str, resource_ids: typing.Optional[typing.Union[str, typing.Sequence[str]]] = None, request_options: typing.Optional[RequestOptions] = None, @@ -723,7 +716,6 @@ async def fetch_evaluation_ids( Fetches evaluation ids for a given resource type and id. Arguments: - app_id (str): The ID of the app for which to fetch evaluations. resource_type (str): The type of resource for which to fetch evaluations. resource_ids List[ObjectId]: The IDs of resource for which to fetch evaluations. @@ -735,7 +727,6 @@ async def fetch_evaluation_ids( Parameters ---------- - app_id : str resource_type : str @@ -763,7 +754,6 @@ async def fetch_evaluation_ids( async def main() -> None: await client.evaluations.fetch_evaluation_ids( - app_id="app_id", resource_type="resource_type", ) @@ -774,7 +764,6 @@ async def main() -> None: "evaluations/by_resource", method="GET", params={ - "app_id": app_id, "resource_type": resource_type, "resource_ids": resource_ids, }, diff --git a/agenta-web/src/lib/helpers/evaluate.ts b/agenta-web/src/lib/helpers/evaluate.ts index 2f167ea021..4fb80b8fb9 100644 --- a/agenta-web/src/lib/helpers/evaluate.ts +++ b/agenta-web/src/lib/helpers/evaluate.ts @@ -240,16 +240,7 @@ export const getVotesPercentage = (record: HumanEvaluationListTableDataType, ind export const checkIfResourceValidForDeletion = async ( data: Omit[0], "appId">, ) => { - let appId - - if (data.resourceType === "testset") { - appId = getAppValues().apps[0]?.app_id - } else { - appId = getAppValues().currentApp?.app_id - } - if (!appId) return false - - const response = await fetchEvaluatonIdsByResource({...data, appId}) + const response = await fetchEvaluatonIdsByResource(data) if (response.data.length > 0) { const name = (data.resourceType === "testset" diff --git a/agenta-web/src/pages/testsets/index.tsx b/agenta-web/src/pages/testsets/index.tsx index bff833049f..53ecfb2ed7 100644 --- a/agenta-web/src/pages/testsets/index.tsx +++ b/agenta-web/src/pages/testsets/index.tsx @@ -178,7 +178,7 @@ const Testset = () => { }, }, { - key: "delete_eval", + key: "delete", label: "Delete", icon: , danger: true, diff --git a/agenta-web/src/services/evaluations/api/index.ts b/agenta-web/src/services/evaluations/api/index.ts index d251d0e261..cfcc47ff20 100644 --- a/agenta-web/src/services/evaluations/api/index.ts +++ b/agenta-web/src/services/evaluations/api/index.ts @@ -242,14 +242,13 @@ export const fetchAllComparisonResults = async (evaluationIds: string[]) => { export const fetchEvaluatonIdsByResource = async ({ resourceIds, resourceType, - appId, }: { resourceIds: string[] resourceType: "testset" | "evaluator_config" | "variant" appId: string }) => { return axios.get(`/api/evaluations/by_resource`, { - params: {resource_ids: resourceIds, resource_type: resourceType, app_id: appId}, + params: {resource_ids: resourceIds, resource_type: resourceType}, paramsSerializer: { indexes: null, //no brackets in query params }, diff --git a/docs/docs/reference/api/fetch-evaluation-ids-evaluations-by-resource-get.api.mdx b/docs/docs/reference/api/fetch-evaluation-ids-evaluations-by-resource-get.api.mdx index 5ab8b95e67..21d0a041c8 100644 --- a/docs/docs/reference/api/fetch-evaluation-ids-evaluations-by-resource-get.api.mdx +++ b/docs/docs/reference/api/fetch-evaluation-ids-evaluations-by-resource-get.api.mdx @@ -44,7 +44,6 @@ import TabItem from "@theme/TabItem"; Fetches evaluation ids for a given resource type and id. Arguments: - app_id (str): The ID of the app for which to fetch evaluations. resource_type (str): The type of resource for which to fetch evaluations. resource_ids List[ObjectId]: The IDs of resource for which to fetch evaluations. @@ -78,11 +77,6 @@ Returns:
    - - Date: Fri, 15 Nov 2024 20:15:30 +0100 Subject: [PATCH 08/18] apply prettier --- .../Playground/AddToTestSetDrawer/AddToTestSetDrawer.tsx | 5 +---- agenta-web/src/components/pages/testset/modals/index.tsx | 4 +--- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/agenta-web/src/components/Playground/AddToTestSetDrawer/AddToTestSetDrawer.tsx b/agenta-web/src/components/Playground/AddToTestSetDrawer/AddToTestSetDrawer.tsx index 2246216901..3815461ccd 100644 --- a/agenta-web/src/components/Playground/AddToTestSetDrawer/AddToTestSetDrawer.tsx +++ b/agenta-web/src/components/Playground/AddToTestSetDrawer/AddToTestSetDrawer.tsx @@ -102,10 +102,7 @@ const AddToTestSetDrawer: React.FC = ({params, isChatVariant, ...props}) const router = useRouter() const {testsets, mutate, isTestsetsLoading, isTestsetsLoadingError} = useLoadTestsetsList() const storedValue = localStorage.getItem(`selectedTestset`)?.replace(/"/g, "") - const [selectedTestset, setSelectedTestset] = useLocalStorage( - `selectedTestset`, - "", - ) + const [selectedTestset, setSelectedTestset] = useLocalStorage(`selectedTestset`, "") useEffect(() => { if (storedValue && testsets.some((testset: testset) => testset._id === storedValue)) { diff --git a/agenta-web/src/components/pages/testset/modals/index.tsx b/agenta-web/src/components/pages/testset/modals/index.tsx index f02fe36ef1..8ea035eafc 100644 --- a/agenta-web/src/components/pages/testset/modals/index.tsx +++ b/agenta-web/src/components/pages/testset/modals/index.tsx @@ -68,9 +68,7 @@ const TestsetModal: React.FC = ({ content: , }, { - content: ( - - ), + content: , }, ] From c81e773c6cd17ff35ddc89adb35536c3a6966534 Mon Sep 17 00:00:00 2001 From: Juan Pablo Vega Date: Fri, 15 Nov 2024 20:17:28 +0100 Subject: [PATCH 09/18] fix props --- .../components/pages/testset/modals/CreateTestsetFromScratch.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/agenta-web/src/components/pages/testset/modals/CreateTestsetFromScratch.tsx b/agenta-web/src/components/pages/testset/modals/CreateTestsetFromScratch.tsx index 12e7add235..b4950d59b6 100644 --- a/agenta-web/src/components/pages/testset/modals/CreateTestsetFromScratch.tsx +++ b/agenta-web/src/components/pages/testset/modals/CreateTestsetFromScratch.tsx @@ -33,7 +33,6 @@ type Props = { setEditTestsetValues: React.Dispatch> setCurrent: React.Dispatch> onCancel: () => void - appId: string } const CreateTestsetFromScratch: React.FC = ({ From 809a3c0f5ad370d3803812b5b21944b1c7415ca4 Mon Sep 17 00:00:00 2001 From: Juan Pablo Vega Date: Fri, 15 Nov 2024 20:20:29 +0100 Subject: [PATCH 10/18] fix fetch by resource --- agenta-web/src/services/evaluations/api/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/agenta-web/src/services/evaluations/api/index.ts b/agenta-web/src/services/evaluations/api/index.ts index cfcc47ff20..8d51d25682 100644 --- a/agenta-web/src/services/evaluations/api/index.ts +++ b/agenta-web/src/services/evaluations/api/index.ts @@ -245,7 +245,6 @@ export const fetchEvaluatonIdsByResource = async ({ }: { resourceIds: string[] resourceType: "testset" | "evaluator_config" | "variant" - appId: string }) => { return axios.get(`/api/evaluations/by_resource`, { params: {resource_ids: resourceIds, resource_type: resourceType}, From dee60a092bb46b44c3f28cbcca763bf836d01030 Mon Sep 17 00:00:00 2001 From: Juan Pablo Vega Date: Sun, 17 Nov 2024 17:02:11 +0100 Subject: [PATCH 11/18] fix create from scratch --- .../src/components/TestSetTable/TestsetTable.tsx | 14 ++++++++------ agenta-web/src/services/testsets/api/index.ts | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/agenta-web/src/components/TestSetTable/TestsetTable.tsx b/agenta-web/src/components/TestSetTable/TestsetTable.tsx index 8edaa31445..4304a5b2e5 100644 --- a/agenta-web/src/components/TestSetTable/TestsetTable.tsx +++ b/agenta-web/src/components/TestSetTable/TestsetTable.tsx @@ -122,12 +122,14 @@ const TestsetTable: React.FC = ({mode}) => { if (writeMode === "edit" && testset_id) { fetchTestset(testset_id as string).then((data) => { setTestsetName(data.name) - setRowData(data.csvdata) - applyColData( - Object.keys(data.csvdata[0]).map((key) => ({ - field: key, - })), - ) + if (data.csvdata.length > 0) { + applyColData( + Object.keys(data.csvdata[0]).map((key) => ({ + field: key, + })), + ) + setRowData(data.csvdata) + } }) } }, [writeMode, testset_id]) diff --git a/agenta-web/src/services/testsets/api/index.ts b/agenta-web/src/services/testsets/api/index.ts index d0e9c2112f..5f4da761b1 100644 --- a/agenta-web/src/services/testsets/api/index.ts +++ b/agenta-web/src/services/testsets/api/index.ts @@ -33,7 +33,7 @@ export const fetchTestsets = async () => { export async function createNewTestset(testsetName: string, testsetData: any) { const response = await axios.post(`${getAgentaApiUrl()}/api/testsets`, { name: testsetName, - csvdata: testsetData, + csvdata: testsetData || [{ "input": null, "correct_answer": null}], }) return response } From fb5e236362fbcb054420c9ecb137ad3424a9d47f Mon Sep 17 00:00:00 2001 From: ashrafchowdury Date: Mon, 18 Nov 2024 11:33:35 +0600 Subject: [PATCH 12/18] fix(cypress): fixed failed frontend tests --- agenta-web/cypress/e2e/eval.evaluations.cy.ts | 2 +- .../cypress/e2e/single-model-test-evaluation.cy.ts | 2 +- agenta-web/cypress/e2e/testset.cy.ts | 10 +++++++--- agenta-web/cypress/support/commands/evaluations.ts | 11 ++++++++++- .../src/components/TestSetTable/TestsetTable.tsx | 4 +++- agenta-web/src/services/testsets/api/index.ts | 2 +- 6 files changed, 23 insertions(+), 8 deletions(-) diff --git a/agenta-web/cypress/e2e/eval.evaluations.cy.ts b/agenta-web/cypress/e2e/eval.evaluations.cy.ts index c04a67bb92..e4f4a49567 100644 --- a/agenta-web/cypress/e2e/eval.evaluations.cy.ts +++ b/agenta-web/cypress/e2e/eval.evaluations.cy.ts @@ -45,7 +45,7 @@ describe("Evaluations CRUD Operations Test", function () { context("Executing Evaluation with different answer column", () => { it("Should successfully rename the testset columns", () => { - cy.visit(`/apps/testsets`) + cy.visit(`/testsets`) cy.location("pathname").should("include", "/testsets") cy.get(".ant-table-row").eq(0).click() cy.wait(1000) diff --git a/agenta-web/cypress/e2e/single-model-test-evaluation.cy.ts b/agenta-web/cypress/e2e/single-model-test-evaluation.cy.ts index 9414bb71d6..0cde7e0d81 100644 --- a/agenta-web/cypress/e2e/single-model-test-evaluation.cy.ts +++ b/agenta-web/cypress/e2e/single-model-test-evaluation.cy.ts @@ -64,7 +64,7 @@ describe("Single Model Test workflow", () => { }) it("Should check the evaluation testset is successfully saved", () => { - cy.visit(`/apps/testsets`) + cy.visit(`/testsets`) cy.url().should("include", "/testsets") cy.get('[data-cy="app-testset-list"]').as("table") cy.get("@table").contains(saved_testset_name).as("tempTestSet").should("be.visible") diff --git a/agenta-web/cypress/e2e/testset.cy.ts b/agenta-web/cypress/e2e/testset.cy.ts index 9085e12089..c1a9a2e147 100644 --- a/agenta-web/cypress/e2e/testset.cy.ts +++ b/agenta-web/cypress/e2e/testset.cy.ts @@ -14,7 +14,7 @@ describe("Testsets crud and UI functionality", () => { context("Testing creation process of testset", () => { beforeEach(() => { // navigate to the new testset page - cy.visit(`/apps/testsets`) + cy.visit(`/testsets`) }) it("Should successfully creates the testset and navigates to the list", () => { @@ -27,8 +27,12 @@ describe("Testsets crud and UI functionality", () => { cy.get('[data-cy="testset-name-input"]').type(testsetName) cy.clickLinkAndWait('[data-cy="create-new-testset-button"]') - cy.get(".ag-row").should("have.length", 3) + cy.get(".ag-row").should("have.length", 1) countries.forEach((country, index) => { + if (index !== 0) { + cy.get('[data-cy="add-new-testset-row"]').click() + } + cy.get(`.ag-center-cols-container .ag-row[row-index="${index}"]`).within(() => { cy.get(".ag-cell").eq(1).type(country.country) cy.get(".ag-cell") @@ -52,7 +56,7 @@ describe("Testsets crud and UI functionality", () => { context("When uploading testset", () => { const testset_name = randString(8) beforeEach(() => { - cy.visit(`/apps/testsets`) + cy.visit(`/testsets`) }) it("Should successfully upload a testset", () => { diff --git a/agenta-web/cypress/support/commands/evaluations.ts b/agenta-web/cypress/support/commands/evaluations.ts index 065651fcfc..cf85dedf19 100644 --- a/agenta-web/cypress/support/commands/evaluations.ts +++ b/agenta-web/cypress/support/commands/evaluations.ts @@ -60,8 +60,17 @@ Cypress.Commands.add("createVariantsAndTestsets", () => { cy.clickLinkAndWait('[data-cy="create-new-testset-button"]') cy.wrap(testsetName).as("testsetName") - cy.get(".ag-row").should("have.length", 3) + cy.get(".ag-row").should("have.length", 1) + cy.get('[data-cy="testset-header-column-edit-button"]').eq(0).click() + cy.get('[data-cy="testset-header-column-edit-input"]').clear() + cy.get('[data-cy="testset-header-column-edit-input"]').type("country") + cy.get('[data-cy="testset-header-column-save-button"]').click() + countries.forEach((country, index) => { + if (index !== 0) { + cy.get('[data-cy="add-new-testset-row"]').click() + } + cy.get(`.ag-center-cols-container .ag-row[row-index="${index}"]`).within(() => { cy.get(".ag-cell").eq(1).type(country.country) cy.get(".ag-cell") diff --git a/agenta-web/src/components/TestSetTable/TestsetTable.tsx b/agenta-web/src/components/TestSetTable/TestsetTable.tsx index 4304a5b2e5..5d36f7deea 100644 --- a/agenta-web/src/components/TestSetTable/TestsetTable.tsx +++ b/agenta-web/src/components/TestSetTable/TestsetTable.tsx @@ -360,7 +360,9 @@ const TestsetTable: React.FC = ({mode}) => { {selectedRow && (
    - + diff --git a/agenta-web/src/services/testsets/api/index.ts b/agenta-web/src/services/testsets/api/index.ts index 5f4da761b1..0a4a0220be 100644 --- a/agenta-web/src/services/testsets/api/index.ts +++ b/agenta-web/src/services/testsets/api/index.ts @@ -33,7 +33,7 @@ export const fetchTestsets = async () => { export async function createNewTestset(testsetName: string, testsetData: any) { const response = await axios.post(`${getAgentaApiUrl()}/api/testsets`, { name: testsetName, - csvdata: testsetData || [{ "input": null, "correct_answer": null}], + csvdata: testsetData || [{input: null, correct_answer: null}], }) return response } From 4ad4b2e5f2f083c02e89fa66743c56d7d49803bd Mon Sep 17 00:00:00 2001 From: ashrafchowdury Date: Mon, 18 Nov 2024 12:25:41 +0600 Subject: [PATCH 13/18] cleanup(frontend): removed unused code --- .../Playground/AddToTestSetDrawer/AddToTestSetDrawer.tsx | 2 -- agenta-web/src/components/Playground/LoadTestsModal.tsx | 2 -- agenta-web/src/components/Sidebar/config.tsx | 2 +- agenta-web/src/components/TestSetTable/TestsetTable.tsx | 1 - .../components/pages/testset/modals/CreateTestsetFromApi.tsx | 2 -- agenta-web/src/pages/testsets/index.tsx | 3 +-- 6 files changed, 2 insertions(+), 10 deletions(-) diff --git a/agenta-web/src/components/Playground/AddToTestSetDrawer/AddToTestSetDrawer.tsx b/agenta-web/src/components/Playground/AddToTestSetDrawer/AddToTestSetDrawer.tsx index 3815461ccd..1768e534a7 100644 --- a/agenta-web/src/components/Playground/AddToTestSetDrawer/AddToTestSetDrawer.tsx +++ b/agenta-web/src/components/Playground/AddToTestSetDrawer/AddToTestSetDrawer.tsx @@ -22,7 +22,6 @@ import { Tooltip, message, } from "antd" -import {useRouter} from "next/router" import React, {useCallback, useEffect, useLayoutEffect, useRef, useState} from "react" import {createUseStyles} from "react-jss" import {useLocalStorage, useUpdateEffect} from "usehooks-ts" @@ -99,7 +98,6 @@ const AddToTestSetDrawer: React.FC = ({params, isChatVariant, ...props}) >(null) const [shouldRender, setShouldRender] = useState(false) const dirty = useRef(false) - const router = useRouter() const {testsets, mutate, isTestsetsLoading, isTestsetsLoadingError} = useLoadTestsetsList() const storedValue = localStorage.getItem(`selectedTestset`)?.replace(/"/g, "") const [selectedTestset, setSelectedTestset] = useLocalStorage(`selectedTestset`, "") diff --git a/agenta-web/src/components/Playground/LoadTestsModal.tsx b/agenta-web/src/components/Playground/LoadTestsModal.tsx index 93d32be7bb..325d608e16 100644 --- a/agenta-web/src/components/Playground/LoadTestsModal.tsx +++ b/agenta-web/src/components/Playground/LoadTestsModal.tsx @@ -1,6 +1,5 @@ import {fetchTestset, useLoadTestsetsList} from "@/services/testsets/api" import {Button, Divider, Modal, Select} from "antd" -import {useRouter} from "next/router" import {PropsWithChildren, useState} from "react" import {createUseStyles} from "react-jss" @@ -24,7 +23,6 @@ const useStyles = createUseStyles({ const LoadTestsModal: React.FC = (props) => { const classes = useStyles() const {onLoad} = props - const router = useRouter() const [isOpen, setIsOpen] = useState(false) const [selectedSet, setSelectedSet] = useState("") diff --git a/agenta-web/src/components/Sidebar/config.tsx b/agenta-web/src/components/Sidebar/config.tsx index efbf7303ff..b03e5a1b44 100644 --- a/agenta-web/src/components/Sidebar/config.tsx +++ b/agenta-web/src/components/Sidebar/config.tsx @@ -73,7 +73,7 @@ export const useSidebarConfig = () => { title: "Observability", link: `/observability`, icon: , - divider: apps.length === 0 ? true : false, + divider: true, }, { key: `${currentApp?.app_name || ""}_key`, diff --git a/agenta-web/src/components/TestSetTable/TestsetTable.tsx b/agenta-web/src/components/TestSetTable/TestsetTable.tsx index 5d36f7deea..42c216760f 100644 --- a/agenta-web/src/components/TestSetTable/TestsetTable.tsx +++ b/agenta-web/src/components/TestSetTable/TestsetTable.tsx @@ -17,7 +17,6 @@ import {NoticeType} from "antd/es/message/interface" import {GenericObject, KeyValuePair} from "@/lib/Types" import TableCellsRenderer from "./TableCellsRenderer" import TableHeaderComponent from "./TableHeaderComponent" -import {useAppsData} from "@/contexts/app.context" type TestsetTableProps = { mode: "edit" diff --git a/agenta-web/src/components/pages/testset/modals/CreateTestsetFromApi.tsx b/agenta-web/src/components/pages/testset/modals/CreateTestsetFromApi.tsx index 50e2aa1648..80326ec18c 100644 --- a/agenta-web/src/components/pages/testset/modals/CreateTestsetFromApi.tsx +++ b/agenta-web/src/components/pages/testset/modals/CreateTestsetFromApi.tsx @@ -5,7 +5,6 @@ import {JSSTheme} from "@/lib/Types" import {PythonOutlined} from "@ant-design/icons" import {ArrowLeft, FileCode, FileTs} from "@phosphor-icons/react" import {Button, Radio, Tabs, Typography} from "antd" -import {useRouter} from "next/router" import {createUseStyles} from "react-jss" import pythonCode from "@/code_snippets/testsets/create_with_json/python" import cURLCode from "@/code_snippets/testsets/create_with_json/curl" @@ -69,7 +68,6 @@ const LanguageCodeBlock = ({selectedLang, codeSnippets}: LanguageCodeBlockProps) const CreateTestsetFromApi: React.FC = ({setCurrent, onCancel}) => { const classes = useStyles() - const router = useRouter() const [uploadType, setUploadType] = useState<"csv" | "json">("csv") const [selectedLang, setSelectedLang] = useState("python") diff --git a/agenta-web/src/pages/testsets/index.tsx b/agenta-web/src/pages/testsets/index.tsx index 53ecfb2ed7..6ccc96f6ad 100644 --- a/agenta-web/src/pages/testsets/index.tsx +++ b/agenta-web/src/pages/testsets/index.tsx @@ -7,12 +7,11 @@ import {JSSTheme, TestSet, testset, TestsetCreationMode} from "@/lib/Types" import {deleteTestsets, useLoadTestsetsList} from "@/services/testsets/api" import {MoreOutlined, PlusOutlined} from "@ant-design/icons" import {Copy, GearSix, Note, PencilSimple, Trash} from "@phosphor-icons/react" -import {Button, Dropdown, Input, message, Spin, Table, Typography} from "antd" +import {Button, Dropdown, Input, Spin, Table, Typography} from "antd" import {ColumnsType} from "antd/es/table/interface" import {useRouter} from "next/router" import {createUseStyles} from "react-jss" import dayjs from "dayjs" -import {useUpdateEffect} from "usehooks-ts" import {useAppsData} from "@/contexts/app.context" const useStyles = createUseStyles((theme: JSSTheme) => ({ From 0c144c6a71c34375de41bde98a3229a88e43bd3c Mon Sep 17 00:00:00 2001 From: Juan Pablo Vega Date: Fri, 22 Nov 2024 17:33:56 +0100 Subject: [PATCH 14/18] prettier --- agenta-web/src/services/testsets/api/index.ts | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/agenta-web/src/services/testsets/api/index.ts b/agenta-web/src/services/testsets/api/index.ts index 40e24303da..14bef8cb45 100644 --- a/agenta-web/src/services/testsets/api/index.ts +++ b/agenta-web/src/services/testsets/api/index.ts @@ -31,9 +31,7 @@ export const useLoadTestsetsList = (appId: string) => { export const fetchTestsets = async () => { const {projectId} = getCurrentProject() - const response = await axios.get( - `${getAgentaApiUrl()}/api/testsets?project_id=${projectId}`, - ) + const response = await axios.get(`${getAgentaApiUrl()}/api/testsets?project_id=${projectId}`) return response.data } @@ -41,14 +39,11 @@ export const fetchTestsets = async () => { export async function createNewTestset(testsetName: string, testsetData: any) { const {projectId} = getCurrentProject() - const response = await axios.post( - `${getAgentaApiUrl()}/api/testsets?project_id=${projectId}`, - { - name: testsetName, - csvdata: testsetData || [{input: null, correct_answer: null}], - }, - ) - + const response = await axios.post(`${getAgentaApiUrl()}/api/testsets?project_id=${projectId}`, { + name: testsetName, + csvdata: testsetData || [{input: null, correct_answer: null}], + }) + return response } From 0bb9a729bef1ed299ae2c8188ed96ad2fd46a26a Mon Sep 17 00:00:00 2001 From: Juan Pablo Vega Date: Fri, 22 Nov 2024 17:36:15 +0100 Subject: [PATCH 15/18] fix merge --- agenta-web/src/services/testsets/api/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agenta-web/src/services/testsets/api/index.ts b/agenta-web/src/services/testsets/api/index.ts index 14bef8cb45..e6fb42f774 100644 --- a/agenta-web/src/services/testsets/api/index.ts +++ b/agenta-web/src/services/testsets/api/index.ts @@ -11,7 +11,7 @@ import {getCurrentProject} from "@/contexts/project.context" // - update: PUT data to server // - delete: DELETE data from server -export const useLoadTestsetsList = (appId: string) => { +export const useLoadTestsetsList = () => { const {projectId} = getCurrentProject() const {data, error, mutate, isLoading} = useSWR( From a8c9a1785e560da9499ea889942a25b8761fbd98 Mon Sep 17 00:00:00 2001 From: Juan Pablo Vega Date: Fri, 22 Nov 2024 19:32:33 +0100 Subject: [PATCH 16/18] fix project_id --- agenta-web/src/services/app-selector/api/index.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/agenta-web/src/services/app-selector/api/index.ts b/agenta-web/src/services/app-selector/api/index.ts index 6fb59ca370..4a146b80bd 100644 --- a/agenta-web/src/services/app-selector/api/index.ts +++ b/agenta-web/src/services/app-selector/api/index.ts @@ -14,10 +14,8 @@ import {waitForAppToStart} from "@/services/api" // - delete: DELETE data from server export const fetchAllTemplates = async () => { - const {projectId} = getCurrentProject() - const response = await axios.get( - `${getAgentaApiUrl()}/api/containers/templates?project_id=${projectId}`, + `${getAgentaApiUrl()}/api/containers/templates`, ) return response.data } From 7bdcb180f3dd7b1c6b1701a83caf4b9bc32360a5 Mon Sep 17 00:00:00 2001 From: Juan Pablo Vega Date: Fri, 22 Nov 2024 20:27:54 +0100 Subject: [PATCH 17/18] fix DEFAULT_UUID --- agenta-web/src/contexts/app.context.tsx | 8 ++++---- agenta-web/src/contexts/project.context.tsx | 2 +- agenta-web/src/services/app-selector/api/index.ts | 4 +--- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/agenta-web/src/contexts/app.context.tsx b/agenta-web/src/contexts/app.context.tsx index 5a66dffcc5..7669655103 100644 --- a/agenta-web/src/contexts/app.context.tsx +++ b/agenta-web/src/contexts/app.context.tsx @@ -45,10 +45,10 @@ const useApps = () => { const {selectedOrg, loading} = useOrgData() const {data, error, isLoading, mutate} = useSWR( !!user - ? `${getAgentaApiUrl()}/api/apps?project_id=${projectId}` + - (isDemo() - ? `&org_id=${selectedOrg?.id}&workspace_id=${selectedOrg?.default_workspace.id}` - : "") + ? `${getAgentaApiUrl()}/api/apps?` + + (!!projectId ? `project_id=${projectId}&` : "") + + (isDemo() ? `workspace_id=${selectedOrg?.default_workspace.id}&` : "") + + (isDemo() ? `org_id=${selectedOrg?.id}&` : "") : null, !!user ? (isDemo() ? (selectedOrg?.id ? axiosFetcher : () => {}) : axiosFetcher) : null, { diff --git a/agenta-web/src/contexts/project.context.tsx b/agenta-web/src/contexts/project.context.tsx index 9791f9d4a8..f3836b04f0 100644 --- a/agenta-web/src/contexts/project.context.tsx +++ b/agenta-web/src/contexts/project.context.tsx @@ -5,7 +5,7 @@ import useStateCallback from "@/hooks/useStateCallback" import {dynamicContext} from "@/lib/helpers/dynamic" import {isDemo} from "@/lib/helpers/utils" -const DEFAULT_UUID = "00000000-0000-0000-0000-000000000000" +const DEFAULT_UUID = null type Project = { workspace_id: string | null diff --git a/agenta-web/src/services/app-selector/api/index.ts b/agenta-web/src/services/app-selector/api/index.ts index 4a146b80bd..f7aa7b63d2 100644 --- a/agenta-web/src/services/app-selector/api/index.ts +++ b/agenta-web/src/services/app-selector/api/index.ts @@ -14,9 +14,7 @@ import {waitForAppToStart} from "@/services/api" // - delete: DELETE data from server export const fetchAllTemplates = async () => { - const response = await axios.get( - `${getAgentaApiUrl()}/api/containers/templates`, - ) + const response = await axios.get(`${getAgentaApiUrl()}/api/containers/templates`) return response.data } From 3169d8c50e3801ce6b381bdb77fdf1ddf104ddca Mon Sep 17 00:00:00 2001 From: Juan Pablo Vega Date: Fri, 22 Nov 2024 20:36:32 +0100 Subject: [PATCH 18/18] fix default uuid check --- agenta-web/src/contexts/app.context.tsx | 6 ++++-- agenta-web/src/contexts/project.context.tsx | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/agenta-web/src/contexts/app.context.tsx b/agenta-web/src/contexts/app.context.tsx index 7669655103..625f435e59 100644 --- a/agenta-web/src/contexts/app.context.tsx +++ b/agenta-web/src/contexts/app.context.tsx @@ -8,7 +8,7 @@ import {dynamicContext} from "@/lib/helpers/dynamic" import {HookAPI} from "antd/es/modal/useModal" import {useLocalStorage} from "usehooks-ts" import {useProfileData} from "./profile.context" -import {useProjectData} from "./project.context" +import {useProjectData, DEFAULT_UUID} from "./project.context" type AppContextType = { currentApp: ListAppsItem | null @@ -42,11 +42,13 @@ const useApps = () => { }) }, []) + const isMockProjectId = projectId === DEFAULT_UUID + const {selectedOrg, loading} = useOrgData() const {data, error, isLoading, mutate} = useSWR( !!user ? `${getAgentaApiUrl()}/api/apps?` + - (!!projectId ? `project_id=${projectId}&` : "") + + (!isMockProjectId ? `project_id=${projectId}&` : "") + (isDemo() ? `workspace_id=${selectedOrg?.default_workspace.id}&` : "") + (isDemo() ? `org_id=${selectedOrg?.id}&` : "") : null, diff --git a/agenta-web/src/contexts/project.context.tsx b/agenta-web/src/contexts/project.context.tsx index f3836b04f0..c3278fae18 100644 --- a/agenta-web/src/contexts/project.context.tsx +++ b/agenta-web/src/contexts/project.context.tsx @@ -5,7 +5,7 @@ import useStateCallback from "@/hooks/useStateCallback" import {dynamicContext} from "@/lib/helpers/dynamic" import {isDemo} from "@/lib/helpers/utils" -const DEFAULT_UUID = null +export const DEFAULT_UUID = "00000000-0000-0000-0000-000000000000" type Project = { workspace_id: string | null