diff --git a/agenta-backend/agenta_backend/routers/evaluation_router.py b/agenta-backend/agenta_backend/routers/evaluation_router.py index 454aedff37..02a4d37a8f 100644 --- a/agenta-backend/agenta_backend/routers/evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/evaluation_router.py @@ -33,7 +33,6 @@ response_model=List[str], ) async def fetch_evaluation_ids( - app_id: str, resource_type: str, request: Request, resource_ids: List[str] = Query(None), @@ -52,11 +51,10 @@ async def fetch_evaluation_ids( List[str]: A list of evaluation ids. """ try: - app = await db_manager.fetch_app_by_id(app_id=app_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=str(app.project_id), + project_id=request.state.project_id, permission=Permission.VIEW_EVALUATION, ) logger.debug( @@ -70,7 +68,9 @@ async def fetch_evaluation_ids( status_code=403, ) evaluations = await db_manager.fetch_evaluations_by_resource( - resource_type, str(app.project_id), resource_ids + resource_type, + request.state.project_id, + resource_ids, ) return list(map(lambda x: str(x.id), evaluations)) except Exception as exc: diff --git a/agenta-backend/agenta_backend/routers/testset_router.py b/agenta-backend/agenta_backend/routers/testset_router.py index 05268749cf..c7d3311af1 100644 --- a/agenta-backend/agenta_backend/routers/testset_router.py +++ b/agenta-backend/agenta_backend/routers/testset_router.py @@ -53,7 +53,6 @@ async def upload_file( upload_type: str = Form(None), file: UploadFile = File(...), testset_name: Optional[str] = File(None), - app_id: str = Form(None), ): """ Uploads a CSV or JSON file and saves its data to MongoDB. @@ -67,11 +66,10 @@ async def upload_file( dict: The result of the upload process. """ - app = await db_manager.fetch_app_by_id(app_id=app_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=str(app.project_id), + project_id=request.state.project_id, permission=Permission.CREATE_TESTSET, ) logger.debug(f"User has Permission to upload Testset: {has_permission}") @@ -114,7 +112,8 @@ async def upload_file( try: testset = await db_manager.create_testset( - app=app, project_id=str(app.project_id), testset_data=document + project_id=request.state.project_id, + testset_data=document, ) return TestSetSimpleResponse( id=str(testset.id), @@ -132,7 +131,6 @@ async def import_testset( request: Request, endpoint: str = Form(None), testset_name: str = Form(None), - app_id: str = Form(None), ): """ Import JSON testset data from an endpoint and save it to MongoDB. @@ -145,11 +143,10 @@ async def import_testset( dict: The result of the import process. """ - app = await db_manager.fetch_app_by_id(app_id=app_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=str(app.project_id), + project_id=request.state.project_id, permission=Permission.CREATE_TESTSET, ) logger.debug(f"User has Permission to import Testset: {has_permission}") @@ -180,7 +177,8 @@ async def import_testset( document["csvdata"].append(row) testset = await db_manager.create_testset( - app=app, project_id=str(app.project_id), testset_data=document + project_id=request.state.project_id, + testset_data=document, ) return TestSetSimpleResponse( id=str(testset.id), @@ -204,30 +202,30 @@ async def import_testset( @router.post( - "/{app_id}/", response_model=TestSetSimpleResponse, operation_id="create_testset" + "/{app_id}", + response_model=TestSetSimpleResponse, + operation_id="deprecating_create_testset", ) +@router.post("/", response_model=TestSetSimpleResponse, operation_id="create_testset") async def create_testset( - app_id: str, csvdata: NewTestset, request: Request, ): """ - Create a testset with given name and app_name, save the testset to MongoDB. + Create a testset with given name, save the testset to MongoDB. Args: name (str): name of the test set. - app_name (str): name of the application. testset (Dict[str, str]): test set data. Returns: str: The id of the test set created. """ - app = await db_manager.fetch_app_by_id(app_id=app_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=str(app.project_id), + project_id=request.state.project_id, permission=Permission.CREATE_TESTSET, ) logger.debug(f"User has Permission to create Testset: {has_permission}") @@ -245,7 +243,8 @@ async def create_testset( "csvdata": csvdata.csvdata, } testset_instance = await db_manager.create_testset( - app=app, project_id=str(app.project_id), testset_data=testset_data + project_id=request.state.project_id, + testset_data=testset_data, ) if testset_instance is not None: return TestSetSimpleResponse( @@ -315,7 +314,6 @@ async def update_testset( @router.get("/", operation_id="get_testsets") async def get_testsets( - app_id: str, request: Request, ) -> List[TestSetOutputResponse]: """ @@ -328,34 +326,52 @@ async def get_testsets( - `HTTPException` with status code 404 if no testsets are found. """ - app = await db_manager.fetch_app_by_id(app_id=app_id) - if isCloudEE(): - has_permission = await check_action_access( - user_uid=request.state.user_id, - project_id=str(app.project_id), - permission=Permission.VIEW_TESTSET, + try: + if isCloudEE(): + has_permission = await check_action_access( + user_uid=request.state.user_id, + project_id=request.state.project_id, + permission=Permission.VIEW_TESTSET, + ) + + logger.debug( + "User has Permission to view Testsets: %s", + has_permission, + ) + + if not has_permission: + error_msg = ( + "You do not have permission to perform this action. " + + "Please contact your organization admin." + ) + logger.error(error_msg) + + return JSONResponse( + status_code=403, + content={"detail": error_msg}, + ) + + testsets = await db_manager.fetch_testsets_by_project_id( + project_id=request.state.project_id, ) - logger.debug(f"User has Permission to view Testsets: {has_permission}") - if not has_permission: - error_msg = f"You do not have permission to perform this action. Please contact your organization admin." - logger.error(error_msg) - return JSONResponse( - {"detail": error_msg}, - status_code=403, + + return [ + TestSetOutputResponse( + _id=str(testset.id), # type: ignore + name=testset.name, + created_at=str(testset.created_at), + updated_at=str(testset.updated_at), ) + for testset in testsets + ] - testsets = await db_manager.fetch_testsets_by_project_id( - project_id=str(app.project_id) - ) - return [ - TestSetOutputResponse( - _id=str(testset.id), # type: ignore - name=testset.name, - created_at=str(testset.created_at), - updated_at=str(testset.updated_at), + except Exception as e: + logger.exception(f"An error occurred: {str(e)}") + + raise HTTPException( + status_code=500, + detail=str(e), ) - for testset in testsets - ] @router.get("/{testset_id}/", operation_id="get_single_testset") diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index f12dabdf5d..f9a9f20e2b 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -1892,38 +1892,6 @@ async def remove_testsets(testset_ids: List[str]): await session.commit() -async def remove_app_testsets(app_id: str): - """Returns a list of testsets owned by an app. - - Args: - app_id (str): The name of the app - - Returns: - int: The number of testsets deleted - """ - - # Find testsets owned by the app - deleted_count: int = 0 - - async with engine.session() as session: - result = await session.execute( - select(TestSetDB).filter_by(app_id=uuid.UUID(app_id)) - ) - testsets = result.scalars().all() - - if len(testsets) == 0: - logger.info(f"No testsets found for app {app_id}") - return 0 - - for testset in testsets: - await session.delete(testset) - deleted_count += 1 - logger.info(f"{deleted_count} testset(s) deleted for app {app_id}") - - await session.commit() - return deleted_count - - async def remove_base_from_db(base_id: str): """ Remove a base from the database. @@ -2086,7 +2054,7 @@ async def fetch_testset_by_id(testset_id: str) -> Optional[TestSetDB]: return testset -async def create_testset(app: AppDB, project_id: str, testset_data: Dict[str, Any]): +async def create_testset(project_id: str, testset_data: Dict[str, Any]): """ Creates a testset. diff --git a/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_testset_router.py b/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_testset_router.py index f8b9e73dc7..850bfa7c06 100644 --- a/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_testset_router.py +++ b/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_testset_router.py @@ -29,34 +29,29 @@ @pytest.mark.asyncio async def test_create_testset(): - async with engine.session() as session: - result = await session.execute( - select(AppDB).filter_by(app_name="app_variant_test") - ) - app = result.scalars().first() - - payload = { - "name": "create_testset_main", - "csvdata": [ - { - "country": "Comoros", - "correct_answer": "The capital of Comoros is Moroni", - }, - { - "country": "Kyrgyzstan", - "correct_answer": "The capital of Kyrgyzstan is Bishkek", - }, - { - "country": "Azerbaijan", - "correct_answer": "The capital of Azerbaijan is Baku", - }, - ], - } - response = await test_client.post( - f"{BACKEND_API_HOST}/testsets/{str(app.id)}/", json=payload - ) - assert response.status_code == 200 - assert response.json()["name"] == payload["name"] + payload = { + "name": "create_testset_main", + "csvdata": [ + { + "country": "Comoros", + "correct_answer": "The capital of Comoros is Moroni", + }, + { + "country": "Kyrgyzstan", + "correct_answer": "The capital of Kyrgyzstan is Bishkek", + }, + { + "country": "Azerbaijan", + "correct_answer": "The capital of Azerbaijan is Baku", + }, + ], + } + response = await test_client.post( + f"{BACKEND_API_HOST}/testsets", + json=payload, + ) + assert response.status_code == 200 + assert response.json()["name"] == payload["name"] @pytest.mark.asyncio @@ -101,18 +96,10 @@ async def test_update_testset(): @pytest.mark.asyncio async def test_get_testsets(): - async with engine.session() as session: - result = await session.execute( - select(AppDB).filter_by(app_name="app_variant_test") - ) - app = result.scalars().first() + response = await test_client.get(f"{BACKEND_API_HOST}/testsets") - response = await test_client.get( - f"{BACKEND_API_HOST}/testsets/?app_id={str(app.id)}" - ) - - assert response.status_code == 200 - assert len(response.json()) == 2 + assert response.status_code == 200 + assert len(response.json()) == 2 @pytest.mark.asyncio() diff --git a/agenta-cli/agenta/client/backend/evaluations/client.py b/agenta-cli/agenta/client/backend/evaluations/client.py index ce3dd7dcca..a8190dd1f2 100644 --- a/agenta-cli/agenta/client/backend/evaluations/client.py +++ b/agenta-cli/agenta/client/backend/evaluations/client.py @@ -26,7 +26,6 @@ def __init__(self, *, client_wrapper: SyncClientWrapper): def fetch_evaluation_ids( self, *, - app_id: str, resource_type: str, resource_ids: typing.Optional[typing.Union[str, typing.Sequence[str]]] = None, request_options: typing.Optional[RequestOptions] = None, @@ -35,7 +34,6 @@ def fetch_evaluation_ids( Fetches evaluation ids for a given resource type and id. Arguments: - app_id (str): The ID of the app for which to fetch evaluations. resource_type (str): The type of resource for which to fetch evaluations. resource_ids List[ObjectId]: The IDs of resource for which to fetch evaluations. @@ -47,8 +45,6 @@ def fetch_evaluation_ids( Parameters ---------- - app_id : str - resource_type : str resource_ids : typing.Optional[typing.Union[str, typing.Sequence[str]]] @@ -70,7 +66,6 @@ def fetch_evaluation_ids( base_url="https://yourhost.com/path/to/api", ) client.evaluations.fetch_evaluation_ids( - app_id="app_id", resource_type="resource_type", ) """ @@ -78,7 +73,6 @@ def fetch_evaluation_ids( "evaluations/by_resource", method="GET", params={ - "app_id": app_id, "resource_type": resource_type, "resource_ids": resource_ids, }, @@ -714,7 +708,6 @@ def __init__(self, *, client_wrapper: AsyncClientWrapper): async def fetch_evaluation_ids( self, *, - app_id: str, resource_type: str, resource_ids: typing.Optional[typing.Union[str, typing.Sequence[str]]] = None, request_options: typing.Optional[RequestOptions] = None, @@ -723,7 +716,6 @@ async def fetch_evaluation_ids( Fetches evaluation ids for a given resource type and id. Arguments: - app_id (str): The ID of the app for which to fetch evaluations. resource_type (str): The type of resource for which to fetch evaluations. resource_ids List[ObjectId]: The IDs of resource for which to fetch evaluations. @@ -735,7 +727,6 @@ async def fetch_evaluation_ids( Parameters ---------- - app_id : str resource_type : str @@ -763,7 +754,6 @@ async def fetch_evaluation_ids( async def main() -> None: await client.evaluations.fetch_evaluation_ids( - app_id="app_id", resource_type="resource_type", ) @@ -774,7 +764,6 @@ async def main() -> None: "evaluations/by_resource", method="GET", params={ - "app_id": app_id, "resource_type": resource_type, "resource_ids": resource_ids, }, diff --git a/agenta-cli/agenta/client/backend/testsets/client.py b/agenta-cli/agenta/client/backend/testsets/client.py index 0ea1f2eae0..fc10205700 100644 --- a/agenta-cli/agenta/client/backend/testsets/client.py +++ b/agenta-cli/agenta/client/backend/testsets/client.py @@ -28,7 +28,6 @@ def upload_file( file: core.File, upload_type: typing.Optional[str] = OMIT, testset_name: typing.Optional[str] = OMIT, - app_id: typing.Optional[str] = OMIT, request_options: typing.Optional[RequestOptions] = None, ) -> TestSetSimpleResponse: """ @@ -51,8 +50,6 @@ def upload_file( testset_name : typing.Optional[str] - app_id : typing.Optional[str] - request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -77,7 +74,6 @@ def upload_file( data={ "upload_type": upload_type, "testset_name": testset_name, - "app_id": app_id, }, files={ "file": file, @@ -173,18 +169,16 @@ def import_testset( def create_testset( self, - app_id: str, *, name: str, csvdata: typing.Sequence[typing.Dict[str, typing.Optional[typing.Any]]], request_options: typing.Optional[RequestOptions] = None, ) -> TestSetSimpleResponse: """ - Create a testset with given name and app_name, save the testset to MongoDB. + Create a testset with given name, save the testset to MongoDB. Args: name (str): name of the test set. - app_name (str): name of the application. testset (Dict[str, str]): test set data. Returns: @@ -192,8 +186,6 @@ def create_testset( Parameters ---------- - app_id : str - name : str csvdata : typing.Sequence[typing.Dict[str, typing.Optional[typing.Any]]] @@ -215,13 +207,12 @@ def create_testset( base_url="https://yourhost.com/path/to/api", ) client.testsets.create_testset( - app_id="app_id", name="name", csvdata=[{"key": "value"}], ) """ _response = self._client_wrapper.httpx_client.request( - f"testsets/{jsonable_encoder(app_id)}", + "testsets", method="POST", json={ "name": name, @@ -405,7 +396,7 @@ def update_testset( raise ApiError(status_code=_response.status_code, body=_response_json) def get_testsets( - self, *, app_id: str, request_options: typing.Optional[RequestOptions] = None + self, *, request_options: typing.Optional[RequestOptions] = None ) -> typing.List[TestSetOutputResponse]: """ Get all testsets. @@ -420,8 +411,6 @@ def get_testsets( Parameters ---------- - app_id : str - request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -438,16 +427,11 @@ def get_testsets( api_key="YOUR_API_KEY", base_url="https://yourhost.com/path/to/api", ) - client.testsets.get_testsets( - app_id="app_id", - ) + client.testsets.get_testsets() """ _response = self._client_wrapper.httpx_client.request( "testsets", method="GET", - params={ - "app_id": app_id, - }, request_options=request_options, ) try: @@ -557,7 +541,6 @@ async def upload_file( file: core.File, upload_type: typing.Optional[str] = OMIT, testset_name: typing.Optional[str] = OMIT, - app_id: typing.Optional[str] = OMIT, request_options: typing.Optional[RequestOptions] = None, ) -> TestSetSimpleResponse: """ @@ -580,8 +563,6 @@ async def upload_file( testset_name : typing.Optional[str] - app_id : typing.Optional[str] - request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -614,7 +595,6 @@ async def main() -> None: data={ "upload_type": upload_type, "testset_name": testset_name, - "app_id": app_id, }, files={ "file": file, @@ -718,18 +698,16 @@ async def main() -> None: async def create_testset( self, - app_id: str, *, name: str, csvdata: typing.Sequence[typing.Dict[str, typing.Optional[typing.Any]]], request_options: typing.Optional[RequestOptions] = None, ) -> TestSetSimpleResponse: """ - Create a testset with given name and app_name, save the testset to MongoDB. + Create a testset with given name, save the testset to MongoDB. Args: name (str): name of the test set. - app_name (str): name of the application. testset (Dict[str, str]): test set data. Returns: @@ -737,8 +715,6 @@ async def create_testset( Parameters ---------- - app_id : str - name : str csvdata : typing.Sequence[typing.Dict[str, typing.Optional[typing.Any]]] @@ -765,7 +741,6 @@ async def create_testset( async def main() -> None: await client.testsets.create_testset( - app_id="app_id", name="name", csvdata=[{"key": "value"}], ) @@ -774,7 +749,7 @@ async def main() -> None: asyncio.run(main()) """ _response = await self._client_wrapper.httpx_client.request( - f"testsets/{jsonable_encoder(app_id)}", + "testsets", method="POST", json={ "name": name, @@ -974,7 +949,7 @@ async def main() -> None: raise ApiError(status_code=_response.status_code, body=_response_json) async def get_testsets( - self, *, app_id: str, request_options: typing.Optional[RequestOptions] = None + self, *, request_options: typing.Optional[RequestOptions] = None ) -> typing.List[TestSetOutputResponse]: """ Get all testsets. @@ -989,8 +964,6 @@ async def get_testsets( Parameters ---------- - app_id : str - request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -1012,9 +985,7 @@ async def get_testsets( async def main() -> None: - await client.testsets.get_testsets( - app_id="app_id", - ) + await client.testsets.get_testsets() asyncio.run(main()) @@ -1022,9 +993,6 @@ async def main() -> None: _response = await self._client_wrapper.httpx_client.request( "testsets", method="GET", - params={ - "app_id": app_id, - }, request_options=request_options, ) try: diff --git a/agenta-web/cypress/e2e/eval.evaluations.cy.ts b/agenta-web/cypress/e2e/eval.evaluations.cy.ts index c04a67bb92..e4f4a49567 100644 --- a/agenta-web/cypress/e2e/eval.evaluations.cy.ts +++ b/agenta-web/cypress/e2e/eval.evaluations.cy.ts @@ -45,7 +45,7 @@ describe("Evaluations CRUD Operations Test", function () { context("Executing Evaluation with different answer column", () => { it("Should successfully rename the testset columns", () => { - cy.visit(`/apps/testsets`) + cy.visit(`/testsets`) cy.location("pathname").should("include", "/testsets") cy.get(".ant-table-row").eq(0).click() cy.wait(1000) diff --git a/agenta-web/cypress/e2e/single-model-test-evaluation.cy.ts b/agenta-web/cypress/e2e/single-model-test-evaluation.cy.ts index 2d8c11d4cf..cb0b73262f 100644 --- a/agenta-web/cypress/e2e/single-model-test-evaluation.cy.ts +++ b/agenta-web/cypress/e2e/single-model-test-evaluation.cy.ts @@ -62,7 +62,7 @@ describe("Single Model Test workflow", () => { }) it("Should check the evaluation testset is successfully saved", () => { - cy.visit(`/apps/testsets`) + cy.visit(`/testsets`) cy.url().should("include", "/testsets") cy.get('[data-cy="app-testset-list"]').as("table") cy.get("@table").contains(saved_testset_name).as("tempTestSet").should("be.visible") diff --git a/agenta-web/cypress/e2e/testset.cy.ts b/agenta-web/cypress/e2e/testset.cy.ts index 9085e12089..c1a9a2e147 100644 --- a/agenta-web/cypress/e2e/testset.cy.ts +++ b/agenta-web/cypress/e2e/testset.cy.ts @@ -14,7 +14,7 @@ describe("Testsets crud and UI functionality", () => { context("Testing creation process of testset", () => { beforeEach(() => { // navigate to the new testset page - cy.visit(`/apps/testsets`) + cy.visit(`/testsets`) }) it("Should successfully creates the testset and navigates to the list", () => { @@ -27,8 +27,12 @@ describe("Testsets crud and UI functionality", () => { cy.get('[data-cy="testset-name-input"]').type(testsetName) cy.clickLinkAndWait('[data-cy="create-new-testset-button"]') - cy.get(".ag-row").should("have.length", 3) + cy.get(".ag-row").should("have.length", 1) countries.forEach((country, index) => { + if (index !== 0) { + cy.get('[data-cy="add-new-testset-row"]').click() + } + cy.get(`.ag-center-cols-container .ag-row[row-index="${index}"]`).within(() => { cy.get(".ag-cell").eq(1).type(country.country) cy.get(".ag-cell") @@ -52,7 +56,7 @@ describe("Testsets crud and UI functionality", () => { context("When uploading testset", () => { const testset_name = randString(8) beforeEach(() => { - cy.visit(`/apps/testsets`) + cy.visit(`/testsets`) }) it("Should successfully upload a testset", () => { diff --git a/agenta-web/cypress/support/commands/evaluations.ts b/agenta-web/cypress/support/commands/evaluations.ts index 17400fe84e..a1294709eb 100644 --- a/agenta-web/cypress/support/commands/evaluations.ts +++ b/agenta-web/cypress/support/commands/evaluations.ts @@ -49,7 +49,7 @@ Cypress.Commands.add("createVariant", () => { Cypress.Commands.add("createVariantsAndTestsets", () => { cy.createVariant() - cy.visit("/apps/testsets") + cy.visit("/testsets") cy.url().should("include", "/testsets") cy.get('[data-cy="create-testset-modal-button"]').click() cy.get(".ant-modal-content").should("exist") @@ -60,8 +60,17 @@ Cypress.Commands.add("createVariantsAndTestsets", () => { cy.clickLinkAndWait('[data-cy="create-new-testset-button"]') cy.wrap(testsetName).as("testsetName") - cy.get(".ag-row").should("have.length", 3) + cy.get(".ag-row").should("have.length", 1) + cy.get('[data-cy="testset-header-column-edit-button"]').eq(0).click() + cy.get('[data-cy="testset-header-column-edit-input"]').clear() + cy.get('[data-cy="testset-header-column-edit-input"]').type("country") + cy.get('[data-cy="testset-header-column-save-button"]').click() + countries.forEach((country, index) => { + if (index !== 0) { + cy.get('[data-cy="add-new-testset-row"]').click() + } + cy.get(`.ag-center-cols-container .ag-row[row-index="${index}"]`).within(() => { cy.get(".ag-cell").eq(1).type(country.country) cy.get(".ag-cell") diff --git a/agenta-web/src/code_snippets/testsets/create_with_upload/curl.ts b/agenta-web/src/code_snippets/testsets/create_with_upload/curl.ts index 6d3cce6844..4070c47273 100644 --- a/agenta-web/src/code_snippets/testsets/create_with_upload/curl.ts +++ b/agenta-web/src/code_snippets/testsets/create_with_upload/curl.ts @@ -1,10 +1,9 @@ import {isDemo} from "@/lib/helpers/utils" -export default function cURLCode(uri: string, appId: string): string { +export default function cURLCode(uri: string): string { return `curl -X POST ${uri} \\ -H 'Content-Type: multipart/form-data' \\ -F 'file=@/path/to/your/file.csv' \\ -F 'testset_name=your_testset_name' \\ --F 'app_id=${appId}' \\ ${!isDemo() ? "" : "-H 'Authorization: your_api_key'"}` } diff --git a/agenta-web/src/code_snippets/testsets/create_with_upload/python.ts b/agenta-web/src/code_snippets/testsets/create_with_upload/python.ts index 3176f462ac..249b367092 100644 --- a/agenta-web/src/code_snippets/testsets/create_with_upload/python.ts +++ b/agenta-web/src/code_snippets/testsets/create_with_upload/python.ts @@ -1,16 +1,15 @@ import {isDemo} from "@/lib/helpers/utils" -export default function pythonCode(uri: string, appId: string): string { +export default function pythonCode(uri: string): string { return `import requests url = '${uri}' file_path = '/path/to/your/file.csv' testset_name = 'your_testset_name' -appId = '${appId}' with open(file_path, 'rb') as file: files = {'file': file} - data = {'testset_name': testset_name, 'app_id': appId} + data = {'testset_name': testset_name} response = requests.post(url, files=files, data=data${ !isDemo() ? "" : ", headers={'Authorization': 'your_api_key'}" }) diff --git a/agenta-web/src/code_snippets/testsets/create_with_upload/typescript.ts b/agenta-web/src/code_snippets/testsets/create_with_upload/typescript.ts index 1816542ffc..d08b2495de 100644 --- a/agenta-web/src/code_snippets/testsets/create_with_upload/typescript.ts +++ b/agenta-web/src/code_snippets/testsets/create_with_upload/typescript.ts @@ -1,7 +1,7 @@ import {isDemo} from "@/lib/helpers/utils" import {js as beautify} from "js-beautify" -export default function tsCode(uri: string, appId: string): string { +export default function tsCode(uri: string): string { const codeString = `import axios from 'axios'; const fs = require('fs'); const FormData = require('form-data'); @@ -9,12 +9,10 @@ export default function tsCode(uri: string, appId: string): string { const url = '${uri}'; const filePath = './cypress/data/countries-genders.csv'; const testsetName = 'tribalafa'; - const appId = '${appId}'; const formData = new FormData(); formData.append('file', fs.createReadStream(filePath)); formData.append('testset_name', testsetName); - formData.append('app_id', appId); const config = { headers: { diff --git a/agenta-web/src/components/HumanEvaluationModal/HumanEvaluationModal.tsx b/agenta-web/src/components/HumanEvaluationModal/HumanEvaluationModal.tsx index 084bf22fbc..2383cac732 100644 --- a/agenta-web/src/components/HumanEvaluationModal/HumanEvaluationModal.tsx +++ b/agenta-web/src/components/HumanEvaluationModal/HumanEvaluationModal.tsx @@ -150,7 +150,7 @@ const HumanEvaluationModal = ({ const appId = router.query.app_id?.toString() || "" - const {testsets, isTestsetsLoadingError} = useLoadTestsetsList(appId) + const {testsets, isTestsetsLoadingError} = useLoadTestsetsList() const [variantsInputs, setVariantsInputs] = useState>({}) @@ -340,7 +340,7 @@ const HumanEvaluationModal = ({ setError({ message: getErrorMessage(err), btnText: "Go to Test sets", - endpoint: `/apps/${appId}/testsets`, + endpoint: `/testsets`, }) } }) diff --git a/agenta-web/src/components/HumanEvaluations/AbTestingEvaluation.tsx b/agenta-web/src/components/HumanEvaluations/AbTestingEvaluation.tsx index 5befb8753c..56b7902dda 100644 --- a/agenta-web/src/components/HumanEvaluations/AbTestingEvaluation.tsx +++ b/agenta-web/src/components/HumanEvaluations/AbTestingEvaluation.tsx @@ -353,7 +353,7 @@ const AbTestingEvaluation = ({viewType}: {viewType: "evaluation" | "overview"}) icon: , onClick: (e) => { e.domEvent.stopPropagation() - router.push(`/apps/testsets/${record.testset._id}`) + router.push(`/testsets/${record.testset._id}`) }, }, {type: "divider"}, diff --git a/agenta-web/src/components/HumanEvaluations/SingleModelEvaluation.tsx b/agenta-web/src/components/HumanEvaluations/SingleModelEvaluation.tsx index 34c217e234..ba6acb2e40 100644 --- a/agenta-web/src/components/HumanEvaluations/SingleModelEvaluation.tsx +++ b/agenta-web/src/components/HumanEvaluations/SingleModelEvaluation.tsx @@ -257,7 +257,7 @@ const SingleModelEvaluation = ({viewType}: {viewType: "evaluation" | "overview"} icon: , onClick: (e) => { e.domEvent.stopPropagation() - router.push(`/apps/testsets/${record.testset._id}`) + router.push(`/testsets/${record.testset._id}`) }, }, {type: "divider"}, diff --git a/agenta-web/src/components/Playground/AddToTestSetDrawer/AddToTestSetDrawer.tsx b/agenta-web/src/components/Playground/AddToTestSetDrawer/AddToTestSetDrawer.tsx index 1d835e28dd..1768e534a7 100644 --- a/agenta-web/src/components/Playground/AddToTestSetDrawer/AddToTestSetDrawer.tsx +++ b/agenta-web/src/components/Playground/AddToTestSetDrawer/AddToTestSetDrawer.tsx @@ -22,7 +22,6 @@ import { Tooltip, message, } from "antd" -import {useRouter} from "next/router" import React, {useCallback, useEffect, useLayoutEffect, useRef, useState} from "react" import {createUseStyles} from "react-jss" import {useLocalStorage, useUpdateEffect} from "usehooks-ts" @@ -99,14 +98,9 @@ const AddToTestSetDrawer: React.FC = ({params, isChatVariant, ...props}) >(null) const [shouldRender, setShouldRender] = useState(false) const dirty = useRef(false) - const router = useRouter() - const appId = router.query.app_id as string - const {testsets, mutate, isTestsetsLoading, isTestsetsLoadingError} = useLoadTestsetsList(appId) - const storedValue = localStorage.getItem(`selectedTestset_${appId}`)?.replace(/"/g, "") - const [selectedTestset, setSelectedTestset] = useLocalStorage( - `selectedTestset_${appId}`, - "", - ) + const {testsets, mutate, isTestsetsLoading, isTestsetsLoadingError} = useLoadTestsetsList() + const storedValue = localStorage.getItem(`selectedTestset`)?.replace(/"/g, "") + const [selectedTestset, setSelectedTestset] = useLocalStorage(`selectedTestset`, "") useEffect(() => { if (storedValue && testsets.some((testset: testset) => testset._id === storedValue)) { @@ -177,7 +171,7 @@ const AddToTestSetDrawer: React.FC = ({params, isChatVariant, ...props}) }) const promise = isNew - ? createNewTestset(appId, name, newRows) + ? createNewTestset(name, newRows) : updateTestset(selectedTestset!, name, [...csvdata, ...newRows]) promise .then(() => { diff --git a/agenta-web/src/components/Playground/LoadTestsModal.tsx b/agenta-web/src/components/Playground/LoadTestsModal.tsx index e3fcedf621..325d608e16 100644 --- a/agenta-web/src/components/Playground/LoadTestsModal.tsx +++ b/agenta-web/src/components/Playground/LoadTestsModal.tsx @@ -1,6 +1,5 @@ import {fetchTestset, useLoadTestsetsList} from "@/services/testsets/api" import {Button, Divider, Modal, Select} from "antd" -import {useRouter} from "next/router" import {PropsWithChildren, useState} from "react" import {createUseStyles} from "react-jss" @@ -24,13 +23,10 @@ const useStyles = createUseStyles({ const LoadTestsModal: React.FC = (props) => { const classes = useStyles() const {onLoad} = props - const router = useRouter() const [isOpen, setIsOpen] = useState(false) const [selectedSet, setSelectedSet] = useState("") - const appId = router.query.app_id as string - - const {testsets, isTestsetsLoading, isTestsetsLoadingError} = useLoadTestsetsList(appId) + const {testsets, isTestsetsLoading, isTestsetsLoadingError} = useLoadTestsetsList() const options = testsets?.map((item: Record) => ({ label: item.name, diff --git a/agenta-web/src/components/SaveTestsetModal/SaveTestsetModal.tsx b/agenta-web/src/components/SaveTestsetModal/SaveTestsetModal.tsx index 02099ea9b1..61fe420962 100644 --- a/agenta-web/src/components/SaveTestsetModal/SaveTestsetModal.tsx +++ b/agenta-web/src/components/SaveTestsetModal/SaveTestsetModal.tsx @@ -1,5 +1,4 @@ import React, {useEffect, useState} from "react" -import {useAppId} from "@/hooks/useAppId" import {Evaluation, EvaluationScenario} from "@/lib/Types" import {EvaluationFlow} from "@/lib/enums" import {createNewTestset} from "@/services/testsets/api" @@ -21,7 +20,6 @@ const SaveTestsetModal: React.FC = ({ onSuccess, ...props }) => { - const appId = useAppId() const [form] = Form.useForm() const [submitLoading, setSubmitLoading] = useState(false) @@ -46,7 +44,7 @@ const SaveTestsetModal: React.FC = ({ } }) - createNewTestset(appId, values.testset_name, newRows) + createNewTestset(values.testset_name, newRows) .then(() => onSuccess(values.testset_name)) .catch(console.error) .finally(() => { diff --git a/agenta-web/src/components/Sidebar/config.tsx b/agenta-web/src/components/Sidebar/config.tsx index 4a633897f6..b741321b96 100644 --- a/agenta-web/src/components/Sidebar/config.tsx +++ b/agenta-web/src/components/Sidebar/config.tsx @@ -59,15 +59,13 @@ export const useSidebarConfig = () => { tooltip: "Create new applications or switch between your existing projects.", link: "/apps", icon: , - divider: apps.length === 0 ? true : false, }, { key: "app-testsets-link", title: "Test Sets", tooltip: "Create and manage testsets for evaluation purposes.", - link: `/apps/testsets`, + link: `/testsets`, icon: , - isHidden: apps.length === 0, }, { key: "app-observability-link", @@ -75,7 +73,6 @@ export const useSidebarConfig = () => { link: `/observability`, icon: , divider: true, - isHidden: apps.length === 0, }, { key: `${currentApp?.app_name || ""}_key`, diff --git a/agenta-web/src/components/TestSetTable/TestsetTable.tsx b/agenta-web/src/components/TestSetTable/TestsetTable.tsx index 10c4a515df..42c216760f 100644 --- a/agenta-web/src/components/TestSetTable/TestsetTable.tsx +++ b/agenta-web/src/components/TestSetTable/TestsetTable.tsx @@ -17,7 +17,6 @@ import {NoticeType} from "antd/es/message/interface" import {GenericObject, KeyValuePair} from "@/lib/Types" import TableCellsRenderer from "./TableCellsRenderer" import TableHeaderComponent from "./TableHeaderComponent" -import {useAppsData} from "@/contexts/app.context" type TestsetTableProps = { mode: "edit" @@ -92,9 +91,6 @@ const TestsetTable: React.FC = ({mode}) => { const router = useRouter() const {appTheme} = useAppTheme() - const {apps, isLoading: isAppsLoading} = useAppsData() - - const appId = apps[0]?.app_id const {testset_id} = router.query useBlockNavigation(unSavedChanges, { @@ -115,13 +111,6 @@ const TestsetTable: React.FC = ({mode}) => { } }, [rowData, testsetName, columnDefs, inputValues]) - useUpdateEffect(() => { - if ((apps.length === 0 || !apps) && !isAppsLoading) { - message.warning("To view the test set, you first need to create an app.") - router.push("/apps") - } - }, [isAppsLoading]) - useEffect(() => { async function applyColData(colData: {field: string}[] = []) { const newColDefs = createNewColDefs(colData) @@ -132,15 +121,17 @@ const TestsetTable: React.FC = ({mode}) => { if (writeMode === "edit" && testset_id) { fetchTestset(testset_id as string).then((data) => { setTestsetName(data.name) - setRowData(data.csvdata) - applyColData( - Object.keys(data.csvdata[0]).map((key) => ({ - field: key, - })), - ) + if (data.csvdata.length > 0) { + applyColData( + Object.keys(data.csvdata[0]).map((key) => ({ + field: key, + })), + ) + setRowData(data.csvdata) + } }) } - }, [writeMode, testset_id, appId]) + }, [writeMode, testset_id]) const handleExportClick = () => { const csvData = convertToCsv( @@ -368,7 +359,9 @@ const TestsetTable: React.FC = ({mode}) => { {selectedRow && (
- + diff --git a/agenta-web/src/components/pages/evaluations/NewEvaluation/NewEvaluationModal.tsx b/agenta-web/src/components/pages/evaluations/NewEvaluation/NewEvaluationModal.tsx index 3ed4d5b8b5..6f2733a58f 100644 --- a/agenta-web/src/components/pages/evaluations/NewEvaluation/NewEvaluationModal.tsx +++ b/agenta-web/src/components/pages/evaluations/NewEvaluation/NewEvaluationModal.tsx @@ -83,7 +83,7 @@ const NewEvaluationModal: React.FC = ({onSuccess, ...props}) => { useEffect(() => { setFetching(true) form.resetFields() - Promise.all([fetchTestsets(appId), fetchVariants(appId)]) + Promise.all([fetchTestsets(), fetchVariants(appId)]) .then(([testSets, variants]) => { setTestSets(testSets) setVariants(variants) diff --git a/agenta-web/src/components/pages/evaluations/autoEvaluation/AutoEvaluation.tsx b/agenta-web/src/components/pages/evaluations/autoEvaluation/AutoEvaluation.tsx index df1ea198e7..2380da611a 100644 --- a/agenta-web/src/components/pages/evaluations/autoEvaluation/AutoEvaluation.tsx +++ b/agenta-web/src/components/pages/evaluations/autoEvaluation/AutoEvaluation.tsx @@ -430,7 +430,7 @@ const AutoEvaluation = () => { icon: , onClick: (e) => { e.domEvent.stopPropagation() - router.push(`/apps/testsets/${record.testset.id}`) + router.push(`/testsets/${record.testset.id}`) }, }, {type: "divider"}, diff --git a/agenta-web/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/EvaluatorsModal.tsx b/agenta-web/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/EvaluatorsModal.tsx index 53d99ac647..dfe43f6c1b 100644 --- a/agenta-web/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/EvaluatorsModal.tsx +++ b/agenta-web/src/components/pages/evaluations/autoEvaluation/EvaluatorsModal/EvaluatorsModal.tsx @@ -76,7 +76,7 @@ const EvaluatorsModal = ({...props}: EvaluatorsModalProps) => { fetchAllEvaluators(), fetchAllEvaluatorConfigs(appId), fetchVariants(appId), - fetchTestsets(appId), + fetchTestsets(), ]).then(([evaluators, configs, variants, testsets]) => { setEvaluators(evaluators) setEvaluatorConfigs(configs) diff --git a/agenta-web/src/components/pages/evaluations/evaluationCompare/EvaluationCompare.tsx b/agenta-web/src/components/pages/evaluations/evaluationCompare/EvaluationCompare.tsx index 56d4eb4469..6b67367368 100644 --- a/agenta-web/src/components/pages/evaluations/evaluationCompare/EvaluationCompare.tsx +++ b/agenta-web/src/components/pages/evaluations/evaluationCompare/EvaluationCompare.tsx @@ -458,7 +458,7 @@ const EvaluationCompareMode: React.FC = () => { Testset: - + {testset?.name || ""} diff --git a/agenta-web/src/components/pages/evaluations/evaluationScenarios/EvaluationScenarios.tsx b/agenta-web/src/components/pages/evaluations/evaluationScenarios/EvaluationScenarios.tsx index 92825ed559..9af52c0e4a 100644 --- a/agenta-web/src/components/pages/evaluations/evaluationScenarios/EvaluationScenarios.tsx +++ b/agenta-web/src/components/pages/evaluations/evaluationScenarios/EvaluationScenarios.tsx @@ -349,7 +349,7 @@ const EvaluationScenarios: React.FC = () => { Testset: - + {evalaution?.testset.name || ""} diff --git a/agenta-web/src/components/pages/overview/automaticEvaluation/AutomaticEvalOverview.tsx b/agenta-web/src/components/pages/overview/automaticEvaluation/AutomaticEvalOverview.tsx index e65db174d9..3a797cac52 100644 --- a/agenta-web/src/components/pages/overview/automaticEvaluation/AutomaticEvalOverview.tsx +++ b/agenta-web/src/components/pages/overview/automaticEvaluation/AutomaticEvalOverview.tsx @@ -370,7 +370,7 @@ const AutomaticEvalOverview = () => { icon: , onClick: (e) => { e.domEvent.stopPropagation() - router.push(`/apps/testsets/${record.testset.id}`) + router.push(`/testsets/${record.testset.id}`) }, }, {type: "divider"}, diff --git a/agenta-web/src/components/pages/testset/modals/CreateTestsetFromApi.tsx b/agenta-web/src/components/pages/testset/modals/CreateTestsetFromApi.tsx index cbdb670424..80326ec18c 100644 --- a/agenta-web/src/components/pages/testset/modals/CreateTestsetFromApi.tsx +++ b/agenta-web/src/components/pages/testset/modals/CreateTestsetFromApi.tsx @@ -5,7 +5,6 @@ import {JSSTheme} from "@/lib/Types" import {PythonOutlined} from "@ant-design/icons" import {ArrowLeft, FileCode, FileTs} from "@phosphor-icons/react" import {Button, Radio, Tabs, Typography} from "antd" -import {useRouter} from "next/router" import {createUseStyles} from "react-jss" import pythonCode from "@/code_snippets/testsets/create_with_json/python" import cURLCode from "@/code_snippets/testsets/create_with_json/curl" @@ -43,7 +42,6 @@ const useStyles = createUseStyles((theme: JSSTheme) => ({ type Props = { setCurrent: React.Dispatch> onCancel: () => void - appId: string } type LanguageCodeBlockProps = { selectedLang: string @@ -68,14 +66,13 @@ const LanguageCodeBlock = ({selectedLang, codeSnippets}: LanguageCodeBlockProps) ) } -const CreateTestsetFromApi: React.FC = ({setCurrent, onCancel, appId}) => { +const CreateTestsetFromApi: React.FC = ({setCurrent, onCancel}) => { const classes = useStyles() - const router = useRouter() const [uploadType, setUploadType] = useState<"csv" | "json">("csv") const [selectedLang, setSelectedLang] = useState("python") const uploadURI = `${getAgentaApiUrl()}/api/testsets/upload` - const jsonURI = `${getAgentaApiUrl()}/api/testsets/${appId}` + const jsonURI = `${getAgentaApiUrl()}/api/testsets` const params = `{ "name": "testset_name",}` @@ -87,9 +84,9 @@ const CreateTestsetFromApi: React.FC = ({setCurrent, onCancel, appId}) => } const csvCodeSnippets: Record = { - python: pythonCodeUpload(uploadURI, appId), - bash: cURLCodeUpload(uploadURI, appId), - typescript: tsCodeUpload(uploadURI, appId), + python: pythonCodeUpload(uploadURI), + bash: cURLCodeUpload(uploadURI), + typescript: tsCodeUpload(uploadURI), } const codeSnippets = uploadType === "csv" ? csvCodeSnippets : jsonCodeSnippets diff --git a/agenta-web/src/components/pages/testset/modals/CreateTestsetFromScratch.tsx b/agenta-web/src/components/pages/testset/modals/CreateTestsetFromScratch.tsx index 7fefb9e377..b4950d59b6 100644 --- a/agenta-web/src/components/pages/testset/modals/CreateTestsetFromScratch.tsx +++ b/agenta-web/src/components/pages/testset/modals/CreateTestsetFromScratch.tsx @@ -33,7 +33,6 @@ type Props = { setEditTestsetValues: React.Dispatch> setCurrent: React.Dispatch> onCancel: () => void - appId: string } const CreateTestsetFromScratch: React.FC = ({ @@ -43,7 +42,6 @@ const CreateTestsetFromScratch: React.FC = ({ setEditTestsetValues, setCurrent, onCancel, - appId, }) => { const classes = useStyles() const router = useRouter() @@ -51,25 +49,15 @@ const CreateTestsetFromScratch: React.FC = ({ mode === "rename" ? (editTestsetValues?.name as string) : "", ) const [isLoading, setIsLoading] = useState(false) - const {mutate} = useLoadTestsetsList(appId) - - const generateInitialRowData = async (): Promise => { - const backendVariants = await fetchVariants(appId) - const variant = backendVariants[0] - const inputParams = await getVariantInputParameters(appId, variant) - const fields = [...inputParams.map((param) => param.name), "correct_answer"] - return Array(3) - .fill({}) - .map(() => fields.reduce((acc, field) => ({...acc, [field]: ""}), {})) - } + const {mutate} = useLoadTestsetsList() const handleCreateTestset = async (data?: KeyValuePair[]) => { setIsLoading(true) try { - const rowData = data || (await generateInitialRowData()) - const response = await createNewTestset(appId, testsetName, rowData) + const rowData = data + const response = await createNewTestset(testsetName, rowData) message.success("Test set created successfully") - router.push(`/apps/testsets/${response.data.id}`) + router.push(`/testsets/${response.data.id}`) } catch (error) { console.error("Error saving test set:", error) message.error("Failed to create Test set. Please try again!") diff --git a/agenta-web/src/components/pages/testset/modals/UploadTestset.tsx b/agenta-web/src/components/pages/testset/modals/UploadTestset.tsx index da1dc8c834..2f636b28da 100644 --- a/agenta-web/src/components/pages/testset/modals/UploadTestset.tsx +++ b/agenta-web/src/components/pages/testset/modals/UploadTestset.tsx @@ -49,10 +49,9 @@ const useStyles = createUseStyles((theme: JSSTheme) => ({ type Props = { setCurrent: React.Dispatch> onCancel: () => void - appId: string } -const UploadTestset: React.FC = ({setCurrent, onCancel, appId}) => { +const UploadTestset: React.FC = ({setCurrent, onCancel}) => { const classes = useStyles() const router = useRouter() const [form] = Form.useForm() @@ -61,7 +60,7 @@ const UploadTestset: React.FC = ({setCurrent, onCancel, appId}) => { const [testsetName, setTestsetName] = useState("") const [uploadLoading, setUploadLoading] = useState(false) const [fileProgress, setFileProgress] = useState({} as UploadFile) - const {mutate} = useLoadTestsetsList(appId) + const {mutate} = useLoadTestsetsList() const onFinish = async (values: any) => { const {file} = values @@ -83,7 +82,6 @@ const UploadTestset: React.FC = ({setCurrent, onCancel, appId}) => { if (testsetName && testsetName.trim() !== "") { formData.append("testset_name", testsetName) } - formData.append("app_id", appId) try { setUploadLoading(true) diff --git a/agenta-web/src/components/pages/testset/modals/index.tsx b/agenta-web/src/components/pages/testset/modals/index.tsx index a59d5b554a..8ea035eafc 100644 --- a/agenta-web/src/components/pages/testset/modals/index.tsx +++ b/agenta-web/src/components/pages/testset/modals/index.tsx @@ -27,7 +27,6 @@ type Props = { setEditTestsetValues: React.Dispatch> current: number setCurrent: React.Dispatch> - appId: string } & React.ComponentProps const TestsetModal: React.FC = ({ @@ -37,7 +36,6 @@ const TestsetModal: React.FC = ({ setEditTestsetValues, current, setCurrent, - appId, ...props }) => { const classes = useStyles() @@ -63,17 +61,14 @@ const TestsetModal: React.FC = ({ onCancel={onCancel} editTestsetValues={editTestsetValues} setEditTestsetValues={setEditTestsetValues} - appId={appId} /> ), }, { - content: , + content: , }, { - content: ( - - ), + content: , }, ] diff --git a/agenta-web/src/contexts/app.context.tsx b/agenta-web/src/contexts/app.context.tsx index 5a66dffcc5..625f435e59 100644 --- a/agenta-web/src/contexts/app.context.tsx +++ b/agenta-web/src/contexts/app.context.tsx @@ -8,7 +8,7 @@ import {dynamicContext} from "@/lib/helpers/dynamic" import {HookAPI} from "antd/es/modal/useModal" import {useLocalStorage} from "usehooks-ts" import {useProfileData} from "./profile.context" -import {useProjectData} from "./project.context" +import {useProjectData, DEFAULT_UUID} from "./project.context" type AppContextType = { currentApp: ListAppsItem | null @@ -42,13 +42,15 @@ const useApps = () => { }) }, []) + const isMockProjectId = projectId === DEFAULT_UUID + const {selectedOrg, loading} = useOrgData() const {data, error, isLoading, mutate} = useSWR( !!user - ? `${getAgentaApiUrl()}/api/apps?project_id=${projectId}` + - (isDemo() - ? `&org_id=${selectedOrg?.id}&workspace_id=${selectedOrg?.default_workspace.id}` - : "") + ? `${getAgentaApiUrl()}/api/apps?` + + (!isMockProjectId ? `project_id=${projectId}&` : "") + + (isDemo() ? `workspace_id=${selectedOrg?.default_workspace.id}&` : "") + + (isDemo() ? `org_id=${selectedOrg?.id}&` : "") : null, !!user ? (isDemo() ? (selectedOrg?.id ? axiosFetcher : () => {}) : axiosFetcher) : null, { diff --git a/agenta-web/src/contexts/project.context.tsx b/agenta-web/src/contexts/project.context.tsx index 9791f9d4a8..c3278fae18 100644 --- a/agenta-web/src/contexts/project.context.tsx +++ b/agenta-web/src/contexts/project.context.tsx @@ -5,7 +5,7 @@ import useStateCallback from "@/hooks/useStateCallback" import {dynamicContext} from "@/lib/helpers/dynamic" import {isDemo} from "@/lib/helpers/utils" -const DEFAULT_UUID = "00000000-0000-0000-0000-000000000000" +export const DEFAULT_UUID = "00000000-0000-0000-0000-000000000000" type Project = { workspace_id: string | null diff --git a/agenta-web/src/lib/helpers/evaluate.ts b/agenta-web/src/lib/helpers/evaluate.ts index 2f167ea021..4fb80b8fb9 100644 --- a/agenta-web/src/lib/helpers/evaluate.ts +++ b/agenta-web/src/lib/helpers/evaluate.ts @@ -240,16 +240,7 @@ export const getVotesPercentage = (record: HumanEvaluationListTableDataType, ind export const checkIfResourceValidForDeletion = async ( data: Omit[0], "appId">, ) => { - let appId - - if (data.resourceType === "testset") { - appId = getAppValues().apps[0]?.app_id - } else { - appId = getAppValues().currentApp?.app_id - } - if (!appId) return false - - const response = await fetchEvaluatonIdsByResource({...data, appId}) + const response = await fetchEvaluatonIdsByResource(data) if (response.data.length > 0) { const name = (data.resourceType === "testset" diff --git a/agenta-web/src/pages/apps/testsets/[testset_id]/index.tsx b/agenta-web/src/pages/testsets/[testset_id]/index.tsx similarity index 100% rename from agenta-web/src/pages/apps/testsets/[testset_id]/index.tsx rename to agenta-web/src/pages/testsets/[testset_id]/index.tsx diff --git a/agenta-web/src/pages/apps/testsets/index.tsx b/agenta-web/src/pages/testsets/index.tsx similarity index 93% rename from agenta-web/src/pages/apps/testsets/index.tsx rename to agenta-web/src/pages/testsets/index.tsx index 7a6cde0221..6ccc96f6ad 100644 --- a/agenta-web/src/pages/apps/testsets/index.tsx +++ b/agenta-web/src/pages/testsets/index.tsx @@ -7,12 +7,11 @@ import {JSSTheme, TestSet, testset, TestsetCreationMode} from "@/lib/Types" import {deleteTestsets, useLoadTestsetsList} from "@/services/testsets/api" import {MoreOutlined, PlusOutlined} from "@ant-design/icons" import {Copy, GearSix, Note, PencilSimple, Trash} from "@phosphor-icons/react" -import {Button, Dropdown, Input, message, Spin, Table, Typography} from "antd" +import {Button, Dropdown, Input, Spin, Table, Typography} from "antd" import {ColumnsType} from "antd/es/table/interface" import {useRouter} from "next/router" import {createUseStyles} from "react-jss" import dayjs from "dayjs" -import {useUpdateEffect} from "usehooks-ts" import {useAppsData} from "@/contexts/app.context" const useStyles = createUseStyles((theme: JSSTheme) => ({ @@ -56,22 +55,14 @@ const Testset = () => { const classes = useStyles() const router = useRouter() const {apps, isLoading: isAppsLoading} = useAppsData() - const appId = apps[0]?.app_id const [selectedRowKeys, setSelectedRowKeys] = useState([]) - const {testsets, isTestsetsLoading, mutate} = useLoadTestsetsList(appId) + const {testsets, isTestsetsLoading, mutate} = useLoadTestsetsList() const [isCreateTestsetModalOpen, setIsCreateTestsetModalOpen] = useState(false) const [searchTerm, setSearchTerm] = useState("") const [testsetCreationMode, setTestsetCreationMode] = useState("create") const [editTestsetValues, setEditTestsetValues] = useState(null) const [current, setCurrent] = useState(0) - useUpdateEffect(() => { - if ((apps.length === 0 || !apps) && !isAppsLoading) { - message.warning("To view the test set, you first need to create an app.") - router.push("/apps") - } - }, [isAppsLoading]) - const rowSelection = { onChange: (selectedRowKeys: React.Key[]) => { setSelectedRowKeys(selectedRowKeys) @@ -157,7 +148,7 @@ const Testset = () => { icon: , onClick: (e) => { e.domEvent.stopPropagation() - router.push(`/apps/testsets/${record._id}`) + router.push(`/testsets/${record._id}`) }, }, { @@ -186,7 +177,7 @@ const Testset = () => { }, }, { - key: "delete_eval", + key: "delete", label: "Delete", icon: , danger: true, @@ -262,7 +253,7 @@ const Testset = () => { pagination={false} onRow={(record) => { return { - onClick: () => router.push(`/apps/testsets/${record._id}`), + onClick: () => router.push(`/testsets/${record._id}`), style: {cursor: "pointer"}, } }} @@ -278,7 +269,6 @@ const Testset = () => { testsetCreationMode={testsetCreationMode} setTestsetCreationMode={setTestsetCreationMode} open={isCreateTestsetModalOpen} - appId={appId} onCancel={() => { setIsCreateTestsetModalOpen(false) }} diff --git a/agenta-web/src/services/app-selector/api/index.ts b/agenta-web/src/services/app-selector/api/index.ts index 6fb59ca370..f7aa7b63d2 100644 --- a/agenta-web/src/services/app-selector/api/index.ts +++ b/agenta-web/src/services/app-selector/api/index.ts @@ -14,11 +14,7 @@ import {waitForAppToStart} from "@/services/api" // - delete: DELETE data from server export const fetchAllTemplates = async () => { - const {projectId} = getCurrentProject() - - const response = await axios.get( - `${getAgentaApiUrl()}/api/containers/templates?project_id=${projectId}`, - ) + const response = await axios.get(`${getAgentaApiUrl()}/api/containers/templates`) return response.data } diff --git a/agenta-web/src/services/evaluations/api/index.ts b/agenta-web/src/services/evaluations/api/index.ts index 81d6bf0f81..6ab6dd3d5f 100644 --- a/agenta-web/src/services/evaluations/api/index.ts +++ b/agenta-web/src/services/evaluations/api/index.ts @@ -274,16 +274,14 @@ export const fetchAllComparisonResults = async (evaluationIds: string[]) => { export const fetchEvaluatonIdsByResource = async ({ resourceIds, resourceType, - appId, }: { resourceIds: string[] resourceType: "testset" | "evaluator_config" | "variant" - appId: string }) => { const {projectId} = getCurrentProject() return axios.get(`/api/evaluations/by_resource?project_id=${projectId}`, { - params: {resource_ids: resourceIds, resource_type: resourceType, app_id: appId}, + params: {resource_ids: resourceIds, resource_type: resourceType}, paramsSerializer: { indexes: null, //no brackets in query params }, diff --git a/agenta-web/src/services/testsets/api/index.ts b/agenta-web/src/services/testsets/api/index.ts index f2a2856ef5..e6fb42f774 100644 --- a/agenta-web/src/services/testsets/api/index.ts +++ b/agenta-web/src/services/testsets/api/index.ts @@ -11,14 +11,11 @@ import {getCurrentProject} from "@/contexts/project.context" // - update: PUT data to server // - delete: DELETE data from server -export const useLoadTestsetsList = (appId: string) => { +export const useLoadTestsetsList = () => { const {projectId} = getCurrentProject() const {data, error, mutate, isLoading} = useSWR( - () => - appId - ? `${getAgentaApiUrl()}/api/testsets?project_id=${projectId}&app_id=${appId}` - : null, + `${getAgentaApiUrl()}/api/testsets?project_id=${projectId}`, axiosFetcher, {revalidateOnFocus: false, shouldRetryOnError: false}, ) @@ -31,25 +28,22 @@ export const useLoadTestsetsList = (appId: string) => { } } -export const fetchTestsets = async (appId: string) => { +export const fetchTestsets = async () => { const {projectId} = getCurrentProject() - const response = await axios.get( - `${getAgentaApiUrl()}/api/testsets?project_id=${projectId}&app_id=${appId}`, - ) + const response = await axios.get(`${getAgentaApiUrl()}/api/testsets?project_id=${projectId}`) + return response.data } -export async function createNewTestset(appId: string, testsetName: string, testsetData: any) { +export async function createNewTestset(testsetName: string, testsetData: any) { const {projectId} = getCurrentProject() - const response = await axios.post( - `${getAgentaApiUrl()}/api/testsets/${appId}?project_id=${projectId}`, - { - name: testsetName, - csvdata: testsetData, - }, - ) + const response = await axios.post(`${getAgentaApiUrl()}/api/testsets?project_id=${projectId}`, { + name: testsetName, + csvdata: testsetData || [{input: null, correct_answer: null}], + }) + return response } diff --git a/cookbook/evaluations_with_sdk.ipynb b/cookbook/evaluations_with_sdk.ipynb index 2a4b0f42ae..01566778e6 100644 --- a/cookbook/evaluations_with_sdk.ipynb +++ b/cookbook/evaluations_with_sdk.ipynb @@ -56,6 +56,7 @@ "\n", "# You can find the application ID in the URL. For example, in the URL https://cloud.agenta.ai/apps/666dde95962bbaffdb0072b5/playground?variant=app.default, the application ID is `666dde95962bbaffdb0072b5`.\n", "from agenta.client.backend.client import AgentaApi\n", + "\n", "# Let's list the applications\n", "client.apps.list_apps()" ] @@ -66,13 +67,12 @@ "metadata": {}, "outputs": [], "source": [ - "\n", "app_id = \"667d8cfad1812781f7e375d9\"\n", "\n", "# You can create the API key under the settings page. If you are using the OSS version, you should keep this as an empty string\n", "api_key = \"EUqJGOUu.xxxx\"\n", "\n", - "# Host. \n", + "# Host.\n", "host = \"https://cloud.agenta.ai\"\n", "\n", "# Initialize the client\n", @@ -96,21 +96,25 @@ "from agenta.client.backend.types.new_testset import NewTestset\n", "\n", "csvdata = [\n", - " {\"country\": \"france\", \"capital\": \"Paris\"},\n", - " {\"country\": \"Germany\", \"capital\": \"paris\"}\n", - " ]\n", + " {\"country\": \"france\", \"capital\": \"Paris\"},\n", + " {\"country\": \"Germany\", \"capital\": \"paris\"},\n", + "]\n", "\n", - "response = client.testsets.create_testset(app_id=app_id, request=NewTestset(name=\"test set\", csvdata=csvdata))\n", + "response = client.testsets.create_testset(\n", + " request=NewTestset(name=\"test set\", csvdata=csvdata)\n", + ")\n", "test_set_id = response.id\n", "\n", "# let's now update it\n", "\n", "csvdata = [\n", - " {\"country\": \"france\", \"capital\": \"Paris\"},\n", - " {\"country\": \"Germany\", \"capital\": \"Berlin\"}\n", - " ]\n", + " {\"country\": \"france\", \"capital\": \"Paris\"},\n", + " {\"country\": \"Germany\", \"capital\": \"Berlin\"},\n", + "]\n", "\n", - "client.testsets.update_testset(testset_id=test_set_id, request=NewTestset(name=\"test set\", csvdata=csvdata))" + "client.testsets.update_testset(\n", + " testset_id=test_set_id, request=NewTestset(name=\"test set\", csvdata=csvdata)\n", + ")" ] }, { @@ -128,7 +132,12 @@ "source": [ "# Create an evaluator that performs an exact match comparison on the 'capital' column\n", "# You can find the list of evaluator keys and evaluators and their configurations in https://github.com/Agenta-AI/agenta/blob/main/agenta-backend/agenta_backend/resources/evaluators/evaluators.py\n", - "response = client.evaluators.create_new_evaluator_config(app_id=app_id, name=\"capital_evaluator\", evaluator_key=\"auto_exact_match\", settings_values={\"correct_answer_key\": \"capital\"})\n", + "response = client.evaluators.create_new_evaluator_config(\n", + " app_id=app_id,\n", + " name=\"capital_evaluator\",\n", + " evaluator_key=\"auto_exact_match\",\n", + " settings_values={\"correct_answer_key\": \"capital\"},\n", + ")\n", "exact_match_eval_id = response.id\n", "\n", "code_snippet = \"\"\"\n", @@ -146,7 +155,12 @@ " return 0.0\n", "\"\"\"\n", "\n", - "response = client.evaluators.create_new_evaluator_config(app_id=app_id, name=\"capital_letter_evaluator\", evaluator_key=\"auto_custom_code_run\", settings_values={\"code\": code_snippet})\n", + "response = client.evaluators.create_new_evaluator_config(\n", + " app_id=app_id,\n", + " name=\"capital_letter_evaluator\",\n", + " evaluator_key=\"auto_custom_code_run\",\n", + " settings_values={\"code\": code_snippet},\n", + ")\n", "letter_match_eval_id = response.id" ] }, @@ -186,13 +200,19 @@ "source": [ "# Run an evaluation\n", "from agenta.client.backend.types.llm_run_rate_limit import LlmRunRateLimit\n", - "response = client.evaluations.create_evaluation(app_id=app_id, variant_ids=[myvariant_id], testset_id=test_set_id, evaluators_configs=[exact_match_eval_id, letter_match_eval_id],\n", - " rate_limit=LlmRunRateLimit(\n", - " batch_size=10, # number of rows to call in parallel\n", - " max_retries=3, # max number of time to retry a failed llm call\n", - " retry_delay=2, # delay before retrying a failed llm call\n", - " delay_between_batches=5, # delay between batches\n", - " ),)\n", + "\n", + "response = client.evaluations.create_evaluation(\n", + " app_id=app_id,\n", + " variant_ids=[myvariant_id],\n", + " testset_id=test_set_id,\n", + " evaluators_configs=[exact_match_eval_id, letter_match_eval_id],\n", + " rate_limit=LlmRunRateLimit(\n", + " batch_size=10, # number of rows to call in parallel\n", + " max_retries=3, # max number of time to retry a failed llm call\n", + " retry_delay=2, # delay before retrying a failed llm call\n", + " delay_between_batches=5, # delay between batches\n", + " ),\n", + ")\n", "print(response)" ] }, @@ -203,7 +223,7 @@ "outputs": [], "source": [ "# check the status\n", - "client.evaluations.fetch_evaluation_status('667d98fbd1812781f7e3761a')" + "client.evaluations.fetch_evaluation_status(\"667d98fbd1812781f7e3761a\")" ] }, { @@ -213,9 +233,12 @@ "outputs": [], "source": [ "# fetch the overall results\n", - "response = client.evaluations.fetch_evaluation_results('667d98fbd1812781f7e3761a')\n", + "response = client.evaluations.fetch_evaluation_results(\"667d98fbd1812781f7e3761a\")\n", "\n", - "results = [(evaluator[\"evaluator_config\"][\"name\"], evaluator[\"result\"]) for evaluator in response[\"results\"]]\n", + "results = [\n", + " (evaluator[\"evaluator_config\"][\"name\"], evaluator[\"result\"])\n", + " for evaluator in response[\"results\"]\n", + "]\n", "# End of Selection" ] }, @@ -226,7 +249,9 @@ "outputs": [], "source": [ "# fetch the detailed results\n", - "client.evaluations.fetch_evaluation_scenarios(evaluations_ids='667d98fbd1812781f7e3761a')" + "client.evaluations.fetch_evaluation_scenarios(\n", + " evaluations_ids=\"667d98fbd1812781f7e3761a\"\n", + ")" ] } ], diff --git a/docs/docs/evaluation/02-create-test-sets.mdx b/docs/docs/evaluation/02-create-test-sets.mdx index 315d4fd3a3..31a90cdcc4 100644 --- a/docs/docs/evaluation/02-create-test-sets.mdx +++ b/docs/docs/evaluation/02-create-test-sets.mdx @@ -85,7 +85,7 @@ Here's an example of such a call: **HTTP Request:** ``` -POST /testsets/{app_id}/ +POST /testsets ``` diff --git a/docs/docs/evaluation/05-sdk-evaluation.mdx b/docs/docs/evaluation/05-sdk-evaluation.mdx index de8acd5174..c8c88886f9 100644 --- a/docs/docs/evaluation/05-sdk-evaluation.mdx +++ b/docs/docs/evaluation/05-sdk-evaluation.mdx @@ -65,7 +65,7 @@ csvdata = [ {"country": "Germany", "capital": "Berlin"} ] -response = client.testsets.create_testset(app_id=app_id, request=NewTestset(name="test set", csvdata=csvdata)) +response = client.testsets.create_testset(request=NewTestset(name="test set", csvdata=csvdata)) test_set_id = response.id ``` diff --git a/docs/docs/reference/api/create-testset.api.mdx b/docs/docs/reference/api/create-testset.api.mdx index 69aca2f6f1..525d153ece 100644 --- a/docs/docs/reference/api/create-testset.api.mdx +++ b/docs/docs/reference/api/create-testset.api.mdx @@ -1,7 +1,7 @@ --- id: create-testset title: "Create Testset" -description: "Create a testset with given name and app_name, save the testset to MongoDB." +description: "Create a testset with given name, save the testset to MongoDB." sidebar_label: "Create Testset" hide_title: true hide_table_of_contents: true @@ -34,18 +34,17 @@ import TabItem from "@theme/TabItem"; -Create a testset with given name and app_name, save the testset to MongoDB. +Create a testset with given name save the testset to MongoDB. Args: name (str): name of the test set. -app_name (str): name of the application. testset (Dict[str, str]): test set data. Returns: diff --git a/docs/docs/reference/api/fetch-evaluation-ids-evaluations-by-resource-get.api.mdx b/docs/docs/reference/api/fetch-evaluation-ids-evaluations-by-resource-get.api.mdx index 663a0ce395..9cded22bc4 100644 --- a/docs/docs/reference/api/fetch-evaluation-ids-evaluations-by-resource-get.api.mdx +++ b/docs/docs/reference/api/fetch-evaluation-ids-evaluations-by-resource-get.api.mdx @@ -44,7 +44,6 @@ import TabItem from "@theme/TabItem"; Fetches evaluation ids for a given resource type and id. Arguments: - app_id (str): The ID of the app for which to fetch evaluations. resource_type (str): The type of resource for which to fetch evaluations. resource_ids List[ObjectId]: The IDs of resource for which to fetch evaluations. @@ -78,11 +77,6 @@ Returns:
    - -