From f7e82251524c1837e13a1b2a426331e3af42a87c Mon Sep 17 00:00:00 2001 From: Abram Date: Sun, 8 Sep 2024 23:11:41 +0100 Subject: [PATCH 01/57] refactor (backend): update scope of testsets, applications, evaluators and evaluations models to be project-specific --- .../agenta_backend/models/db_models.py | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/agenta-backend/agenta_backend/models/db_models.py b/agenta-backend/agenta_backend/models/db_models.py index bfec0c5eb4..5f3c8a2edb 100644 --- a/agenta-backend/agenta_backend/models/db_models.py +++ b/agenta-backend/agenta_backend/models/db_models.py @@ -58,6 +58,12 @@ class ProjectDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) + app = relationship("AppDB", cascade="all, delete-orphan", backref="project") + evaluator_config = relationship( + "EvaluatorConfigDB", cascade="all, delete-orphan", backref="project" + ) + testset = relationship("TestSetDB", cascade="all, delete-orphan", backref="project") + class ImageDB(Base): __tablename__ = "docker_images" @@ -95,7 +101,9 @@ class AppDB(Base): nullable=False, ) app_name = Column(String) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) created_at = Column( DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) @@ -103,11 +111,9 @@ class AppDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - user = relationship("UserDB") variant = relationship( "AppVariantDB", cascade="all, delete-orphan", back_populates="app" ) - testset = relationship("TestSetDB", cascade="all, delete-orphan", backref="app") deployment = relationship( "DeploymentDB", cascade="all, delete-orphan", back_populates="app" ) @@ -358,9 +364,10 @@ class TestSetDB(Base): nullable=False, ) name = Column(String) - app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) csvdata = Column(mutable_json_type(dbtype=JSONB, nested=True)) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) created_at = Column( DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) @@ -368,7 +375,7 @@ class TestSetDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - user = relationship("UserDB") + project = relationship("ProjectDB") class EvaluatorConfigDB(Base): @@ -382,8 +389,9 @@ class EvaluatorConfigDB(Base): nullable=False, ) - app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="SET NULL")) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) name = Column(String) evaluator_key = Column(String) settings_values = Column(mutable_json_type(dbtype=JSONB, nested=True), default=dict) @@ -394,7 +402,7 @@ class EvaluatorConfigDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - user = relationship("UserDB") + project = relationship("ProjectDB") class HumanEvaluationVariantDB(Base): @@ -543,7 +551,9 @@ class EvaluationDB(Base): nullable=False, ) app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) status = Column(mutable_json_type(dbtype=JSONB, nested=True)) # Result testset_id = Column( UUID(as_uuid=True), ForeignKey("testsets.id", ondelete="SET NULL") @@ -564,7 +574,7 @@ class EvaluationDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - user = relationship("UserDB") + project = relationship("ProjectDB") testset = relationship("TestSetDB") variant = relationship("AppVariantDB") variant_revision = relationship("AppVariantRevisionsDB") From 8cfc13113edbf82f788c25c7c7031fe96d22602c Mon Sep 17 00:00:00 2001 From: Abram Date: Sun, 8 Sep 2024 23:35:33 +0100 Subject: [PATCH 02/57] chore (backend): resolve sqlalchemy.exc.ArgumentError after registering relationship mapper in models --- agenta-backend/agenta_backend/models/db_models.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/agenta-backend/agenta_backend/models/db_models.py b/agenta-backend/agenta_backend/models/db_models.py index 5f3c8a2edb..320e766063 100644 --- a/agenta-backend/agenta_backend/models/db_models.py +++ b/agenta-backend/agenta_backend/models/db_models.py @@ -402,8 +402,6 @@ class EvaluatorConfigDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - project = relationship("ProjectDB") - class HumanEvaluationVariantDB(Base): __tablename__ = "human_evaluation_variants" From 06406f1b3d4ea255a2f0f850cb998e1466e00bf7 Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 12 Sep 2024 08:13:38 +0100 Subject: [PATCH 03/57] refactor (backend): scope project_id to entities aside from testsets, applications, & evaluators --- .../agenta_backend/models/db_models.py | 95 +++++++++++-------- 1 file changed, 56 insertions(+), 39 deletions(-) diff --git a/agenta-backend/agenta_backend/models/db_models.py b/agenta-backend/agenta_backend/models/db_models.py index 320e766063..e19deab7af 100644 --- a/agenta-backend/agenta_backend/models/db_models.py +++ b/agenta-backend/agenta_backend/models/db_models.py @@ -18,6 +18,9 @@ from agenta_backend.models.shared_models import TemplateType +CASCADE_ALL_DELETE = "all, delete-orphan" + + class UserDB(Base): __tablename__ = "users" @@ -58,11 +61,12 @@ class ProjectDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - app = relationship("AppDB", cascade="all, delete-orphan", backref="project") + image = relationship("ImageDB", cascade=CASCADE_ALL_DELETE, backref="project") + app = relationship("AppDB", cascade=CASCADE_ALL_DELETE, backref="project") evaluator_config = relationship( - "EvaluatorConfigDB", cascade="all, delete-orphan", backref="project" + "EvaluatorConfigDB", cascade=CASCADE_ALL_DELETE, backref="project" ) - testset = relationship("TestSetDB", cascade="all, delete-orphan", backref="project") + testset = relationship("TestSetDB", cascade=CASCADE_ALL_DELETE, backref="project") class ImageDB(Base): @@ -80,8 +84,9 @@ class ImageDB(Base): docker_id = Column(String, nullable=True, index=True) tags = Column(String, nullable=True) deletable = Column(Boolean, default=True) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) - user = relationship("UserDB") + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) created_at = Column( DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) @@ -112,19 +117,14 @@ class AppDB(Base): ) variant = relationship( - "AppVariantDB", cascade="all, delete-orphan", back_populates="app" + "AppVariantDB", cascade=CASCADE_ALL_DELETE, back_populates="app" ) deployment = relationship( - "DeploymentDB", cascade="all, delete-orphan", back_populates="app" - ) - base = relationship( - "VariantBaseDB", cascade="all, delete-orphan", back_populates="app" - ) - evaluation = relationship( - "EvaluationDB", cascade="all, delete-orphan", backref="app" + "DeploymentDB", cascade=CASCADE_ALL_DELETE, back_populates="app" ) + evaluation = relationship("EvaluationDB", cascade=CASCADE_ALL_DELETE, backref="app") human_evaluation = relationship( - "HumanEvaluationDB", cascade="all, delete-orphan", backref="app" + "HumanEvaluationDB", cascade=CASCADE_ALL_DELETE, backref="app" ) @@ -139,7 +139,9 @@ class DeploymentDB(Base): nullable=False, ) app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) container_name = Column(String) container_id = Column(String) uri = Column(String) @@ -151,7 +153,7 @@ class DeploymentDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - user = relationship("UserDB") + project = relationship("ProjectDB") app = relationship("AppDB", back_populates="deployment") @@ -165,8 +167,9 @@ class VariantBaseDB(Base): unique=True, nullable=False, ) - app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) base_name = Column(String) image_id = Column( UUID(as_uuid=True), ForeignKey("docker_images.id", ondelete="SET NULL") @@ -181,10 +184,9 @@ class VariantBaseDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - user = relationship("UserDB") image = relationship("ImageDB") deployment = relationship("DeploymentDB") - app = relationship("AppDB", back_populates="base") + project = relationship("ProjectDB") class AppVariantDB(Base): @@ -205,7 +207,9 @@ class AppVariantDB(Base): ForeignKey("docker_images.id", ondelete="SET NULL"), nullable=True, ) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) modified_by_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) base_name = Column(String) base_id = Column(UUID(as_uuid=True), ForeignKey("bases.id")) @@ -222,12 +226,12 @@ class AppVariantDB(Base): image = relationship("ImageDB") app = relationship("AppDB", back_populates="variant") - user = relationship("UserDB", foreign_keys=[user_id]) + project = relationship("ProjectDB") modified_by = relationship("UserDB", foreign_keys=[modified_by_id]) base = relationship("VariantBaseDB") variant_revision = relationship( "AppVariantRevisionsDB", - cascade="all, delete-orphan", + cascade=CASCADE_ALL_DELETE, backref="variant_revision", ) @@ -246,6 +250,9 @@ class AppVariantRevisionsDB(Base): UUID(as_uuid=True), ForeignKey("app_variants.id", ondelete="CASCADE") ) revision = Column(Integer) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) modified_by_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) base_id = Column(UUID(as_uuid=True), ForeignKey("bases.id")) config_name = Column(String, nullable=False) @@ -259,6 +266,7 @@ class AppVariantRevisionsDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) + project = relationship("ProjectDB") modified_by = relationship("UserDB") base = relationship("VariantBaseDB") @@ -278,7 +286,9 @@ class AppEnvironmentDB(Base): ) app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) name = Column(String) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) revision = Column(Integer) deployed_app_variant_id = Column( UUID(as_uuid=True), ForeignKey("app_variants.id", ondelete="SET NULL") @@ -293,9 +303,9 @@ class AppEnvironmentDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - user = relationship("UserDB") + project = relationship("ProjectDB") environment_revisions = relationship( - "AppEnvironmentRevisionDB", cascade="all, delete-orphan", backref="environment" + "AppEnvironmentRevisionDB", cascade=CASCADE_ALL_DELETE, backref="environment" ) deployed_app_variant = relationship("AppVariantDB") deployed_app_variant_revision = relationship("AppVariantRevisionsDB") @@ -314,6 +324,9 @@ class AppEnvironmentRevisionDB(Base): environment_id = Column( UUID(as_uuid=True), ForeignKey("environments.id", ondelete="CASCADE") ) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) revision = Column(Integer) modified_by_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) deployed_app_variant_revision_id = Column( @@ -326,6 +339,7 @@ class AppEnvironmentRevisionDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) + project = relationship("ProjectDB") modified_by = relationship("UserDB") @@ -375,8 +389,6 @@ class TestSetDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - project = relationship("ProjectDB") - class EvaluatorConfigDB(Base): __tablename__ = "evaluators_configs" @@ -440,7 +452,9 @@ class HumanEvaluationDB(Base): nullable=False, ) app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) status = Column(String) evaluation_type = Column(String) testset_id = Column(UUID(as_uuid=True), ForeignKey("testsets.id")) @@ -451,16 +465,15 @@ class HumanEvaluationDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - user = relationship("UserDB") testset = relationship("TestSetDB") evaluation_variant = relationship( "HumanEvaluationVariantDB", - cascade="all, delete-orphan", + cascade=CASCADE_ALL_DELETE, backref="human_evaluation", ) evaluation_scenario = relationship( "HumanEvaluationScenarioDB", - cascade="all, delete-orphan", + cascade=CASCADE_ALL_DELETE, backref="evaluation_scenario", ) @@ -475,7 +488,9 @@ class HumanEvaluationScenarioDB(Base): unique=True, nullable=False, ) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) evaluation_id = Column( UUID(as_uuid=True), ForeignKey("human_evaluations.id", ondelete="CASCADE") ) @@ -578,15 +593,15 @@ class EvaluationDB(Base): variant_revision = relationship("AppVariantRevisionsDB") aggregated_results = relationship( "EvaluationAggregatedResultDB", - cascade="all, delete-orphan", + cascade=CASCADE_ALL_DELETE, backref="evaluation", ) evaluation_scenarios = relationship( - "EvaluationScenarioDB", cascade="all, delete-orphan", backref="evaluation" + "EvaluationScenarioDB", cascade=CASCADE_ALL_DELETE, backref="evaluation" ) evaluator_configs = relationship( "EvaluationEvaluatorConfigDB", - cascade="all, delete-orphan", + cascade=CASCADE_ALL_DELETE, backref="evaluation", ) @@ -623,7 +638,9 @@ class EvaluationScenarioDB(Base): unique=True, nullable=False, ) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) evaluation_id = Column( UUID(as_uuid=True), ForeignKey("evaluations.id", ondelete="CASCADE") ) @@ -650,11 +667,11 @@ class EvaluationScenarioDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - user = relationship("UserDB") + project = relationship("ProjectDB") variant = relationship("AppVariantDB") results = relationship( "EvaluationScenarioResultDB", - cascade="all, delete-orphan", + cascade=CASCADE_ALL_DELETE, backref="evaluation_scenario", ) From af0497bc1575952bb4deaca84c4edd96e3beecf6 Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 12 Sep 2024 16:03:23 +0100 Subject: [PATCH 04/57] minor refactor (migration): ensure migration points to the appropriate down_revision head --- .../postgres/versions/c5ae28e37102_created_projects_table.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agenta-backend/agenta_backend/migrations/postgres/versions/c5ae28e37102_created_projects_table.py b/agenta-backend/agenta_backend/migrations/postgres/versions/c5ae28e37102_created_projects_table.py index 4ede21f140..cd4a35761f 100644 --- a/agenta-backend/agenta_backend/migrations/postgres/versions/c5ae28e37102_created_projects_table.py +++ b/agenta-backend/agenta_backend/migrations/postgres/versions/c5ae28e37102_created_projects_table.py @@ -1,7 +1,7 @@ """Created projects table Revision ID: c5ae28e37102 -Revises: b80c708c21bb +Revises: 5c29a64204f4 Create Date: 2024-08-29 13:15:06.934841 """ @@ -14,7 +14,7 @@ # revision identifiers, used by Alembic. revision: str = "c5ae28e37102" -down_revision: Union[str, None] = "b80c708c21bb" +down_revision: Union[str, None] = "5c29a64204f4" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None From 96aea7a704e7a3114e2b82e8cbe30ca6f9501554 Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 12 Sep 2024 16:10:49 +0100 Subject: [PATCH 05/57] refactor (backend): remove project_id from db entities --- .../agenta_backend/models/db_models.py | 125 +++++++----------- 1 file changed, 49 insertions(+), 76 deletions(-) diff --git a/agenta-backend/agenta_backend/models/db_models.py b/agenta-backend/agenta_backend/models/db_models.py index 834cc8594f..70f6700a71 100644 --- a/agenta-backend/agenta_backend/models/db_models.py +++ b/agenta-backend/agenta_backend/models/db_models.py @@ -18,9 +18,6 @@ from agenta_backend.models.shared_models import TemplateType -CASCADE_ALL_DELETE = "all, delete-orphan" - - class UserDB(Base): __tablename__ = "users" @@ -52,7 +49,7 @@ class ProjectDB(Base): unique=True, nullable=False, ) - project_name = Column(String, nullable=False, unique=True) + project_name = Column(String, nullable=False) is_default = Column(Boolean, default=False) created_at = Column( DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) @@ -61,13 +58,6 @@ class ProjectDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - image = relationship("ImageDB", cascade=CASCADE_ALL_DELETE, backref="project") - app = relationship("AppDB", cascade=CASCADE_ALL_DELETE, backref="project") - evaluator_config = relationship( - "EvaluatorConfigDB", cascade=CASCADE_ALL_DELETE, backref="project" - ) - testset = relationship("TestSetDB", cascade=CASCADE_ALL_DELETE, backref="project") - class ImageDB(Base): __tablename__ = "docker_images" @@ -84,9 +74,8 @@ class ImageDB(Base): docker_id = Column(String, nullable=True, index=True) tags = Column(String, nullable=True) deletable = Column(Boolean, default=True) - project_id = Column( - UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") - ) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) + user = relationship("UserDB") created_at = Column( DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) @@ -106,10 +95,7 @@ class AppDB(Base): nullable=False, ) app_name = Column(String) - project_id = Column( - UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") - ) - modified_by_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) created_at = Column( DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) @@ -117,16 +103,22 @@ class AppDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - modified_by = relationship("UserDB", foreign_keys=[modified_by_id]) + user = relationship("UserDB") variant = relationship( - "AppVariantDB", cascade=CASCADE_ALL_DELETE, back_populates="app" + "AppVariantDB", cascade="all, delete-orphan", back_populates="app" ) + testset = relationship("TestSetDB", cascade="all, delete-orphan", backref="app") deployment = relationship( - "DeploymentDB", cascade=CASCADE_ALL_DELETE, back_populates="app" + "DeploymentDB", cascade="all, delete-orphan", back_populates="app" + ) + base = relationship( + "VariantBaseDB", cascade="all, delete-orphan", back_populates="app" + ) + evaluation = relationship( + "EvaluationDB", cascade="all, delete-orphan", backref="app" ) - evaluation = relationship("EvaluationDB", cascade=CASCADE_ALL_DELETE, backref="app") human_evaluation = relationship( - "HumanEvaluationDB", cascade=CASCADE_ALL_DELETE, backref="app" + "HumanEvaluationDB", cascade="all, delete-orphan", backref="app" ) @@ -141,9 +133,7 @@ class DeploymentDB(Base): nullable=False, ) app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) - project_id = Column( - UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") - ) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) container_name = Column(String) container_id = Column(String) uri = Column(String) @@ -155,7 +145,7 @@ class DeploymentDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - project = relationship("ProjectDB") + user = relationship("UserDB") app = relationship("AppDB", back_populates="deployment") @@ -169,9 +159,8 @@ class VariantBaseDB(Base): unique=True, nullable=False, ) - project_id = Column( - UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") - ) + app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) base_name = Column(String) image_id = Column( UUID(as_uuid=True), ForeignKey("docker_images.id", ondelete="SET NULL") @@ -186,9 +175,10 @@ class VariantBaseDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) + user = relationship("UserDB") image = relationship("ImageDB") deployment = relationship("DeploymentDB") - project = relationship("ProjectDB") + app = relationship("AppDB", back_populates="base") class AppVariantDB(Base): @@ -209,9 +199,7 @@ class AppVariantDB(Base): ForeignKey("docker_images.id", ondelete="SET NULL"), nullable=True, ) - project_id = Column( - UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") - ) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) modified_by_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) base_name = Column(String) base_id = Column(UUID(as_uuid=True), ForeignKey("bases.id")) @@ -228,12 +216,12 @@ class AppVariantDB(Base): image = relationship("ImageDB") app = relationship("AppDB", back_populates="variant") - project = relationship("ProjectDB") + user = relationship("UserDB", foreign_keys=[user_id]) modified_by = relationship("UserDB", foreign_keys=[modified_by_id]) base = relationship("VariantBaseDB") variant_revision = relationship( "AppVariantRevisionsDB", - cascade=CASCADE_ALL_DELETE, + cascade="all, delete-orphan", backref="variant_revision", ) @@ -252,9 +240,6 @@ class AppVariantRevisionsDB(Base): UUID(as_uuid=True), ForeignKey("app_variants.id", ondelete="CASCADE") ) revision = Column(Integer) - project_id = Column( - UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") - ) modified_by_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) base_id = Column(UUID(as_uuid=True), ForeignKey("bases.id")) config_name = Column(String, nullable=False) @@ -268,7 +253,6 @@ class AppVariantRevisionsDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - project = relationship("ProjectDB") modified_by = relationship("UserDB") base = relationship("VariantBaseDB") @@ -288,9 +272,7 @@ class AppEnvironmentDB(Base): ) app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) name = Column(String) - project_id = Column( - UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") - ) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) revision = Column(Integer) deployed_app_variant_id = Column( UUID(as_uuid=True), ForeignKey("app_variants.id", ondelete="SET NULL") @@ -305,9 +287,9 @@ class AppEnvironmentDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - project = relationship("ProjectDB") + user = relationship("UserDB") environment_revisions = relationship( - "AppEnvironmentRevisionDB", cascade=CASCADE_ALL_DELETE, backref="environment" + "AppEnvironmentRevisionDB", cascade="all, delete-orphan", backref="environment" ) deployed_app_variant = relationship("AppVariantDB") deployed_app_variant_revision = relationship("AppVariantRevisionsDB") @@ -326,9 +308,6 @@ class AppEnvironmentRevisionDB(Base): environment_id = Column( UUID(as_uuid=True), ForeignKey("environments.id", ondelete="CASCADE") ) - project_id = Column( - UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") - ) revision = Column(Integer) modified_by_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) deployed_app_variant_revision_id = Column( @@ -341,7 +320,6 @@ class AppEnvironmentRevisionDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - project = relationship("ProjectDB") modified_by = relationship("UserDB") @@ -380,10 +358,9 @@ class TestSetDB(Base): nullable=False, ) name = Column(String) - project_id = Column( - UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") - ) + app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) csvdata = Column(mutable_json_type(dbtype=JSONB, nested=True)) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) created_at = Column( DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) @@ -391,6 +368,8 @@ class TestSetDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) + user = relationship("UserDB") + class EvaluatorConfigDB(Base): __tablename__ = "evaluators_configs" @@ -403,9 +382,8 @@ class EvaluatorConfigDB(Base): nullable=False, ) - project_id = Column( - UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") - ) + app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="SET NULL")) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) name = Column(String) evaluator_key = Column(String) settings_values = Column(mutable_json_type(dbtype=JSONB, nested=True), default=dict) @@ -416,6 +394,8 @@ class EvaluatorConfigDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) + user = relationship("UserDB") + class HumanEvaluationVariantDB(Base): __tablename__ = "human_evaluation_variants" @@ -454,9 +434,7 @@ class HumanEvaluationDB(Base): nullable=False, ) app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) - project_id = Column( - UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") - ) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) status = Column(String) evaluation_type = Column(String) testset_id = Column(UUID(as_uuid=True), ForeignKey("testsets.id")) @@ -467,15 +445,16 @@ class HumanEvaluationDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) + user = relationship("UserDB") testset = relationship("TestSetDB") evaluation_variant = relationship( "HumanEvaluationVariantDB", - cascade=CASCADE_ALL_DELETE, + cascade="all, delete-orphan", backref="human_evaluation", ) evaluation_scenario = relationship( "HumanEvaluationScenarioDB", - cascade=CASCADE_ALL_DELETE, + cascade="all, delete-orphan", backref="evaluation_scenario", ) @@ -490,9 +469,7 @@ class HumanEvaluationScenarioDB(Base): unique=True, nullable=False, ) - project_id = Column( - UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") - ) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) evaluation_id = Column( UUID(as_uuid=True), ForeignKey("human_evaluations.id", ondelete="CASCADE") ) @@ -566,9 +543,7 @@ class EvaluationDB(Base): nullable=False, ) app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) - project_id = Column( - UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") - ) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) status = Column(mutable_json_type(dbtype=JSONB, nested=True)) # Result testset_id = Column( UUID(as_uuid=True), ForeignKey("testsets.id", ondelete="SET NULL") @@ -589,21 +564,21 @@ class EvaluationDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - project = relationship("ProjectDB") + user = relationship("UserDB") testset = relationship("TestSetDB") variant = relationship("AppVariantDB") variant_revision = relationship("AppVariantRevisionsDB") aggregated_results = relationship( "EvaluationAggregatedResultDB", - cascade=CASCADE_ALL_DELETE, + cascade="all, delete-orphan", backref="evaluation", ) evaluation_scenarios = relationship( - "EvaluationScenarioDB", cascade=CASCADE_ALL_DELETE, backref="evaluation" + "EvaluationScenarioDB", cascade="all, delete-orphan", backref="evaluation" ) evaluator_configs = relationship( "EvaluationEvaluatorConfigDB", - cascade=CASCADE_ALL_DELETE, + cascade="all, delete-orphan", backref="evaluation", ) @@ -640,9 +615,7 @@ class EvaluationScenarioDB(Base): unique=True, nullable=False, ) - project_id = Column( - UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") - ) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) evaluation_id = Column( UUID(as_uuid=True), ForeignKey("evaluations.id", ondelete="CASCADE") ) @@ -669,11 +642,11 @@ class EvaluationScenarioDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - project = relationship("ProjectDB") + user = relationship("UserDB") variant = relationship("AppVariantDB") results = relationship( "EvaluationScenarioResultDB", - cascade=CASCADE_ALL_DELETE, + cascade="all, delete-orphan", backref="evaluation_scenario", ) @@ -690,4 +663,4 @@ class IDsMappingDB(Base): ) table_name = Column(String, nullable=False) objectid = Column(String, nullable=False) - uuid = Column(UUID(as_uuid=True), nullable=False) + uuid = Column(UUID(as_uuid=True), nullable=False) \ No newline at end of file From 31cdcddfc40b39d107b5f581044f5701ffbbb706 Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 12 Sep 2024 16:59:13 +0100 Subject: [PATCH 06/57] refactor (backend): make user_id column optional in DB entities and create Alembic migration for project_id scoping --- ...et_user_id_column_in_db_entities_to_be_.py | 68 +++++++++++++++++++ .../agenta_backend/models/db_models.py | 30 ++++---- 2 files changed, 83 insertions(+), 15 deletions(-) create mode 100644 agenta-backend/agenta_backend/migrations/postgres/versions/6cfe239894fb_set_user_id_column_in_db_entities_to_be_.py diff --git a/agenta-backend/agenta_backend/migrations/postgres/versions/6cfe239894fb_set_user_id_column_in_db_entities_to_be_.py b/agenta-backend/agenta_backend/migrations/postgres/versions/6cfe239894fb_set_user_id_column_in_db_entities_to_be_.py new file mode 100644 index 0000000000..8f742b7915 --- /dev/null +++ b/agenta-backend/agenta_backend/migrations/postgres/versions/6cfe239894fb_set_user_id_column_in_db_entities_to_be_.py @@ -0,0 +1,68 @@ +"""Set user_id column in db entities to be optional --- prep for project_id scoping + +Revision ID: 6cfe239894fb +Revises: 362gbs21a2ee +Create Date: 2024-09-12 15:25:29.462793 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "6cfe239894fb" +down_revision: Union[str, None] = "362gbs21a2ee" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column("docker_images", "user_id", existing_type=sa.UUID, nullable=True) + op.alter_column("app_db", "user_id", existing_type=sa.UUID, nullable=True) + op.alter_column("deployments", "user_id", existing_type=sa.UUID, nullable=True) + op.alter_column("bases", "user_id", existing_type=sa.UUID, nullable=True) + op.alter_column("app_variants", "user_id", existing_type=sa.UUID, nullable=True) + op.alter_column("environments", "user_id", existing_type=sa.UUID, nullable=True) + op.alter_column("testsets", "user_id", existing_type=sa.UUID, nullable=True) + op.alter_column( + "evaluators_configs", "user_id", existing_type=sa.UUID, nullable=True + ) + op.alter_column( + "human_evaluations", "user_id", existing_type=sa.UUID, nullable=True + ) + op.alter_column( + "human_evaluations_scenarios", "user_id", existing_type=sa.UUID, nullable=True + ) + op.alter_column("evaluations", "user_id", existing_type=sa.UUID, nullable=True) + op.alter_column( + "evaluation_scenarios", "user_id", existing_type=sa.UUID, nullable=True + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column("docker_images", "user_id", existing_type=sa.UUID, nullable=False) + op.alter_column("app_db", "user_id", existing_type=sa.UUID, nullable=False) + op.alter_column("deployments", "user_id", existing_type=sa.UUID, nullable=False) + op.alter_column("bases", "user_id", existing_type=sa.UUID, nullable=False) + op.alter_column("app_variants", "user_id", existing_type=sa.UUID, nullable=False) + op.alter_column("environments", "user_id", existing_type=sa.UUID, nullable=False) + op.alter_column("testsets", "user_id", existing_type=sa.UUID, nullable=False) + op.alter_column( + "evaluators_configs", "user_id", existing_type=sa.UUID, nullable=False + ) + op.alter_column( + "human_evaluations", "user_id", existing_type=sa.UUID, nullable=False + ) + op.alter_column( + "human_evaluations_scenarios", "user_id", existing_type=sa.UUID, nullable=False + ) + op.alter_column("evaluations", "user_id", existing_type=sa.UUID, nullable=False) + op.alter_column( + "evaluation_scenarios", "user_id", existing_type=sa.UUID, nullable=False + ) + # ### end Alembic commands ### diff --git a/agenta-backend/agenta_backend/models/db_models.py b/agenta-backend/agenta_backend/models/db_models.py index 70f6700a71..06dcdcebaf 100644 --- a/agenta-backend/agenta_backend/models/db_models.py +++ b/agenta-backend/agenta_backend/models/db_models.py @@ -74,8 +74,7 @@ class ImageDB(Base): docker_id = Column(String, nullable=True, index=True) tags = Column(String, nullable=True) deletable = Column(Boolean, default=True) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) - user = relationship("UserDB") + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) created_at = Column( DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) @@ -95,7 +94,8 @@ class AppDB(Base): nullable=False, ) app_name = Column(String) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) + modified_by_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) created_at = Column( DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) @@ -103,7 +103,7 @@ class AppDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - user = relationship("UserDB") + user = relationship("UserDB", foreign_keys=[user_id]) variant = relationship( "AppVariantDB", cascade="all, delete-orphan", back_populates="app" ) @@ -133,7 +133,7 @@ class DeploymentDB(Base): nullable=False, ) app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) container_name = Column(String) container_id = Column(String) uri = Column(String) @@ -160,7 +160,7 @@ class VariantBaseDB(Base): nullable=False, ) app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) base_name = Column(String) image_id = Column( UUID(as_uuid=True), ForeignKey("docker_images.id", ondelete="SET NULL") @@ -199,7 +199,7 @@ class AppVariantDB(Base): ForeignKey("docker_images.id", ondelete="SET NULL"), nullable=True, ) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) modified_by_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) base_name = Column(String) base_id = Column(UUID(as_uuid=True), ForeignKey("bases.id")) @@ -272,7 +272,7 @@ class AppEnvironmentDB(Base): ) app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) name = Column(String) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) revision = Column(Integer) deployed_app_variant_id = Column( UUID(as_uuid=True), ForeignKey("app_variants.id", ondelete="SET NULL") @@ -360,7 +360,7 @@ class TestSetDB(Base): name = Column(String) app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) csvdata = Column(mutable_json_type(dbtype=JSONB, nested=True)) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) created_at = Column( DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) @@ -383,7 +383,7 @@ class EvaluatorConfigDB(Base): ) app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="SET NULL")) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) name = Column(String) evaluator_key = Column(String) settings_values = Column(mutable_json_type(dbtype=JSONB, nested=True), default=dict) @@ -434,7 +434,7 @@ class HumanEvaluationDB(Base): nullable=False, ) app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) status = Column(String) evaluation_type = Column(String) testset_id = Column(UUID(as_uuid=True), ForeignKey("testsets.id")) @@ -469,7 +469,7 @@ class HumanEvaluationScenarioDB(Base): unique=True, nullable=False, ) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) evaluation_id = Column( UUID(as_uuid=True), ForeignKey("human_evaluations.id", ondelete="CASCADE") ) @@ -543,7 +543,7 @@ class EvaluationDB(Base): nullable=False, ) app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) status = Column(mutable_json_type(dbtype=JSONB, nested=True)) # Result testset_id = Column( UUID(as_uuid=True), ForeignKey("testsets.id", ondelete="SET NULL") @@ -615,7 +615,7 @@ class EvaluationScenarioDB(Base): unique=True, nullable=False, ) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) evaluation_id = Column( UUID(as_uuid=True), ForeignKey("evaluations.id", ondelete="CASCADE") ) @@ -663,4 +663,4 @@ class IDsMappingDB(Base): ) table_name = Column(String, nullable=False) objectid = Column(String, nullable=False) - uuid = Column(UUID(as_uuid=True), nullable=False) \ No newline at end of file + uuid = Column(UUID(as_uuid=True), nullable=False) From e5b8e1f436dff7518bab703a21fcf2d7e2d0af69 Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 12 Sep 2024 22:54:39 +0100 Subject: [PATCH 07/57] chore (backend): added helper function to check if unique constraint exists in a table --- .../migrations/postgres/utils.py | 33 +++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/agenta-backend/agenta_backend/migrations/postgres/utils.py b/agenta-backend/agenta_backend/migrations/postgres/utils.py index 3666175a3e..a327e32f55 100644 --- a/agenta-backend/agenta_backend/migrations/postgres/utils.py +++ b/agenta-backend/agenta_backend/migrations/postgres/utils.py @@ -5,13 +5,14 @@ import click import asyncpg + +from sqlalchemy import inspect, text, Engine from sqlalchemy.exc import ProgrammingError +from sqlalchemy.ext.asyncio import create_async_engine, AsyncEngine from alembic import command from alembic.config import Config -from sqlalchemy import inspect, text from alembic.script import ScriptDirectory -from sqlalchemy.ext.asyncio import create_async_engine, AsyncEngine from agenta_backend.utils.common import isCloudEE, isCloudDev @@ -173,3 +174,31 @@ async def check_if_templates_table_exist(): await engine.dispose() return True + + +def unique_constraint_exists( + engine: Engine, table_name: str, constraint_name: str +) -> bool: + """ + The function checks if a unique constraint with a specific name exists on a table in a PostgreSQL + database. + + Args: + - engine (Engine): instance of a database engine that represents a connection to a database. + - table_name (str): name of the table to check the existence of the unique constraint. + - constraint_name (str): name of the unique constraint to check for existence. + + Returns: + - returns a boolean value indicating whether a unique constraint with the specified `constraint_name` exists in the table. + """ + + with engine.connect() as conn: + result = conn.execute( + text( + f""" + SELECT conname FROM pg_constraint + WHERE conname = '{constraint_name}' AND conrelid = '{table_name}'::regclass; + """ + ) + ) + return result.fetchone() is not None From 8d41eee33859fb695cf96dc81b685abfe6ce9c38 Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 12 Sep 2024 22:55:15 +0100 Subject: [PATCH 08/57] refactor (backend): scope project_id to db models entities and create alembic migration --- ..._scope_project_id_to_db_models_entities.py | 357 ++++++++++++++++++ .../agenta_backend/models/db_models.py | 119 +++--- 2 files changed, 430 insertions(+), 46 deletions(-) create mode 100644 agenta-backend/agenta_backend/migrations/postgres/versions/c00a326c625a_scope_project_id_to_db_models_entities.py diff --git a/agenta-backend/agenta_backend/migrations/postgres/versions/c00a326c625a_scope_project_id_to_db_models_entities.py b/agenta-backend/agenta_backend/migrations/postgres/versions/c00a326c625a_scope_project_id_to_db_models_entities.py new file mode 100644 index 0000000000..801cb7ffbf --- /dev/null +++ b/agenta-backend/agenta_backend/migrations/postgres/versions/c00a326c625a_scope_project_id_to_db_models_entities.py @@ -0,0 +1,357 @@ +"""scope project_id to db models/entities + +Revision ID: c00a326c625a +Revises: 6cfe239894fb +Create Date: 2024-09-12 20:34:16.175845 + +""" + +import os +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +from agenta_backend.migrations.postgres import utils + + +# revision identifiers, used by Alembic. +revision: str = "c00a326c625a" +down_revision: Union[str, None] = "6cfe239894fb" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + engine = sa.create_engine(os.getenv("POSTGRES_URI")) + op.add_column("app_db", sa.Column("project_id", sa.UUID(), nullable=True)) + op.drop_constraint("app_db_user_id_fkey", "app_db", type_="foreignkey") + op.create_foreign_key( + "app_db_projects_fkey", + "app_db", + "projects", + ["project_id"], + ["id"], + ondelete="CASCADE", + ) + op.drop_column("app_db", "user_id") + op.add_column( + "app_variant_revisions", sa.Column("project_id", sa.UUID(), nullable=True) + ) + op.create_foreign_key( + "app_variant_revisions_projects_fkey", + "app_variant_revisions", + "projects", + ["project_id"], + ["id"], + ondelete="CASCADE", + ) + op.add_column("app_variants", sa.Column("project_id", sa.UUID(), nullable=True)) + op.drop_constraint("app_variants_user_id_fkey", "app_variants", type_="foreignkey") + op.create_foreign_key( + "app_variants_projects_fkey", + "app_variants", + "projects", + ["project_id"], + ["id"], + ondelete="CASCADE", + ) + op.drop_column("app_variants", "user_id") + op.add_column("bases", sa.Column("project_id", sa.UUID(), nullable=True)) + op.drop_constraint("bases_app_id_fkey", "bases", type_="foreignkey") + op.drop_constraint("bases_user_id_fkey", "bases", type_="foreignkey") + op.create_foreign_key( + "bases_projects_fkey", + "bases", + "projects", + ["project_id"], + ["id"], + ondelete="CASCADE", + ) + op.drop_column("bases", "app_id") + op.drop_column("bases", "user_id") + op.add_column("deployments", sa.Column("project_id", sa.UUID(), nullable=True)) + op.drop_constraint("deployments_user_id_fkey", "deployments", type_="foreignkey") + op.create_foreign_key( + "deployments_projects_fkey", + "deployments", + "projects", + ["project_id"], + ["id"], + ondelete="CASCADE", + ) + op.drop_column("deployments", "user_id") + op.add_column("docker_images", sa.Column("project_id", sa.UUID(), nullable=True)) + op.drop_constraint( + "docker_images_user_id_fkey", "docker_images", type_="foreignkey" + ) + op.create_foreign_key( + "docker_images_projects_fkey", + "docker_images", + "projects", + ["project_id"], + ["id"], + ondelete="CASCADE", + ) + op.drop_column("docker_images", "user_id") + op.add_column("environments", sa.Column("project_id", sa.UUID(), nullable=True)) + op.drop_constraint("environments_user_id_fkey", "environments", type_="foreignkey") + op.create_foreign_key( + "environments_projects_fkey", + "environments", + "projects", + ["project_id"], + ["id"], + ondelete="CASCADE", + ) + op.drop_column("environments", "user_id") + op.add_column( + "environments_revisions", sa.Column("project_id", sa.UUID(), nullable=True) + ) + op.create_foreign_key( + "environments_revisions_projects_fkey", + "environments_revisions", + "projects", + ["project_id"], + ["id"], + ondelete="CASCADE", + ) + op.add_column( + "evaluation_scenarios", sa.Column("project_id", sa.UUID(), nullable=True) + ) + op.drop_constraint( + "evaluation_scenarios_user_id_fkey", "evaluation_scenarios", type_="foreignkey" + ) + op.create_foreign_key( + "evaluation_scenarios_projects_fkey", + "evaluation_scenarios", + "projects", + ["project_id"], + ["id"], + ondelete="CASCADE", + ) + op.drop_column("evaluation_scenarios", "user_id") + op.add_column("evaluations", sa.Column("project_id", sa.UUID(), nullable=True)) + op.drop_constraint("evaluations_user_id_fkey", "evaluations", type_="foreignkey") + op.create_foreign_key( + "evaluations_projects_fkey", + "evaluations", + "projects", + ["project_id"], + ["id"], + ondelete="CASCADE", + ) + op.drop_column("evaluations", "user_id") + op.add_column( + "evaluators_configs", sa.Column("project_id", sa.UUID(), nullable=True) + ) + op.drop_constraint( + "evaluators_configs_user_id_fkey", "evaluators_configs", type_="foreignkey" + ) + op.drop_constraint( + "evaluators_configs_app_id_fkey", "evaluators_configs", type_="foreignkey" + ) + op.create_foreign_key( + "evaluators_configs_projects_fkey", + "evaluators_configs", + "projects", + ["project_id"], + ["id"], + ondelete="CASCADE", + ) + op.drop_column("evaluators_configs", "app_id") + op.drop_column("evaluators_configs", "user_id") + op.add_column( + "human_evaluations", sa.Column("project_id", sa.UUID(), nullable=True) + ) + op.drop_constraint( + "human_evaluations_user_id_fkey", "human_evaluations", type_="foreignkey" + ) + op.create_foreign_key( + "human_evaluations_projects_fkey", + "human_evaluations", + "projects", + ["project_id"], + ["id"], + ondelete="CASCADE", + ) + op.drop_column("human_evaluations", "user_id") + op.add_column( + "human_evaluations_scenarios", sa.Column("project_id", sa.UUID(), nullable=True) + ) + op.drop_constraint( + "human_evaluations_scenarios_user_id_fkey", + "human_evaluations_scenarios", + type_="foreignkey", + ) + op.create_foreign_key( + "human_evaluations_scenarios_projects_fkey", + "human_evaluations_scenarios", + "projects", + ["project_id"], + ["id"], + ondelete="CASCADE", + ) + op.drop_column("human_evaluations_scenarios", "user_id") + op.alter_column("projects", "is_default", existing_type=sa.BOOLEAN(), nullable=True) + op.add_column("testsets", sa.Column("project_id", sa.UUID(), nullable=True)) + if not utils.unique_constraint_exists(engine, "testsets", "testsets_user_id_fkey"): + op.drop_constraint("testsets_user_id_fkey", "testsets", type_="foreignkey") + op.drop_constraint("testsets_app_id_fkey", "testsets", type_="foreignkey") + + op.create_foreign_key( + "testsets_projects_fkey", + "testsets", + "projects", + ["project_id"], + ["id"], + ondelete="CASCADE", + ) + op.drop_column("testsets", "app_id") + op.drop_column("testsets", "user_id") + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "testsets", sa.Column("user_id", sa.UUID(), autoincrement=False, nullable=True) + ) + op.add_column( + "testsets", sa.Column("app_id", sa.UUID(), autoincrement=False, nullable=True) + ) + op.create_foreign_key( + "testsets_app_id_fkey", + "testsets", + "app_db", + ["app_id"], + ["id"], + ondelete="CASCADE", + ) + op.create_foreign_key( + "testsets_user_id_fkey", "testsets", "users", ["user_id"], ["id"] + ) + op.drop_column("testsets", "project_id") + op.alter_column( + "projects", "is_default", existing_type=sa.BOOLEAN(), nullable=False + ) + op.add_column( + "human_evaluations_scenarios", + sa.Column("user_id", sa.UUID(), autoincrement=False, nullable=True), + ) + op.create_foreign_key( + "human_evaluations_scenarios_user_id_fkey", + "human_evaluations_scenarios", + "users", + ["user_id"], + ["id"], + ) + op.drop_column("human_evaluations_scenarios", "project_id") + op.add_column( + "human_evaluations", + sa.Column("user_id", sa.UUID(), autoincrement=False, nullable=True), + ) + op.create_foreign_key( + "human_evaluations_user_id_fkey", + "human_evaluations", + "users", + ["user_id"], + ["id"], + ) + op.drop_column("human_evaluations", "project_id") + op.add_column( + "evaluators_configs", + sa.Column("user_id", sa.UUID(), autoincrement=False, nullable=True), + ) + op.add_column( + "evaluators_configs", + sa.Column("app_id", sa.UUID(), autoincrement=False, nullable=True), + ) + op.create_foreign_key( + "evaluators_configs_app_id_fkey", + "evaluators_configs", + "app_db", + ["app_id"], + ["id"], + ondelete="SET NULL", + ) + op.create_foreign_key( + "evaluators_configs_user_id_fkey", + "evaluators_configs", + "users", + ["user_id"], + ["id"], + ) + op.drop_column("evaluators_configs", "project_id") + op.add_column( + "evaluations", + sa.Column("user_id", sa.UUID(), autoincrement=False, nullable=True), + ) + op.create_foreign_key( + "evaluations_user_id_fkey", "evaluations", "users", ["user_id"], ["id"] + ) + op.drop_column("evaluations", "project_id") + op.add_column( + "evaluation_scenarios", + sa.Column("user_id", sa.UUID(), autoincrement=False, nullable=True), + ) + op.create_foreign_key( + "evaluation_scenarios_user_id_fkey", + "evaluation_scenarios", + "users", + ["user_id"], + ["id"], + ) + op.drop_column("evaluation_scenarios", "project_id") + op.drop_column("environments_revisions", "project_id") + op.add_column( + "environments", + sa.Column("user_id", sa.UUID(), autoincrement=False, nullable=True), + ) + op.create_foreign_key( + "environments_user_id_fkey", "environments", "users", ["user_id"], ["id"] + ) + op.drop_column("environments", "project_id") + op.add_column( + "docker_images", + sa.Column("user_id", sa.UUID(), autoincrement=False, nullable=True), + ) + op.create_foreign_key( + "docker_images_user_id_fkey", "docker_images", "users", ["user_id"], ["id"] + ) + op.drop_column("docker_images", "project_id") + op.add_column( + "deployments", + sa.Column("user_id", sa.UUID(), autoincrement=False, nullable=True), + ) + op.create_foreign_key( + "deployments_user_id_fkey", "deployments", "users", ["user_id"], ["id"] + ) + op.drop_column("deployments", "project_id") + op.add_column( + "bases", sa.Column("user_id", sa.UUID(), autoincrement=False, nullable=True) + ) + op.add_column( + "bases", sa.Column("app_id", sa.UUID(), autoincrement=False, nullable=True) + ) + op.create_foreign_key("bases_user_id_fkey", "bases", "users", ["user_id"], ["id"]) + op.create_foreign_key( + "bases_app_id_fkey", "bases", "app_db", ["app_id"], ["id"], ondelete="CASCADE" + ) + op.drop_column("bases", "project_id") + op.add_column( + "app_variants", + sa.Column("user_id", sa.UUID(), autoincrement=False, nullable=True), + ) + op.create_foreign_key( + "app_variants_user_id_fkey", "app_variants", "users", ["user_id"], ["id"] + ) + op.drop_column("app_variants", "project_id") + op.drop_column("app_variant_revisions", "project_id") + op.add_column( + "app_db", sa.Column("user_id", sa.UUID(), autoincrement=False, nullable=True) + ) + op.create_foreign_key("app_db_user_id_fkey", "app_db", "users", ["user_id"], ["id"]) + op.drop_column("app_db", "project_id") + # ### end Alembic commands ### diff --git a/agenta-backend/agenta_backend/models/db_models.py b/agenta-backend/agenta_backend/models/db_models.py index 06dcdcebaf..cb50121e28 100644 --- a/agenta-backend/agenta_backend/models/db_models.py +++ b/agenta-backend/agenta_backend/models/db_models.py @@ -18,6 +18,9 @@ from agenta_backend.models.shared_models import TemplateType +CASCADE_ALL_DELETE = "all, delete-orphan" + + class UserDB(Base): __tablename__ = "users" @@ -58,6 +61,13 @@ class ProjectDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) + image = relationship("ImageDB", cascade=CASCADE_ALL_DELETE, backref="project") + app = relationship("AppDB", cascade=CASCADE_ALL_DELETE, backref="project") + evaluator_config = relationship( + "EvaluatorConfigDB", cascade=CASCADE_ALL_DELETE, backref="project" + ) + testset = relationship("TestSetDB", cascade=CASCADE_ALL_DELETE, backref="project") + class ImageDB(Base): __tablename__ = "docker_images" @@ -74,7 +84,9 @@ class ImageDB(Base): docker_id = Column(String, nullable=True, index=True) tags = Column(String, nullable=True) deletable = Column(Boolean, default=True) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) created_at = Column( DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) @@ -94,7 +106,9 @@ class AppDB(Base): nullable=False, ) app_name = Column(String) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) modified_by_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) created_at = Column( DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) @@ -103,22 +117,16 @@ class AppDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - user = relationship("UserDB", foreign_keys=[user_id]) + modified_by = relationship("UserDB", foreign_keys=[modified_by_id]) variant = relationship( - "AppVariantDB", cascade="all, delete-orphan", back_populates="app" + "AppVariantDB", cascade=CASCADE_ALL_DELETE, back_populates="app" ) - testset = relationship("TestSetDB", cascade="all, delete-orphan", backref="app") deployment = relationship( - "DeploymentDB", cascade="all, delete-orphan", back_populates="app" - ) - base = relationship( - "VariantBaseDB", cascade="all, delete-orphan", back_populates="app" - ) - evaluation = relationship( - "EvaluationDB", cascade="all, delete-orphan", backref="app" + "DeploymentDB", cascade=CASCADE_ALL_DELETE, back_populates="app" ) + evaluation = relationship("EvaluationDB", cascade=CASCADE_ALL_DELETE, backref="app") human_evaluation = relationship( - "HumanEvaluationDB", cascade="all, delete-orphan", backref="app" + "HumanEvaluationDB", cascade=CASCADE_ALL_DELETE, backref="app" ) @@ -133,7 +141,9 @@ class DeploymentDB(Base): nullable=False, ) app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) container_name = Column(String) container_id = Column(String) uri = Column(String) @@ -145,7 +155,7 @@ class DeploymentDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - user = relationship("UserDB") + project = relationship("ProjectDB") app = relationship("AppDB", back_populates="deployment") @@ -159,8 +169,9 @@ class VariantBaseDB(Base): unique=True, nullable=False, ) - app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) base_name = Column(String) image_id = Column( UUID(as_uuid=True), ForeignKey("docker_images.id", ondelete="SET NULL") @@ -175,10 +186,9 @@ class VariantBaseDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - user = relationship("UserDB") image = relationship("ImageDB") deployment = relationship("DeploymentDB") - app = relationship("AppDB", back_populates="base") + project = relationship("ProjectDB") class AppVariantDB(Base): @@ -199,7 +209,9 @@ class AppVariantDB(Base): ForeignKey("docker_images.id", ondelete="SET NULL"), nullable=True, ) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) modified_by_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) base_name = Column(String) base_id = Column(UUID(as_uuid=True), ForeignKey("bases.id")) @@ -216,12 +228,12 @@ class AppVariantDB(Base): image = relationship("ImageDB") app = relationship("AppDB", back_populates="variant") - user = relationship("UserDB", foreign_keys=[user_id]) + project = relationship("ProjectDB") modified_by = relationship("UserDB", foreign_keys=[modified_by_id]) base = relationship("VariantBaseDB") variant_revision = relationship( "AppVariantRevisionsDB", - cascade="all, delete-orphan", + cascade=CASCADE_ALL_DELETE, backref="variant_revision", ) @@ -240,6 +252,9 @@ class AppVariantRevisionsDB(Base): UUID(as_uuid=True), ForeignKey("app_variants.id", ondelete="CASCADE") ) revision = Column(Integer) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) modified_by_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) base_id = Column(UUID(as_uuid=True), ForeignKey("bases.id")) config_name = Column(String, nullable=False) @@ -253,6 +268,7 @@ class AppVariantRevisionsDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) + project = relationship("ProjectDB") modified_by = relationship("UserDB") base = relationship("VariantBaseDB") @@ -272,7 +288,9 @@ class AppEnvironmentDB(Base): ) app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) name = Column(String) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) revision = Column(Integer) deployed_app_variant_id = Column( UUID(as_uuid=True), ForeignKey("app_variants.id", ondelete="SET NULL") @@ -287,9 +305,9 @@ class AppEnvironmentDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - user = relationship("UserDB") + project = relationship("ProjectDB") environment_revisions = relationship( - "AppEnvironmentRevisionDB", cascade="all, delete-orphan", backref="environment" + "AppEnvironmentRevisionDB", cascade=CASCADE_ALL_DELETE, backref="environment" ) deployed_app_variant = relationship("AppVariantDB") deployed_app_variant_revision = relationship("AppVariantRevisionsDB") @@ -308,6 +326,9 @@ class AppEnvironmentRevisionDB(Base): environment_id = Column( UUID(as_uuid=True), ForeignKey("environments.id", ondelete="CASCADE") ) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) revision = Column(Integer) modified_by_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) deployed_app_variant_revision_id = Column( @@ -320,6 +341,7 @@ class AppEnvironmentRevisionDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) + project = relationship("ProjectDB") modified_by = relationship("UserDB") @@ -358,9 +380,10 @@ class TestSetDB(Base): nullable=False, ) name = Column(String) - app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) csvdata = Column(mutable_json_type(dbtype=JSONB, nested=True)) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) created_at = Column( DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) @@ -368,8 +391,6 @@ class TestSetDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - user = relationship("UserDB") - class EvaluatorConfigDB(Base): __tablename__ = "evaluators_configs" @@ -382,8 +403,9 @@ class EvaluatorConfigDB(Base): nullable=False, ) - app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="SET NULL")) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) name = Column(String) evaluator_key = Column(String) settings_values = Column(mutable_json_type(dbtype=JSONB, nested=True), default=dict) @@ -394,8 +416,6 @@ class EvaluatorConfigDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - user = relationship("UserDB") - class HumanEvaluationVariantDB(Base): __tablename__ = "human_evaluation_variants" @@ -434,7 +454,9 @@ class HumanEvaluationDB(Base): nullable=False, ) app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) status = Column(String) evaluation_type = Column(String) testset_id = Column(UUID(as_uuid=True), ForeignKey("testsets.id")) @@ -445,16 +467,15 @@ class HumanEvaluationDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - user = relationship("UserDB") testset = relationship("TestSetDB") evaluation_variant = relationship( "HumanEvaluationVariantDB", - cascade="all, delete-orphan", + cascade=CASCADE_ALL_DELETE, backref="human_evaluation", ) evaluation_scenario = relationship( "HumanEvaluationScenarioDB", - cascade="all, delete-orphan", + cascade=CASCADE_ALL_DELETE, backref="evaluation_scenario", ) @@ -469,7 +490,9 @@ class HumanEvaluationScenarioDB(Base): unique=True, nullable=False, ) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) evaluation_id = Column( UUID(as_uuid=True), ForeignKey("human_evaluations.id", ondelete="CASCADE") ) @@ -543,7 +566,9 @@ class EvaluationDB(Base): nullable=False, ) app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) status = Column(mutable_json_type(dbtype=JSONB, nested=True)) # Result testset_id = Column( UUID(as_uuid=True), ForeignKey("testsets.id", ondelete="SET NULL") @@ -564,21 +589,21 @@ class EvaluationDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - user = relationship("UserDB") + project = relationship("ProjectDB") testset = relationship("TestSetDB") variant = relationship("AppVariantDB") variant_revision = relationship("AppVariantRevisionsDB") aggregated_results = relationship( "EvaluationAggregatedResultDB", - cascade="all, delete-orphan", + cascade=CASCADE_ALL_DELETE, backref="evaluation", ) evaluation_scenarios = relationship( - "EvaluationScenarioDB", cascade="all, delete-orphan", backref="evaluation" + "EvaluationScenarioDB", cascade=CASCADE_ALL_DELETE, backref="evaluation" ) evaluator_configs = relationship( "EvaluationEvaluatorConfigDB", - cascade="all, delete-orphan", + cascade=CASCADE_ALL_DELETE, backref="evaluation", ) @@ -615,7 +640,9 @@ class EvaluationScenarioDB(Base): unique=True, nullable=False, ) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) + project_id = Column( + UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") + ) evaluation_id = Column( UUID(as_uuid=True), ForeignKey("evaluations.id", ondelete="CASCADE") ) @@ -642,11 +669,11 @@ class EvaluationScenarioDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) - user = relationship("UserDB") + project = relationship("ProjectDB") variant = relationship("AppVariantDB") results = relationship( "EvaluationScenarioResultDB", - cascade="all, delete-orphan", + cascade=CASCADE_ALL_DELETE, backref="evaluation_scenario", ) From 1bae734c308810d47e1acfe8601aab776917e23c Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 12 Sep 2024 23:19:59 +0100 Subject: [PATCH 09/57] refactor (backend): include default project_id in authentication middleware (auth_context) --- .../agenta_backend/services/auth_helper.py | 13 +++++++++++-- .../agenta_backend/services/db_manager.py | 16 ++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/agenta-backend/agenta_backend/services/auth_helper.py b/agenta-backend/agenta_backend/services/auth_helper.py index 263de7c4e6..d48c75357c 100644 --- a/agenta-backend/agenta_backend/services/auth_helper.py +++ b/agenta-backend/agenta_backend/services/auth_helper.py @@ -1,5 +1,7 @@ from fastapi import Request, HTTPException +from agenta_backend.services.db_manager import fetch_default_project, NoResultFound + class SessionContainer(object): """dummy class""" @@ -19,13 +21,20 @@ def inner_function(): async def authentication_middleware(request: Request, call_next): try: - # Initialize the user_id attribute in the request state if it doesn't exist if not hasattr(request.state, "user_id"): user_uid_id = "0" setattr(request.state, "user_id", user_uid_id) + if not hasattr(request.state, "project_id"): + project = await fetch_default_project() + if project is None: + raise NoResultFound("Default project not found.") + + setattr(request.state, "project_id", str(project.id)) + # Call the next middleware or route handler response = await call_next(request) return response except Exception as e: - raise HTTPException(status_code=401, detail=str(e)) + status_code = e.status_code if hasattr(e, "status_code") else 500 + raise HTTPException(status_code=status_code, detail=str(e)) diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 699406a324..878965c904 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -28,6 +28,7 @@ AppDB_ as AppDB, UserDB_ as UserDB, ImageDB_ as ImageDB, + ProjectDB_ as ProjectDB, TestSetDB_ as TestSetDB, AppVariantDB_ as AppVariantDB, EvaluationDB_ as EvaluationDB, @@ -49,6 +50,7 @@ AppDB, UserDB, ImageDB, + ProjectDB, TestSetDB, AppVariantDB, EvaluationDB, @@ -3219,3 +3221,17 @@ async def fetch_corresponding_object_uuid(table_name: str, object_id: str) -> st ) object_mapping = result.scalars().first() return str(object_mapping.uuid) + + +async def fetch_default_project() -> ProjectDB: + """ + Fetch the default project from the database. + + Returns: + ProjectDB: The default project instance. + """ + + async with db_engine.get_session() as session: + result = await session.execute(select(ProjectDB).filter_by(is_default=True)) + default_project = result.scalars().first() + return default_project From 43fc826a84087adc502c9c096900168120a357be Mon Sep 17 00:00:00 2001 From: Abram Date: Fri, 13 Sep 2024 08:05:58 +0100 Subject: [PATCH 10/57] feat (migrations): add custom data migration to manage project_id in DB entities and integrate with Alembic --- .../postgres/data_migrations/projects.py | 146 ++++++++++++++++-- ...65_add_default_project_to_scoped_model_.py | 36 +++++ 2 files changed, 172 insertions(+), 10 deletions(-) create mode 100644 agenta-backend/agenta_backend/migrations/postgres/versions/55bdd2e9a465_add_default_project_to_scoped_model_.py diff --git a/agenta-backend/agenta_backend/migrations/postgres/data_migrations/projects.py b/agenta-backend/agenta_backend/migrations/postgres/data_migrations/projects.py index d07f6d1e3e..fa4b183fcf 100644 --- a/agenta-backend/agenta_backend/migrations/postgres/data_migrations/projects.py +++ b/agenta-backend/agenta_backend/migrations/postgres/data_migrations/projects.py @@ -1,13 +1,49 @@ import os import traceback +from typing import Sequence import click from sqlalchemy.future import select from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker - -from agenta_backend.models.db_models import ProjectDB +from sqlalchemy.orm import sessionmaker, Session + +from agenta_backend.models.db_models import ( + ProjectDB, + AppDB, + AppVariantDB, + AppVariantRevisionsDB, + VariantBaseDB, + DeploymentDB, + ImageDB, + AppEnvironmentDB, + AppEnvironmentRevisionDB, + EvaluationScenarioDB, + EvaluationDB, + EvaluatorConfigDB, + HumanEvaluationDB, + HumanEvaluationScenarioDB, + TestSetDB, +) + + +BATCH_SIZE = 1000 +MODELS = [ + AppDB, + AppVariantDB, + AppVariantRevisionsDB, + VariantBaseDB, + DeploymentDB, + ImageDB, + AppEnvironmentDB, + AppEnvironmentRevisionDB, + EvaluationScenarioDB, + EvaluationDB, + EvaluatorConfigDB, + HumanEvaluationDB, + HumanEvaluationScenarioDB, + TestSetDB, +] def get_default_projects(session): @@ -15,6 +51,15 @@ def get_default_projects(session): return query.scalars().all() +def check_for_multiple_default_projects(session: Session) -> Sequence[ProjectDB]: + default_projects = get_default_projects(session) + if len(default_projects) >= 1: + raise ValueError( + "Multiple default projects found. Please ensure only one exists." + ) + return default_projects + + def create_default_project(): PROJECT_NAME = "Default" engine = create_engine(os.getenv("POSTGRES_URI")) @@ -22,12 +67,7 @@ def create_default_project(): with sync_session() as session: try: - default_projects = get_default_projects(session) - if len(default_projects) >= 1: - raise ValueError( - "Multiple default projects found. Please ensure only one exists." - ) - + default_projects = check_for_multiple_default_projects(session) if len(default_projects) == 0: new_project = ProjectDB(project_name=PROJECT_NAME, is_default=True) session.add(new_project) @@ -35,7 +75,12 @@ def create_default_project(): except Exception as e: session.rollback() - click.echo(click.style(f"ERROR: {traceback.format_exc()}", fg="red")) + click.echo( + click.style( + f"ERROR creating default project: {traceback.format_exc()}", + fg="red", + ) + ) raise e @@ -65,3 +110,84 @@ def remove_default_project(): session.rollback() click.echo(click.style(f"ERROR: {traceback.format_exc()}", fg="red")) raise e + + +def add_project_id_to_db_entities(): + engine = create_engine(os.getenv("POSTGRES_URI")) + sync_session = sessionmaker(engine, expire_on_commit=False) + + with sync_session() as session: + try: + default_project = check_for_multiple_default_projects(session)[0] + for model in MODELS: + offset = 0 + while True: + records = ( + session.execute( + select(model) + .where(model.project_id == None) + .offset(offset) + .limit(BATCH_SIZE) + ) + .scalars() + .all() + ) + if not records: + break + + # Update records with default project_id + for record in records: + record.project_id = default_project.id + + session.commit() + offset += BATCH_SIZE + + except Exception as e: + session.rollback() + click.echo( + click.style( + f"ERROR adding project_id to db entities: {traceback.format_exc()}", + fg="red", + ) + ) + raise e + + +def remove_project_id_from_db_entities(): + engine = create_engine(os.getenv("POSTGRES_URI")) + sync_session = sessionmaker(engine, expire_on_commit=False) + + with sync_session() as session: + try: + for model in MODELS: + offset = 0 + while True: + records = ( + session.execute( + select(model) + .where(model.project_id == None) + .offset(offset) + .limit(BATCH_SIZE) + ) + .scalars() + .all() + ) + if not records: + break + + # Update records project_id column with None + for record in records: + record.project_id = None + + session.commit() + offset += BATCH_SIZE + + except Exception as e: + session.rollback() + click.echo( + click.style( + f"ERROR removing project_id to db entities: {traceback.format_exc()}", + fg="red", + ) + ) + raise e diff --git a/agenta-backend/agenta_backend/migrations/postgres/versions/55bdd2e9a465_add_default_project_to_scoped_model_.py b/agenta-backend/agenta_backend/migrations/postgres/versions/55bdd2e9a465_add_default_project_to_scoped_model_.py new file mode 100644 index 0000000000..3cb0054b14 --- /dev/null +++ b/agenta-backend/agenta_backend/migrations/postgres/versions/55bdd2e9a465_add_default_project_to_scoped_model_.py @@ -0,0 +1,36 @@ +"""add default project to scoped model entities + +Revision ID: 55bdd2e9a465 +Revises: c00a326c625a +Create Date: 2024-09-12 21:56:38.701088 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +from agenta_backend.migrations.postgres.data_migrations.projects import ( + add_project_id_to_db_entities, + remove_project_id_from_db_entities, +) + + +# revision identifiers, used by Alembic. +revision: str = "55bdd2e9a465" +down_revision: Union[str, None] = "c00a326c625a" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### custom command ### + add_project_id_to_db_entities() + # ### end custom command ### + + +def downgrade() -> None: + # ### custom command ### + remove_project_id_from_db_entities() + # ### end custom command ### From 0d5f922ea1573f11794340e0517a8ea28d4ae479 Mon Sep 17 00:00:00 2001 From: Abram Date: Fri, 13 Sep 2024 09:20:46 +0100 Subject: [PATCH 11/57] minor refactor (backend): fix condition when checking for multiple default projects --- .../migrations/postgres/data_migrations/projects.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/agenta-backend/agenta_backend/migrations/postgres/data_migrations/projects.py b/agenta-backend/agenta_backend/migrations/postgres/data_migrations/projects.py index fa4b183fcf..c10d333f28 100644 --- a/agenta-backend/agenta_backend/migrations/postgres/data_migrations/projects.py +++ b/agenta-backend/agenta_backend/migrations/postgres/data_migrations/projects.py @@ -53,7 +53,7 @@ def get_default_projects(session): def check_for_multiple_default_projects(session: Session) -> Sequence[ProjectDB]: default_projects = get_default_projects(session) - if len(default_projects) >= 1: + if len(default_projects) > 1: raise ValueError( "Multiple default projects found. Please ensure only one exists." ) @@ -90,17 +90,13 @@ def remove_default_project(): with sync_session() as session: try: - default_projects = get_default_projects(session) + default_projects = check_for_multiple_default_projects(session) if len(default_projects) == 0: click.echo( click.style("No default project found to remove.", fg="yellow") ) return - if len(default_projects) > 1: - raise ValueError( - "Multiple default projects found. Please ensure only one exists." - ) session.delete(default_projects[0]) session.commit() From 7e5993f6da32e7957d692d3054a6db245cadf9e1 Mon Sep 17 00:00:00 2001 From: Abram Date: Fri, 13 Sep 2024 09:40:19 +0100 Subject: [PATCH 12/57] chore (style): format projects with black@23.12.0 --- .../migrations/postgres/data_migrations/projects.py | 1 - 1 file changed, 1 deletion(-) diff --git a/agenta-backend/agenta_backend/migrations/postgres/data_migrations/projects.py b/agenta-backend/agenta_backend/migrations/postgres/data_migrations/projects.py index c10d333f28..e589f2084f 100644 --- a/agenta-backend/agenta_backend/migrations/postgres/data_migrations/projects.py +++ b/agenta-backend/agenta_backend/migrations/postgres/data_migrations/projects.py @@ -97,7 +97,6 @@ def remove_default_project(): ) return - session.delete(default_projects[0]) session.commit() click.echo(click.style("Default project removed successfully.", fg="green")) From e6eecfc4799a154e54354069c43bc42219ed511d Mon Sep 17 00:00:00 2001 From: Abram Date: Fri, 13 Sep 2024 09:41:54 +0100 Subject: [PATCH 13/57] refactor (backend): scope 'project_id' in API router endpoints --- .../agenta_backend/routers/app_router.py | 19 ++++++++++-- .../agenta_backend/routers/bases_router.py | 1 + .../agenta_backend/routers/configs_router.py | 14 +++++++-- .../routers/container_router.py | 4 +++ .../routers/environment_router.py | 2 ++ .../routers/evaluation_router.py | 21 +++++++++++-- .../routers/evaluators_router.py | 31 +++++++++++++++---- .../routers/human_evaluation_router.py | 12 ++++++- .../agenta_backend/routers/testset_router.py | 7 +++++ .../agenta_backend/routers/variants_router.py | 27 +++++++++++++--- 10 files changed, 119 insertions(+), 19 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index bede0cb241..b16808f212 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -90,6 +90,7 @@ async def list_app_variants( app_id: str, request: Request, + project_id: Optional[str] = None, ): """ Retrieve a list of app variants for a given app ID. @@ -139,6 +140,7 @@ async def get_variant_by_env( app_id: str, environment: str, request: Request, + project_id: Optional[str] = None, ): """ Retrieve the app variant based on the provided app_id and environment. @@ -196,6 +198,7 @@ async def get_variant_by_env( async def create_app( payload: CreateApp, request: Request, + project_id: Optional[str] = None, ) -> CreateAppOutput: """ Create a new app for a user or organization. @@ -291,6 +294,7 @@ async def update_app( app_id: str, payload: UpdateApp, request: Request, + project_id: Optional[str] = None, ) -> UpdateAppOutput: """ Update an app for a user or organization. @@ -337,6 +341,7 @@ async def list_apps( app_name: Optional[str] = None, org_id: Optional[str] = None, workspace_id: Optional[str] = None, + project_id: Optional[str] = None, ) -> List[App]: """ Retrieve a list of apps filtered by app_name and org_id. @@ -370,6 +375,7 @@ async def add_variant_from_image( app_id: str, payload: AddVariantFromImagePayload, request: Request, + project_id: Optional[str] = None, ): """ Add a new variant to an app based on a Docker image. @@ -441,7 +447,11 @@ async def add_variant_from_image( @router.delete("/{app_id}/", operation_id="remove_app") -async def remove_app(app_id: str, request: Request): +async def remove_app( + app_id: str, + request: Request, + project_id: Optional[str] = None, +): """Remove app, all its variant, containers and images Arguments: @@ -482,6 +492,7 @@ async def remove_app(app_id: str, request: Request): async def create_app_and_variant_from_template( payload: CreateAppVariant, request: Request, + project_id: Optional[str] = None, ) -> AppVariantResponse: """ Create an app and variant from a template. @@ -666,6 +677,7 @@ async def create_app_and_variant_from_template( async def list_environments( app_id: str, request: Request, + project_id: Optional[str] = None, ): """ Retrieve a list of environments for a given app ID. @@ -710,7 +722,10 @@ async def list_environments( response_model=EnvironmentOutputExtended, ) async def list_app_environment_revisions( - request: Request, app_id: str, environment_name + request: Request, + app_id: str, + environment_name, + project_id: Optional[str] = None, ): logger.debug("getting environment " + environment_name) user_org_workspace_data: dict = await get_user_org_and_workspace_id( diff --git a/agenta-backend/agenta_backend/routers/bases_router.py b/agenta-backend/agenta_backend/routers/bases_router.py index 6e98a06656..d983cea0a2 100644 --- a/agenta-backend/agenta_backend/routers/bases_router.py +++ b/agenta-backend/agenta_backend/routers/bases_router.py @@ -24,6 +24,7 @@ async def list_bases( request: Request, app_id: str, base_name: Optional[str] = None, + project_id: Optional[str] = None, ) -> List[BaseOutput]: """ Retrieve a list of bases filtered by app_id and base_name. diff --git a/agenta-backend/agenta_backend/routers/configs_router.py b/agenta-backend/agenta_backend/routers/configs_router.py index 03afc928b2..c06cdff62d 100644 --- a/agenta-backend/agenta_backend/routers/configs_router.py +++ b/agenta-backend/agenta_backend/routers/configs_router.py @@ -28,6 +28,7 @@ async def save_config( payload: SaveConfigPayload, request: Request, + project_id: Optional[str] = None, ): try: base_db = await db_manager.fetch_base_by_id(payload.base_id) @@ -102,6 +103,7 @@ async def get_config( base_id: str, config_name: Optional[str] = None, environment_name: Optional[str] = None, + project_id: Optional[str] = None, ): try: base_db = await db_manager.fetch_base_by_id(base_id) @@ -195,7 +197,11 @@ async def get_config( "/deployment/{deployment_revision_id}/", operation_id="get_config_deployment_revision", ) -async def get_config_deployment_revision(request: Request, deployment_revision_id: str): +async def get_config_deployment_revision( + request: Request, + deployment_revision_id: str, + project_id: Optional[str] = None, +): try: environment_revision = await db_manager.fetch_app_environment_revision( deployment_revision_id @@ -231,7 +237,11 @@ async def get_config_deployment_revision(request: Request, deployment_revision_i "/deployment/{deployment_revision_id}/revert/", operation_id="revert_deployment_revision", ) -async def revert_deployment_revision(request: Request, deployment_revision_id: str): +async def revert_deployment_revision( + request: Request, + deployment_revision_id: str, + project_id: Optional[str] = None, +): environment_revision = await db_manager.fetch_app_environment_revision( deployment_revision_id ) diff --git a/agenta-backend/agenta_backend/routers/container_router.py b/agenta-backend/agenta_backend/routers/container_router.py index 4dfea8a01d..54d3bf2045 100644 --- a/agenta-backend/agenta_backend/routers/container_router.py +++ b/agenta-backend/agenta_backend/routers/container_router.py @@ -51,6 +51,7 @@ async def build_image( base_name: str, tar_file: UploadFile, request: Request, + project_id: Optional[str] = None, ) -> Image: """ Builds a Docker image from a tar file containing the application code. @@ -97,6 +98,7 @@ async def build_image( async def restart_docker_container( payload: RestartAppContainer, request: Request, + project_id: Optional[str] = None, ) -> dict: """Restart docker container. @@ -121,6 +123,7 @@ async def restart_docker_container( @router.get("/templates/", operation_id="container_templates") async def container_templates( request: Request, + project_id: Optional[str] = None, ) -> Union[List[Template], str]: """ Returns a list of templates available for creating new containers. @@ -144,6 +147,7 @@ async def construct_app_container_url( request: Request, base_id: Optional[str] = None, variant_id: Optional[str] = None, + project_id: Optional[str] = None, ) -> URI: """ Constructs the URL for an app container based on the provided base_id or variant_id. diff --git a/agenta-backend/agenta_backend/routers/environment_router.py b/agenta-backend/agenta_backend/routers/environment_router.py index 3fe69188ea..87e101d3d1 100644 --- a/agenta-backend/agenta_backend/routers/environment_router.py +++ b/agenta-backend/agenta_backend/routers/environment_router.py @@ -1,4 +1,5 @@ import logging +from typing import Optional from fastapi.responses import JSONResponse from fastapi import Request, HTTPException @@ -20,6 +21,7 @@ async def deploy_to_environment( payload: DeployToEnvironmentPayload, request: Request, + project_id: Optional[str] = None, ): """Deploys a given variant to an environment diff --git a/agenta-backend/agenta_backend/routers/evaluation_router.py b/agenta-backend/agenta_backend/routers/evaluation_router.py index e7d57b196c..709aabea64 100644 --- a/agenta-backend/agenta_backend/routers/evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/evaluation_router.py @@ -1,6 +1,6 @@ import random import logging -from typing import Any, List +from typing import Any, List, Optional from fastapi.responses import JSONResponse from fastapi import HTTPException, Request, status, Response, Query @@ -38,6 +38,7 @@ async def fetch_evaluation_ids( resource_type: str, request: Request, resource_ids: List[str] = Query(None), + project_id: Optional[str] = None, ): """Fetches evaluation ids for a given resource type and id. @@ -85,6 +86,7 @@ async def fetch_evaluation_ids( async def create_evaluation( payload: NewEvaluation, request: Request, + project_id: Optional[str] = None, ): """Creates a new comparison table document Raises: @@ -162,7 +164,11 @@ async def create_evaluation( @router.get("/{evaluation_id}/status/", operation_id="fetch_evaluation_status") -async def fetch_evaluation_status(evaluation_id: str, request: Request): +async def fetch_evaluation_status( + evaluation_id: str, + request: Request, + project_id: Optional[str] = None, +): """Fetches the status of the evaluation. Args: @@ -198,7 +204,11 @@ async def fetch_evaluation_status(evaluation_id: str, request: Request): @router.get("/{evaluation_id}/results/", operation_id="fetch_evaluation_results") -async def fetch_evaluation_results(evaluation_id: str, request: Request): +async def fetch_evaluation_results( + evaluation_id: str, + request: Request, + project_id: Optional[str] = None, +): """Fetches the results of the evaluation Args: @@ -244,6 +254,7 @@ async def fetch_evaluation_results(evaluation_id: str, request: Request): async def fetch_evaluation_scenarios( evaluation_id: str, request: Request, + project_id: Optional[str] = None, ): """Fetches evaluation scenarios for a given evaluation ID. @@ -300,6 +311,7 @@ async def fetch_evaluation_scenarios( async def fetch_list_evaluations( app_id: str, request: Request, + project_id: Optional[str] = None, ): """Fetches a list of evaluations, optionally filtered by an app ID. @@ -346,6 +358,7 @@ async def fetch_list_evaluations( async def fetch_evaluation( evaluation_id: str, request: Request, + project_id: Optional[str] = None, ): """Fetches a single evaluation based on its ID. @@ -390,6 +403,7 @@ async def fetch_evaluation( async def delete_evaluations( payload: DeleteEvaluation, request: Request, + project_id: Optional[str] = None, ): """ Delete specific comparison tables based on their unique IDs. @@ -440,6 +454,7 @@ async def delete_evaluations( async def fetch_evaluation_scenarios( evaluations_ids: str, request: Request, + project_id: Optional[str] = None, ): """Fetches evaluation scenarios for a given evaluation ID. diff --git a/agenta-backend/agenta_backend/routers/evaluators_router.py b/agenta-backend/agenta_backend/routers/evaluators_router.py index 8806035247..e2340c0a0d 100644 --- a/agenta-backend/agenta_backend/routers/evaluators_router.py +++ b/agenta-backend/agenta_backend/routers/evaluators_router.py @@ -1,6 +1,6 @@ import logging -from typing import List +from typing import List, Optional from fastapi import HTTPException, Request from fastapi.responses import JSONResponse @@ -48,7 +48,11 @@ async def get_evaluators_endpoint(): @router.get("/configs/", response_model=List[EvaluatorConfig]) -async def get_evaluator_configs(app_id: str, request: Request): +async def get_evaluator_configs( + app_id: str, + request: Request, + project_id: Optional[str] = None, +): """Endpoint to fetch evaluator configurations for a specific app. Args: @@ -83,7 +87,11 @@ async def get_evaluator_configs(app_id: str, request: Request): @router.get("/configs/{evaluator_config_id}/", response_model=EvaluatorConfig) -async def get_evaluator_config(evaluator_config_id: str, request: Request): +async def get_evaluator_config( + evaluator_config_id: str, + request: Request, + project_id: Optional[str] = None, +): """Endpoint to fetch evaluator configurations for a specific app. Returns: @@ -119,7 +127,11 @@ async def get_evaluator_config(evaluator_config_id: str, request: Request): @router.post("/configs/", response_model=EvaluatorConfig) -async def create_new_evaluator_config(payload: NewEvaluatorConfig, request: Request): +async def create_new_evaluator_config( + payload: NewEvaluatorConfig, + request: Request, + project_id: Optional[str] = None, +): """Endpoint to fetch evaluator configurations for a specific app. Args: @@ -171,7 +183,10 @@ async def create_new_evaluator_config(payload: NewEvaluatorConfig, request: Requ @router.put("/configs/{evaluator_config_id}/", response_model=EvaluatorConfig) async def update_evaluator_config( - evaluator_config_id: str, payload: UpdateEvaluatorConfig, request: Request + evaluator_config_id: str, + payload: UpdateEvaluatorConfig, + request: Request, + project_id: Optional[str] = None, ): """Endpoint to update evaluator configurations for a specific app. @@ -217,7 +232,11 @@ async def update_evaluator_config( @router.delete("/configs/{evaluator_config_id}/", response_model=bool) -async def delete_evaluator_config(evaluator_config_id: str, request: Request): +async def delete_evaluator_config( + evaluator_config_id: str, + request: Request, + project_id: Optional[str] = None, +): """Endpoint to delete a specific evaluator configuration. Args: diff --git a/agenta-backend/agenta_backend/routers/human_evaluation_router.py b/agenta-backend/agenta_backend/routers/human_evaluation_router.py index 16672358d9..e6cd8f4ac7 100644 --- a/agenta-backend/agenta_backend/routers/human_evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/human_evaluation_router.py @@ -1,6 +1,6 @@ import random import logging -from typing import List, Dict +from typing import List, Dict, Optional from fastapi import HTTPException, Body, Request, status, Response from agenta_backend.models import converters @@ -46,6 +46,7 @@ async def create_evaluation( payload: NewHumanEvaluation, request: Request, + project_id: Optional[str] = None, ): """Creates a new comparison table document Raises: @@ -104,6 +105,7 @@ async def create_evaluation( async def fetch_list_human_evaluations( app_id: str, request: Request, + project_id: Optional[str] = None, ): """Fetches a list of evaluations, optionally filtered by an app ID. @@ -139,6 +141,7 @@ async def fetch_list_human_evaluations( async def fetch_human_evaluation( evaluation_id: str, request: Request, + project_id: Optional[str] = None, ): """Fetches a single evaluation based on its ID. @@ -181,6 +184,7 @@ async def fetch_human_evaluation( async def fetch_evaluation_scenarios( evaluation_id: str, request: Request, + project_id: Optional[str] = None, ): """Fetches evaluation scenarios for a given evaluation ID. @@ -236,6 +240,7 @@ async def update_human_evaluation( request: Request, evaluation_id: str, update_data: HumanEvaluationUpdate = Body(...), + project_id: Optional[str] = None, ): """Updates an evaluation's status. @@ -292,6 +297,7 @@ async def update_evaluation_scenario_router( evaluation_type: EvaluationType, payload: HumanEvaluationScenarioUpdate, request: Request, + project_id: Optional[str] = None, ): """Updates an evaluation scenario's vote or score based on its type. @@ -351,6 +357,7 @@ async def update_evaluation_scenario_router( async def get_evaluation_scenario_score_router( evaluation_scenario_id: str, request: Request, + project_id: Optional[str] = None, ) -> Dict[str, str]: """ Fetch the score of a specific evaluation scenario. @@ -399,6 +406,7 @@ async def update_evaluation_scenario_score_router( evaluation_scenario_id: str, payload: EvaluationScenarioScoreUpdate, request: Request, + project_id: Optional[str] = None, ): """Updates the score of an evaluation scenario. @@ -454,6 +462,7 @@ async def update_evaluation_scenario_score_router( async def fetch_results( evaluation_id: str, request: Request, + project_id: Optional[str] = None, ): """Fetch all the results for one the comparison table @@ -505,6 +514,7 @@ async def fetch_results( async def delete_evaluations( delete_evaluations: DeleteEvaluation, request: Request, + project_id: Optional[str] = None, ): """ Delete specific comparison tables based on their unique IDs. diff --git a/agenta-backend/agenta_backend/routers/testset_router.py b/agenta-backend/agenta_backend/routers/testset_router.py index b6691f4965..650619b0ef 100644 --- a/agenta-backend/agenta_backend/routers/testset_router.py +++ b/agenta-backend/agenta_backend/routers/testset_router.py @@ -53,6 +53,7 @@ async def upload_file( file: UploadFile = File(...), testset_name: Optional[str] = File(None), app_id: str = Form(None), + project_id: Optional[str] = None, ): """ Uploads a CSV or JSON file and saves its data to MongoDB. @@ -132,6 +133,7 @@ async def import_testset( endpoint: str = Form(None), testset_name: str = Form(None), app_id: str = Form(None), + project_id: Optional[str] = None, ): """ Import JSON testset data from an endpoint and save it to MongoDB. @@ -208,6 +210,7 @@ async def create_testset( app_id: str, csvdata: NewTestset, request: Request, + project_id: Optional[str] = None, ): """ Create a testset with given name and app_name, save the testset to MongoDB. @@ -261,6 +264,7 @@ async def update_testset( testset_id: str, csvdata: NewTestset, request: Request, + project_id: Optional[str] = None, ): """ Update a testset with given id, update the testset in MongoDB. @@ -314,6 +318,7 @@ async def update_testset( async def get_testsets( app_id: str, request: Request, + project_id: Optional[str] = None, ) -> List[TestSetOutputResponse]: """ Get all testsets. @@ -358,6 +363,7 @@ async def get_testsets( async def get_single_testset( testset_id: str, request: Request, + project_id: Optional[str] = None, ): """ Fetch a specific testset in a MongoDB collection using its _id. @@ -398,6 +404,7 @@ async def get_single_testset( async def delete_testsets( payload: DeleteTestsets, request: Request, + project_id: Optional[str] = None, ): """ Delete specific testsets based on their unique IDs. diff --git a/agenta-backend/agenta_backend/routers/variants_router.py b/agenta-backend/agenta_backend/routers/variants_router.py index e3bf09c396..6415078cb7 100644 --- a/agenta-backend/agenta_backend/routers/variants_router.py +++ b/agenta-backend/agenta_backend/routers/variants_router.py @@ -1,5 +1,3 @@ -import os -import inspect import logging from typing import Any, Optional, Union, List @@ -52,6 +50,7 @@ async def add_variant_from_base_and_config( payload: AddVariantFromBasePayload, request: Request, + project_id: Optional[str] = None, ) -> Union[AppVariantResponse, Any]: """Add a new variant based on an existing one. Same as POST /config @@ -124,6 +123,7 @@ async def add_variant_from_base_and_config( async def remove_variant( variant_id: str, request: Request, + project_id: Optional[str] = None, ): """Remove a variant from the server. In the case it's the last variant using the image, stop the container and remove the image. @@ -176,6 +176,7 @@ async def update_variant_parameters( request: Request, variant_id: str, payload: UpdateVariantParameterPayload = Body(...), + project_id: Optional[str] = None, ): """ Updates the parameters for an app variant. @@ -239,6 +240,7 @@ async def update_variant_image( variant_id: str, image: Image, request: Request, + project_id: Optional[str] = None, ): """ Updates the image used in an app variant. @@ -312,6 +314,7 @@ async def start_variant( variant_id: str, action: VariantAction, env_vars: Optional[DockerEnvVars] = None, + project_id: Optional[str] = None, ) -> URI: """ Start a variant of an app. @@ -364,7 +367,11 @@ async def start_variant( @router.get("/{variant_id}/logs/", operation_id="retrieve_variant_logs") -async def retrieve_variant_logs(variant_id: str, request: Request): +async def retrieve_variant_logs( + variant_id: str, + request: Request, + project_id: Optional[str] = None, +): try: app_variant = await db_manager.fetch_app_variant_by_id(variant_id) deployment = await db_manager.get_deployment_by_appid(str(app_variant.app.id)) @@ -383,6 +390,7 @@ async def retrieve_variant_logs(variant_id: str, request: Request): async def get_variant( variant_id: str, request: Request, + project_id: Optional[str] = None, ): logger.debug("getting variant " + variant_id) try: @@ -416,7 +424,11 @@ async def get_variant( operation_id="get_variant_revisions", response_model=List[AppVariantRevision], ) -async def get_variant_revisions(variant_id: str, request: Request): +async def get_variant_revisions( + variant_id: str, + request: Request, + project_id: Optional[str] = None, +): logger.debug("getting variant revisions: ", variant_id) try: app_variant = await db_manager.fetch_app_variant_by_id( @@ -454,7 +466,12 @@ async def get_variant_revisions(variant_id: str, request: Request): operation_id="get_variant_revision", response_model=AppVariantRevision, ) -async def get_variant_revision(variant_id: str, revision_number: int, request: Request): +async def get_variant_revision( + variant_id: str, + revision_number: int, + request: Request, + project_id: Optional[str] = None, +): logger.debug("getting variant revision: ", variant_id, revision_number) try: assert ( From 0beaf89a38bc8f6f43b19493657e1c106f4a4f09 Mon Sep 17 00:00:00 2001 From: Abram Date: Fri, 13 Sep 2024 16:41:03 +0100 Subject: [PATCH 14/57] minor refactor (backend): revert app_id column removal from bases table --- .../c00a326c625a_scope_project_id_to_db_models_entities.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/agenta-backend/agenta_backend/migrations/postgres/versions/c00a326c625a_scope_project_id_to_db_models_entities.py b/agenta-backend/agenta_backend/migrations/postgres/versions/c00a326c625a_scope_project_id_to_db_models_entities.py index 801cb7ffbf..2d4df676e9 100644 --- a/agenta-backend/agenta_backend/migrations/postgres/versions/c00a326c625a_scope_project_id_to_db_models_entities.py +++ b/agenta-backend/agenta_backend/migrations/postgres/versions/c00a326c625a_scope_project_id_to_db_models_entities.py @@ -69,7 +69,6 @@ def upgrade() -> None: ["id"], ondelete="CASCADE", ) - op.drop_column("bases", "app_id") op.drop_column("bases", "user_id") op.add_column("deployments", sa.Column("project_id", sa.UUID(), nullable=True)) op.drop_constraint("deployments_user_id_fkey", "deployments", type_="foreignkey") @@ -332,9 +331,6 @@ def downgrade() -> None: op.add_column( "bases", sa.Column("user_id", sa.UUID(), autoincrement=False, nullable=True) ) - op.add_column( - "bases", sa.Column("app_id", sa.UUID(), autoincrement=False, nullable=True) - ) op.create_foreign_key("bases_user_id_fkey", "bases", "users", ["user_id"], ["id"]) op.create_foreign_key( "bases_app_id_fkey", "bases", "app_db", ["app_id"], ["id"], ondelete="CASCADE" From 667c40d028610a55cf2fed93fc6853c6d658af54 Mon Sep 17 00:00:00 2001 From: Abram Date: Fri, 13 Sep 2024 16:42:05 +0100 Subject: [PATCH 15/57] feat (backend): implement utility function get project_id from request state or request query_param --- .../agenta_backend/utils/project_utils.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 agenta-backend/agenta_backend/utils/project_utils.py diff --git a/agenta-backend/agenta_backend/utils/project_utils.py b/agenta-backend/agenta_backend/utils/project_utils.py new file mode 100644 index 0000000000..d31a4cdc46 --- /dev/null +++ b/agenta-backend/agenta_backend/utils/project_utils.py @@ -0,0 +1,30 @@ +from typing import Optional + +from fastapi import Request + + +def get_project_id(request: Request, project_id: Optional[str] = None) -> str: + """ + Retrieve the project_id from the request or use the default from the request state. + + Args: + request (Request): The current request object containing state information. + project_id (Optional[str]): The provided project_id from the API endpoint. + + Returns: + str: The project_id to use for the operation. + + Raises: + ValueError: If no project_id is provided and no default is found in the request state. + """ + + if project_id is not None: + return project_id + + default_project_id: str = getattr(request.state, "project_id", None) + if default_project_id is None: + raise ValueError( + "No project_id provided and no default project_id found in the request state." + ) + + return default_project_id From 9068ae6d4d465ff325b579c865637e82c544dd29 Mon Sep 17 00:00:00 2001 From: Abram Date: Fri, 13 Sep 2024 16:42:55 +0100 Subject: [PATCH 16/57] refactor (backend): make use of utility function to get default project_id and make use of it in db querying --- .../agenta_backend/models/api/api_models.py | 3 +- .../models/api/evaluation_model.py | 7 +- .../agenta_backend/models/converters.py | 19 +- .../agenta_backend/models/db_models.py | 2 + .../agenta_backend/routers/app_router.py | 60 +- .../agenta_backend/routers/configs_router.py | 13 +- .../routers/container_router.py | 11 +- .../routers/environment_router.py | 6 + .../routers/evaluation_router.py | 8 +- .../routers/evaluators_router.py | 41 +- .../routers/human_evaluation_router.py | 109 ++-- .../agenta_backend/routers/testset_router.py | 38 +- .../agenta_backend/routers/variants_router.py | 23 +- .../agenta_backend/services/app_manager.py | 138 ++-- .../agenta_backend/services/db_manager.py | 588 ++++++++---------- .../services/deployment_manager.py | 11 +- .../services/evaluation_service.py | 52 +- .../services/evaluator_manager.py | 27 +- 18 files changed, 550 insertions(+), 606 deletions(-) diff --git a/agenta-backend/agenta_backend/models/api/api_models.py b/agenta-backend/agenta_backend/models/api/api_models.py index 353ab3e47d..dc2eea46f0 100644 --- a/agenta-backend/agenta_backend/models/api/api_models.py +++ b/agenta-backend/agenta_backend/models/api/api_models.py @@ -100,8 +100,8 @@ class AppVariantResponse(BaseModel): app_name: str variant_id: str variant_name: str + project_id: str parameters: Optional[Dict[str, Any]] - user_id: str base_name: str base_id: str config_name: str @@ -138,6 +138,7 @@ class AppVariantOutputExtended(BaseModel): class EnvironmentOutput(BaseModel): name: str app_id: str + project_id: str deployed_app_variant_id: Optional[str] deployed_variant_name: Optional[str] deployed_app_variant_revision_id: Optional[str] diff --git a/agenta-backend/agenta_backend/models/api/evaluation_model.py b/agenta-backend/agenta_backend/models/api/evaluation_model.py index e12d471489..f79ebba169 100644 --- a/agenta-backend/agenta_backend/models/api/evaluation_model.py +++ b/agenta-backend/agenta_backend/models/api/evaluation_model.py @@ -17,6 +17,7 @@ class Evaluator(BaseModel): class EvaluatorConfig(BaseModel): id: str name: str + project_id: str evaluator_key: str settings_values: Optional[Dict[str, Any]] = None created_at: str @@ -63,8 +64,7 @@ class AppOutput(BaseModel): class Evaluation(BaseModel): id: str app_id: str - user_id: str - user_username: str + project_id: str variant_ids: List[str] variant_names: List[str] variant_revision_ids: List[str] @@ -122,8 +122,7 @@ class HumanEvaluationScenarioOutput(BaseModel): class HumanEvaluation(BaseModel): id: str app_id: str - user_id: str - user_username: str + project_id: str evaluation_type: str variant_ids: List[str] variant_names: List[str] diff --git a/agenta-backend/agenta_backend/models/converters.py b/agenta-backend/agenta_backend/models/converters.py index 41b3ae0caa..b6b994deb8 100644 --- a/agenta-backend/agenta_backend/models/converters.py +++ b/agenta-backend/agenta_backend/models/converters.py @@ -107,6 +107,7 @@ async def human_evaluation_db_to_simple_evaluation_output( return SimpleEvaluationOutput( id=str(human_evaluation_db.id), app_id=str(human_evaluation_db.app_id), + project_id=str(human_evaluation_db.project_id), status=human_evaluation_db.status, # type: ignore evaluation_type=human_evaluation_db.evaluation_type, # type: ignore variant_ids=[ @@ -131,8 +132,7 @@ async def evaluation_db_to_pydantic( return Evaluation( id=str(evaluation_db.id), app_id=str(evaluation_db.app_id), - user_id=str(evaluation_db.user_id), - user_username=evaluation_db.user.username or "", + project_id=str(evaluation_db.project_id), status=evaluation_db.status, variant_ids=[str(evaluation_db.variant_id)], variant_revision_ids=[str(evaluation_db.variant_revision_id)], @@ -179,8 +179,7 @@ async def human_evaluation_db_to_pydantic( return HumanEvaluation( id=str(evaluation_db.id), app_id=str(evaluation_db.app_id), - user_id=str(evaluation_db.user_id), - user_username=evaluation_db.user.username or "", + project_id=str(evaluation_db.project_id), status=evaluation_db.status, # type: ignore evaluation_type=evaluation_db.evaluation_type, # type: ignore variant_ids=variants_ids, @@ -301,9 +300,9 @@ async def app_variant_db_to_output(app_variant_db: AppVariantDB) -> AppVariantRe variant_response = AppVariantResponse( app_id=str(app_variant_db.app_id), app_name=str(app_variant_db.app.app_name), + project_id=str(app_variant_db.project_id), variant_name=app_variant_db.variant_name, # type: ignore variant_id=str(app_variant_db.id), - user_id=str(app_variant_db.user_id), parameters=app_variant_db.config_parameters, # type: ignore base_name=app_variant_db.base_name, # type: ignore base_id=str(app_variant_db.base_id), @@ -367,7 +366,7 @@ async def environment_db_to_output( ) if deployed_app_variant_id: deployed_app_variant = await db_manager.get_app_variant_instance_by_id( - deployed_app_variant_id + deployed_app_variant_id, str(environment_db.project_id) ) deployed_variant_name = deployed_app_variant.variant_name revision = deployed_app_variant.revision @@ -378,6 +377,7 @@ async def environment_db_to_output( environment_output = EnvironmentOutput( name=environment_db.name, app_id=str(environment_db.app_id), + project_id=str(environment_db.project_id), deployed_app_variant_id=deployed_app_variant_id, deployed_variant_name=deployed_variant_name, deployed_app_variant_revision_id=str( @@ -385,10 +385,6 @@ async def environment_db_to_output( ), revision=revision, ) - - if isCloudEE(): - environment_output.organization_id = str(environment_db.organization_id) - environment_output.workspace_id = str(environment_db.workspace_id) return environment_output @@ -513,12 +509,13 @@ def user_db_to_pydantic(user_db: UserDB) -> User: uid=user_db.uid, username=user_db.username, email=user_db.email, - ).dict(exclude_unset=True) + ).model_dump(exclude_unset=True) def evaluator_config_db_to_pydantic(evaluator_config: EvaluatorConfigDB): return EvaluatorConfig( id=str(evaluator_config.id), + project_id=str(evaluator_config.project_id), name=evaluator_config.name, evaluator_key=evaluator_config.evaluator_key, settings_values=evaluator_config.settings_values, diff --git a/agenta-backend/agenta_backend/models/db_models.py b/agenta-backend/agenta_backend/models/db_models.py index cb50121e28..339f554eb5 100644 --- a/agenta-backend/agenta_backend/models/db_models.py +++ b/agenta-backend/agenta_backend/models/db_models.py @@ -169,6 +169,7 @@ class VariantBaseDB(Base): unique=True, nullable=False, ) + app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="CASCADE")) project_id = Column( UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE") ) @@ -186,6 +187,7 @@ class VariantBaseDB(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) + app = relationship("AppDB") image = relationship("ImageDB") deployment = relationship("DeploymentDB") project = relationship("ProjectDB") diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index b16808f212..60cafa630a 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -8,6 +8,7 @@ from fastapi import HTTPException, Request from agenta_backend.models import converters +from agenta_backend.utils import project_utils from agenta_backend.utils.common import ( isEE, isCloudProd, @@ -103,6 +104,9 @@ async def list_app_variants( List[AppVariantResponse]: A list of app variants for the given app ID. """ try: + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -118,7 +122,9 @@ async def list_app_variants( status_code=403, ) - app_variants = await db_manager.list_app_variants(app_id=app_id) + app_variants = await db_manager.list_app_variants( + app_id=app_id, project_id=project_id + ) return [ await converters.app_variant_db_to_output(app_variant) for app_variant in app_variants @@ -358,12 +364,10 @@ async def list_apps( HTTPException: If there was an error retrieving the list of apps. """ try: - apps = await db_manager.list_apps( - user_uid=request.state.user_id, - app_name=app_name, - org_id=org_id, - workspace_id=workspace_id, + project_id = project_utils.get_project_id( + request=request, project_id=project_id ) + apps = await db_manager.list_apps(project_id=project_id, app_name=app_name) return apps except Exception as e: logger.exception(f"An error occurred: {str(e)}") @@ -458,7 +462,10 @@ async def remove_app( app -- App to remove """ try: - app = await db_manager.fetch_app_by_id(app_id) + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) + app = await db_manager.fetch_app_by_id(app_id, project_id) if isCloudEE(): has_permission = await check_action_access( @@ -474,7 +481,7 @@ async def remove_app( status_code=403, ) - await app_manager.remove_app(app) + await app_manager.remove_app(app, project_id) except DockerException as e: detail = f"Docker error while trying to remove the app: {str(e)}" logger.exception(f"Docker error while trying to remove the app: {str(e)}") @@ -510,6 +517,9 @@ async def create_app_and_variant_from_template( try: logger.debug("Start: Creating app and variant from template") + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) if isCloudEE(): # Get user and org id logger.debug("Step 1: Getting user and organization ID") @@ -528,8 +538,7 @@ async def create_app_and_variant_from_template( logger.debug("Step 3: Checking user has permission to create app") has_permission = await check_rbac_permission( user_org_workspace_data=user_org_workspace_data, - workspace_id=payload.workspace_id, - organization_id=payload.organization_id, + project_id=project_id, permission=Permission.CREATE_APPLICATION, ) logger.debug( @@ -549,12 +558,7 @@ async def create_app_and_variant_from_template( else f"Step 1: Checking if app {payload.app_name} already exists" ) app_name = payload.app_name.lower() - app = await db_manager.fetch_app_by_name_and_parameters( - app_name, - request.state.user_id, - payload.organization_id if isCloudEE() else None, # type: ignore - payload.workspace_id if isCloudEE() else None, # type: ignore - ) + app = await db_manager.fetch_app_by_name_and_parameters(app_name, project_id) if app is not None: raise Exception( f"App with name {app_name} already exists", @@ -566,12 +570,7 @@ async def create_app_and_variant_from_template( else "Step 2: Creating new app and initializing environments" ) if app is None: - app = await db_manager.create_app_and_envs( - app_name, - request.state.user_id, - payload.organization_id if isCloudEE() else None, # type: ignore - payload.workspace_id if isCloudEE() else None, # type: ignore - ) + app = await db_manager.create_app_and_envs(app_name, project_id) logger.debug( "Step 6: Retrieve template from db" @@ -589,6 +588,7 @@ async def create_app_and_variant_from_template( ) app_variant_db = await app_manager.add_variant_based_on_image( app=app, + project_id=project_id, variant_name="app.default", docker_id_or_template_uri=( # type: ignore template_db.template_uri if isCloudProd() else template_db.digest @@ -606,12 +606,9 @@ async def create_app_and_variant_from_template( else "Step 5: Creating testset for app variant" ) await db_manager.add_testset_to_app_variant( - app_id=str(app.id), - org_id=payload.organization_id if isCloudEE() else None, # type: ignore - workspace_id=payload.workspace_id if isCloudEE() else None, # type: ignore template_name=template_db.name, # type: ignore app_name=app.app_name, # type: ignore - user_uid=request.state.user_id, + project_id=project_id, ) logger.debug( @@ -619,7 +616,7 @@ async def create_app_and_variant_from_template( if isCloudEE() else "Step 6: We create ready-to use evaluators" ) - await evaluator_manager.create_ready_to_use_evaluators(app=app) + await evaluator_manager.create_ready_to_use_evaluators(project_id=project_id) logger.debug( "Step 10: Starting variant and injecting environment variables" @@ -656,7 +653,7 @@ async def create_app_and_variant_from_template( envvars[key] = os.environ[key] else: envvars = {} if payload.env_vars is None else payload.env_vars - await app_manager.start_variant(app_variant_db, envvars) + await app_manager.start_variant(app_variant_db, project_id, envvars) logger.debug("End: Successfully created app and variant") return await converters.app_variant_db_to_output(app_variant_db) @@ -691,6 +688,9 @@ async def list_environments( """ logger.debug(f"Listing environments for app: {app_id}") try: + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -706,7 +706,9 @@ async def list_environments( status_code=403, ) - environments_db = await db_manager.list_environments(app_id=app_id) + environments_db = await db_manager.list_environments( + app_id=app_id, project_id=project_id + ) logger.debug(f"environments_db: {environments_db}") return [ await converters.environment_db_to_output(env) for env in environments_db diff --git a/agenta-backend/agenta_backend/routers/configs_router.py b/agenta-backend/agenta_backend/routers/configs_router.py index c06cdff62d..fc27c9465f 100644 --- a/agenta-backend/agenta_backend/routers/configs_router.py +++ b/agenta-backend/agenta_backend/routers/configs_router.py @@ -3,8 +3,9 @@ from typing import Optional from fastapi.responses import JSONResponse from fastapi import Request, HTTPException -from agenta_backend.utils.common import APIRouter, isCloudEE +from agenta_backend.utils import project_utils +from agenta_backend.utils.common import APIRouter, isCloudEE from agenta_backend.models.api.api_models import ( SaveConfigPayload, GetConfigResponse, @@ -31,7 +32,10 @@ async def save_config( project_id: Optional[str] = None, ): try: - base_db = await db_manager.fetch_base_by_id(payload.base_id) + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) + base_db = await db_manager.fetch_base_by_id(payload.base_id, project_id) if isCloudEE(): has_permission = await check_action_access( @@ -47,7 +51,7 @@ async def save_config( status_code=403, ) - variants_db = await db_manager.list_variants_for_base(base_db) + variants_db = await db_manager.list_variants_for_base(base_db, project_id) variant_to_overwrite = None for variant_db in variants_db: if variant_db.config_name == payload.config_name: @@ -61,12 +65,14 @@ async def save_config( app_variant_id=str(variant_to_overwrite.id), parameters=payload.parameters, user_uid=request.state.user_id, + project_id=project_id, ) logger.debug("Deploying to production environment") await db_manager.deploy_to_environment( environment_name="production", variant_id=str(variant_to_overwrite.id), + project_id=project_id, user_uid=request.state.user_id, ) else: @@ -83,6 +89,7 @@ async def save_config( new_config_name=payload.config_name, parameters=payload.parameters, user_uid=request.state.user_id, + project_id=project_id, ) except HTTPException as e: diff --git a/agenta-backend/agenta_backend/routers/container_router.py b/agenta-backend/agenta_backend/routers/container_router.py index 54d3bf2045..8a97e3d747 100644 --- a/agenta-backend/agenta_backend/routers/container_router.py +++ b/agenta-backend/agenta_backend/routers/container_router.py @@ -1,10 +1,10 @@ -import uuid import logging from typing import List, Optional, Union from fastapi.responses import JSONResponse from fastapi import Request, UploadFile, HTTPException +from agenta_backend.utils import project_utils from agenta_backend.services import db_manager from agenta_backend.utils.common import ( APIRouter, @@ -166,8 +166,9 @@ async def construct_app_container_url( # assert that one of base_id or variant_id is provided assert base_id or variant_id, "Please provide either base_id or variant_id" + project_id = project_utils.get_project_id(request=request, project_id=project_id) if base_id: - object_db = await db_manager.fetch_base_by_id(base_id) + object_db = await db_manager.fetch_base_by_id(base_id, project_id) elif variant_id and variant_id != "None": # NOTE: Backward Compatibility # --------------------------- @@ -178,7 +179,7 @@ async def construct_app_container_url( # This change ensures that users can still view their evaluations; however, # they will no longer be able to access a deployment URL for the deleted variant. # Therefore, we ensure that variant_id is not "None". - object_db = await db_manager.fetch_app_variant_by_id(variant_id) + object_db = await db_manager.fetch_app_variant_by_id(variant_id, project_id) else: # NOTE: required for backward compatibility object_db = None @@ -198,11 +199,11 @@ async def construct_app_container_url( try: if getattr(object_db, "deployment_id", None): # this is a base deployment = await db_manager.get_deployment_by_id( - str(object_db.deployment_id) # type: ignore + str(object_db.deployment_id), project_id # type: ignore ) elif getattr(object_db, "base_id", None): # this is a variant deployment = await db_manager.get_deployment_by_id( - str(object_db.base.deployment_id) # type: ignore + str(object_db.base.deployment_id), project_id # type: ignore ) else: raise HTTPException( diff --git a/agenta-backend/agenta_backend/routers/environment_router.py b/agenta-backend/agenta_backend/routers/environment_router.py index 87e101d3d1..2e88e2ea30 100644 --- a/agenta-backend/agenta_backend/routers/environment_router.py +++ b/agenta-backend/agenta_backend/routers/environment_router.py @@ -4,6 +4,7 @@ from fastapi.responses import JSONResponse from fastapi import Request, HTTPException +from agenta_backend.utils import project_utils from agenta_backend.services import db_manager, app_manager from agenta_backend.utils.common import APIRouter, isCloudEE from agenta_backend.models.api.api_models import DeployToEnvironmentPayload @@ -34,6 +35,9 @@ async def deploy_to_environment( HTTPException: If the deployment fails. """ try: + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -53,6 +57,7 @@ async def deploy_to_environment( await db_manager.deploy_to_environment( environment_name=payload.environment_name, variant_id=payload.variant_id, + project_id=project_id, user_uid=request.state.user_id, ) @@ -61,6 +66,7 @@ async def deploy_to_environment( user_uid=request.state.user_id, object_id=payload.variant_id, object_type="variant", + project_id=project_id, ) logger.debug("Successfully updated last_modified_by app information") except Exception as e: diff --git a/agenta-backend/agenta_backend/routers/evaluation_router.py b/agenta-backend/agenta_backend/routers/evaluation_router.py index 709aabea64..ae36ab2c63 100644 --- a/agenta-backend/agenta_backend/routers/evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/evaluation_router.py @@ -6,6 +6,7 @@ from fastapi import HTTPException, Request, status, Response, Query from agenta_backend.models import converters +from agenta_backend.utils import project_utils from agenta_backend.tasks.evaluations import evaluate from agenta_backend.utils.common import APIRouter, isCloudEE from agenta_backend.models.api.evaluation_model import ( @@ -322,7 +323,10 @@ async def fetch_list_evaluations( List[Evaluation]: A list of evaluations. """ try: - app = await db_manager.fetch_app_by_id(app_id) + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) + app = await db_manager.fetch_app_by_id(app_id, project_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -340,7 +344,7 @@ async def fetch_list_evaluations( status_code=403, ) - return await evaluation_service.fetch_list_evaluations(app) + return await evaluation_service.fetch_list_evaluations(app, project_id) except Exception as exc: import traceback diff --git a/agenta-backend/agenta_backend/routers/evaluators_router.py b/agenta-backend/agenta_backend/routers/evaluators_router.py index e2340c0a0d..f215a2e710 100644 --- a/agenta-backend/agenta_backend/routers/evaluators_router.py +++ b/agenta-backend/agenta_backend/routers/evaluators_router.py @@ -4,6 +4,7 @@ from fastapi import HTTPException, Request from fastapi.responses import JSONResponse +from agenta_backend.utils import project_utils from agenta_backend.utils.common import APIRouter, isCloudEE from agenta_backend.services import evaluator_manager, db_manager, app_manager @@ -63,6 +64,9 @@ async def get_evaluator_configs( """ try: + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -78,7 +82,7 @@ async def get_evaluator_configs( status_code=403, ) - evaluators_configs = await evaluator_manager.get_evaluators_configs(app_id) + evaluators_configs = await evaluator_manager.get_evaluators_configs(project_id) return evaluators_configs except Exception as e: raise HTTPException( @@ -141,6 +145,9 @@ async def create_new_evaluator_config( EvaluatorConfigDB: Evaluator configuration api model. """ try: + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -157,20 +164,11 @@ async def create_new_evaluator_config( ) evaluator_config = await evaluator_manager.create_evaluator_config( - app_id=payload.app_id, + project_id=project_id, name=payload.name, evaluator_key=payload.evaluator_key, settings_values=payload.settings_values, ) - - # Update last_modified_by app information - await app_manager.update_last_modified_by( - user_uid=request.state.user_id, - object_id=payload.app_id, - object_type="app", - ) - logger.debug("Successfully updated last_modified_by app information") - return evaluator_config except Exception as e: import traceback @@ -195,6 +193,9 @@ async def update_evaluator_config( """ try: + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -211,16 +212,8 @@ async def update_evaluator_config( ) evaluators_configs = await evaluator_manager.update_evaluator_config( - evaluator_config_id=evaluator_config_id, updates=payload.dict() - ) - - # Update last_modified_by app information - await app_manager.update_last_modified_by( - user_uid=request.state.user_id, - object_id=evaluator_config_id, - object_type="evaluator_config", + evaluator_config_id=evaluator_config_id, updates=payload.model_dump() ) - logger.debug("Successfully updated last_modified_by app information") return evaluators_configs except Exception as e: import traceback @@ -261,14 +254,6 @@ async def delete_evaluator_config( status_code=403, ) - # Update last_modified_by app information - await app_manager.update_last_modified_by( - user_uid=request.state.user_id, - object_id=evaluator_config_id, - object_type="evaluator_config", - ) - logger.debug("Successfully updated last_modified_by app information") - success = await evaluator_manager.delete_evaluator_config(evaluator_config_id) return success except Exception as e: diff --git a/agenta-backend/agenta_backend/routers/human_evaluation_router.py b/agenta-backend/agenta_backend/routers/human_evaluation_router.py index e6cd8f4ac7..c785ec9a7f 100644 --- a/agenta-backend/agenta_backend/routers/human_evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/human_evaluation_router.py @@ -4,6 +4,7 @@ from fastapi import HTTPException, Body, Request, status, Response from agenta_backend.models import converters +from agenta_backend.utils import project_utils from agenta_backend.services import results_service from agenta_backend.services import evaluation_service from agenta_backend.services import db_manager, app_manager @@ -55,7 +56,12 @@ async def create_evaluation( _description_ """ try: - app = await db_manager.fetch_app_by_id(app_id=payload.app_id) + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) + app = await db_manager.fetch_app_by_id( + app_id=payload.app_id, project_id=project_id + ) if app is None: raise HTTPException(status_code=404, detail="App not found") @@ -74,17 +80,8 @@ async def create_evaluation( ) new_human_evaluation_db = await evaluation_service.create_new_human_evaluation( - payload, request.state.user_id - ) - - # Update last_modified_by app information - await app_manager.update_last_modified_by( - user_uid=request.state.user_id, - object_id=payload.app_id, - object_type="app", + payload, request.state.user_id, project_id ) - logger.debug("Successfully updated last_modified_by app information") - return await converters.human_evaluation_db_to_simple_evaluation_output( new_human_evaluation_db ) @@ -117,6 +114,9 @@ async def fetch_list_human_evaluations( """ try: + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -131,7 +131,7 @@ async def fetch_list_human_evaluations( status_code=403, ) - return await evaluation_service.fetch_list_human_evaluations(app_id) + return await evaluation_service.fetch_list_human_evaluations(app_id, project_id) except Exception as e: status_code = e.status_code if hasattr(e, "status_code") else 500 # type: ignore raise HTTPException(status_code=status_code, detail=str(e)) from e @@ -152,7 +152,12 @@ async def fetch_human_evaluation( HumanEvaluation: The fetched evaluation. """ try: - human_evaluation = await db_manager.fetch_human_evaluation_by_id(evaluation_id) + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) + human_evaluation = await db_manager.fetch_human_evaluation_by_id( + evaluation_id, project_id + ) if not human_evaluation: raise HTTPException(status_code=404, detail="Evaluation not found") @@ -199,7 +204,12 @@ async def fetch_evaluation_scenarios( """ try: - human_evaluation = await db_manager.fetch_human_evaluation_by_id(evaluation_id) + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) + human_evaluation = await db_manager.fetch_human_evaluation_by_id( + evaluation_id, project_id + ) if human_evaluation is None: raise HTTPException( status_code=404, @@ -252,7 +262,12 @@ async def update_human_evaluation( """ try: - human_evaluation = await db_manager.fetch_human_evaluation_by_id(evaluation_id) + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) + human_evaluation = await db_manager.fetch_human_evaluation_by_id( + evaluation_id, project_id + ) if not human_evaluation: raise HTTPException(status_code=404, detail="Evaluation not found") @@ -270,15 +285,6 @@ async def update_human_evaluation( ) await update_human_evaluation_service(human_evaluation, update_data) - - # Update last_modified_by app information - await app_manager.update_last_modified_by( - user_uid=request.state.user_id, - object_id=str(human_evaluation.app_id), - object_type="app", - ) - logger.debug("Successfully updated last_modified_by app information") - return Response(status_code=status.HTTP_204_NO_CONTENT) except KeyError: @@ -308,8 +314,11 @@ async def update_evaluation_scenario_router( None: 204 No Content status code upon successful update. """ try: + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) evaluation_scenario_db = await db_manager.fetch_human_evaluation_scenario_by_id( - evaluation_scenario_id + evaluation_scenario_id, project_id ) if evaluation_scenario_db is None: raise HTTPException( @@ -335,15 +344,6 @@ async def update_evaluation_scenario_router( payload, evaluation_type, ) - - # Update last_modified_by app information - await app_manager.update_last_modified_by( - user_uid=request.state.user_id, - object_id=str(evaluation_scenario_db.evaluation_id), - object_type="human_evaluation", - ) - logger.debug("Successfully updated last_modified_by app information") - return Response(status_code=status.HTTP_204_NO_CONTENT) except UpdateEvaluationScenarioError as e: import traceback @@ -370,8 +370,11 @@ async def get_evaluation_scenario_score_router( Dictionary containing the scenario ID and its score. """ try: + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) evaluation_scenario = db_manager.fetch_evaluation_scenario_by_id( - evaluation_scenario_id + evaluation_scenario_id, project_id ) if evaluation_scenario is None: raise HTTPException( @@ -417,8 +420,11 @@ async def update_evaluation_scenario_score_router( None: 204 No Content status code upon successful update. """ try: + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) evaluation_scenario = await db_manager.fetch_evaluation_scenario_by_id( - evaluation_scenario_id + evaluation_scenario_id, project_id ) if evaluation_scenario is None: raise HTTPException( @@ -441,17 +447,8 @@ async def update_evaluation_scenario_score_router( await db_manager.update_human_evaluation_scenario( evaluation_scenario_id=str(evaluation_scenario.id), # type: ignore - values_to_update=payload.dict(), - ) - - # Update last_modified_by app information - await app_manager.update_last_modified_by( - user_uid=request.state.user_id, - object_id=str(evaluation_scenario.evaluation_id), - object_type="human_evaluation", + values_to_update=payload.model_dump(), ) - logger.debug("Successfully updated last_modified_by app information") - return Response(status_code=status.HTTP_204_NO_CONTENT) except Exception as e: status_code = e.status_code if hasattr(e, "status_code") else 500 # type: ignore @@ -474,7 +471,12 @@ async def fetch_results( """ try: - evaluation = await db_manager.fetch_human_evaluation_by_id(evaluation_id) + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) + evaluation = await db_manager.fetch_human_evaluation_by_id( + evaluation_id, project_id + ) if evaluation is None: raise HTTPException( status_code=404, @@ -527,6 +529,9 @@ async def delete_evaluations( """ try: + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) if isCloudEE(): evaluation_id = random.choice(delete_evaluations.evaluations_ids) has_permission = await check_action_access( @@ -542,16 +547,8 @@ async def delete_evaluations( status_code=403, ) - # Update last_modified_by app information - await app_manager.update_last_modified_by( - user_uid=request.state.user_id, - object_id=random.choice(delete_evaluations.evaluations_ids), - object_type="human_evaluation", - ) - logger.debug("Successfully updated last_modified_by app information") - await evaluation_service.delete_human_evaluations( - delete_evaluations.evaluations_ids + delete_evaluations.evaluations_ids, project_id ) return Response(status_code=status.HTTP_204_NO_CONTENT) except Exception as e: diff --git a/agenta-backend/agenta_backend/routers/testset_router.py b/agenta-backend/agenta_backend/routers/testset_router.py index 650619b0ef..ea50d3c9dc 100644 --- a/agenta-backend/agenta_backend/routers/testset_router.py +++ b/agenta-backend/agenta_backend/routers/testset_router.py @@ -11,6 +11,7 @@ from fastapi.responses import JSONResponse from fastapi import HTTPException, UploadFile, File, Form, Request +from agenta_backend.utils import project_utils from agenta_backend.services import db_manager from agenta_backend.utils.common import APIRouter, isCloudEE from agenta_backend.models.converters import testset_db_to_pydantic @@ -67,7 +68,8 @@ async def upload_file( dict: The result of the upload process. """ - app = await db_manager.fetch_app_by_id(app_id=app_id) + project_id = project_utils.get_project_id(request=request, project_id=project_id) + app = await db_manager.fetch_app_by_id(app_id=app_id, project_id=project_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -114,7 +116,7 @@ async def upload_file( try: testset = await db_manager.create_testset( - app=app, user_uid=request.state.user_id, testset_data=document + app=app, project_id=project_id, testset_data=document ) return TestSetSimpleResponse( id=str(testset.id), @@ -145,7 +147,9 @@ async def import_testset( Returns: dict: The result of the import process. """ - app = await db_manager.fetch_app_by_id(app_id=app_id) + + project_id = project_utils.get_project_id(request=request, project_id=project_id) + app = await db_manager.fetch_app_by_id(app_id=app_id, project_id=project_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -180,7 +184,7 @@ async def import_testset( document["csvdata"].append(row) testset = await db_manager.create_testset( - app=app, user_uid=request.state.user_id, testset_data=document + app=app, project_id=project_id, testset_data=document ) return TestSetSimpleResponse( id=str(testset.id), @@ -224,7 +228,8 @@ async def create_testset( str: The id of the test set created. """ - app = await db_manager.fetch_app_by_id(app_id=app_id) + project_id = project_utils.get_project_id(request=request, project_id=project_id) + app = await db_manager.fetch_app_by_id(app_id=app_id, project_id=project_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -246,7 +251,7 @@ async def create_testset( "csvdata": csvdata.csvdata, } testset_instance = await db_manager.create_testset( - app=app, user_uid=request.state.user_id, testset_data=testset_data + app=app, project_id=project_id, testset_data=testset_data ) if testset_instance is not None: return TestSetSimpleResponse( @@ -277,7 +282,10 @@ async def update_testset( str: The id of the test set updated. """ - testset = await db_manager.fetch_testset_by_id(testset_id=testset_id) + project_id = project_utils.get_project_id(request=request, project_id=project_id) + testset = await db_manager.fetch_testset_by_id( + testset_id=testset_id, project_id=project_id + ) if testset is None: raise HTTPException(status_code=404, detail="testset not found") @@ -329,7 +337,9 @@ async def get_testsets( Raises: - `HTTPException` with status code 404 if no testsets are found. """ - app = await db_manager.fetch_app_by_id(app_id=app_id) + + project_id = project_utils.get_project_id(request=request, project_id=project_id) + app = await db_manager.fetch_app_by_id(app_id=app_id, project_id=project_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -345,10 +355,7 @@ async def get_testsets( status_code=403, ) - if app is None: - raise HTTPException(status_code=404, detail="App not found") - - testsets = await db_manager.fetch_testsets_by_app_id(app_id=app_id) + testsets = await db_manager.fetch_testsets_by_app_id(project_id=project_id) return [ TestSetOutputResponse( _id=str(testset.id), # type: ignore @@ -376,7 +383,12 @@ async def get_single_testset( """ try: - test_set = await db_manager.fetch_testset_by_id(testset_id=testset_id) + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) + test_set = await db_manager.fetch_testset_by_id( + testset_id=testset_id, project_id=project_id + ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, diff --git a/agenta-backend/agenta_backend/routers/variants_router.py b/agenta-backend/agenta_backend/routers/variants_router.py index 6415078cb7..e9c6fe1403 100644 --- a/agenta-backend/agenta_backend/routers/variants_router.py +++ b/agenta-backend/agenta_backend/routers/variants_router.py @@ -6,6 +6,7 @@ from fastapi import HTTPException, Request, Body from agenta_backend.models import converters +from agenta_backend.utils import project_utils from agenta_backend.utils.common import APIRouter, isCloudEE from agenta_backend.services import ( app_manager, @@ -68,7 +69,9 @@ async def add_variant_from_base_and_config( try: logger.debug("Initiating process to add a variant based on a previous one.") logger.debug(f"Received payload: {payload}") - + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) base_db = await db_manager.fetch_base_by_id(payload.base_id) # Check user has permission to add variant @@ -95,6 +98,7 @@ async def add_variant_from_base_and_config( new_config_name=payload.new_config_name, parameters=payload.parameters, user_uid=request.state.user_id, + project_id=project_id, ) logger.debug(f"Successfully added new variant: {db_app_variant}") @@ -103,11 +107,12 @@ async def add_variant_from_base_and_config( user_uid=request.state.user_id, object_id=str(db_app_variant.app_id), object_type="app", + project_id=project_id, ) logger.debug("Successfully updated last_modified_by app information") app_variant_db = await db_manager.get_app_variant_instance_by_id( - str(db_app_variant.id) + str(db_app_variant.id), project_id ) return await converters.app_variant_db_to_output(app_variant_db) @@ -134,7 +139,11 @@ async def remove_variant( Raises: HTTPException: If there is a problem removing the app variant """ + try: + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -156,10 +165,13 @@ async def remove_variant( user_uid=request.state.user_id, object_id=variant_id, object_type="variant", + project_id=project_id, ) logger.debug("Successfully updated last_modified_by app information") - await app_manager.terminate_and_remove_app_variant(app_variant_id=variant_id) + await app_manager.terminate_and_remove_app_variant( + project_id=project_id, app_variant_id=variant_id + ) except DockerException as e: detail = f"Docker error while trying to remove the app variant: {str(e)}" raise HTTPException(status_code=500, detail=detail) @@ -193,6 +205,9 @@ async def update_variant_parameters( JSONResponse: A JSON response containing the updated app variant parameters. """ try: + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -215,6 +230,7 @@ async def update_variant_parameters( app_variant_id=variant_id, parameters=payload.parameters, user_uid=request.state.user_id, + project_id=project_id, ) # Update last_modified_by app information @@ -222,6 +238,7 @@ async def update_variant_parameters( user_uid=request.state.user_id, object_id=variant_id, object_type="variant", + project_id=project_id, ) logger.debug("Successfully updated last_modified_by app information") except ValueError as e: diff --git a/agenta-backend/agenta_backend/services/app_manager.py b/agenta-backend/agenta_backend/services/app_manager.py index 337b1b675d..9838610f68 100644 --- a/agenta-backend/agenta_backend/services/app_manager.py +++ b/agenta-backend/agenta_backend/services/app_manager.py @@ -54,6 +54,7 @@ async def start_variant( db_app_variant: AppVariantDB, + project_id: str, env_vars: Optional[DockerEnvVars] = None, ) -> URI: """ @@ -64,6 +65,7 @@ async def start_variant( Args: app_variant (AppVariant): The app variant for which a container is to be started. + project_id (str): The ID of the project the app variant belongs to. env_vars (DockerEnvVars): (optional) The environment variables to be passed to the container. Returns: @@ -81,8 +83,6 @@ async def start_variant( db_app_variant.image.docker_id, db_app_variant.image.tags, db_app_variant.app.app_name, - str(db_app_variant.organization_id) if isCloudEE() else None, - str(db_app_variant.workspace_id) if isCloudEE() else None, ) logger.debug("App name is %s", db_app_variant.app.app_name) # update the env variables @@ -104,6 +104,7 @@ async def start_variant( if isCloudEE(): api_key = await api_key_service.create_api_key( str(db_app_variant.user.uid), + project_id=project_id, workspace_id=str(db_app_variant.workspace_id), expiration_date=None, hidden=True, @@ -111,7 +112,7 @@ async def start_variant( env_vars.update({"AGENTA_API_KEY": api_key}) deployment = await deployment_manager.start_service( - app_variant_db=db_app_variant, env_vars=env_vars + app_variant_db=db_app_variant, project_id=project_id, env_vars=env_vars ) await db_manager.update_base( @@ -185,9 +186,7 @@ async def update_variant_image( async def update_last_modified_by( - user_uid: str, - object_id: str, - object_type: str, + user_uid: str, object_id: str, object_type: str, project_id: str ) -> None: """Updates the last_modified_by field in the app variant table. @@ -195,41 +194,26 @@ async def update_last_modified_by( object_id (str): The object ID to update. object_type (str): The type of object to update. user_uid (str): The user UID to update. + project_id (str): The project ID. """ async def get_appdb_str_by_id(object_id: str, object_type: str) -> str: if object_type == "app": return object_id elif object_type == "variant": - app_variant_db = await db_manager.fetch_app_variant_by_id(object_id) + app_variant_db = await db_manager.fetch_app_variant_by_id( + object_id, project_id + ) if app_variant_db is None: raise db_manager.NoResultFound(f"Variant with id {object_id} not found") return str(app_variant_db.app_id) - elif object_type == "evaluation": - evaluation_db = await db_manager.fetch_evaluation_by_id(object_id) - if evaluation_db is None: - raise db_manager.NoResultFound( - f"Evaluation with id {object_id} not found" - ) - return str(evaluation_db.app_id) - elif object_type == "human_evaluation": - human_evaluation_db = await db_manager.fetch_human_evaluation_by_id( - object_id - ) - if human_evaluation_db is None: - raise db_manager.NoResultFound( - f"Human Evaluation with id {object_id} not found" - ) - return str(human_evaluation_db.app_id) - elif object_type == "evaluator_config": - evaluator_config_db = await db_manager.fetch_evaluator_config(object_id) - if evaluator_config_db is None: + elif object_type == "deployment": + deployment_db = await db_manager.get_deployment_by_id(object_id, project_id) + if deployment_db is None: raise db_manager.NoResultFound( - f"Evaluator Config with id {str(object_id)} not found" + f"Deployment with id {object_id} not found" ) - return str(evaluator_config_db.app_id) - else: - raise ValueError(f"Unsupported object type: {object_type}") + return str(deployment_db.app_id) user = await db_manager.get_user(user_uid=user_uid) app_id = await get_appdb_str_by_id(object_id=object_id, object_type=object_type) @@ -243,7 +227,9 @@ async def get_appdb_str_by_id(object_id: str, object_type: str) -> str: async def terminate_and_remove_app_variant( - app_variant_id: Optional[str] = None, app_variant_db: Optional[AppVariantDB] = None + project_id: str, + app_variant_id: Optional[str] = None, + app_variant_db: Optional[AppVariantDB] = None, ) -> None: """ Removes app variant from the database. If it's the last one using an image, performs additional operations: @@ -252,6 +238,7 @@ async def terminate_and_remove_app_variant( - Removes the image from the registry. Args: + project_id (str): The ID of the project variant_id (srt): The app variant to remove. Raises: @@ -267,7 +254,9 @@ async def terminate_and_remove_app_variant( ), "Only one of app_variant_id or app_variant_db must be provided" if app_variant_id: - app_variant_db = await db_manager.fetch_app_variant_by_id(app_variant_id) + app_variant_db = await db_manager.fetch_app_variant_by_id( + app_variant_id, project_id + ) logger.debug(f"Fetched app variant {app_variant_db}") app_id = str(app_variant_db.app_id) # type: ignore @@ -278,14 +267,16 @@ async def terminate_and_remove_app_variant( try: is_last_variant_for_image = await db_manager.check_is_last_variant_for_image( - app_variant_db + str(app_variant_db.base_id), project_id ) if is_last_variant_for_image: base_db = await db_manager.fetch_base_by_id( - base_id=str(app_variant_db.base_id) + base_id=str(app_variant_db.base_id), project_id=project_id ) if not base_db: - raise + raise db_manager.NoResultFound( + f"Variant base with the ID {str(app_variant_db.base_id)} not found" + ) image = base_db.image logger.debug(f"is_last_variant_for_image {image}") @@ -298,7 +289,7 @@ async def terminate_and_remove_app_variant( logger.debug("_stop_and_delete_app_container") try: deployment = await db_manager.get_deployment_by_id( - str(base_db.deployment_id) + str(base_db.deployment_id), project_id ) except Exception as e: logger.error(f"Failed to get deployment {e}") @@ -320,22 +311,22 @@ async def terminate_and_remove_app_variant( except RuntimeError as e: logger.error(f"Failed to remove image {image} {e}") finally: - await db_manager.remove_image(image) + await db_manager.remove_image(image, project_id) # remove app variant - await db_manager.remove_app_variant_from_db(app_variant_db) + await db_manager.remove_app_variant_from_db(app_variant_db, project_id) else: # remove variant + config logger.debug("remove_app_variant_from_db") - await db_manager.remove_app_variant_from_db(app_variant_db) + await db_manager.remove_app_variant_from_db(app_variant_db, project_id) - app_variants = await db_manager.list_app_variants(app_id) + app_variants = await db_manager.list_app_variants(app_id, project_id) logger.debug(f"Count of app variants available: {len(app_variants)}") if ( len(app_variants) == 0 ): # remove app related resources if the length of the app variants hit 0 logger.debug("remove_app_related_resources") - await remove_app_related_resources(app_id) + await remove_app_related_resources(app_id, project_id) except Exception as e: logger.error( f"An error occurred while deleting app variant {app_variant_db.app.app_name}/{app_variant_db.variant_name}: {str(e)}" @@ -343,18 +334,19 @@ async def terminate_and_remove_app_variant( raise e from None -async def remove_app_related_resources(app_id: str): +async def remove_app_related_resources(app_id: str, project_id: str): """Removes associated tables with an app after its deletion. When an app or its last variant is deleted, this function ensures that all related resources such as environments and testsets are also deleted. Args: - app_name: The name of the app whose associated resources are to be removed. + app_name (str): The name of the app whose associated resources are to be removed. + project_id (str: The ID of the project) """ try: - await db_manager.remove_app_by_id(app_id) + await db_manager.remove_app_by_id(app_id, project_id) logger.info(f"Successfully remove app object {app_id}.") except Exception as e: logger.error( @@ -363,13 +355,14 @@ async def remove_app_related_resources(app_id: str): raise e from None -async def remove_app(app: AppDB): +async def remove_app(app: AppDB, project_id: str): """Removes all app variants from db, if it is the last one using an image, then deletes the image from the db, shutdowns the container, deletes it and remove the image from the registry - Arguments: - app_name -- the app name to remove + Args: + app (AppDB): The application instance to remove from database. + project_id (str): The ID of the project. """ if app is None: @@ -377,17 +370,19 @@ async def remove_app(app: AppDB): logger.error(error_msg) raise ValueError(error_msg) - app_variants = await db_manager.list_app_variants(str(app.id)) + app_variants = await db_manager.list_app_variants(str(app.id), project_id) try: for app_variant_db in app_variants: - await terminate_and_remove_app_variant(app_variant_db=app_variant_db) + await terminate_and_remove_app_variant( + project_id=project_id, app_variant_db=app_variant_db + ) logger.info( f"Successfully deleted app variant {app_variant_db.app.app_name}/{app_variant_db.variant_name}." ) if len(app_variants) == 0: logger.debug("remove_app_related_resources") - await remove_app_related_resources(str(app.id)) + await remove_app_related_resources(str(app.id), project_id) except Exception as e: # Failsafe: in case something went wrong, @@ -403,14 +398,15 @@ async def remove_app(app: AppDB): async def update_variant_parameters( - app_variant_id: str, parameters: Dict[str, Any], user_uid: str + app_variant_id: str, parameters: Dict[str, Any], user_uid: str, project_id: str ): """Updates the parameters for app variant in the database. Arguments: - app_variant -- the app variant to update - parameters -- the parameters to update - user_uid -- the user uid + app_variant (str): The app variant to update + parameters (dict): The parameters to update + user_uid (str): The UID of the user + project_id (str): The ID of the project """ assert app_variant_id is not None, "app_variant_id must be provided" @@ -418,7 +414,10 @@ async def update_variant_parameters( try: await db_manager.update_variant_parameters( - app_variant_id=app_variant_id, parameters=parameters, user_uid=user_uid + app_variant_id=app_variant_id, + parameters=parameters, + project_id=project_id, + user_uid=user_uid, ) except Exception as e: logger.error(f"Error updating app variant {app_variant_id}") @@ -427,6 +426,7 @@ async def update_variant_parameters( async def add_variant_based_on_image( app: AppDB, + project_id: str, variant_name: str, docker_id_or_template_uri: str, user_uid: str, @@ -440,6 +440,7 @@ async def add_variant_based_on_image( Args: app (AppDB): The app to add the variant to. + project_id (str): The ID of the project. variant_name (str): The name of the new variant. docker_id (str): The ID of the Docker image to use for the new variant. tags (str): The tags associated with the Docker image. @@ -475,7 +476,9 @@ async def add_variant_based_on_image( # Check if app variant already exists logger.debug("Step 2: Checking if app variant already exists") - variants = await db_manager.list_app_variants_for_app_id(app_id=str(app.id)) + variants = await db_manager.list_app_variants_for_app_id( + app_id=str(app.id), project_id=project_id + ) already_exists = any(av for av in variants if av.variant_name == variant_name) # type: ignore if already_exists: logger.error("App variant with the same name already exists") @@ -492,9 +495,7 @@ async def add_variant_based_on_image( ) else: db_image = await db_manager.get_orga_image_instance_by_docker_id( - docker_id=docker_id_or_template_uri, - organization_id=str(app.organization_id) if isCloudEE() else None, # type: ignore - workspace_id=str(app.workspace_id) if isCloudEE() else None, # type: ignore + docker_id=docker_id_or_template_uri, project_id=project_id ) # Create new image if not exists @@ -503,22 +504,18 @@ async def add_variant_based_on_image( if parsed_url.scheme and parsed_url.netloc: db_image = await db_manager.create_image( image_type="zip", + project_id=project_id, template_uri=docker_id_or_template_uri, deletable=not (is_template_image), - user=user_instance, - organization=str(app.organization_id) if isCloudEE() else None, # noqa - workspace=str(app.workspace_id) if isCloudEE() else None, # noqa ) else: docker_id = docker_id_or_template_uri db_image = await db_manager.create_image( image_type="image", + project_id=project_id, docker_id=docker_id, tags=tags, deletable=not (is_template_image), - user=user_instance, - organization=str(app.organization_id) if isCloudEE() else None, # noqa - workspace=str(app.workspace_id) if isCloudEE() else None, # noqa ) # Create config @@ -535,9 +532,7 @@ async def add_variant_based_on_image( ] # TODO: Change this in SDK2 to directly use base_name db_base = await db_manager.create_new_variant_base( app=app, - organization=str(app.organization_id) if isCloudEE() else None, # noqa - workspace=str(app.workspace_id) if isCloudEE() else None, # noqa - user=user_instance, + project_id=project_id, base_name=base_name, # the first variant always has default base image=db_image, ) @@ -546,14 +541,13 @@ async def add_variant_based_on_image( logger.debug("Step 7: Creating app variant") db_app_variant = await db_manager.create_new_app_variant( app=app, + user=user_instance, variant_name=variant_name, + project_id=project_id, image=db_image, - user=user_instance, - organization=str(app.organization_id) if isCloudEE() else None, # noqa - workspace=str(app.workspace_id) if isCloudEE() else None, # noqa - base_name=base_name, base=db_base, config=config_db, + base_name=base_name, ) logger.debug("End: Successfully created db_app_variant: %s", db_app_variant) return db_app_variant diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 878965c904..8e141da4cf 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -95,27 +95,19 @@ async def add_testset_to_app_variant( - app_id: str, - template_name: str, - app_name: str, - user_uid: str, - org_id: Optional[str] = None, - workspace_id: Optional[str] = None, + template_name: str, app_name: str, project_id: str ): """Add testset to app variant. + Args: - app_id (str): The id of the app org_id (str): The id of the organization template_name (str): The name of the app template image app_name (str): The name of the app - user_uid (str): The uid of the user + project_id (str): The ID of the project """ async with db_engine.get_session() as session: try: - app_db = await get_app_instance_by_id(app_id) - user_db = await get_user(user_uid) - json_path = os.path.join( PARENT_DIRECTORY, "resources", @@ -131,22 +123,9 @@ async def add_testset_to_app_variant( } testset_db = TestSetDB( **testset, - app_id=app_db.id, - user_id=user_db.id, + project_id=uuid.UUID(project_id), ) - if isCloudEE(): - # assert that if organization is provided, workspace_id is also provided, and vice versa - assert ( - org_id is not None and workspace_id is not None - ), "organization and workspace must be provided together" - - organization_db = await db_manager_ee.get_organization(org_id) # type: ignore - workspace_db = await db_manager_ee.get_workspace(workspace_id) # type: ignore - - testset_db.organization_id = organization_db.id - testset_db.workspace_id = workspace_db.id - session.add(testset_db) await session.commit() await session.refresh(testset_db) @@ -173,7 +152,7 @@ async def get_image_by_id(image_id: str) -> ImageDB: return image -async def fetch_app_by_id(app_id: str) -> AppDB: +async def fetch_app_by_id(app_id: str, project_id: str) -> AppDB: """Fetches an app by its ID. Args: @@ -183,26 +162,23 @@ async def fetch_app_by_id(app_id: str) -> AppDB: assert app_id is not None, "app_id cannot be None" app_uuid = await get_object_uuid(object_id=app_id, table_name="app_db") async with db_engine.get_session() as session: - base_query = select(AppDB).filter_by(id=uuid.UUID(app_uuid)) - if isCloudEE(): - base_query = base_query.options( - joinedload(AppDB.workspace).joinedload(WorkspaceDB.members), # type: ignore - joinedload(AppDB.organization), - ) - + base_query = select(AppDB).filter_by( + id=uuid.UUID(app_uuid), project_id=uuid.UUID(project_id) + ) result = await session.execute(base_query) app = result.unique().scalars().first() return app async def fetch_app_variant_by_id( - app_variant_id: str, + app_variant_id: str, project_id: str ) -> Optional[AppVariantDB]: """ Fetches an app variant by its ID. Args: app_variant_id (str): The ID of the app variant to fetch. + project_id (str): The ID of the project the app variant belongs to. Returns: AppVariantDB: The fetched app variant, or None if no app variant was found. @@ -217,16 +193,18 @@ async def fetch_app_variant_by_id( if isCloudEE(): query = base_query.options( joinedload(AppVariantDB.organization), - joinedload(AppVariantDB.user.of_type(UserDB)).load_only(UserDB.uid), # type: ignore joinedload(AppVariantDB.image.of_type(ImageDB)).load_only(ImageDB.docker_id, ImageDB.tags), # type: ignore ) else: query = base_query.options( - joinedload(AppVariantDB.user).load_only(UserDB.uid), # type: ignore joinedload(AppVariantDB.image).load_only(ImageDB.docker_id, ImageDB.tags), # type: ignore ) - result = await session.execute(query.filter_by(id=uuid.UUID(app_variant_id))) + result = await session.execute( + query.filter_by( + id=uuid.UUID(app_variant_id), project_id=uuid.UUID(project_id) + ) + ) app_variant = result.scalars().first() return app_variant @@ -278,7 +256,7 @@ async def fetch_app_variant_by_base_id_and_config_name( async def fetch_app_variant_revision_by_variant( - app_variant_id: str, revision: int + app_variant_id: str, project_id: str, revision: int ) -> AppVariantRevisionsDB: """Fetches app variant revision by variant id and revision @@ -296,7 +274,9 @@ async def fetch_app_variant_revision_by_variant( async with db_engine.get_session() as session: result = await session.execute( select(AppVariantRevisionsDB).filter_by( - variant_id=uuid.UUID(app_variant_id), revision=revision + variant_id=uuid.UUID(app_variant_id), + project_id=uuid.UUID(project_id), + revision=revision, ) ) app_variant_revision = result.scalars().first() @@ -307,11 +287,14 @@ async def fetch_app_variant_revision_by_variant( return app_variant_revision -async def fetch_base_by_id(base_id: str) -> Optional[VariantBaseDB]: +async def fetch_base_by_id(base_id: str, project_id: str) -> Optional[VariantBaseDB]: """ Fetches a base by its ID. + Args: base_id (str): The ID of the base to fetch. + project_id (str): The ID of the project. + Returns: VariantBaseDB: The fetched base, or None if no base was found. """ @@ -324,7 +307,7 @@ async def fetch_base_by_id(base_id: str) -> Optional[VariantBaseDB]: .options( joinedload(VariantBaseDB.image), joinedload(VariantBaseDB.deployment) ) - .filter_by(id=uuid.UUID(base_uuid)) + .filter_by(id=uuid.UUID(base_uuid), project_id=uuid.UUID(project_id)) ) base = result.scalars().first() return base @@ -355,20 +338,16 @@ async def fetch_app_variant_by_name_and_appid( async def create_new_variant_base( app: AppDB, - user: UserDB, + project_id: str, base_name: str, image: ImageDB, - organization=None, - workspace=None, ) -> VariantBaseDB: """Create a new base. Args: base_name (str): The name of the base. image (ImageDB): The image of the base. - user (UserDB): The User Object creating the variant. + project_id (str): The ID of the project app (AppDB): The associated App Object. - organization (OrganizationDB): The Organization the variant belongs to. - workspace (WorkspaceDB): The Workspace the variant belongs to. Returns: VariantBaseDB: The created base. """ @@ -377,20 +356,11 @@ async def create_new_variant_base( async with db_engine.get_session() as session: base = VariantBaseDB( app_id=app.id, - user_id=user.id, + project_id=uuid.UUID(project_id), base_name=base_name, image_id=image.id, ) - if isCloudEE(): - # assert that if organization is provided, workspace_id is also provided, and vice versa - assert ( - organization is not None and workspace is not None - ), "organization and workspace must be provided together" - - base.organization_id = uuid.UUID(organization) - base.workspace_id = uuid.UUID(workspace) - session.add(base) await session.commit() await session.refresh(base) @@ -422,19 +392,22 @@ async def create_new_app_variant( app: AppDB, user: UserDB, variant_name: str, + project_id: str, image: ImageDB, base: VariantBaseDB, config: ConfigDB, base_name: str, - organization=None, - workspace=None, ) -> AppVariantDB: """Create a new variant. + Args: variant_name (str): The name of the variant. + project_id (str): The ID of the project. image (ImageDB): The image of the variant. base (VariantBaseDB): The base of the variant. config (ConfigDB): The config of the variant. + base_name (str): The name of the variant base. + Returns: AppVariantDB: The created variant. """ @@ -446,7 +419,7 @@ async def create_new_app_variant( async with db_engine.get_session() as session: variant = AppVariantDB( app_id=app.id, - user_id=user.id, + project_id=uuid.UUID(project_id), modified_by_id=user.id, revision=0, variant_name=variant_name, @@ -457,21 +430,11 @@ async def create_new_app_variant( config_parameters=config.parameters, ) - if isCloudEE(): - # assert that if organization is provided, workspace_id is also provided, and vice versa - assert ( - organization is not None and workspace is not None - ), "organization and workspace must be provided together" - - variant.organization_id = uuid.UUID(organization) - variant.workspace_id = uuid.UUID(workspace) - session.add(variant) attributes_to_refresh = [ "app", "image", - "user", "base", ] if isCloudEE(): @@ -486,6 +449,7 @@ async def create_new_app_variant( variant_revision = AppVariantRevisionsDB( variant_id=variant.id, revision=0, + project_id=uuid.UUID(project_id), modified_by_id=user.id, base_id=base.id, config_name=config.config_name, @@ -501,22 +465,20 @@ async def create_new_app_variant( async def create_image( image_type: str, - user: UserDB, + project_id: str, deletable: bool, - organization=None, - workspace=None, template_uri: Optional[str] = None, docker_id: Optional[str] = None, tags: Optional[str] = None, ) -> ImageDB: """Create a new image. Args: + image_type (str): The type of image to create. + project_id (str): The ID of the project. docker_id (str): The ID of the image. - tags (str): The tags of the image. - user (UserDB): The user that the image belongs to. deletable (bool): Whether the image can be deleted. - organization (OrganizationDB): The organization that the image belongs to. - workspace (WorkspaceDB): The workspace that the image belongs to. + tags (str): The tags of the image. + Returns: ImageDB: The created image. """ @@ -539,7 +501,7 @@ async def create_image( async with db_engine.get_session() as session: image = ImageDB( deletable=deletable, - user_id=user.id, + project_id=uuid.UUID(project_id), ) image_types = {"zip": TemplateType.ZIP.value, "image": TemplateType.IMAGE.value} @@ -555,15 +517,6 @@ async def create_image( image.tags = tags # type: ignore image.docker_id = docker_id # type: ignore - if isCloudEE(): - # assert that if organization is provided, workspace_id is also provided, and vice versa - assert ( - organization is not None and workspace is not None - ), "organization and workspace must be provided together" - - image.organization_id = uuid.UUID(organization) - image.workspace_id = uuid.UUID(workspace) - session.add(image) await session.commit() await session.refresh(image) @@ -573,24 +526,22 @@ async def create_image( async def create_deployment( app_id: str, - user_id: str, + project_id: str, container_name: str, container_id: str, uri: str, status: str, - organization=None, - workspace=None, ) -> DeploymentDB: """Create a new deployment. + Args: app (str): The app to create the deployment for. - organization (OrganizationDB): The organization that the deployment belongs to. - workspace (WorkspaceDB): The Workspace that the deployment belongs to. - user (str): The user that the deployment belongs to. + project_id (str): The ID of the project to create the deployment for. container_name (str): The name of the container. container_id (str): The ID of the container. uri (str): The URI of the container. status (str): The status of the container. + Returns: DeploymentDB: The created deployment. """ @@ -599,17 +550,13 @@ async def create_deployment( try: deployment = DeploymentDB( app_id=uuid.UUID(app_id), - user_id=uuid.UUID(user_id), + project_id=uuid.UUID(project_id), container_name=container_name, container_id=container_id, uri=uri, status=status, ) - if isCloudEE(): - deployment.organization_id = uuid.UUID(organization) - deployment.workspace_id = uuid.UUID(workspace) - session.add(deployment) await session.commit() await session.refresh(deployment) @@ -619,20 +566,13 @@ async def create_deployment( raise Exception(f"Error while creating deployment: {e}") -async def create_app_and_envs( - app_name: str, - user_uid: str, - organization_id: Optional[str] = None, - workspace_id: Optional[str] = None, -) -> AppDB: +async def create_app_and_envs(app_name: str, project_id: str) -> AppDB: """ Create a new app with the given name and organization ID. Args: app_name (str): The name of the app to create. - user_uid (str): The UID of the user that the app belongs to. - organization_id (str): The ID of the organization that the app belongs to. - workspace_id (str): The ID of the workspace that the app belongs to. + project_id (str): The ID of the project. Returns: AppDB: The created app. @@ -641,36 +581,20 @@ async def create_app_and_envs( ValueError: If an app with the same name already exists. """ - user = await get_user(user_uid) - app = await fetch_app_by_name_and_parameters( - app_name, - user_uid, - organization_id, - workspace_id, - ) + app = await fetch_app_by_name_and_parameters(app_name, project_id) if app is not None: raise ValueError("App with the same name already exists") async with db_engine.get_session() as session: - app = AppDB(app_name=app_name, user_id=user.id) - - if isCloudEE(): - # assert that if organization_id is provided, workspace_id is also provided, and vice versa - assert ( - organization_id is not None and workspace_id is not None - ), "org_id and workspace_id must be provided together" - - organization_db = await db_manager_ee.get_organization(organization_id) # type: ignore - workspace_db = await db_manager_ee.get_workspace(workspace_id) # type: ignore - - app.organization_id = organization_db.id - app.workspace_id = workspace_db.id + app = AppDB(app_name=app_name, project_id=uuid.UUID(project_id)) session.add(app) await session.commit() await session.refresh(app) - await initialize_environments(session=session, app_db=app) + await initialize_environments( + session=session, app_db=app, project_id=project_id + ) return app @@ -695,13 +619,12 @@ async def update_app(app_id: str, values_to_update: dict) -> None: await session.commit() -async def get_deployment_by_id( - deployment_id: str, -) -> DeploymentDB: +async def get_deployment_by_id(deployment_id: str, project_id: str) -> DeploymentDB: """Get the deployment object from the database with the provided id. Arguments: deployment_id (str): The deployment id + project_id (str): The ID of the project Returns: DeploymentDB: instance of deployment object @@ -709,7 +632,9 @@ async def get_deployment_by_id( async with db_engine.get_session() as session: result = await session.execute( - select(DeploymentDB).filter_by(id=uuid.UUID(deployment_id)) + select(DeploymentDB).filter_by( + id=uuid.UUID(deployment_id), project_id=uuid.UUID(project_id) + ) ) deployment = result.scalars().first() return deployment @@ -734,13 +659,14 @@ async def get_deployment_by_appid(app_id: str) -> DeploymentDB: return deployment -async def list_app_variants_for_app_id( - app_id: str, -): +async def list_app_variants_for_app_id(app_id: str, project_id: str): """ Lists all the app variants from the db + Args: - app_name: if specified, only returns the variants for the app name + app_name (str): if specified, only returns the variants for the app name + project_id (str): The ID of the project. + Returns: List[AppVariant]: List of AppVariant objects """ @@ -748,7 +674,9 @@ async def list_app_variants_for_app_id( assert app_id is not None, "app_id cannot be None" async with db_engine.get_session() as session: result = await session.execute( - select(AppVariantDB).filter_by(app_id=uuid.UUID(app_id)) + select(AppVariantDB).filter_by( + app_id=uuid.UUID(app_id), project_id=uuid.UUID(project_id) + ) ) app_variants = result.scalars().all() return app_variants @@ -776,11 +704,14 @@ async def list_bases_for_app_id(app_id: str, base_name: Optional[str] = None): return bases -async def list_variants_for_base(base: VariantBaseDB): +async def list_variants_for_base(base: VariantBaseDB, project_id: str): """ Lists all the app variants from the db for a base + Args: - base: if specified, only returns the variants for the base + base (VariantBaseDB): if specified, only returns the variants for the base + project_id (str): The ID of the project + Returns: List[AppVariant]: List of AppVariant objects """ @@ -789,7 +720,7 @@ async def list_variants_for_base(base: VariantBaseDB): async with db_engine.get_session() as session: result = await session.execute( select(AppVariantDB) - .filter_by(base_id=base.id) + .filter_by(base_id=base.id, project_id=uuid.UUID(project_id)) .order_by(AppVariantDB.variant_name.asc()) ) app_variants = result.scalars().all() @@ -902,34 +833,22 @@ async def get_users_by_ids(user_ids: List): async def get_orga_image_instance_by_docker_id( - docker_id: str, - organization_id: Optional[str] = None, - workspace_id: Optional[str] = None, + docker_id: str, project_id: str ) -> ImageDB: """Get the image object from the database with the provided id. Arguments: - organization_id (str): The organization unique identifier docker_id (str): The image id + project_id (str): The ID of project. Returns: ImageDB: instance of image object """ async with db_engine.get_session() as session: - query = select(ImageDB).filter_by(docker_id=docker_id) - - if isCloudEE(): - # assert that if organization is provided, workspace_id is also provided, and vice versa - assert ( - organization_id is not None and workspace_id is not None - ), "organization and workspace must be provided together" - - query = query.filter_by( - organization_id=uuid.UUID(organization_id), - workspace_id=workspace_id, - ) - + query = select(ImageDB).filter_by( + docker_id=docker_id, project_id=uuid.UUID(project_id) + ) result = await session.execute(query) image = result.scalars().first() return image @@ -973,7 +892,7 @@ async def get_orga_image_instance_by_uri( return image -async def get_app_instance_by_id(app_id: str) -> AppDB: +async def get_app_instance_by_id(app_id: str, project_id: str) -> AppDB: """Get the app object from the database with the provided id. Arguments: @@ -984,7 +903,11 @@ async def get_app_instance_by_id(app_id: str) -> AppDB: """ async with db_engine.get_session() as session: - result = await session.execute(select(AppDB).filter_by(id=uuid.UUID(app_id))) + result = await session.execute( + select(AppDB).filter_by( + id=uuid.UUID(app_id), project_id=uuid.UUID(project_id) + ) + ) app = result.scalars().first() return app @@ -994,6 +917,7 @@ async def add_variant_from_base_and_config( new_config_name: str, parameters: Dict[str, Any], user_uid: str, + project_id: str, ) -> AppVariantDB: """ Add a new variant to the database based on an existing base and a new configuration. @@ -1003,19 +927,22 @@ async def add_variant_from_base_and_config( new_config_name (str): The name of the new configuration to use for the new variant. parameters (Dict[str, Any]): The parameters to use for the new configuration. user_uid (str): The UID of the user + project_id (str): The ID of the project Returns: AppVariantDB: The newly created app variant. """ new_variant_name = f"{base_db.base_name}.{new_config_name}" - previous_app_variant_db = await find_previous_variant_from_base_id(str(base_db.id)) + previous_app_variant_db = await find_previous_variant_from_base_id( + str(base_db.id), project_id + ) if previous_app_variant_db is None: logger.error("Failed to find the previous app variant in the database.") raise HTTPException(status_code=404, detail="Previous app variant not found") logger.debug(f"Located previous variant: {previous_app_variant_db}") - app_variant_for_base = await list_variants_for_base(base_db) + app_variant_for_base = await list_variants_for_base(base_db, project_id) already_exists = any( av for av in app_variant_for_base if av.config_name == new_config_name # type: ignore @@ -1029,10 +956,10 @@ async def add_variant_from_base_and_config( app_id=previous_app_variant_db.app_id, variant_name=new_variant_name, image_id=base_db.image_id, - user_id=user_db.id, modified_by_id=user_db.id, revision=1, base_name=base_db.base_name, + project_id=uuid.UUID(project_id), base_id=base_db.id, config_name=new_config_name, config_parameters=parameters, @@ -1050,6 +977,7 @@ async def add_variant_from_base_and_config( variant_id=db_app_variant.id, revision=1, modified_by_id=user_db.id, + project_id=uuid.UUID(project_id), base_id=base_db.id, config_name=new_config_name, config_parameters=parameters, @@ -1063,10 +991,8 @@ async def add_variant_from_base_and_config( async def list_apps( - user_uid: str, + project_id: str, app_name: Optional[str] = None, - org_id: Optional[str] = None, - workspace_id: Optional[str] = None, ): """ Lists all the unique app names and their IDs from the database @@ -1078,37 +1004,18 @@ async def list_apps( List[App] """ - user = await get_user(user_uid) - assert user is not None, "User is None" - if app_name is not None: app_db = await fetch_app_by_name_and_parameters( - app_name=app_name, - user_uid=user_uid, - organization_id=org_id, - workspace_id=workspace_id, + app_name=app_name, project_id=project_id ) return [converters.app_db_to_pydantic(app_db)] - elif org_id is not None or workspace_id is not None: - if not isCloudEE(): - raise HTTPException( - status_code=400, - detail={ - "error": "organization and/or workspace is only available in Cloud and EE" - }, - ) - - # assert that if org_id is provided, workspace_id is also provided, and vice versa - assert ( - org_id is not None and workspace_id is not None - ), "org_id and workspace_id must be provided together" + elif isCloudEE(): if isCloudEE(): user_org_workspace_data = await get_user_org_and_workspace_id(user_uid) # type: ignore has_permission = await check_rbac_permission( # type: ignore user_org_workspace_data=user_org_workspace_data, - workspace_id=workspace_id, - organization_id=org_id, + project_id=project_id, permission=Permission.VIEW_APPLICATION, # type: ignore ) logger.debug(f"User has Permission to list apps: {has_permission}") @@ -1120,26 +1027,28 @@ async def list_apps( async with db_engine.get_session() as session: result = await session.execute( - select(AppDB).filter_by( - organization_id=org_id, - workspace_id=workspace_id, - ) + select(AppDB).filter_by(project_id=uuid.UUID(project_id)) ) apps = result.unique().scalars().all() return [converters.app_db_to_pydantic(app) for app in apps] else: async with db_engine.get_session() as session: - result = await session.execute(select(AppDB).filter_by(user_id=user.id)) + result = await session.execute( + select(AppDB).filter_by(project_id=uuid.UUID(project_id)) + ) apps = result.unique().scalars().all() return [converters.app_db_to_pydantic(app) for app in apps] -async def list_app_variants(app_id: str): +async def list_app_variants(app_id: str, project_id: str): """ Lists all the app variants from the db + Args: - app_name: if specified, only returns the variants for the app name + app_name (str): if specified, only returns the variants for the app name + project_id (str): The ID of the project + Returns: List[AppVariant]: List of AppVariant objects """ @@ -1152,13 +1061,15 @@ async def list_app_variants(app_id: str): joinedload(AppVariantDB.app.of_type(AppDB)).load_only(AppDB.id, AppDB.app_name), # type: ignore joinedload(AppVariantDB.base.of_type(VariantBaseDB)).joinedload(VariantBaseDB.deployment.of_type(DeploymentDB)).load_only(DeploymentDB.uri), # type: ignore ) - .filter_by(app_id=uuid.UUID(app_uuid)) + .filter_by(app_id=uuid.UUID(app_uuid), project_id=uuid.UUID(project_id)) ) app_variants = result.scalars().all() return app_variants -async def check_is_last_variant_for_image(db_app_variant: AppVariantDB) -> bool: +async def check_is_last_variant_for_image( + variant_base_id: str, project_id: str +) -> bool: """Checks whether the input variant is the sole variant that uses its linked image. NOTE: This is a helpful function to determine whether to delete the image when removing a variant. Usually many variants will use the same image (these variants would have been created using the UI). We only delete the image and shutdown the container if the variant is the last one using the image @@ -1171,14 +1082,9 @@ async def check_is_last_variant_for_image(db_app_variant: AppVariantDB) -> bool: """ async with db_engine.get_session() as session: - query = select(AppVariantDB).filter_by(base_id=db_app_variant.base_id) - - if isCloudEE(): - query = query.filter( - AppVariantDB.organization_id == db_app_variant.organization_id, - AppVariantDB.workspace_id == db_app_variant.workspace_id, - ) - + query = select(AppVariantDB).filter_by( + base_id=uuid.UUID(variant_base_id), project_id=uuid.UUID(project_id) + ) count_result = await session.execute( query.with_only_columns(func.count()) # type: ignore ) @@ -1226,19 +1132,22 @@ async def list_deployments(app_id: str): return environments -async def remove_app_variant_from_db(app_variant_db: AppVariantDB): +async def remove_app_variant_from_db(app_variant_db: AppVariantDB, project_id: str): """Remove an app variant from the db the logic for removing the image is in app_manager.py - Arguments: - app_variant -- AppVariant to remove + Args: + app_variant (AppVariantDB): the application variant to remove + project_id (str): The ID of the project """ logger.debug("Removing app variant") assert app_variant_db is not None, "app_variant_db is missing" logger.debug("list_app_variants_revisions_by_variant") - app_variant_revisions = await list_app_variant_revisions_by_variant(app_variant_db) + app_variant_revisions = await list_app_variant_revisions_by_variant( + app_variant_db, project_id + ) async with db_engine.get_session() as session: # Delete all the revisions associated with the variant @@ -1251,7 +1160,7 @@ async def remove_app_variant_from_db(app_variant_db: AppVariantDB): async def deploy_to_environment( - environment_name: str, variant_id: str, **user_org_data + environment_name: str, variant_id: str, project_id: str, **user_org_data ): """ Deploys an app variant to a specified environment. @@ -1259,6 +1168,7 @@ async def deploy_to_environment( Args: environment_name (str): The name of the environment to deploy the app variant to. variant_id (str): The ID of the app variant to deploy. + project_id (str): The ID of the project. Raises: ValueError: If the app variant is not found or if the environment is not found or if the app variant is already @@ -1267,9 +1177,9 @@ async def deploy_to_environment( None """ - app_variant_db = await fetch_app_variant_by_id(variant_id) + app_variant_db = await fetch_app_variant_by_id(variant_id, project_id) app_variant_revision_db = await fetch_app_variant_revision_by_variant( - app_variant_id=variant_id, revision=app_variant_db.revision # type: ignore + app_variant_id=variant_id, project_id=project_id, revision=app_variant_db.revision # type: ignore ) if app_variant_db is None: raise ValueError("App variant not found") @@ -1285,7 +1195,9 @@ async def deploy_to_environment( # Find the environment for the given app name and user result = await session.execute( select(AppEnvironmentDB).filter_by( - app_id=app_variant_db.app_id, name=environment_name + app_id=app_variant_db.app_id, + project_id=uuid.UUID(project_id), + name=environment_name, ) ) environment_db = result.scalars().first() @@ -1303,6 +1215,7 @@ async def deploy_to_environment( session, environment_db, user, + project_id, deployed_app_variant_revision=app_variant_revision_db, deployment=deployment, ) @@ -1448,19 +1361,20 @@ async def update_app_environment_deployed_variant_revision( await session.refresh(app_environment) -async def list_environments(app_id: str, **kwargs: dict): +async def list_environments(app_id: str, project_id: str, **kwargs: dict): """ List all environments for a given app ID. Args: app_id (str): The ID of the app to list environments for. + project_id (str): The ID of the project to list environments for. Returns: List[AppEnvironmentDB]: A list of AppEnvironmentDB objects representing the environments for the given app ID. """ logging.debug("Listing environments for app %s", app_id) - app_instance = await fetch_app_by_id(app_id=app_id) + app_instance = await fetch_app_by_id(app_id=app_id, project_id=project_id) if app_instance is None: logging.error(f"App with id {app_id} not found") raise ValueError("App not found") @@ -1476,20 +1390,21 @@ async def list_environments(app_id: str, **kwargs: dict): AppVariantRevisionsDB.config_parameters, # type: ignore ) ) - .filter_by(app_id=uuid.UUID(app_id)) + .filter_by(app_id=uuid.UUID(app_id), project_id=uuid.UUID(project_id)) ) environments_db = result.scalars().all() return environments_db async def initialize_environments( - session: AsyncSession, app_db: AppDB + session: AsyncSession, app_db: AppDB, project_id: str ) -> List[AppEnvironmentDB]: """ Initializes the environments for the app with the given database. Args: app_db (AppDB): The database for the app. + project_id (str): The ID of the project. Returns: List[AppEnvironmentDB]: A list of the initialized environments. @@ -1497,13 +1412,15 @@ async def initialize_environments( environments = [] for env_name in ["development", "staging", "production"]: - env = await create_environment(session=session, name=env_name, app_db=app_db) + env = await create_environment( + session=session, name=env_name, app_db=app_db, project_id=project_id + ) environments.append(env) return environments async def create_environment( - session: AsyncSession, name: str, app_db: AppDB + session: AsyncSession, name: str, app_db: AppDB, project_id: str ) -> AppEnvironmentDB: """ Creates a new environment in the database. @@ -1511,19 +1428,16 @@ async def create_environment( Args: name (str): The name of the environment. app_db (AppDB): The AppDB object representing the app that the environment belongs to. + project_id (str): The ID of the project. Returns: AppEnvironmentDB: The newly created AppEnvironmentDB object. """ environment_db = AppEnvironmentDB( - app_id=app_db.id, name=name, user_id=app_db.user_id, revision=0 + app_id=app_db.id, name=name, project_id=uuid.UUID(project_id), revision=0 ) - if isCloudEE(): - environment_db.organization_id = app_db.organization_id - environment_db.workspace_id = app_db.workspace_id - session.add(environment_db) await session.commit() await session.refresh(environment_db) @@ -1532,13 +1446,18 @@ async def create_environment( async def create_environment_revision( - session: AsyncSession, environment: AppEnvironmentDB, user: UserDB, **kwargs: dict + session: AsyncSession, + environment: AppEnvironmentDB, + user: UserDB, + project_id: str, + **kwargs: dict, ): """Creates a new environment revision. Args: environment (AppEnvironmentDB): The environment to create a revision for. user (UserDB): The user that made the deployment. + project_id (str): The ID of the project. """ assert environment is not None, "environment cannot be None" @@ -1548,6 +1467,7 @@ async def create_environment_revision( environment_id=environment.id, revision=environment.revision, modified_by_id=user.id, + project_id=uuid.UUID(project_id), ) if kwargs: @@ -1574,27 +1494,26 @@ async def create_environment_revision( if deployment is not None: environment_revision.deployment_id = deployment.id # type: ignore - if isCloudEE(): - environment_revision.organization_id = environment.organization_id - environment_revision.workspace_id = environment.workspace_id - session.add(environment_revision) async def list_app_variant_revisions_by_variant( - app_variant: AppVariantDB, + app_variant: AppVariantDB, project_id: str ): """Returns list of app variant revision for the given app variant Args: app_variant (AppVariantDB): The app variant to retrieve environments for. + project_id (str): The ID of the project. Returns: List[AppVariantRevisionsDB]: A list of AppVariantRevisionsDB objects. """ async with db_engine.get_session() as session: - base_query = select(AppVariantRevisionsDB).filter_by(variant_id=app_variant.id) + base_query = select(AppVariantRevisionsDB).filter_by( + variant_id=app_variant.id, project_id=uuid.UUID(project_id) + ) if isCloudEE(): base_query = base_query.options( joinedload(AppVariantRevisionsDB.modified_by.of_type(UserDB)).load_only( @@ -1638,12 +1557,13 @@ async def fetch_app_variant_revision(app_variant: str, revision_number: int): return app_variant_revisions -async def remove_image(image: ImageDB): +async def remove_image(image: ImageDB, project_id: str): """ Removes an image from the database. Args: image (ImageDB): The image to remove from the database. + project_id (str): The ID of the project the image belongs to. Raises: ValueError: If the image is None. @@ -1656,7 +1576,9 @@ async def remove_image(image: ImageDB): raise ValueError("Image is None") async with db_engine.get_session() as session: - result = await session.execute(select(ImageDB).filter_by(id=image.id)) + result = await session.execute( + select(ImageDB).filter_by(id=image.id, project_id=uuid.UUID(project_id)) + ) image = result.scalars().first() await session.delete(image) @@ -1759,12 +1681,13 @@ async def remove_base_from_db(base_id: str): await session.commit() -async def remove_app_by_id(app_id: str): +async def remove_app_by_id(app_id: str, project_id: str): """ Removes an app instance from the database by its ID. Args: app_id (str): The ID of the app instance to remove. + project_id (str): The ID of the project. Raises: AssertionError: If app_id is None or if the app instance could not be found. @@ -1775,7 +1698,11 @@ async def remove_app_by_id(app_id: str): assert app_id is not None, "app_id cannot be None" async with db_engine.get_session() as session: - result = await session.execute(select(AppDB).filter_by(id=uuid.UUID(app_id))) + result = await session.execute( + select(AppDB).filter_by( + id=uuid.UUID(app_id), project_id=uuid.UUID(project_id) + ) + ) app_db = result.scalars().first() if not app_db: raise NoResultFound(f"App with id {app_id} not found") @@ -1785,7 +1712,7 @@ async def remove_app_by_id(app_id: str): async def update_variant_parameters( - app_variant_id: str, parameters: Dict[str, Any], user_uid: str + app_variant_id: str, parameters: Dict[str, Any], project_id: str, user_uid: str ) -> None: """ Update the parameters of an app variant in the database. @@ -1793,6 +1720,7 @@ async def update_variant_parameters( Args: app_variant_id (str): The app variant ID. parameters (Dict[str, Any]): The new parameters to set for the app variant. + project_id (str): The ID of the project. user_uid (str): The UID of the user that is updating the app variant. Raises: @@ -1802,7 +1730,9 @@ async def update_variant_parameters( user = await get_user(user_uid) async with db_engine.get_session() as session: result = await session.execute( - select(AppVariantDB).filter_by(id=uuid.UUID(app_variant_id)) + select(AppVariantDB).filter_by( + id=uuid.UUID(app_variant_id), project_id=uuid.UUID(project_id) + ) ) app_variant_db = result.scalars().first() if not app_variant_db: @@ -1826,6 +1756,7 @@ async def update_variant_parameters( variant_id=app_variant_db.id, revision=app_variant_db.revision, modified_by_id=user.id, + project_id=uuid.UUID(project_id), base_id=app_variant_db.base_id, config_name=app_variant_db.config_name, config_parameters=app_variant_db.config_parameters, @@ -1835,11 +1766,14 @@ async def update_variant_parameters( await session.commit() -async def get_app_variant_instance_by_id(variant_id: str) -> AppVariantDB: +async def get_app_variant_instance_by_id( + variant_id: str, project_id: str +) -> AppVariantDB: """Get the app variant object from the database with the provided id. Arguments: variant_id (str): The app variant unique identifier + project_id (str): The ID of the project Returns: AppVariantDB: instance of app variant object @@ -1852,13 +1786,13 @@ async def get_app_variant_instance_by_id(variant_id: str) -> AppVariantDB: joinedload(AppVariantDB.app.of_type(AppDB)).load_only(AppDB.id, AppDB.app_name), # type: ignore joinedload(AppVariantDB.base.of_type(VariantBaseDB)).joinedload(VariantBaseDB.deployment.of_type(DeploymentDB)).load_only(DeploymentDB.uri), # type: ignore ) - .filter_by(id=uuid.UUID(variant_id)) + .filter_by(id=uuid.UUID(variant_id), project_id=uuid.UUID(project_id)), ) app_variant_db = result.scalars().first() return app_variant_db -async def fetch_testset_by_id(testset_id: str) -> Optional[TestSetDB]: +async def fetch_testset_by_id(testset_id: str, project_id: str) -> Optional[TestSetDB]: """Fetches a testset by its ID. Args: testset_id (str): The ID of the testset to fetch. @@ -1875,30 +1809,30 @@ async def fetch_testset_by_id(testset_id: str) -> Optional[TestSetDB]: raise ValueError(f"testset_id {testset_id} is not a valid UUID") from e async with db_engine.get_session() as session: - result = await session.execute(select(TestSetDB).filter_by(id=testset_uuid)) + result = await session.execute( + select(TestSetDB).filter_by( + id=testset_uuid, project_id=uuid.UUID(project_id) + ) + ) testset = result.scalars().first() return testset -async def create_testset(app: AppDB, user_uid: str, testset_data: Dict[str, Any]): +async def create_testset(app: AppDB, project_id: str, testset_data: Dict[str, Any]): """ Creates a testset. Args: app (AppDB): The app object - user_uid (str): The user uID + project_id (str): The ID of the project testset_data (dict): The data of the testset to create with Returns: returns the newly created TestsetDB """ - user = await get_user(user_uid=user_uid) async with db_engine.get_session() as session: - testset_db = TestSetDB(**testset_data, app_id=app.id, user_id=user.id) - if isCloudEE(): - testset_db.organization_id = app.organization_id - testset_db.workspace_id = app.workspace_id + testset_db = TestSetDB(**testset_data, project_id=uuid.UUID(project_id)) session.add(testset_db) await session.commit() @@ -1930,24 +1864,27 @@ async def update_testset(testset_id: str, values_to_update: dict) -> None: await session.refresh(testset) -async def fetch_testsets_by_app_id(app_id: str): +async def fetch_testsets_by_app_id(project_id: str): """Fetches all testsets for a given app. + Args: - app_id (str): The ID of the app to fetch testsets for. + project_id (str): The ID of the project. + Returns: List[TestSetDB]: The fetched testsets. """ - assert app_id is not None, "app_id cannot be None" async with db_engine.get_session() as session: result = await session.execute( - select(TestSetDB).filter_by(app_id=uuid.UUID(app_id)) + select(TestSetDB).filter_by(project_id=uuid.UUID(project_id)) ) testsets = result.scalars().all() return testsets -async def fetch_evaluation_by_id(evaluation_id: str) -> Optional[EvaluationDB]: +async def fetch_evaluation_by_id( + evaluation_id: str, project_id: str +) -> Optional[EvaluationDB]: """Fetches a evaluation by its ID. Args: @@ -1959,17 +1896,18 @@ async def fetch_evaluation_by_id(evaluation_id: str) -> Optional[EvaluationDB]: assert evaluation_id is not None, "evaluation_id cannot be None" async with db_engine.get_session() as session: - base_query = select(EvaluationDB).filter_by(id=uuid.UUID(evaluation_id)) + base_query = select(EvaluationDB).filter_by( + id=uuid.UUID(evaluation_id), project_id=uuid.UUID(project_id) + ) if isCloudEE(): query = base_query.options( - joinedload(EvaluationDB.user.of_type(UserDB)).load_only(UserDB.id, UserDB.username), # type: ignore joinedload(EvaluationDB.testset.of_type(TestSetDB)).load_only(TestSetDB.id, TestSetDB.name), # type: ignore ) else: query = base_query.options( - joinedload(EvaluationDB.user).load_only(UserDB.username), # type: ignore joinedload(EvaluationDB.testset).load_only(TestSetDB.id, TestSetDB.name), # type: ignore ) + result = await session.execute( query.options( joinedload(EvaluationDB.variant.of_type(AppVariantDB)).load_only(AppVariantDB.id, AppVariantDB.variant_name), # type: ignore @@ -1985,7 +1923,7 @@ async def fetch_evaluation_by_id(evaluation_id: str) -> Optional[EvaluationDB]: return evaluation -async def list_human_evaluations(app_id: str): +async def list_human_evaluations(app_id: str, project_id: str): """ Fetches human evaluations belonging to an App. @@ -1996,17 +1934,15 @@ async def list_human_evaluations(app_id: str): async with db_engine.get_session() as session: base_query = ( select(HumanEvaluationDB) - .filter_by(app_id=uuid.UUID(app_id)) + .filter_by(app_id=uuid.UUID(app_id), project_id=uuid.UUID(project_id)) .filter(HumanEvaluationDB.testset_id.isnot(None)) ) if isCloudEE(): query = base_query.options( - joinedload(HumanEvaluationDB.user.of_type(UserDB)).load_only(UserDB.id, UserDB.username), # type: ignore joinedload(HumanEvaluationDB.testset.of_type(TestSetDB)).load_only(TestSetDB.id, TestSetDB.name), # type: ignore ) else: query = base_query.options( - joinedload(HumanEvaluationDB.user).load_only(UserDB.id, UserDB.username), # type: ignore joinedload(HumanEvaluationDB.testset).load_only(TestSetDB.id, TestSetDB.name), # type: ignore ) result = await session.execute(query) @@ -2016,7 +1952,7 @@ async def list_human_evaluations(app_id: str): async def create_human_evaluation( app: AppDB, - user_id: str, + project_id: str, status: str, evaluation_type: str, testset_id: str, @@ -2027,7 +1963,7 @@ async def create_human_evaluation( Args: app (AppDB: The app object - user_id (id): The ID of the user + project_id (str): The ID of the project status (str): The status of the evaluation evaluation_type (str): The evaluation type testset_id (str): The ID of the evaluation testset @@ -2037,14 +1973,11 @@ async def create_human_evaluation( async with db_engine.get_session() as session: human_evaluation = HumanEvaluationDB( app_id=app.id, - user_id=uuid.UUID(user_id), + project_id=uuid.UUID(project_id), status=status, evaluation_type=evaluation_type, testset_id=testset_id, ) - if isCloudEE(): - human_evaluation.organization_id = str(app.organization_id) - human_evaluation.workspace_id = str(app.workspace_id) session.add(human_evaluation) await session.commit() @@ -2052,7 +1985,9 @@ async def create_human_evaluation( # create variants for human evaluation await create_human_evaluation_variants( - human_evaluation_id=str(human_evaluation.id), variants_ids=variants_ids + human_evaluation_id=str(human_evaluation.id), + variants_ids=variants_ids, + project_id=project_id, ) return human_evaluation @@ -2092,7 +2027,7 @@ async def fetch_human_evaluation_variants(human_evaluation_id: str): async def create_human_evaluation_variants( - human_evaluation_id: str, variants_ids: List[str] + human_evaluation_id: str, variants_ids: List[str], project_id: str ): """ Creates human evaluation variants. @@ -2100,18 +2035,21 @@ async def create_human_evaluation_variants( Args: human_evaluation_id (str): The human evaluation identifier variants_ids (List[str]): The variants identifiers + project_id (str): The project ID """ variants_dict = {} for variant_id in variants_ids: - variant = await fetch_app_variant_by_id(app_variant_id=variant_id) + variant = await fetch_app_variant_by_id( + app_variant_id=variant_id, project_id=project_id + ) if variant: variants_dict[variant_id] = variant variants_revisions_dict = {} for variant_id, variant in variants_dict.items(): variant_revision = await fetch_app_variant_revision_by_variant( - app_variant_id=str(variant.id), revision=variant.revision # type: ignore + app_variant_id=str(variant.id), project_id=project_id, revision=variant.revision # type: ignore ) if variant_revision: variants_revisions_dict[variant_id] = variant_revision @@ -2134,7 +2072,7 @@ async def create_human_evaluation_variants( async def fetch_human_evaluation_by_id( - evaluation_id: str, + evaluation_id: str, project_id: str ) -> Optional[HumanEvaluationDB]: """Fetches a evaluation by its ID. Args: @@ -2145,15 +2083,15 @@ async def fetch_human_evaluation_by_id( assert evaluation_id is not None, "evaluation_id cannot be None" async with db_engine.get_session() as session: - base_query = select(HumanEvaluationDB).filter_by(id=uuid.UUID(evaluation_id)) + base_query = select(HumanEvaluationDB).filter_by( + id=uuid.UUID(evaluation_id), project_id=uuid.UUID(project_id) + ) if isCloudEE(): query = base_query.options( - joinedload(HumanEvaluationDB.user.of_type(UserDB)).load_only(UserDB.id, UserDB.username), # type: ignore joinedload(HumanEvaluationDB.testset.of_type(TestSetDB)).load_only(TestSetDB.id, TestSetDB.name), # type: ignore ) else: query = base_query.options( - joinedload(HumanEvaluationDB.user).load_only(UserDB.id, UserDB.username), # type: ignore joinedload(HumanEvaluationDB.testset).load_only(TestSetDB.id, TestSetDB.name), # type: ignore ) result = await session.execute(query) @@ -2188,7 +2126,7 @@ async def update_human_evaluation(evaluation_id: str, values_to_update: dict): await session.refresh(human_evaluation) -async def delete_human_evaluation(evaluation_id: str): +async def delete_human_evaluation(evaluation_id: str, project_id: str): """Delete the evaluation by its ID. Args: @@ -2198,7 +2136,9 @@ async def delete_human_evaluation(evaluation_id: str): assert evaluation_id is not None, "evaluation_id cannot be None" async with db_engine.get_session() as session: result = await session.execute( - select(HumanEvaluationDB).filter_by(id=uuid.UUID(evaluation_id)) + select(HumanEvaluationDB).filter_by( + id=uuid.UUID(evaluation_id), project_id=uuid.UUID(project_id) + ) ) evaluation = result.scalars().first() if not evaluation: @@ -2210,8 +2150,7 @@ async def delete_human_evaluation(evaluation_id: str): async def create_human_evaluation_scenario( inputs: List[HumanEvaluationScenarioInput], - user_id: str, - app: AppDB, + project_id: str, evaluation_id: str, evaluation_extend: Dict[str, Any], ): @@ -2220,8 +2159,6 @@ async def create_human_evaluation_scenario( Args: inputs (List[HumanEvaluationScenarioInput]): The inputs. - user_id (str): The user ID. - app (AppDB): The app object. evaluation_id (str): The evaluation identifier. evaluation_extend (Dict[str, any]): An extended required payload for the evaluation scenario. Contains score, vote, and correct_answer. """ @@ -2229,16 +2166,12 @@ async def create_human_evaluation_scenario( async with db_engine.get_session() as session: evaluation_scenario = HumanEvaluationScenarioDB( **evaluation_extend, - user_id=uuid.UUID(user_id), + project_id=uuid.UUID(project_id), evaluation_id=uuid.UUID(evaluation_id), - inputs=[input.dict() for input in inputs], + inputs=[input.model_dump() for input in inputs], outputs=[], ) - if isCloudEE(): - evaluation_scenario.organization_id = str(app.organization_id) - evaluation_scenario.workspace_id = str(app.workspace_id) - session.add(evaluation_scenario) await session.commit() @@ -2319,11 +2252,14 @@ async def fetch_evaluation_scenarios(evaluation_id: str): async def fetch_evaluation_scenario_by_id( - evaluation_scenario_id: str, + evaluation_scenario_id: str, project_id: str ) -> Optional[EvaluationScenarioDB]: """Fetches and evaluation scenario by its ID. + Args: evaluation_scenario_id (str): The ID of the evaluation scenario to fetch. + project_id (str): The project ID to use in fetching the evaluation scenario. + Returns: EvaluationScenarioDB: The fetched evaluation scenario, or None if no evaluation scenario was found. """ @@ -2331,18 +2267,23 @@ async def fetch_evaluation_scenario_by_id( assert evaluation_scenario_id is not None, "evaluation_scenario_id cannot be None" async with db_engine.get_session() as session: result = await session.execute( - select(EvaluationScenarioDB).filter_by(id=uuid.UUID(evaluation_scenario_id)) + select(EvaluationScenarioDB).filter_by( + id=uuid.UUID(evaluation_scenario_id), project_id=uuid.UUID(project_id) + ) ) evaluation_scenario = result.scalars().first() return evaluation_scenario async def fetch_human_evaluation_scenario_by_id( - evaluation_scenario_id: str, + evaluation_scenario_id: str, project_id: str ) -> Optional[HumanEvaluationScenarioDB]: """Fetches and evaluation scenario by its ID. + Args: evaluation_scenario_id (str): The ID of the evaluation scenario to fetch. + project_id (str): The project ID to use in fetching the human evaluation. + Returns: EvaluationScenarioDB: The fetched evaluation scenario, or None if no evaluation scenario was found. """ @@ -2351,7 +2292,7 @@ async def fetch_human_evaluation_scenario_by_id( async with db_engine.get_session() as session: result = await session.execute( select(HumanEvaluationScenarioDB).filter_by( - id=uuid.UUID(evaluation_scenario_id) + id=uuid.UUID(evaluation_scenario_id), project_id=uuid.UUID(project_id) ) ) evaluation_scenario = result.scalars().first() @@ -2380,12 +2321,13 @@ async def fetch_human_evaluation_scenario_by_evaluation_id( async def find_previous_variant_from_base_id( - base_id: str, + base_id: str, project_id: str ) -> Optional[AppVariantDB]: """Find the previous variant from a base id. Args: base_id (str): The base id to search for. + project_id (str): The ID of the project. Returns: Optional[AppVariantDB]: The previous variant, or None if no previous variant was found. @@ -2395,7 +2337,7 @@ async def find_previous_variant_from_base_id( async with db_engine.get_session() as session: result = await session.execute( select(AppVariantDB) - .filter_by(base_id=uuid.UUID(base_id)) + .filter_by(base_id=uuid.UUID(base_id), project_id=uuid.UUID(project_id)) .order_by(AppVariantDB.created_at.desc()) ) last_variant = result.scalars().first() @@ -2620,41 +2562,21 @@ async def update_app_variant( return app_variant -async def fetch_app_by_name_and_parameters( - app_name: str, - user_uid: str, - organization_id: Optional[str] = None, - workspace_id: Optional[str] = None, -): +async def fetch_app_by_name_and_parameters(app_name: str, project_id: str): """Fetch an app by its name, organization id, and workspace_id. Args: app_name (str): The name of the app - organization_id (str): The ID of the app organization - workspace_id (str): The ID of the app workspace + project_id (str): The ID of the project Returns: AppDB: the instance of the app """ async with db_engine.get_session() as session: - base_query = select(AppDB).filter_by(app_name=app_name) - - if isCloudEE(): - # assert that if organization is provided, workspace_id is also provided, and vice versa - assert ( - organization_id is not None and workspace_id is not None - ), "organization_id and workspace_id must be provided together" - - query = base_query.filter_by( - organization_id=uuid.UUID(organization_id), - workspace_id=workspace_id, - ) - else: - query = base_query.join(UserDB, AppDB.user_id == UserDB.id).filter( - UserDB.uid == user_uid - ) - + query = select(AppDB).filter_by( + app_name=app_name, project_id=uuid.UUID(project_id) + ) result = await session.execute(query) app_db = result.unique().scalars().first() return app_db @@ -2710,23 +2632,24 @@ async def create_new_evaluation( return evaluation -async def list_evaluations(app_id: str): +async def list_evaluations(app_id: str, project_id: str): """Retrieves evaluations of the specified app from the db. Args: app_id (str): The ID of the app + project_id (str): The ID of the project """ async with db_engine.get_session() as session: - base_query = select(EvaluationDB).filter_by(app_id=uuid.UUID(app_id)) + base_query = select(EvaluationDB).filter_by( + app_id=uuid.UUID(app_id), project_id=uuid.UUID(project_id) + ) if isCloudEE(): query = base_query.options( - joinedload(EvaluationDB.user.of_type(UserDB)).load_only(UserDB.id, UserDB.username), # type: ignore joinedload(EvaluationDB.testset.of_type(TestSetDB)).load_only(TestSetDB.id, TestSetDB.name), # type: ignore ) else: query = base_query.options( - joinedload(EvaluationDB.user).load_only(UserDB.id, UserDB.username), # type: ignore joinedload(EvaluationDB.testset).load_only(TestSetDB.id, TestSetDB.name), # type: ignore ) @@ -2952,32 +2875,37 @@ async def fetch_eval_aggregated_results(evaluation_id: str): return aggregated_results -async def fetch_evaluators_configs(app_id: str): +async def fetch_evaluators_configs(project_id: str): """Fetches a list of evaluator configurations from the database. Returns: List[EvaluatorConfigDB]: A list of evaluator configuration objects. """ - assert app_id is not None, "evaluation_id cannot be None" async with db_engine.get_session() as session: result = await session.execute( - select(EvaluatorConfigDB).filter_by(app_id=uuid.UUID(app_id)) + select(EvaluatorConfigDB).filter_by(project_id=uuid.UUID(project_id)) ) evaluators_configs = result.scalars().all() return evaluators_configs -async def fetch_evaluator_config(evaluator_config_id: str): +async def fetch_evaluator_config(evaluator_config_id: str, project_id: str): """Fetch evaluator configurations from the database. + Args: + evaluator_config_id (str): The ID of the evaluator configuration. + project_id (str): The ID of the project. + Returns: EvaluatorConfigDB: the evaluator configuration object. """ async with db_engine.get_session() as session: result = await session.execute( - select(EvaluatorConfigDB).filter_by(id=uuid.UUID(evaluator_config_id)) + select(EvaluatorConfigDB).filter_by( + id=uuid.UUID(evaluator_config_id), project_id=uuid.UUID(project_id) + ) ) evaluator_config = result.scalars().first() return evaluator_config @@ -3033,8 +2961,7 @@ async def fetch_evaluator_config_by_appId( async def create_evaluator_config( - app: AppDB, - user_id: str, + project_id: str, name: str, evaluator_key: str, settings_values: Optional[Dict[str, Any]] = None, @@ -3043,17 +2970,12 @@ async def create_evaluator_config( async with db_engine.get_session() as session: new_evaluator_config = EvaluatorConfigDB( - app_id=app.id, - user_id=uuid.UUID(user_id), + project_id=uuid.UUID(project_id), name=name, evaluator_key=evaluator_key, settings_values=settings_values, ) - if isCloudEE(): - new_evaluator_config.organization_id = app.organization_id - new_evaluator_config.workspace_id = app.workspace_id - session.add(new_evaluator_config) await session.commit() await session.refresh(new_evaluator_config) diff --git a/agenta-backend/agenta_backend/services/deployment_manager.py b/agenta-backend/agenta_backend/services/deployment_manager.py index b44c320cf0..5bfbdc4c07 100644 --- a/agenta-backend/agenta_backend/services/deployment_manager.py +++ b/agenta-backend/agenta_backend/services/deployment_manager.py @@ -16,13 +16,14 @@ async def start_service( - app_variant_db: AppVariantDB, env_vars: Dict[str, str] + app_variant_db: AppVariantDB, project_id: str, env_vars: Dict[str, str] ) -> DeploymentDB: """ Start a service. Args: app_variant_db (AppVariantDB): The app variant to start. + project_id (str): The ID of the project the app variant belongs to. env_vars (Dict[str, str]): The environment variables to pass to the container. Returns: @@ -33,8 +34,8 @@ async def start_service( uri_path = f"{app_variant_db.organization.id}/{app_variant_db.app.app_name}/{app_variant_db.base_name}" container_name = f"{app_variant_db.app.app_name}-{app_variant_db.base_name}-{app_variant_db.organization.id}" else: - uri_path = f"{app_variant_db.user.id}/{app_variant_db.app.app_name}/{app_variant_db.base_name}" - container_name = f"{app_variant_db.app.app_name}-{app_variant_db.base_name}-{app_variant_db.user.id}" + uri_path = f"{app_variant_db.project_id}/{app_variant_db.app.app_name}/{app_variant_db.base_name}" + container_name = f"{app_variant_db.app.app_name}-{app_variant_db.base_name}-{app_variant_db.project_id}" logger.debug("Starting service with the following parameters:") logger.debug(f"image_name: {app_variant_db.image.tags}") @@ -59,13 +60,11 @@ async def start_service( deployment = await db_manager.create_deployment( app_id=str(app_variant_db.app.id), - user_id=str(app_variant_db.user.id), + project_id=project_id, container_name=container_name, container_id=container_id, uri=uri, status="running", - organization=str(app_variant_db.organization_id) if isCloudEE() else None, - workspace=str(app_variant_db.workspace_id) if isCloudEE() else None, ) return deployment diff --git a/agenta-backend/agenta_backend/services/evaluation_service.py b/agenta-backend/agenta_backend/services/evaluation_service.py index d15a4cf1df..c3de5769e0 100644 --- a/agenta-backend/agenta_backend/services/evaluation_service.py +++ b/agenta-backend/agenta_backend/services/evaluation_service.py @@ -67,10 +67,9 @@ class UpdateEvaluationScenarioError(Exception): async def prepare_csvdata_and_create_evaluation_scenario( csvdata: List[Dict[str, str]], payload_inputs: List[str], + project_id: str, evaluation_type: EvaluationType, new_evaluation: HumanEvaluationDB, - user: UserDB, - app: AppDB, ): """ Prepares CSV data and creates evaluation scenarios based on the inputs, evaluation @@ -79,10 +78,9 @@ async def prepare_csvdata_and_create_evaluation_scenario( Args: csvdata: A list of dictionaries representing the CSV data. payload_inputs: A list of strings representing the names of the inputs in the variant. + project_id (str): The ID of the project evaluation_type: The type of evaluation new_evaluation: The instance of EvaluationDB - user: The owner of the evaluation scenario - app: The app the evaluation is going to belong to """ for datum in csvdata: @@ -94,7 +92,7 @@ async def prepare_csvdata_and_create_evaluation_scenario( ] except KeyError: await db_manager.delete_human_evaluation( - evaluation_id=str(new_evaluation.id) + evaluation_id=str(new_evaluation.id), project_id=project_id ) msg = f""" Columns in the test set should match the names of the inputs in the variant. @@ -121,8 +119,7 @@ async def prepare_csvdata_and_create_evaluation_scenario( } await db_manager.create_human_evaluation_scenario( inputs=list_of_scenario_input, - user_id=str(user.id), - app=app, + project_id=project_id, evaluation_id=str(new_evaluation.id), evaluation_extend=evaluation_scenario_extend_payload, ) @@ -141,7 +138,7 @@ async def update_human_evaluation_service( # Update the evaluation await db_manager.update_human_evaluation( - evaluation_id=str(evaluation.id), values_to_update=update_payload.dict() + evaluation_id=str(evaluation.id), values_to_update=update_payload.model_dump() ) @@ -213,7 +210,7 @@ async def update_human_evaluation_scenario( """ values_to_update = {} - payload = evaluation_scenario_data.dict(exclude_unset=True) + payload = evaluation_scenario_data.model_dump(exclude_unset=True) if "score" in payload and evaluation_type == EvaluationType.single_model_test: values_to_update["score"] = str(payload["score"]) @@ -226,7 +223,7 @@ async def update_human_evaluation_scenario( HumanEvaluationScenarioOutput( variant_id=output["variant_id"], variant_output=output["variant_output"], - ).dict() + ).model_dump() for output in payload["outputs"] ] values_to_update["outputs"] = new_outputs @@ -236,7 +233,7 @@ async def update_human_evaluation_scenario( HumanEvaluationScenarioInput( input_name=input_item["input_name"], input_value=input_item["input_value"], - ).dict() + ).model_dump() for input_item in payload["inputs"] ] values_to_update["inputs"] = new_inputs @@ -273,20 +270,21 @@ def _extend_with_correct_answer(evaluation_type: EvaluationType, row: dict): return correct_answer -async def fetch_list_evaluations( - app: AppDB, -) -> List[Evaluation]: +async def fetch_list_evaluations(app: AppDB, project_id: str) -> List[Evaluation]: """ Fetches a list of evaluations based on the provided filtering criteria. Args: app (AppDB): An app to filter the evaluations. + project_id (str): The ID of the project Returns: List[Evaluation]: A list of evaluations. """ - evaluations_db = await db_manager.list_evaluations(app_id=str(app.id)) + evaluations_db = await db_manager.list_evaluations( + app_id=str(app.id), project_id=project_id + ) return [ await converters.evaluation_db_to_pydantic(evaluation) for evaluation in evaluations_db @@ -294,19 +292,22 @@ async def fetch_list_evaluations( async def fetch_list_human_evaluations( - app_id: str, + app_id: str, project_id: str ) -> List[HumanEvaluation]: """ Fetches a list of evaluations based on the provided filtering criteria. Args: app_id (Optional[str]): An optional app ID to filter the evaluations. + project_id (str): The ID of the project. Returns: List[Evaluation]: A list of evaluations. """ - evaluations_db = await db_manager.list_human_evaluations(app_id=app_id) + evaluations_db = await db_manager.list_human_evaluations( + app_id=app_id, project_id=project_id + ) return [ await converters.human_evaluation_db_to_pydantic(evaluation) for evaluation in evaluations_db @@ -327,19 +328,22 @@ async def fetch_human_evaluation(human_evaluation_db) -> HumanEvaluation: return await converters.human_evaluation_db_to_pydantic(human_evaluation_db) -async def delete_human_evaluations(evaluation_ids: List[str]) -> None: +async def delete_human_evaluations(evaluation_ids: List[str], project_id: str) -> None: """ Delete evaluations by their IDs. Args: evaluation_ids (List[str]): A list of evaluation IDs. + project_id (str): The ID of the project. Raises: NoResultFound: If evaluation not found or access denied. """ for evaluation_id in evaluation_ids: - await db_manager.delete_human_evaluation(evaluation_id=evaluation_id) + await db_manager.delete_human_evaluation( + evaluation_id=evaluation_id, project_id=project_id + ) async def delete_evaluations(evaluation_ids: List[str]) -> None: @@ -357,7 +361,7 @@ async def delete_evaluations(evaluation_ids: List[str]) -> None: async def create_new_human_evaluation( - payload: NewHumanEvaluation, user_uid: str + payload: NewHumanEvaluation, user_uid: str, project_id: str ) -> HumanEvaluationDB: """ Create a new evaluation based on the provided payload and additional arguments. @@ -365,13 +369,14 @@ async def create_new_human_evaluation( Args: payload (NewEvaluation): The evaluation payload. user_uid (str): The user_uid of the user + project_id (str): The ID of the project Returns: HumanEvaluationDB """ user = await db_manager.get_user(user_uid) - app = await db_manager.fetch_app_by_id(app_id=payload.app_id) + app = await db_manager.fetch_app_by_id(app_id=payload.app_id, project_id=project_id) if app is None: raise HTTPException( status_code=404, @@ -380,7 +385,7 @@ async def create_new_human_evaluation( human_evaluation = await db_manager.create_human_evaluation( app=app, - user_id=str(user.id), + project_id=project_id, status=payload.status, evaluation_type=payload.evaluation_type, testset_id=payload.testset_id, @@ -394,10 +399,9 @@ async def create_new_human_evaluation( await prepare_csvdata_and_create_evaluation_scenario( human_evaluation.testset.csvdata, payload.inputs, + project_id, payload.evaluation_type, human_evaluation, - user, - app, ) return human_evaluation diff --git a/agenta-backend/agenta_backend/services/evaluator_manager.py b/agenta-backend/agenta_backend/services/evaluator_manager.py index 586c59b282..7eb32a45ff 100644 --- a/agenta-backend/agenta_backend/services/evaluator_manager.py +++ b/agenta-backend/agenta_backend/services/evaluator_manager.py @@ -31,17 +31,18 @@ def get_evaluators() -> List[Evaluator]: return [Evaluator(**evaluator_dict) for evaluator_dict in evaluators_as_dict] -async def get_evaluators_configs(app_id: str) -> List[EvaluatorConfig]: +async def get_evaluators_configs(project_id: str) -> List[EvaluatorConfig]: """ Get evaluators configs by app_id. Args: - app_id (str): The ID of the app. + project_id (str): The ID of the project. Returns: List[EvaluatorConfig]: A list of evaluator configuration objects. """ - evaluator_configs_db = await db_manager.fetch_evaluators_configs(app_id) + + evaluator_configs_db = await db_manager.fetch_evaluators_configs(project_id) return [ evaluator_config_db_to_pydantic(evaluator_config_db) for evaluator_config_db in evaluator_configs_db @@ -62,7 +63,7 @@ async def get_evaluator_config(evaluator_config: EvaluatorConfig) -> EvaluatorCo async def create_evaluator_config( - app_id: str, + project_id: str, name: str, evaluator_key: str, settings_values: Optional[Dict[str, Any]] = None, @@ -71,7 +72,6 @@ async def create_evaluator_config( Create a new evaluator configuration for an app. Args: - app_id (str): The ID of the app. name (str): The name of the evaluator config. evaluator_key (str): The key of the evaluator. settings_values (Optional[Dict[str, Any]]): Additional settings for the evaluator. @@ -79,11 +79,9 @@ async def create_evaluator_config( Returns: EvaluatorConfigDB: The newly created evaluator configuration object. """ - app = await db_manager.fetch_app_by_id(app_id) evaluator_config = await db_manager.create_evaluator_config( - app=app, - user_id=str(app.user_id), + project_id=project_id, name=name, evaluator_key=evaluator_key, settings_values=settings_values, @@ -123,7 +121,7 @@ async def delete_evaluator_config(evaluator_config_id: str) -> bool: return await db_manager.delete_evaluator_config(evaluator_config_id) -async def create_ready_to_use_evaluators(app: AppDB): +async def create_ready_to_use_evaluators(project_id: str): """ Create configurations for all evaluators that are marked for direct use. @@ -131,13 +129,11 @@ async def create_ready_to_use_evaluators(app: AppDB): out those marked for direct use, and creates configuration entries for them in the database using the database manager. - Parameters: - - evaluator_manager: The manager object responsible for handling evaluators. - - db_manager: The database manager object used for database operations. - - app: The application context, containing details like organization and user. + Args: + project_id (str): The ID of the project. Returns: - Nothing. The function works by side effect, modifying the database. + Nothing. The function works by side effect, modifying the database. """ direct_use_evaluators = [ @@ -160,8 +156,7 @@ async def create_ready_to_use_evaluators(app: AppDB): evaluator, "key" ), f"'name' and 'key' does not exist in the evaluator: {evaluator}" await db_manager.create_evaluator_config( - app=app, - user_id=str(app.user_id), + project_id=project_id, name=evaluator.name, evaluator_key=evaluator.key, settings_values=settings_values, From b275e1516bd20a720f93b9f233a0cc5d0cc9d129 Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 16 Sep 2024 09:01:46 +0100 Subject: [PATCH 17/57] refactor (backend): - scope evaluation router endpoints and parts of app router endpoints - update business logic managers to include project_id - update app_variant_response type in agenta-cli/ to include project_id. --- .../agenta_backend/routers/app_router.py | 52 ++------ .../routers/container_router.py | 14 ++- .../routers/evaluation_router.py | 54 +++++++-- .../agenta_backend/routers/variants_router.py | 42 +++++-- .../agenta_backend/services/app_manager.py | 28 ++--- .../services/container_manager.py | 11 +- .../agenta_backend/services/db_manager.py | 113 +++++++++--------- .../services/evaluation_service.py | 35 +++--- .../agenta_backend/tasks/evaluations.py | 37 +++--- .../backend/types/app_variant_response.py | 4 +- 10 files changed, 212 insertions(+), 178 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index 60cafa630a..79568c6d5a 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -220,6 +220,9 @@ async def create_app( HTTPException: If there is an error creating the app or the user does not have permission to access the app. """ try: + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) if isCloudEE(): api_key_from_headers = request.headers.get("Authorization") if api_key_from_headers is not None: @@ -238,37 +241,9 @@ async def create_app( detail="Failed to get user org and workspace data", ) - if payload.organization_id: - organization_id = payload.organization_id - organization = await db_manager_ee.get_organization(organization_id) - else: - organization = await get_user_own_org( - user_org_workspace_data["uid"] - ) - organization_id = str(organization.id) - - if not organization: - raise HTTPException( - status_code=400, - detail="User Organization not found", - ) - - if payload.workspace_id: - workspace_id = payload.workspace_id - workspace = db_manager_ee.get_workspace(workspace_id) - else: - workspace = await get_org_default_workspace(organization) - - if not workspace: - raise HTTPException( - status_code=400, - detail="User Organization not found", - ) - has_permission = await check_rbac_permission( user_org_workspace_data=user_org_workspace_data, - workspace_id=str(workspace.id), - organization=organization, + project_id=project_id, permission=Permission.CREATE_APPLICATION, ) logger.debug( @@ -283,12 +258,7 @@ async def create_app( except Exception as e: raise HTTPException(status_code=500, detail=str(e)) - app_db = await db_manager.create_app_and_envs( - payload.app_name, - request.state.user_id, - organization_id if isCloudEE() else None, - str(workspace.id) if isCloudEE() else None, - ) + app_db = await db_manager.create_app_and_envs(payload.app_name, project_id) return CreateAppOutput(app_id=str(app_db.id), app_name=str(app_db.app_name)) except Exception as e: logger.exception(f"An error occurred: {str(e)}") @@ -396,6 +366,7 @@ async def add_variant_from_image( dict: The newly added variant. """ + project_id = project_utils.get_project_id(request=request, project_id=project_id) if not isCloudEE(): image = Image( type="image", @@ -411,12 +382,12 @@ async def add_variant_from_image( raise HTTPException(status_code=404, detail="Image not found") try: - app = await db_manager.fetch_app_by_id(app_id) + app = await db_manager.fetch_app_by_id(app_id, project_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=app, + project_id=project_id, permission=Permission.CREATE_APPLICATION, ) logger.debug( @@ -431,6 +402,7 @@ async def add_variant_from_image( variant_db = await app_manager.add_variant_based_on_image( app=app, + project_id=project_id, variant_name=payload.variant_name, docker_id_or_template_uri=payload.docker_id, tags=payload.tags, @@ -439,10 +411,12 @@ async def add_variant_from_image( is_template_image=False, user_uid=request.state.user_id, ) - app_variant_db = await db_manager.fetch_app_variant_by_id(str(variant_db.id)) + app_variant_db = await db_manager.fetch_app_variant_by_id( + str(variant_db.id), project_id + ) logger.debug("Step 8: We create ready-to use evaluators") - await evaluator_manager.create_ready_to_use_evaluators(app=app) + await evaluator_manager.create_ready_to_use_evaluators(project_id=project_id) return await converters.app_variant_db_to_output(app_variant_db) except Exception as e: diff --git a/agenta-backend/agenta_backend/routers/container_router.py b/agenta-backend/agenta_backend/routers/container_router.py index 8a97e3d747..b64fee1176 100644 --- a/agenta-backend/agenta_backend/routers/container_router.py +++ b/agenta-backend/agenta_backend/routers/container_router.py @@ -66,13 +66,16 @@ async def build_image( Image: The Docker image that was built. """ try: - app_db = await db_manager.fetch_app_by_id(app_id) + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) + app_db = await db_manager.fetch_app_by_id(app_id, project_id) # Check app access if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=app_db, + project_id=project_id, permission=Permission.CREATE_APPLICATION, ) if not has_permission: @@ -106,10 +109,13 @@ async def restart_docker_container( payload (RestartAppContainer) -- the required data (app_name and variant_name) """ logger.debug(f"Restarting container for variant {payload.variant_id}") - app_variant_db = await db_manager.fetch_app_variant_by_id(payload.variant_id) + project_id = project_utils.get_project_id(request=request, project_id=project_id) + app_variant_db = await db_manager.fetch_app_variant_by_id( + payload.variant_id, project_id + ) try: deployment = await db_manager.get_deployment_by_id( - app_variant_db.base.deployment + app_variant_db.base.deployment, project_id ) container_id = deployment.container_id diff --git a/agenta-backend/agenta_backend/routers/evaluation_router.py b/agenta-backend/agenta_backend/routers/evaluation_router.py index ae36ab2c63..a8c60c9d7e 100644 --- a/agenta-backend/agenta_backend/routers/evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/evaluation_router.py @@ -55,11 +55,13 @@ async def fetch_evaluation_ids( List[str]: A list of evaluation ids. """ try: + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=app_id, - object_type="app", + project_id=project_id, permission=Permission.VIEW_EVALUATION, ) logger.debug( @@ -73,7 +75,7 @@ async def fetch_evaluation_ids( status_code=403, ) evaluations = await db_manager.fetch_evaluations_by_resource( - resource_type, resource_ids + resource_type, project_id, resource_ids ) return list(map(lambda x: str(x.id), evaluations)) except Exception as exc: @@ -96,14 +98,19 @@ async def create_evaluation( _description_ """ try: - app = await db_manager.fetch_app_by_id(app_id=payload.app_id) + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) + app = await db_manager.fetch_app_by_id( + app_id=payload.app_id, project_id=project_id + ) if app is None: raise HTTPException(status_code=404, detail="App not found") if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=app, + project_id=project_id, permission=Permission.CREATE_EVALUATION, ) logger.debug(f"User has permission to create evaluation: {has_permission}") @@ -126,13 +133,14 @@ async def create_evaluation( for variant_id in payload.variant_ids: evaluation = await evaluation_service.create_new_evaluation( app_id=payload.app_id, + project_id=project_id, variant_id=variant_id, - evaluator_config_ids=payload.evaluators_configs, testset_id=payload.testset_id, ) evaluate.delay( app_id=payload.app_id, + project_id=project_id, variant_id=variant_id, evaluators_config_ids=payload.evaluators_configs, testset_id=payload.testset_id, @@ -147,6 +155,7 @@ async def create_evaluation( user_uid=request.state.user_id, object_id=payload.app_id, object_type="app", + project_id=project_id, ) logger.debug("Successfully updated last_modified_by app information") @@ -181,7 +190,10 @@ async def fetch_evaluation_status( """ try: - evaluation = await db_manager.fetch_evaluation_by_id(evaluation_id) + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) + evaluation = await db_manager.fetch_evaluation_by_id(evaluation_id, project_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -221,7 +233,12 @@ async def fetch_evaluation_results( """ try: - evaluation = await db_manager.fetch_evaluation_by_id(evaluation_id) + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) + evaluation = await db_manager.fetch_evaluation_by_id( + evaluation_id, project_id=project_id + ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -270,7 +287,10 @@ async def fetch_evaluation_scenarios( """ try: - evaluation = await db_manager.fetch_evaluation_by_id(evaluation_id) + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) + evaluation = await db_manager.fetch_evaluation_by_id(evaluation_id, project_id) if not evaluation: raise HTTPException( status_code=404, detail=f"Evaluation with id {evaluation_id} not found" @@ -373,7 +393,10 @@ async def fetch_evaluation( Evaluation: The fetched evaluation. """ try: - evaluation = await db_manager.fetch_evaluation_by_id(evaluation_id) + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) + evaluation = await db_manager.fetch_evaluation_by_id(evaluation_id, project_id) if not evaluation: raise HTTPException( status_code=404, detail=f"Evaluation with id {evaluation_id} not found" @@ -420,6 +443,9 @@ async def delete_evaluations( """ try: + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) if isCloudEE(): evaluation_id = random.choice(payload.evaluations_ids) has_permission = await check_action_access( @@ -442,10 +468,11 @@ async def delete_evaluations( user_uid=request.state.user_id, object_id=random.choice(payload.evaluations_ids), object_type="evaluation", + project_id=project_id, ) logger.debug("Successfully updated last_modified_by app information") - await evaluation_service.delete_evaluations(payload.evaluations_ids) + await evaluation_service.delete_evaluations(payload.evaluations_ids, project_id) return Response(status_code=status.HTTP_204_NO_CONTENT) except Exception as exc: raise HTTPException(status_code=500, detail=str(exc)) @@ -472,6 +499,9 @@ async def fetch_evaluation_scenarios( List[EvaluationScenario]: A list of evaluation scenarios. """ try: + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) evaluations_ids_list = evaluations_ids.split(",") if isCloudEE(): @@ -494,7 +524,7 @@ async def fetch_evaluation_scenarios( ) eval_scenarios = await evaluation_service.compare_evaluations_scenarios( - evaluations_ids_list + evaluations_ids_list, project_id ) return eval_scenarios diff --git a/agenta-backend/agenta_backend/routers/variants_router.py b/agenta-backend/agenta_backend/routers/variants_router.py index e9c6fe1403..9c7224b3bb 100644 --- a/agenta-backend/agenta_backend/routers/variants_router.py +++ b/agenta-backend/agenta_backend/routers/variants_router.py @@ -273,14 +273,17 @@ async def update_variant_image( JSONResponse: A JSON response indicating whether the update was successful or not. """ try: + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) db_app_variant = await db_manager.fetch_app_variant_by_id( - app_variant_id=variant_id + app_variant_id=variant_id, project_id=project_id ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=db_app_variant, + project_id=project_id, permission=Permission.CREATE_APPLICATION, ) logger.debug( @@ -295,7 +298,7 @@ async def update_variant_image( ) await app_manager.update_variant_image( - db_app_variant, image, request.state.user_id + db_app_variant, project_id, image, request.state.user_id ) # Update last_modified_by app information @@ -303,6 +306,7 @@ async def update_variant_image( user_uid=request.state.user_id, object_id=str(db_app_variant.app_id), object_type="app", + project_id=project_id, ) logger.debug("Successfully updated last_modified_by app information") except ValueError as e: @@ -348,13 +352,17 @@ async def start_variant( Raises: HTTPException: If the app container cannot be started. """ - app_variant_db = await db_manager.fetch_app_variant_by_id(app_variant_id=variant_id) + + project_id = project_utils.get_project_id(request=request, project_id=project_id) + app_variant_db = await db_manager.fetch_app_variant_by_id( + app_variant_id=variant_id, project_id=project_id + ) # Check user has permission to start variant if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=app_variant_db, + project_id=project_id, permission=Permission.CREATE_APPLICATION, ) logger.debug(f"User has Permission to start variant: {has_permission}") @@ -372,12 +380,13 @@ async def start_variant( envvars = {} if env_vars is None else env_vars.env_vars if action.action == VariantActionEnum.START: - url: URI = await app_manager.start_variant(app_variant_db, envvars) + url: URI = await app_manager.start_variant(app_variant_db, project_id, envvars) # Deploy to production await db_manager.deploy_to_environment( environment_name="production", variant_id=str(app_variant_db.id), + project_id=project_id, user_uid=request.state.user_id, ) return url @@ -411,14 +420,17 @@ async def get_variant( ): logger.debug("getting variant " + variant_id) try: + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) app_variant = await db_manager.fetch_app_variant_by_id( - app_variant_id=variant_id + app_variant_id=variant_id, project_id=project_id ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=app_variant, + project_id=project_id, permission=Permission.VIEW_APPLICATION, ) logger.debug(f"User has Permission to get variant: {has_permission}") @@ -448,8 +460,11 @@ async def get_variant_revisions( ): logger.debug("getting variant revisions: ", variant_id) try: + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) app_variant = await db_manager.fetch_app_variant_by_id( - app_variant_id=variant_id + app_variant_id=variant_id, project_id=project_id ) if isCloudEE(): @@ -468,7 +483,7 @@ async def get_variant_revisions( ) app_variant_revisions = await db_manager.list_app_variant_revisions_by_variant( - app_variant=app_variant + app_variant=app_variant, project_id=project_id ) return await converters.app_variant_db_revisions_to_output( app_variant_revisions @@ -491,17 +506,20 @@ async def get_variant_revision( ): logger.debug("getting variant revision: ", variant_id, revision_number) try: + project_id = project_utils.get_project_id( + request=request, project_id=project_id + ) assert ( variant_id != "undefined" ), "Variant id is required to retrieve variant revision" app_variant = await db_manager.fetch_app_variant_by_id( - app_variant_id=variant_id + app_variant_id=variant_id, project_id=project_id ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=app_variant, + project_id=project_id, permission=Permission.VIEW_APPLICATION, ) logger.debug(f"User has Permission to get variant: {has_permission}") diff --git a/agenta-backend/agenta_backend/services/app_manager.py b/agenta-backend/agenta_backend/services/app_manager.py index 9838610f68..5de31f2060 100644 --- a/agenta-backend/agenta_backend/services/app_manager.py +++ b/agenta-backend/agenta_backend/services/app_manager.py @@ -134,21 +134,25 @@ async def start_variant( async def update_variant_image( - app_variant_db: AppVariantDB, image: Image, user_uid: str + app_variant_db: AppVariantDB, project_id: str, image: Image, user_uid: str ): """Updates the image for app variant in the database. Arguments: - app_variant -- the app variant to update - image -- the image to update + app_variant (AppVariantDB): the app variant to update + project_id (str): The ID of the project + image (Image): the image to update + user_uid (str): The ID of the user updating the image """ valid_image = await deployment_manager.validate_image(image) if not valid_image: raise ValueError("Image could not be found in registry.") - base = await db_manager.fetch_base_by_id(str(app_variant_db.base_id)) - deployment = await db_manager.get_deployment_by_id(str(base.deployment_id)) + base = await db_manager.fetch_base_by_id(str(app_variant_db.base_id), project_id) + deployment = await db_manager.get_deployment_by_id( + str(base.deployment_id), project_id + ) await deployment_manager.stop_and_delete_service(deployment) await db_manager.remove_deployment(str(deployment.id)) @@ -156,25 +160,21 @@ async def update_variant_image( if isOss(): await deployment_manager.remove_image(base.image) - await db_manager.remove_image(base.image) + await db_manager.remove_image(base.image, project_id) # Create a new image instance db_image = await db_manager.create_image( image_type="image", + project_id=project_id, tags=image.tags, docker_id=image.docker_id, - user=app_variant_db.user, deletable=True, - organization=( - str(app_variant_db.organization_id) if isCloudEE() else None - ), # noqa - workspace=str(app_variant_db.workspace_id) if isCloudEE() else None, # noqa ) # Update base with new image await db_manager.update_base(str(app_variant_db.base_id), image_id=db_image.id) # Update variant to remove configuration await db_manager.update_variant_parameters( - str(app_variant_db.id), parameters={}, user_uid=user_uid + str(app_variant_db.id), parameters={}, project_id=project_id, user_uid=user_uid ) # Update variant with new image app_variant_db = await db_manager.update_app_variant( @@ -182,7 +182,7 @@ async def update_variant_image( ) # Start variant - await start_variant(app_variant_db) + await start_variant(app_variant_db, project_id) async def update_last_modified_by( @@ -389,7 +389,7 @@ async def remove_app(app: AppDB, project_id: str): # delete app and its related resources try: logger.debug("remove_app_related_resources") - await remove_app_related_resources(str(app.id)) + await remove_app_related_resources(str(app.id), project_id) except Exception as e: logger.error( f"An error occurred while deleting app {app.id} and its associated resources: {str(e)}" diff --git a/agenta-backend/agenta_backend/services/container_manager.py b/agenta-backend/agenta_backend/services/container_manager.py index 7575c1efa3..9a8751a3e6 100644 --- a/agenta-backend/agenta_backend/services/container_manager.py +++ b/agenta-backend/agenta_backend/services/container_manager.py @@ -30,7 +30,7 @@ async def build_image(app_db: AppDB, base_name: str, tar_file: UploadFile) -> Image: app_name = app_db.app_name - user_id = str(app_db.user_id) + project_id = str(app_db.project_id) image_name = f"agentaai/{app_name.lower()}_{base_name.lower()}:latest" # Get event loop @@ -47,6 +47,7 @@ async def build_image(app_db: AppDB, base_name: str, tar_file: UploadFile) -> Im tar_path = temp_dir / tar_file.filename with tar_path.open("wb") as buffer: buffer.write(await tar_file.read()) + future = loop.run_in_executor( thread_pool, build_image_job, @@ -56,7 +57,7 @@ async def build_image(app_db: AppDB, base_name: str, tar_file: UploadFile) -> Im tar_path, image_name, temp_dir, - user_id, + project_id, ), ) image_result = await asyncio.wrap_future(future) @@ -69,7 +70,7 @@ def build_image_job( tar_path: Path, image_name: str, temp_dir: Path, - user_id: str, + project_id: str, ) -> Image: """Business logic for building a docker image from a tar file @@ -86,7 +87,7 @@ def build_image_job( image that will be built. It is used as the tag for the image temp_dir -- The `temp_dir` parameter is a `Path` object that represents the temporary directory where the contents of the tar file will be extracted - user_id -- The id of the user that owns the app + project_id -- The id of the project that owns the app Raises: HTTPException: _description_ @@ -108,7 +109,7 @@ def build_image_job( image, build_log = client.images.build( path=str(temp_dir), tag=image_name, - buildargs={"ROOT_PATH": f"/{user_id}/{app_name}/{base_name}"}, + buildargs={"ROOT_PATH": f"/{project_id}/{app_name}/{base_name}"}, rm=True, dockerfile=dockerfile, pull=True, diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 8e141da4cf..9cf529ba8d 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -2230,12 +2230,13 @@ async def fetch_human_evaluation_scenarios(evaluation_id: str): return evaluation_scenarios -async def fetch_evaluation_scenarios(evaluation_id: str): +async def fetch_evaluation_scenarios(evaluation_id: str, project_id: str): """ Fetches evaluation scenarios. Args: evaluation_id (str): The evaluation identifier + project_id (str): The ID of the project Returns: The evaluation scenarios. @@ -2244,7 +2245,9 @@ async def fetch_evaluation_scenarios(evaluation_id: str): async with db_engine.get_session() as session: result = await session.execute( select(EvaluationScenarioDB) - .filter_by(evaluation_id=uuid.UUID(evaluation_id)) + .filter_by( + evaluation_id=uuid.UUID(evaluation_id), project_id=uuid.UUID(project_id) + ) .options(joinedload(EvaluationScenarioDB.results)) ) evaluation_scenarios = result.unique().scalars().all() @@ -2545,18 +2548,14 @@ async def update_app_variant( if hasattr(app_variant, key): setattr(app_variant, key, value) - relationships_to_load_in_session = [ - "user", - "app", - "image", - "base", - ] - if isCloudEE(): - relationships_to_load_in_session.append("organization") - await session.commit() await session.refresh( - app_variant, attribute_names=relationships_to_load_in_session + app_variant, + attribute_names=[ + "app", + "image", + "base", + ], ) return app_variant @@ -2584,13 +2583,11 @@ async def fetch_app_by_name_and_parameters(app_name: str, project_id: str): async def create_new_evaluation( app: AppDB, - user_id: str, + project_id: str, testset: TestSetDB, status: Result, variant: str, variant_revision: str, - organization=None, - workspace=None, ) -> EvaluationDB: """Create a new evaluation scenario. Returns: @@ -2600,28 +2597,18 @@ async def create_new_evaluation( async with db_engine.get_session() as session: evaluation = EvaluationDB( app_id=app.id, - user_id=uuid.UUID(user_id), + project_id=uuid.UUID(project_id), testset_id=testset.id, - status=status.dict(), + status=status.model_dump(), variant_id=uuid.UUID(variant), variant_revision_id=uuid.UUID(variant_revision), ) - if isCloudEE(): - # assert that if organization is provided, workspace is also provided, and vice versa - assert ( - organization is not None and workspace is not None - ), "organization and workspace must be provided together" - - evaluation.organization_id = uuid.UUID(organization) # type: ignore - evaluation.workspace_id = uuid.UUID(workspace) # type: ignore - session.add(evaluation) await session.commit() await session.refresh( evaluation, attribute_names=[ - "user", "testset", "variant", "variant_revision", @@ -2668,13 +2655,16 @@ async def list_evaluations(app_id: str, project_id: str): return evaluations -async def fetch_evaluations_by_resource(resource_type: str, resource_ids: List[str]): +async def fetch_evaluations_by_resource( + resource_type: str, project_id: str, resource_ids: List[str] +): """ Fetches an evaluations by resource. Args: - resource_type: The resource type - resource_ids: The resource identifiers + resource_type (str): The resource type + project_id (str): The ID of the project + resource_ids (List[str]): The resource identifiers Returns: The evaluations by resource. @@ -2689,13 +2679,19 @@ async def fetch_evaluations_by_resource(resource_type: str, resource_ids: List[s if resource_type == "variant": result_evaluations = await session.execute( select(EvaluationDB) - .filter(EvaluationDB.variant_id.in_(ids)) + .filter( + EvaluationDB.variant_id.in_(ids), + EvaluationDB.project_id == uuid.UUID(project_id), + ) .options(load_only(EvaluationDB.id)) # type: ignore ) result_human_evaluations = await session.execute( select(HumanEvaluationDB) .join(HumanEvaluationVariantDB) - .filter(HumanEvaluationVariantDB.variant_id.in_(ids)) + .filter( + HumanEvaluationVariantDB.variant_id.in_(ids), + EvaluationDB.project_id == uuid.UUID(project_id), + ) .options(load_only(HumanEvaluationDB.id)) # type: ignore ) res_evaluations = result_evaluations.scalars().all() @@ -2705,12 +2701,18 @@ async def fetch_evaluations_by_resource(resource_type: str, resource_ids: List[s if resource_type == "testset": result_evaluations = await session.execute( select(EvaluationDB) - .filter(EvaluationDB.testset_id.in_(ids)) + .filter( + EvaluationDB.testset_id.in_(ids), + EvaluationDB.project_id == uuid.UUID(project_id), + ) .options(load_only(EvaluationDB.id)) # type: ignore ) result_human_evaluations = await session.execute( select(HumanEvaluationDB) - .filter(HumanEvaluationDB.testset_id.in_(ids)) + .filter( + HumanEvaluationDB.testset_id.in_(ids), + EvaluationDB.project_id == uuid.UUID(project_id), + ) .options(load_only(HumanEvaluationDB.id)) # type: ignore ) res_evaluations = result_evaluations.scalars().all() @@ -2721,7 +2723,10 @@ async def fetch_evaluations_by_resource(resource_type: str, resource_ids: List[s query = ( select(EvaluationDB) .join(EvaluationDB.evaluator_configs) - .filter(EvaluationEvaluatorConfigDB.evaluator_config_id.in_(ids)) + .filter( + EvaluationEvaluatorConfigDB.evaluator_config_id.in_(ids), + EvaluationDB.project_id == uuid.UUID(project_id), + ) ) result = await session.execute(query) res = result.scalars().all() @@ -2733,15 +2738,19 @@ async def fetch_evaluations_by_resource(resource_type: str, resource_ids: List[s ) -async def delete_evaluations(evaluation_ids: List[str]) -> None: +async def delete_evaluations(evaluation_ids: List[str], project_id: str) -> None: """Delete evaluations based on the ids provided from the db. Args: evaluations_ids (list[str]): The IDs of the evaluation + project_id (str): The ID of the project """ async with db_engine.get_session() as session: - query = select(EvaluationDB).where(EvaluationDB.id.in_(evaluation_ids)) + query = select(EvaluationDB).where( + EvaluationDB.id.in_(evaluation_ids), + EvaluationDB.project_id == uuid.UUID(project_id), + ) result = await session.execute(query) evaluations = result.scalars().all() for evaluation in evaluations: @@ -2750,7 +2759,7 @@ async def delete_evaluations(evaluation_ids: List[str]) -> None: async def create_new_evaluation_scenario( - user_id: str, + project_id: str, evaluation_id: str, variant_id: str, inputs: List[EvaluationScenarioInput], @@ -2759,8 +2768,6 @@ async def create_new_evaluation_scenario( is_pinned: Optional[bool], note: Optional[str], results: List[EvaluationScenarioResult], - organization=None, - workspace=None, ) -> EvaluationScenarioDB: """Create a new evaluation scenario. @@ -2770,13 +2777,13 @@ async def create_new_evaluation_scenario( async with db_engine.get_session() as session: evaluation_scenario = EvaluationScenarioDB( - user_id=uuid.UUID(user_id), + project_id=uuid.UUID(project_id), evaluation_id=uuid.UUID(evaluation_id), variant_id=uuid.UUID(variant_id), - inputs=[input.dict() for input in inputs], - outputs=[output.dict() for output in outputs], + inputs=[input.model_dump() for input in inputs], + outputs=[output.model_dump() for output in outputs], correct_answers=( - [correct_answer.dict() for correct_answer in correct_answers] + [correct_answer.model_dump() for correct_answer in correct_answers] if correct_answers is not None else [] ), @@ -2784,15 +2791,6 @@ async def create_new_evaluation_scenario( note=note, ) - if isCloudEE(): - # assert that if organization is provided, workspace is also provided, and vice versa - assert ( - organization is not None and workspace is not None - ), "organization and workspace must be provided together" - - evaluation_scenario.organization_id = organization # type: ignore - evaluation_scenario.workspace_id = workspace # type: ignore - session.add(evaluation_scenario) await session.commit() await session.refresh(evaluation_scenario) @@ -2802,7 +2800,7 @@ async def create_new_evaluation_scenario( evaluation_scenario_result = EvaluationScenarioResultDB( evaluation_scenario_id=evaluation_scenario.id, evaluator_config_id=uuid.UUID(result.evaluator_config), - result=result.result.dict(), + result=result.result.model_dump(), ) session.add(evaluation_scenario_result) @@ -3039,13 +3037,14 @@ async def delete_evaluator_config(evaluator_config_id: str) -> bool: async def update_evaluation( - evaluation_id: str, updates: Dict[str, Any] + evaluation_id: str, project_id: str, updates: Dict[str, Any] ) -> EvaluationDB: """ Update an evaluator configuration in the database with the provided id. Arguments: evaluation_id (str): The ID of the evaluator configuration to be updated. + project_id (str): The ID of the project. updates (Dict[str, Any]): The updates to apply to the evaluator configuration. Returns: @@ -3054,7 +3053,9 @@ async def update_evaluation( async with db_engine.get_session() as session: result = await session.execute( - select(EvaluationDB).filter_by(id=uuid.UUID(evaluation_id)) + select(EvaluationDB).filter_by( + id=uuid.UUID(evaluation_id), project_id=uuid.UUID(project_id) + ) ) evaluation = result.scalars().first() for key, value in updates.items(): diff --git a/agenta-backend/agenta_backend/services/evaluation_service.py b/agenta-backend/agenta_backend/services/evaluation_service.py index c3de5769e0..9b98e3be48 100644 --- a/agenta-backend/agenta_backend/services/evaluation_service.py +++ b/agenta-backend/agenta_backend/services/evaluation_service.py @@ -142,19 +142,22 @@ async def update_human_evaluation_service( ) -async def fetch_evaluation_scenarios_for_evaluation(evaluation_id: str): +async def fetch_evaluation_scenarios_for_evaluation( + evaluation_id: str, project_id: str +): """ Fetch evaluation scenarios for a given evaluation ID. Args: evaluation_id (str): The ID of the evaluation. + project_id (str): The ID of the project. Returns: List[EvaluationScenario]: A list of evaluation scenarios. """ evaluation_scenarios = await db_manager.fetch_evaluation_scenarios( - evaluation_id=evaluation_id + evaluation_id=evaluation_id, project_id=project_id ) return [ await converters.evaluation_scenario_db_to_pydantic( @@ -408,8 +411,8 @@ async def create_new_human_evaluation( async def create_new_evaluation( app_id: str, + project_id: str, variant_id: str, - evaluator_config_ids: List[str], testset_id: str, ) -> Evaluation: """ @@ -417,32 +420,34 @@ async def create_new_evaluation( Args: app_id (str): The ID of the app. + project_id (str): The ID of the project. variant_id (str): The ID of the variant. - evaluator_config_ids (List[str]): The IDs of the evaluator configurations. testset_id (str): The ID of the testset. Returns: Evaluation: The newly created evaluation. """ - app = await db_manager.fetch_app_by_id(app_id=app_id) - testset = await db_manager.fetch_testset_by_id(testset_id=testset_id) - variant_db = await db_manager.get_app_variant_instance_by_id(variant_id=variant_id) + app = await db_manager.fetch_app_by_id(app_id=app_id, project_id=project_id) + testset = await db_manager.fetch_testset_by_id( + testset_id=testset_id, project_id=project_id + ) + variant_db = await db_manager.get_app_variant_instance_by_id( + variant_id=variant_id, project_id=project_id + ) variant_revision = await db_manager.fetch_app_variant_revision_by_variant( - app_variant_id=variant_id, revision=variant_db.revision # type: ignore + app_variant_id=variant_id, project_id=project_id, revision=variant_db.revision # type: ignore ) evaluation_db = await db_manager.create_new_evaluation( app=app, - user_id=str(app.user_id), + project_id=project_id, testset=testset, status=Result( value=EvaluationStatusEnum.EVALUATION_INITIALIZED, type="status", error=None ), variant=variant_id, variant_revision=str(variant_revision.id), - organization=str(app.organization_id) if isCloudEE() else None, - workspace=str(app.workspace_id) if isCloudEE() else None, ) return await converters.evaluation_db_to_pydantic(evaluation_db) @@ -462,10 +467,8 @@ async def retrieve_evaluation_results(evaluation_id: str) -> List[dict]: return await converters.aggregated_result_to_pydantic(evaluation.aggregated_results) -async def compare_evaluations_scenarios( - evaluations_ids: List[str], -): - evaluation = await db_manager.fetch_evaluation_by_id(evaluations_ids[0]) +async def compare_evaluations_scenarios(evaluations_ids: List[str], project_id: str): + evaluation = await db_manager.fetch_evaluation_by_id(evaluations_ids[0], project_id) testset = evaluation.testset unique_testset_datapoints = remove_duplicates(testset.csvdata) formatted_inputs = extract_inputs_values_from_testset(unique_testset_datapoints) @@ -475,7 +478,7 @@ async def compare_evaluations_scenarios( for evaluation_id in evaluations_ids: eval_scenarios = await fetch_evaluation_scenarios_for_evaluation( - evaluation_id=evaluation_id + evaluation_id=evaluation_id, project_id=project_id ) all_scenarios.append(eval_scenarios) diff --git a/agenta-backend/agenta_backend/tasks/evaluations.py b/agenta-backend/agenta_backend/tasks/evaluations.py index b563bdc202..10619ea8b2 100644 --- a/agenta-backend/agenta_backend/tasks/evaluations.py +++ b/agenta-backend/agenta_backend/tasks/evaluations.py @@ -38,16 +38,12 @@ ) from agenta_backend.services.evaluator_manager import get_evaluators -if isCloudEE(): - from agenta_backend.commons.models.db_models import AppDB_ as AppDB -else: - from agenta_backend.models.db_models import AppDB # Set logger logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) -# Fetch all evaluators and precompute ground truth keys +# Fetch all evaluators and pre-compute ground truth keys all_evaluators = get_evaluators() ground_truth_keys_dict = { evaluator.key: [ @@ -63,6 +59,7 @@ def evaluate( self, app_id: str, + project_id: str, variant_id: str, evaluators_config_ids: List[str], testset_id: str, @@ -76,6 +73,7 @@ def evaluate( Args: self: The task instance. app_id (str): The ID of the app. + project_id (str): The ID of the project. variant_id (str): The ID of the app variant. evaluators_config_ids (List[str]): The IDs of the evaluators configurations to be used. testset_id (str): The ID of the testset. @@ -94,6 +92,7 @@ def evaluate( loop.run_until_complete( update_evaluation( evaluation_id, + project_id, { "status": Result( type="status", value=EvaluationStatusEnum.EVALUATION_STARTED @@ -103,22 +102,26 @@ def evaluate( ) # 1. Fetch data from the database - app = loop.run_until_complete(fetch_app_by_id(app_id)) - app_variant_db = loop.run_until_complete(fetch_app_variant_by_id(variant_id)) + app = loop.run_until_complete(fetch_app_by_id(app_id, project_id)) + app_variant_db = loop.run_until_complete( + fetch_app_variant_by_id(variant_id, project_id) + ) assert ( app_variant_db is not None ), f"App variant with id {variant_id} not found!" app_variant_parameters = app_variant_db.config_parameters - testset_db = loop.run_until_complete(fetch_testset_by_id(testset_id)) + testset_db = loop.run_until_complete( + fetch_testset_by_id(testset_id, project_id) + ) evaluator_config_dbs = [] for evaluator_config_id in evaluators_config_ids: evaluator_config = loop.run_until_complete( - fetch_evaluator_config(evaluator_config_id) + fetch_evaluator_config(evaluator_config_id, project_id) ) evaluator_config_dbs.append(evaluator_config) deployment_db = loop.run_until_complete( - get_deployment_by_id(str(app_variant_db.base.deployment_id)) + get_deployment_by_id(str(app_variant_db.base.deployment_id), project_id) ) uri = deployment_manager.get_deployment_uri(uri=deployment_db.uri) # type: ignore @@ -169,7 +172,7 @@ def evaluate( ] logger.debug(f"Inputs: {inputs}") - # 2. We skip the iteration if error invking the llm-app + # 2. We skip the iteration if error invoking the llm-app if app_output.result.error: print("There is an error when invoking the llm app so we need to skip") error_results = [ @@ -189,7 +192,7 @@ def evaluate( loop.run_until_complete( create_new_evaluation_scenario( - user_id=str(app.user_id), + project_id=project_id, evaluation_id=evaluation_id, variant_id=variant_id, inputs=inputs, @@ -209,8 +212,6 @@ def evaluate( is_pinned=False, note="", results=error_results, - organization=str(app.organization_id) if isCloudEE() else None, - workspace=str(app.workspace_id) if isCloudEE() else None, ) ) continue @@ -268,7 +269,7 @@ def evaluate( # 4. We save the result of the eval scenario in the db loop.run_until_complete( create_new_evaluation_scenario( - user_id=str(app.user_id), + project_id=project_id, evaluation_id=evaluation_id, variant_id=variant_id, inputs=inputs, @@ -285,8 +286,6 @@ def evaluate( is_pinned=False, note="", results=evaluators_results, - organization=str(app.organization_id) if isCloudEE() else None, - workspace=str(app.workspace_id) if isCloudEE() else None, ) ) @@ -303,6 +302,7 @@ def evaluate( loop.run_until_complete( update_evaluation( evaluation_id, + project_id, { "average_latency": average_latency.model_dump(), "average_cost": average_cost.model_dump(), @@ -317,6 +317,7 @@ def evaluate( loop.run_until_complete( update_evaluation( evaluation_id, + project_id, { "status": Result( type="status", @@ -359,6 +360,7 @@ def evaluate( loop.run_until_complete( update_evaluation( evaluation_id=evaluation_id, + project_id=project_id, updates={"status": evaluation_status.model_dump()}, ) ) @@ -369,6 +371,7 @@ def evaluate( loop.run_until_complete( update_evaluation( evaluation_id, + project_id, { "status": Result( type="status", diff --git a/agenta-cli/agenta/client/backend/types/app_variant_response.py b/agenta-cli/agenta/client/backend/types/app_variant_response.py index 79c821ba11..70c819544d 100644 --- a/agenta-cli/agenta/client/backend/types/app_variant_response.py +++ b/agenta-cli/agenta/client/backend/types/app_variant_response.py @@ -12,7 +12,7 @@ class AppVariantResponse(UniversalBaseModel): variant_id: str variant_name: str parameters: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = None - user_id: str + project_id: str base_name: str base_id: str config_name: str @@ -21,8 +21,6 @@ class AppVariantResponse(UniversalBaseModel): created_at: typing.Optional[str] = None updated_at: typing.Optional[str] = None modified_by_id: typing.Optional[str] = None - organization_id: typing.Optional[str] = None - workspace_id: typing.Optional[str] = None if IS_PYDANTIC_V2: model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( From f06e2a86a05ef20c4a6d4d46e7a1c1220b2959ad Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 16 Sep 2024 12:43:35 +0100 Subject: [PATCH 18/57] feat (migrations): created custom migration logic to update evaluators with app_name --- .../postgres/data_migrations/applications.py | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 agenta-backend/agenta_backend/migrations/postgres/data_migrations/applications.py diff --git a/agenta-backend/agenta_backend/migrations/postgres/data_migrations/applications.py b/agenta-backend/agenta_backend/migrations/postgres/data_migrations/applications.py new file mode 100644 index 0000000000..ac0c626264 --- /dev/null +++ b/agenta-backend/agenta_backend/migrations/postgres/data_migrations/applications.py @@ -0,0 +1,63 @@ +import os +import uuid +import traceback +from typing import Optional + + +import click +from sqlalchemy.future import select +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker, Session + +from agenta_backend.models.db_models import ( + AppDB, + EvaluatorConfigDB, +) + + +BATCH_SIZE = 1000 + + +def get_app_db(session: Session, app_id: str) -> Optional[AppDB]: + query = session.execute(select(AppDB).filter_by(id=uuid.UUID(app_id))) + return query.scalars().first() + + +def update_evaluators_with_app_name(): + engine = create_engine(os.getenv("POSTGRES_URI")) + sync_session = sessionmaker(engine, expire_on_commit=False) + + with sync_session() as session: + try: + offset = 0 + while True: + records = ( + session.execute( + select(EvaluatorConfigDB).offset(offset).limit(BATCH_SIZE) + ) + .scalars() + .all() + ) + if not records: + break + + # Update records with app_name as prefix + for record in records: + evaluator_config_app = get_app_db( + session=session, app_id=str(record.app_id) + ) + if evaluator_config_app: + record.name = f"{record.name} ({evaluator_config_app.app_name})" + + session.commit() + offset += BATCH_SIZE + + except Exception as e: + session.rollback() + click.echo( + click.style( + f"ERROR updating evaluator config names: {traceback.format_exc()}", + fg="red", + ) + ) + raise e From 37b97298f1c13f77f2a21f89831814442c46b3b6 Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 16 Sep 2024 13:36:27 +0100 Subject: [PATCH 19/57] refactor (migration): reorder migration script application for consistency --- ..._update_evaluators_names_with_app_name_.py | 32 +++++++++++++++++++ ..._scope_project_id_to_db_models_entities.py | 4 +-- 2 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 agenta-backend/agenta_backend/migrations/postgres/versions/22d29365f5fc_update_evaluators_names_with_app_name_.py diff --git a/agenta-backend/agenta_backend/migrations/postgres/versions/22d29365f5fc_update_evaluators_names_with_app_name_.py b/agenta-backend/agenta_backend/migrations/postgres/versions/22d29365f5fc_update_evaluators_names_with_app_name_.py new file mode 100644 index 0000000000..f817b805d3 --- /dev/null +++ b/agenta-backend/agenta_backend/migrations/postgres/versions/22d29365f5fc_update_evaluators_names_with_app_name_.py @@ -0,0 +1,32 @@ +"""Update evaluators names with app name as prefix + +Revision ID: 22d29365f5fc +Revises: 6cfe239894fb +Create Date: 2024-09-16 11:38:33.886908 + +""" + +from typing import Sequence, Union + +from agenta_backend.migrations.postgres.data_migrations.applications import ( + update_evaluators_with_app_name, +) + + +# revision identifiers, used by Alembic. +revision: str = "22d29365f5fc" +down_revision: Union[str, None] = "6cfe239894fb" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### custom command ### + update_evaluators_with_app_name() + # ### end custom command ### + + +def downgrade() -> None: + # ### custom command ### + pass + # ### end custom command ### diff --git a/agenta-backend/agenta_backend/migrations/postgres/versions/c00a326c625a_scope_project_id_to_db_models_entities.py b/agenta-backend/agenta_backend/migrations/postgres/versions/c00a326c625a_scope_project_id_to_db_models_entities.py index 2d4df676e9..0bf5ab761d 100644 --- a/agenta-backend/agenta_backend/migrations/postgres/versions/c00a326c625a_scope_project_id_to_db_models_entities.py +++ b/agenta-backend/agenta_backend/migrations/postgres/versions/c00a326c625a_scope_project_id_to_db_models_entities.py @@ -1,7 +1,7 @@ """scope project_id to db models/entities Revision ID: c00a326c625a -Revises: 6cfe239894fb +Revises: 22d29365f5fc Create Date: 2024-09-12 20:34:16.175845 """ @@ -17,7 +17,7 @@ # revision identifiers, used by Alembic. revision: str = "c00a326c625a" -down_revision: Union[str, None] = "6cfe239894fb" +down_revision: Union[str, None] = "22d29365f5fc" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None From 94e356d809b9fcba9ff4b1710ea490322eb5942f Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 16 Sep 2024 17:01:45 +0100 Subject: [PATCH 20/57] feat (backend): added AppDB and EvaluatorConfigDB to deprecated models file --- .../models/deprecated_models.py | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 agenta-backend/agenta_backend/models/deprecated_models.py diff --git a/agenta-backend/agenta_backend/models/deprecated_models.py b/agenta-backend/agenta_backend/models/deprecated_models.py new file mode 100644 index 0000000000..4bd4373b4e --- /dev/null +++ b/agenta-backend/agenta_backend/models/deprecated_models.py @@ -0,0 +1,61 @@ +from datetime import datetime, timezone + +import uuid_utils.compat as uuid +from sqlalchemy import ( + Column, + String, + DateTime, + ForeignKey, +) +from sqlalchemy_json import mutable_json_type +from sqlalchemy.dialects.postgresql import UUID, JSONB +from sqlalchemy.ext.declarative import declarative_base + + +DeprecatedBase = declarative_base() + + +class DeprecatedAppDB(DeprecatedBase): + __tablename__ = "app_db" + __table_args__ = {"extend_existing": True} + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid7, + unique=True, + nullable=False, + ) + app_name = Column(String) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) + modified_by_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) + created_at = Column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) + ) + updated_at = Column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) + ) + + +class DeprecatedEvaluatorConfigDB(DeprecatedBase): + __tablename__ = "evaluators_configs" + __table_args__ = {"extend_existing": True} + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid7, + unique=True, + nullable=False, + ) + + app_id = Column(UUID(as_uuid=True), ForeignKey("app_db.id", ondelete="SET NULL")) + name = Column(String) + evaluator_key = Column(String) + settings_values = Column(mutable_json_type(dbtype=JSONB, nested=True), default=dict) + created_at = Column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) + ) + updated_at = Column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) + ) From ec6041e5bd85bb7d8ce4c970e7416af86b516eb0 Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 16 Sep 2024 17:04:39 +0100 Subject: [PATCH 21/57] refactor (migrations): use deprecated models to ensure correct table schema --- .../postgres/data_migrations/applications.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/agenta-backend/agenta_backend/migrations/postgres/data_migrations/applications.py b/agenta-backend/agenta_backend/migrations/postgres/data_migrations/applications.py index ac0c626264..d0e76a8f85 100644 --- a/agenta-backend/agenta_backend/migrations/postgres/data_migrations/applications.py +++ b/agenta-backend/agenta_backend/migrations/postgres/data_migrations/applications.py @@ -9,17 +9,17 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker, Session -from agenta_backend.models.db_models import ( - AppDB, - EvaluatorConfigDB, +from agenta_backend.models.deprecated_models import ( + DeprecatedEvaluatorConfigDB, + DeprecatedAppDB, ) BATCH_SIZE = 1000 -def get_app_db(session: Session, app_id: str) -> Optional[AppDB]: - query = session.execute(select(AppDB).filter_by(id=uuid.UUID(app_id))) +def get_app_db(session: Session, app_id: str) -> Optional[DeprecatedAppDB]: + query = session.execute(select(DeprecatedAppDB).filter_by(id=uuid.UUID(app_id))) return query.scalars().first() @@ -33,7 +33,9 @@ def update_evaluators_with_app_name(): while True: records = ( session.execute( - select(EvaluatorConfigDB).offset(offset).limit(BATCH_SIZE) + select(DeprecatedEvaluatorConfigDB) + .offset(offset) + .limit(BATCH_SIZE) ) .scalars() .all() @@ -46,7 +48,7 @@ def update_evaluators_with_app_name(): evaluator_config_app = get_app_db( session=session, app_id=str(record.app_id) ) - if evaluator_config_app: + if record.app_id is not None and evaluator_config_app is not None: record.name = f"{record.name} ({evaluator_config_app.app_name})" session.commit() From f6d998b9266d5bd93222418701d78a51fc63aa3d Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 16 Sep 2024 17:08:18 +0100 Subject: [PATCH 22/57] refactor (backend): replace get_project_id functionality with retrieve_project_id_from_request --- .../agenta_backend/utils/project_utils.py | 59 +++++++++++++------ 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/agenta-backend/agenta_backend/utils/project_utils.py b/agenta-backend/agenta_backend/utils/project_utils.py index d31a4cdc46..798b304ac7 100644 --- a/agenta-backend/agenta_backend/utils/project_utils.py +++ b/agenta-backend/agenta_backend/utils/project_utils.py @@ -1,30 +1,53 @@ +import json +import logging from typing import Optional from fastapi import Request -def get_project_id(request: Request, project_id: Optional[str] = None) -> str: +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +async def retrieve_project_id_from_request(request: Request) -> Optional[str]: """ - Retrieve the project_id from the request or use the default from the request state. + Retrieves the `project_id` from an incoming HTTP request. + + This function attempts to extract the `project_id` from various parts of the request: + 1. Path parameters + 2. Query parameters + 3. Request body (assuming it is JSON) Args: - request (Request): The current request object containing state information. - project_id (Optional[str]): The provided project_id from the API endpoint. + request (Request): The FastAPI `Request` object from which to extract the `project_id`. Returns: - str: The project_id to use for the operation. - - Raises: - ValueError: If no project_id is provided and no default is found in the request state. + Optional[str]: The extracted `project_id` if found; otherwise, `None`. """ - if project_id is not None: - return project_id - - default_project_id: str = getattr(request.state, "project_id", None) - if default_project_id is None: - raise ValueError( - "No project_id provided and no default project_id found in the request state." - ) - - return default_project_id + logger.info("Retrieving project_id from request...") + + project_id_from_path_params = request.path_params.get("project_id") + if project_id_from_path_params: + logger.info("Project ID found in path params") + return project_id_from_path_params + + project_id_from_query_params = request.query_params.get("project_id") + if project_id_from_query_params: + logger.info("Project ID found in query params") + return project_id_from_query_params + + try: + request_body = await request.body() + if request_body: + project_id_response_str = request_body.decode("utf-8") + response_json = json.loads(project_id_response_str) + project_id_from_request_body = response_json.get("project_id") + if project_id_from_request_body: + logger.info("Project ID found in request body") + return project_id_from_request_body + except (json.JSONDecodeError, UnicodeDecodeError) as e: + logger.error(f"Error decoding request body: {e}") + + logger.info("No project ID found in the request") + return None From 97b0b917bed3565fbcfb53aaf964a70f4525c174 Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 16 Sep 2024 17:09:12 +0100 Subject: [PATCH 23/57] refactor (backend): include application name when creating evaluators --- agenta-backend/agenta_backend/routers/app_router.py | 8 ++++++-- .../agenta_backend/routers/evaluators_router.py | 4 ++++ agenta-backend/agenta_backend/services/db_manager.py | 5 +++-- .../agenta_backend/services/evaluator_manager.py | 8 +++++++- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index 79568c6d5a..8e46084a7b 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -416,7 +416,9 @@ async def add_variant_from_image( ) logger.debug("Step 8: We create ready-to use evaluators") - await evaluator_manager.create_ready_to_use_evaluators(project_id=project_id) + await evaluator_manager.create_ready_to_use_evaluators( + app_name=app.app_name, project_id=project_id + ) return await converters.app_variant_db_to_output(app_variant_db) except Exception as e: @@ -590,7 +592,9 @@ async def create_app_and_variant_from_template( if isCloudEE() else "Step 6: We create ready-to use evaluators" ) - await evaluator_manager.create_ready_to_use_evaluators(project_id=project_id) + await evaluator_manager.create_ready_to_use_evaluators( + app_name=app.app_name, project_id=project_id + ) logger.debug( "Step 10: Starting variant and injecting environment variables" diff --git a/agenta-backend/agenta_backend/routers/evaluators_router.py b/agenta-backend/agenta_backend/routers/evaluators_router.py index f215a2e710..92a27efd37 100644 --- a/agenta-backend/agenta_backend/routers/evaluators_router.py +++ b/agenta-backend/agenta_backend/routers/evaluators_router.py @@ -148,6 +148,9 @@ async def create_new_evaluator_config( project_id = project_utils.get_project_id( request=request, project_id=project_id ) + app_db = await db_manager.get_app_instance_by_id( + app_id=payload.app_id, project_id=project_id + ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -165,6 +168,7 @@ async def create_new_evaluator_config( evaluator_config = await evaluator_manager.create_evaluator_config( project_id=project_id, + app_name=app_db.app_name, name=payload.name, evaluator_key=payload.evaluator_key, settings_values=payload.settings_values, diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 9cf529ba8d..264bcf2e69 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -2819,7 +2819,7 @@ async def update_evaluation_with_aggregated_results( aggregated_result = EvaluationAggregatedResultDB( evaluation_id=uuid.UUID(evaluation_id), evaluator_config_id=uuid.UUID(result.evaluator_config), - result=result.result.dict(), + result=result.result.model_dump(), ) session.add(aggregated_result) @@ -2960,6 +2960,7 @@ async def fetch_evaluator_config_by_appId( async def create_evaluator_config( project_id: str, + app_name: str, name: str, evaluator_key: str, settings_values: Optional[Dict[str, Any]] = None, @@ -2969,7 +2970,7 @@ async def create_evaluator_config( async with db_engine.get_session() as session: new_evaluator_config = EvaluatorConfigDB( project_id=uuid.UUID(project_id), - name=name, + name=f"{name} ({app_name})", evaluator_key=evaluator_key, settings_values=settings_values, ) diff --git a/agenta-backend/agenta_backend/services/evaluator_manager.py b/agenta-backend/agenta_backend/services/evaluator_manager.py index 7eb32a45ff..3ebbc6d8c4 100644 --- a/agenta-backend/agenta_backend/services/evaluator_manager.py +++ b/agenta-backend/agenta_backend/services/evaluator_manager.py @@ -64,6 +64,7 @@ async def get_evaluator_config(evaluator_config: EvaluatorConfig) -> EvaluatorCo async def create_evaluator_config( project_id: str, + app_name: str, name: str, evaluator_key: str, settings_values: Optional[Dict[str, Any]] = None, @@ -72,6 +73,8 @@ async def create_evaluator_config( Create a new evaluator configuration for an app. Args: + project_id (str): The ID of the project. + app_name (str): The name of the app. name (str): The name of the evaluator config. evaluator_key (str): The key of the evaluator. settings_values (Optional[Dict[str, Any]]): Additional settings for the evaluator. @@ -82,6 +85,7 @@ async def create_evaluator_config( evaluator_config = await db_manager.create_evaluator_config( project_id=project_id, + app_name=app_name, name=name, evaluator_key=evaluator_key, settings_values=settings_values, @@ -121,7 +125,7 @@ async def delete_evaluator_config(evaluator_config_id: str) -> bool: return await db_manager.delete_evaluator_config(evaluator_config_id) -async def create_ready_to_use_evaluators(project_id: str): +async def create_ready_to_use_evaluators(app_name: str, project_id: str): """ Create configurations for all evaluators that are marked for direct use. @@ -130,6 +134,7 @@ async def create_ready_to_use_evaluators(project_id: str): in the database using the database manager. Args: + app_name (str): The name of the application. project_id (str): The ID of the project. Returns: @@ -157,6 +162,7 @@ async def create_ready_to_use_evaluators(project_id: str): ), f"'name' and 'key' does not exist in the evaluator: {evaluator}" await db_manager.create_evaluator_config( project_id=project_id, + app_name=app_name, name=evaluator.name, evaluator_key=evaluator.key, settings_values=settings_values, From d2fb567356d35e01cdb9ed7889579128a18d274b Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 16 Sep 2024 17:10:56 +0100 Subject: [PATCH 24/57] refactor (backend): improve authentication middleware to: - set project_id from request (if it exists) to request.state - otherwise, set the default project id to request.state - improve try-except-finally to set default user_id --- .../agenta_backend/services/auth_helper.py | 30 ++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/agenta-backend/agenta_backend/services/auth_helper.py b/agenta-backend/agenta_backend/services/auth_helper.py index d48c75357c..27b3fbb811 100644 --- a/agenta-backend/agenta_backend/services/auth_helper.py +++ b/agenta-backend/agenta_backend/services/auth_helper.py @@ -1,8 +1,15 @@ +import logging + from fastapi import Request, HTTPException +from agenta_backend.utils.project_utils import retrieve_project_id_from_request from agenta_backend.services.db_manager import fetch_default_project, NoResultFound +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + class SessionContainer(object): """dummy class""" @@ -21,20 +28,29 @@ def inner_function(): async def authentication_middleware(request: Request, call_next): try: - if not hasattr(request.state, "user_id"): - user_uid_id = "0" - setattr(request.state, "user_id", user_uid_id) - - if not hasattr(request.state, "project_id"): - project = await fetch_default_project() + # Retrieve project_id from request + project_id_from_request = await retrieve_project_id_from_request( + request=request + ) + + # Set project_id if found or fetch default + if project_id_from_request and not hasattr(request.state, "project_id"): + setattr(request.state, "project_id", project_id_from_request) + elif not project_id_from_request: + project = await fetch_default_project() # Fetch the default project if project is None: raise NoResultFound("Default project not found.") - setattr(request.state, "project_id", str(project.id)) # Call the next middleware or route handler response = await call_next(request) return response except Exception as e: + # Handle exceptions, set status code status_code = e.status_code if hasattr(e, "status_code") else 500 raise HTTPException(status_code=status_code, detail=str(e)) + finally: + # Set default user_id if not already set + if not hasattr(request.state, "user_id"): + logger.warning("user_id not found in request.state, setting to default '0'") + setattr(request.state, "user_id", "0") From 4c3ba31cc14ecaec919354852b8d92ca35f42182 Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 16 Sep 2024 18:02:17 +0100 Subject: [PATCH 25/57] refactor (backend): clean up route endpoints to make use of project_id from request.state --- .../agenta_backend/routers/app_router.py | 88 +++++++------------ .../agenta_backend/routers/bases_router.py | 1 - .../agenta_backend/routers/configs_router.py | 28 +++--- .../routers/container_router.py | 30 +++---- .../routers/environment_router.py | 10 +-- .../routers/evaluation_router.py | 76 ++++++---------- .../routers/evaluators_router.py | 24 ++--- .../routers/human_evaluation_router.py | 66 +++----------- .../agenta_backend/routers/testset_router.py | 47 +++++----- .../agenta_backend/routers/variants_router.py | 80 ++++++----------- .../agenta_backend/services/auth_helper.py | 13 +-- 11 files changed, 162 insertions(+), 301 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index 8e46084a7b..b54deb6daa 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -8,7 +8,6 @@ from fastapi import HTTPException, Request from agenta_backend.models import converters -from agenta_backend.utils import project_utils from agenta_backend.utils.common import ( isEE, isCloudProd, @@ -91,7 +90,6 @@ async def list_app_variants( app_id: str, request: Request, - project_id: Optional[str] = None, ): """ Retrieve a list of app variants for a given app ID. @@ -104,9 +102,6 @@ async def list_app_variants( List[AppVariantResponse]: A list of app variants for the given app ID. """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -123,7 +118,7 @@ async def list_app_variants( ) app_variants = await db_manager.list_app_variants( - app_id=app_id, project_id=project_id + app_id=app_id, project_id=request.state.project_id ) return [ await converters.app_variant_db_to_output(app_variant) @@ -146,7 +141,6 @@ async def get_variant_by_env( app_id: str, environment: str, request: Request, - project_id: Optional[str] = None, ): """ Retrieve the app variant based on the provided app_id and environment. @@ -204,7 +198,6 @@ async def get_variant_by_env( async def create_app( payload: CreateApp, request: Request, - project_id: Optional[str] = None, ) -> CreateAppOutput: """ Create a new app for a user or organization. @@ -220,9 +213,6 @@ async def create_app( HTTPException: If there is an error creating the app or the user does not have permission to access the app. """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) if isCloudEE(): api_key_from_headers = request.headers.get("Authorization") if api_key_from_headers is not None: @@ -243,7 +233,7 @@ async def create_app( has_permission = await check_rbac_permission( user_org_workspace_data=user_org_workspace_data, - project_id=project_id, + project_id=request.state.project_id, permission=Permission.CREATE_APPLICATION, ) logger.debug( @@ -258,7 +248,9 @@ async def create_app( except Exception as e: raise HTTPException(status_code=500, detail=str(e)) - app_db = await db_manager.create_app_and_envs(payload.app_name, project_id) + app_db = await db_manager.create_app_and_envs( + payload.app_name, request.state.project_id + ) return CreateAppOutput(app_id=str(app_db.id), app_name=str(app_db.app_name)) except Exception as e: logger.exception(f"An error occurred: {str(e)}") @@ -270,7 +262,6 @@ async def update_app( app_id: str, payload: UpdateApp, request: Request, - project_id: Optional[str] = None, ) -> UpdateAppOutput: """ Update an app for a user or organization. @@ -288,7 +279,7 @@ async def update_app( """ try: - app = await db_manager.fetch_app_by_id(app_id) + app = await db_manager.fetch_app_by_id(app_id, request.state.project_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -315,9 +306,6 @@ async def update_app( async def list_apps( request: Request, app_name: Optional[str] = None, - org_id: Optional[str] = None, - workspace_id: Optional[str] = None, - project_id: Optional[str] = None, ) -> List[App]: """ Retrieve a list of apps filtered by app_name and org_id. @@ -334,10 +322,9 @@ async def list_apps( HTTPException: If there was an error retrieving the list of apps. """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id + apps = await db_manager.list_apps( + project_id=request.state.project_id, app_name=app_name ) - apps = await db_manager.list_apps(project_id=project_id, app_name=app_name) return apps except Exception as e: logger.exception(f"An error occurred: {str(e)}") @@ -349,7 +336,6 @@ async def add_variant_from_image( app_id: str, payload: AddVariantFromImagePayload, request: Request, - project_id: Optional[str] = None, ): """ Add a new variant to an app based on a Docker image. @@ -366,7 +352,6 @@ async def add_variant_from_image( dict: The newly added variant. """ - project_id = project_utils.get_project_id(request=request, project_id=project_id) if not isCloudEE(): image = Image( type="image", @@ -382,12 +367,12 @@ async def add_variant_from_image( raise HTTPException(status_code=404, detail="Image not found") try: - app = await db_manager.fetch_app_by_id(app_id, project_id) + app = await db_manager.fetch_app_by_id(app_id, request.state.project_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=project_id, + project_id=request.state.project_id, permission=Permission.CREATE_APPLICATION, ) logger.debug( @@ -402,7 +387,7 @@ async def add_variant_from_image( variant_db = await app_manager.add_variant_based_on_image( app=app, - project_id=project_id, + project_id=request.state.project_id, variant_name=payload.variant_name, docker_id_or_template_uri=payload.docker_id, tags=payload.tags, @@ -412,12 +397,12 @@ async def add_variant_from_image( user_uid=request.state.user_id, ) app_variant_db = await db_manager.fetch_app_variant_by_id( - str(variant_db.id), project_id + str(variant_db.id), request.state.project_id ) logger.debug("Step 8: We create ready-to use evaluators") await evaluator_manager.create_ready_to_use_evaluators( - app_name=app.app_name, project_id=project_id + app_name=app.app_name, project_id=request.state.project_id ) return await converters.app_variant_db_to_output(app_variant_db) @@ -430,7 +415,6 @@ async def add_variant_from_image( async def remove_app( app_id: str, request: Request, - project_id: Optional[str] = None, ): """Remove app, all its variant, containers and images @@ -438,10 +422,7 @@ async def remove_app( app -- App to remove """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) - app = await db_manager.fetch_app_by_id(app_id, project_id) + app = await db_manager.fetch_app_by_id(app_id, request.state.project_id) if isCloudEE(): has_permission = await check_action_access( @@ -457,7 +438,7 @@ async def remove_app( status_code=403, ) - await app_manager.remove_app(app, project_id) + await app_manager.remove_app(app, request.state.project_id) except DockerException as e: detail = f"Docker error while trying to remove the app: {str(e)}" logger.exception(f"Docker error while trying to remove the app: {str(e)}") @@ -475,7 +456,6 @@ async def remove_app( async def create_app_and_variant_from_template( payload: CreateAppVariant, request: Request, - project_id: Optional[str] = None, ) -> AppVariantResponse: """ Create an app and variant from a template. @@ -493,9 +473,6 @@ async def create_app_and_variant_from_template( try: logger.debug("Start: Creating app and variant from template") - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) if isCloudEE(): # Get user and org id logger.debug("Step 1: Getting user and organization ID") @@ -503,18 +480,16 @@ async def create_app_and_variant_from_template( request.state.user_id ) - logger.debug( - "Step 2: Checking that workspace ID and organization ID are provided" - ) - if payload.organization_id is None or payload.workspace_id is None: + logger.debug("Step 2: Checking that Project ID is provided") + if request.state.project_id is None: raise Exception( - "Organization ID and Workspace ID must be provided to create app from template", + "Project ID must be provided to create app from template", ) logger.debug("Step 3: Checking user has permission to create app") has_permission = await check_rbac_permission( user_org_workspace_data=user_org_workspace_data, - project_id=project_id, + project_id=request.state.project_id, permission=Permission.CREATE_APPLICATION, ) logger.debug( @@ -534,7 +509,9 @@ async def create_app_and_variant_from_template( else f"Step 1: Checking if app {payload.app_name} already exists" ) app_name = payload.app_name.lower() - app = await db_manager.fetch_app_by_name_and_parameters(app_name, project_id) + app = await db_manager.fetch_app_by_name_and_parameters( + app_name, request.state.project_id + ) if app is not None: raise Exception( f"App with name {app_name} already exists", @@ -546,7 +523,9 @@ async def create_app_and_variant_from_template( else "Step 2: Creating new app and initializing environments" ) if app is None: - app = await db_manager.create_app_and_envs(app_name, project_id) + app = await db_manager.create_app_and_envs( + app_name, request.state.project_id + ) logger.debug( "Step 6: Retrieve template from db" @@ -564,7 +543,7 @@ async def create_app_and_variant_from_template( ) app_variant_db = await app_manager.add_variant_based_on_image( app=app, - project_id=project_id, + project_id=request.state.project_id, variant_name="app.default", docker_id_or_template_uri=( # type: ignore template_db.template_uri if isCloudProd() else template_db.digest @@ -584,7 +563,7 @@ async def create_app_and_variant_from_template( await db_manager.add_testset_to_app_variant( template_name=template_db.name, # type: ignore app_name=app.app_name, # type: ignore - project_id=project_id, + project_id=request.state.project_id, ) logger.debug( @@ -593,7 +572,7 @@ async def create_app_and_variant_from_template( else "Step 6: We create ready-to use evaluators" ) await evaluator_manager.create_ready_to_use_evaluators( - app_name=app.app_name, project_id=project_id + app_name=app.app_name, project_id=request.state.project_id ) logger.debug( @@ -631,7 +610,9 @@ async def create_app_and_variant_from_template( envvars[key] = os.environ[key] else: envvars = {} if payload.env_vars is None else payload.env_vars - await app_manager.start_variant(app_variant_db, project_id, envvars) + await app_manager.start_variant( + app_variant_db, request.state.project_id, envvars + ) logger.debug("End: Successfully created app and variant") return await converters.app_variant_db_to_output(app_variant_db) @@ -652,7 +633,6 @@ async def create_app_and_variant_from_template( async def list_environments( app_id: str, request: Request, - project_id: Optional[str] = None, ): """ Retrieve a list of environments for a given app ID. @@ -666,9 +646,6 @@ async def list_environments( """ logger.debug(f"Listing environments for app: {app_id}") try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -685,7 +662,7 @@ async def list_environments( ) environments_db = await db_manager.list_environments( - app_id=app_id, project_id=project_id + app_id=app_id, project_id=request.state.project_id ) logger.debug(f"environments_db: {environments_db}") return [ @@ -705,7 +682,6 @@ async def list_app_environment_revisions( request: Request, app_id: str, environment_name, - project_id: Optional[str] = None, ): logger.debug("getting environment " + environment_name) user_org_workspace_data: dict = await get_user_org_and_workspace_id( diff --git a/agenta-backend/agenta_backend/routers/bases_router.py b/agenta-backend/agenta_backend/routers/bases_router.py index d983cea0a2..6e98a06656 100644 --- a/agenta-backend/agenta_backend/routers/bases_router.py +++ b/agenta-backend/agenta_backend/routers/bases_router.py @@ -24,7 +24,6 @@ async def list_bases( request: Request, app_id: str, base_name: Optional[str] = None, - project_id: Optional[str] = None, ) -> List[BaseOutput]: """ Retrieve a list of bases filtered by app_id and base_name. diff --git a/agenta-backend/agenta_backend/routers/configs_router.py b/agenta-backend/agenta_backend/routers/configs_router.py index fc27c9465f..538b46c466 100644 --- a/agenta-backend/agenta_backend/routers/configs_router.py +++ b/agenta-backend/agenta_backend/routers/configs_router.py @@ -4,7 +4,7 @@ from fastapi.responses import JSONResponse from fastapi import Request, HTTPException -from agenta_backend.utils import project_utils + from agenta_backend.utils.common import APIRouter, isCloudEE from agenta_backend.models.api.api_models import ( SaveConfigPayload, @@ -29,13 +29,11 @@ async def save_config( payload: SaveConfigPayload, request: Request, - project_id: Optional[str] = None, ): try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id + base_db = await db_manager.fetch_base_by_id( + payload.base_id, request.state.project_id ) - base_db = await db_manager.fetch_base_by_id(payload.base_id, project_id) if isCloudEE(): has_permission = await check_action_access( @@ -51,7 +49,9 @@ async def save_config( status_code=403, ) - variants_db = await db_manager.list_variants_for_base(base_db, project_id) + variants_db = await db_manager.list_variants_for_base( + base_db, request.state.project_id + ) variant_to_overwrite = None for variant_db in variants_db: if variant_db.config_name == payload.config_name: @@ -65,14 +65,14 @@ async def save_config( app_variant_id=str(variant_to_overwrite.id), parameters=payload.parameters, user_uid=request.state.user_id, - project_id=project_id, + project_id=request.state.project_id, ) logger.debug("Deploying to production environment") await db_manager.deploy_to_environment( environment_name="production", variant_id=str(variant_to_overwrite.id), - project_id=project_id, + project_id=request.state.project_id, user_uid=request.state.user_id, ) else: @@ -89,7 +89,7 @@ async def save_config( new_config_name=payload.config_name, parameters=payload.parameters, user_uid=request.state.user_id, - project_id=project_id, + project_id=request.state.project_id, ) except HTTPException as e: @@ -110,7 +110,6 @@ async def get_config( base_id: str, config_name: Optional[str] = None, environment_name: Optional[str] = None, - project_id: Optional[str] = None, ): try: base_db = await db_manager.fetch_base_by_id(base_id) @@ -133,7 +132,8 @@ async def get_config( # in case environment_name is provided, find the variant deployed if environment_name: app_environments = await db_manager.list_environments( - app_id=str(base_db.app_id) # type: ignore + app_id=str(base_db.app_id), # type: ignore + project_id=request.state.project_id, ) found_variant_revision = next( ( @@ -160,7 +160,9 @@ async def get_config( "parameters": found_variant_revision.config_parameters, } elif config_name: - variants_db = await db_manager.list_variants_for_base(base_db) + variants_db = await db_manager.list_variants_for_base( + base_db, request.state.project_id + ) found_variant = next( ( variant_db @@ -207,7 +209,6 @@ async def get_config( async def get_config_deployment_revision( request: Request, deployment_revision_id: str, - project_id: Optional[str] = None, ): try: environment_revision = await db_manager.fetch_app_environment_revision( @@ -247,7 +248,6 @@ async def get_config_deployment_revision( async def revert_deployment_revision( request: Request, deployment_revision_id: str, - project_id: Optional[str] = None, ): environment_revision = await db_manager.fetch_app_environment_revision( deployment_revision_id diff --git a/agenta-backend/agenta_backend/routers/container_router.py b/agenta-backend/agenta_backend/routers/container_router.py index b64fee1176..ed8a737a32 100644 --- a/agenta-backend/agenta_backend/routers/container_router.py +++ b/agenta-backend/agenta_backend/routers/container_router.py @@ -4,7 +4,7 @@ from fastapi.responses import JSONResponse from fastapi import Request, UploadFile, HTTPException -from agenta_backend.utils import project_utils + from agenta_backend.services import db_manager from agenta_backend.utils.common import ( APIRouter, @@ -51,7 +51,6 @@ async def build_image( base_name: str, tar_file: UploadFile, request: Request, - project_id: Optional[str] = None, ) -> Image: """ Builds a Docker image from a tar file containing the application code. @@ -66,16 +65,13 @@ async def build_image( Image: The Docker image that was built. """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) - app_db = await db_manager.fetch_app_by_id(app_id, project_id) + app_db = await db_manager.fetch_app_by_id(app_id, request.state.project_id) # Check app access if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=project_id, + project_id=request.state.project_id, permission=Permission.CREATE_APPLICATION, ) if not has_permission: @@ -101,7 +97,6 @@ async def build_image( async def restart_docker_container( payload: RestartAppContainer, request: Request, - project_id: Optional[str] = None, ) -> dict: """Restart docker container. @@ -109,13 +104,13 @@ async def restart_docker_container( payload (RestartAppContainer) -- the required data (app_name and variant_name) """ logger.debug(f"Restarting container for variant {payload.variant_id}") - project_id = project_utils.get_project_id(request=request, project_id=project_id) + app_variant_db = await db_manager.fetch_app_variant_by_id( - payload.variant_id, project_id + payload.variant_id, request.state.project_id ) try: deployment = await db_manager.get_deployment_by_id( - app_variant_db.base.deployment, project_id + app_variant_db.base.deployment, request.state.project_id ) container_id = deployment.container_id @@ -129,7 +124,6 @@ async def restart_docker_container( @router.get("/templates/", operation_id="container_templates") async def container_templates( request: Request, - project_id: Optional[str] = None, ) -> Union[List[Template], str]: """ Returns a list of templates available for creating new containers. @@ -153,7 +147,6 @@ async def construct_app_container_url( request: Request, base_id: Optional[str] = None, variant_id: Optional[str] = None, - project_id: Optional[str] = None, ) -> URI: """ Constructs the URL for an app container based on the provided base_id or variant_id. @@ -172,9 +165,8 @@ async def construct_app_container_url( # assert that one of base_id or variant_id is provided assert base_id or variant_id, "Please provide either base_id or variant_id" - project_id = project_utils.get_project_id(request=request, project_id=project_id) if base_id: - object_db = await db_manager.fetch_base_by_id(base_id, project_id) + object_db = await db_manager.fetch_base_by_id(base_id, request.state.project_id) elif variant_id and variant_id != "None": # NOTE: Backward Compatibility # --------------------------- @@ -185,7 +177,9 @@ async def construct_app_container_url( # This change ensures that users can still view their evaluations; however, # they will no longer be able to access a deployment URL for the deleted variant. # Therefore, we ensure that variant_id is not "None". - object_db = await db_manager.fetch_app_variant_by_id(variant_id, project_id) + object_db = await db_manager.fetch_app_variant_by_id( + variant_id, request.state.project_id + ) else: # NOTE: required for backward compatibility object_db = None @@ -205,11 +199,11 @@ async def construct_app_container_url( try: if getattr(object_db, "deployment_id", None): # this is a base deployment = await db_manager.get_deployment_by_id( - str(object_db.deployment_id), project_id # type: ignore + str(object_db.deployment_id), request.state.project_id # type: ignore ) elif getattr(object_db, "base_id", None): # this is a variant deployment = await db_manager.get_deployment_by_id( - str(object_db.base.deployment_id), project_id # type: ignore + str(object_db.base.deployment_id), request.state.project_id # type: ignore ) else: raise HTTPException( diff --git a/agenta-backend/agenta_backend/routers/environment_router.py b/agenta-backend/agenta_backend/routers/environment_router.py index 2e88e2ea30..d93a860742 100644 --- a/agenta-backend/agenta_backend/routers/environment_router.py +++ b/agenta-backend/agenta_backend/routers/environment_router.py @@ -4,7 +4,7 @@ from fastapi.responses import JSONResponse from fastapi import Request, HTTPException -from agenta_backend.utils import project_utils + from agenta_backend.services import db_manager, app_manager from agenta_backend.utils.common import APIRouter, isCloudEE from agenta_backend.models.api.api_models import DeployToEnvironmentPayload @@ -22,7 +22,6 @@ async def deploy_to_environment( payload: DeployToEnvironmentPayload, request: Request, - project_id: Optional[str] = None, ): """Deploys a given variant to an environment @@ -35,9 +34,6 @@ async def deploy_to_environment( HTTPException: If the deployment fails. """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -57,7 +53,7 @@ async def deploy_to_environment( await db_manager.deploy_to_environment( environment_name=payload.environment_name, variant_id=payload.variant_id, - project_id=project_id, + project_id=request.state.project_id, user_uid=request.state.user_id, ) @@ -66,7 +62,7 @@ async def deploy_to_environment( user_uid=request.state.user_id, object_id=payload.variant_id, object_type="variant", - project_id=project_id, + project_id=request.state.project_id, ) logger.debug("Successfully updated last_modified_by app information") except Exception as e: diff --git a/agenta-backend/agenta_backend/routers/evaluation_router.py b/agenta-backend/agenta_backend/routers/evaluation_router.py index a8c60c9d7e..51125ac4f8 100644 --- a/agenta-backend/agenta_backend/routers/evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/evaluation_router.py @@ -6,7 +6,7 @@ from fastapi import HTTPException, Request, status, Response, Query from agenta_backend.models import converters -from agenta_backend.utils import project_utils + from agenta_backend.tasks.evaluations import evaluate from agenta_backend.utils.common import APIRouter, isCloudEE from agenta_backend.models.api.evaluation_model import ( @@ -39,7 +39,6 @@ async def fetch_evaluation_ids( resource_type: str, request: Request, resource_ids: List[str] = Query(None), - project_id: Optional[str] = None, ): """Fetches evaluation ids for a given resource type and id. @@ -55,13 +54,10 @@ async def fetch_evaluation_ids( List[str]: A list of evaluation ids. """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=project_id, + project_id=request.state.project_id, permission=Permission.VIEW_EVALUATION, ) logger.debug( @@ -75,7 +71,7 @@ async def fetch_evaluation_ids( status_code=403, ) evaluations = await db_manager.fetch_evaluations_by_resource( - resource_type, project_id, resource_ids + resource_type, request.state.project_id, resource_ids ) return list(map(lambda x: str(x.id), evaluations)) except Exception as exc: @@ -89,7 +85,6 @@ async def fetch_evaluation_ids( async def create_evaluation( payload: NewEvaluation, request: Request, - project_id: Optional[str] = None, ): """Creates a new comparison table document Raises: @@ -98,11 +93,8 @@ async def create_evaluation( _description_ """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) app = await db_manager.fetch_app_by_id( - app_id=payload.app_id, project_id=project_id + app_id=payload.app_id, project_id=request.state.project_id ) if app is None: raise HTTPException(status_code=404, detail="App not found") @@ -110,7 +102,7 @@ async def create_evaluation( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=project_id, + project_id=request.state.project_id, permission=Permission.CREATE_EVALUATION, ) logger.debug(f"User has permission to create evaluation: {has_permission}") @@ -133,14 +125,14 @@ async def create_evaluation( for variant_id in payload.variant_ids: evaluation = await evaluation_service.create_new_evaluation( app_id=payload.app_id, - project_id=project_id, + project_id=request.state.project_id, variant_id=variant_id, testset_id=payload.testset_id, ) evaluate.delay( app_id=payload.app_id, - project_id=project_id, + project_id=request.state.project_id, variant_id=variant_id, evaluators_config_ids=payload.evaluators_configs, testset_id=payload.testset_id, @@ -155,7 +147,7 @@ async def create_evaluation( user_uid=request.state.user_id, object_id=payload.app_id, object_type="app", - project_id=project_id, + project_id=request.state.project_id, ) logger.debug("Successfully updated last_modified_by app information") @@ -177,7 +169,6 @@ async def create_evaluation( async def fetch_evaluation_status( evaluation_id: str, request: Request, - project_id: Optional[str] = None, ): """Fetches the status of the evaluation. @@ -190,10 +181,9 @@ async def fetch_evaluation_status( """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id + evaluation = await db_manager.fetch_evaluation_by_id( + evaluation_id, request.state.project_id ) - evaluation = await db_manager.fetch_evaluation_by_id(evaluation_id, project_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -220,7 +210,6 @@ async def fetch_evaluation_status( async def fetch_evaluation_results( evaluation_id: str, request: Request, - project_id: Optional[str] = None, ): """Fetches the results of the evaluation @@ -233,11 +222,8 @@ async def fetch_evaluation_results( """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) evaluation = await db_manager.fetch_evaluation_by_id( - evaluation_id, project_id=project_id + evaluation_id, project_id=request.state.project_id ) if isCloudEE(): has_permission = await check_action_access( @@ -272,7 +258,6 @@ async def fetch_evaluation_results( async def fetch_evaluation_scenarios( evaluation_id: str, request: Request, - project_id: Optional[str] = None, ): """Fetches evaluation scenarios for a given evaluation ID. @@ -287,10 +272,9 @@ async def fetch_evaluation_scenarios( """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id + evaluation = await db_manager.fetch_evaluation_by_id( + evaluation_id, request.state.project_id ) - evaluation = await db_manager.fetch_evaluation_by_id(evaluation_id, project_id) if not evaluation: raise HTTPException( status_code=404, detail=f"Evaluation with id {evaluation_id} not found" @@ -315,7 +299,7 @@ async def fetch_evaluation_scenarios( eval_scenarios = ( await evaluation_service.fetch_evaluation_scenarios_for_evaluation( - evaluation_id=str(evaluation.id) + evaluation_id=str(evaluation.id), project_id=request.state.project_id ) ) return eval_scenarios @@ -332,7 +316,6 @@ async def fetch_evaluation_scenarios( async def fetch_list_evaluations( app_id: str, request: Request, - project_id: Optional[str] = None, ): """Fetches a list of evaluations, optionally filtered by an app ID. @@ -343,10 +326,7 @@ async def fetch_list_evaluations( List[Evaluation]: A list of evaluations. """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) - app = await db_manager.fetch_app_by_id(app_id, project_id) + app = await db_manager.fetch_app_by_id(app_id, request.state.project_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -364,7 +344,9 @@ async def fetch_list_evaluations( status_code=403, ) - return await evaluation_service.fetch_list_evaluations(app, project_id) + return await evaluation_service.fetch_list_evaluations( + app, request.state.project_id + ) except Exception as exc: import traceback @@ -382,7 +364,6 @@ async def fetch_list_evaluations( async def fetch_evaluation( evaluation_id: str, request: Request, - project_id: Optional[str] = None, ): """Fetches a single evaluation based on its ID. @@ -393,10 +374,9 @@ async def fetch_evaluation( Evaluation: The fetched evaluation. """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id + evaluation = await db_manager.fetch_evaluation_by_id( + evaluation_id, request.state.project_id ) - evaluation = await db_manager.fetch_evaluation_by_id(evaluation_id, project_id) if not evaluation: raise HTTPException( status_code=404, detail=f"Evaluation with id {evaluation_id} not found" @@ -430,7 +410,6 @@ async def fetch_evaluation( async def delete_evaluations( payload: DeleteEvaluation, request: Request, - project_id: Optional[str] = None, ): """ Delete specific comparison tables based on their unique IDs. @@ -443,9 +422,6 @@ async def delete_evaluations( """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) if isCloudEE(): evaluation_id = random.choice(payload.evaluations_ids) has_permission = await check_action_access( @@ -468,11 +444,13 @@ async def delete_evaluations( user_uid=request.state.user_id, object_id=random.choice(payload.evaluations_ids), object_type="evaluation", - project_id=project_id, + project_id=request.state.project_id, ) logger.debug("Successfully updated last_modified_by app information") - await evaluation_service.delete_evaluations(payload.evaluations_ids, project_id) + await evaluation_service.delete_evaluations( + payload.evaluations_ids, request.state.project_id + ) return Response(status_code=status.HTTP_204_NO_CONTENT) except Exception as exc: raise HTTPException(status_code=500, detail=str(exc)) @@ -485,7 +463,6 @@ async def delete_evaluations( async def fetch_evaluation_scenarios( evaluations_ids: str, request: Request, - project_id: Optional[str] = None, ): """Fetches evaluation scenarios for a given evaluation ID. @@ -499,9 +476,6 @@ async def fetch_evaluation_scenarios( List[EvaluationScenario]: A list of evaluation scenarios. """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) evaluations_ids_list = evaluations_ids.split(",") if isCloudEE(): @@ -524,7 +498,7 @@ async def fetch_evaluation_scenarios( ) eval_scenarios = await evaluation_service.compare_evaluations_scenarios( - evaluations_ids_list, project_id + evaluations_ids_list, request.state.project_id ) return eval_scenarios diff --git a/agenta-backend/agenta_backend/routers/evaluators_router.py b/agenta-backend/agenta_backend/routers/evaluators_router.py index 92a27efd37..2a4c63a642 100644 --- a/agenta-backend/agenta_backend/routers/evaluators_router.py +++ b/agenta-backend/agenta_backend/routers/evaluators_router.py @@ -4,7 +4,7 @@ from fastapi import HTTPException, Request from fastapi.responses import JSONResponse -from agenta_backend.utils import project_utils + from agenta_backend.utils.common import APIRouter, isCloudEE from agenta_backend.services import evaluator_manager, db_manager, app_manager @@ -52,7 +52,6 @@ async def get_evaluators_endpoint(): async def get_evaluator_configs( app_id: str, request: Request, - project_id: Optional[str] = None, ): """Endpoint to fetch evaluator configurations for a specific app. @@ -64,9 +63,6 @@ async def get_evaluator_configs( """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -82,7 +78,9 @@ async def get_evaluator_configs( status_code=403, ) - evaluators_configs = await evaluator_manager.get_evaluators_configs(project_id) + evaluators_configs = await evaluator_manager.get_evaluators_configs( + request.state.project_id + ) return evaluators_configs except Exception as e: raise HTTPException( @@ -94,7 +92,6 @@ async def get_evaluator_configs( async def get_evaluator_config( evaluator_config_id: str, request: Request, - project_id: Optional[str] = None, ): """Endpoint to fetch evaluator configurations for a specific app. @@ -134,7 +131,6 @@ async def get_evaluator_config( async def create_new_evaluator_config( payload: NewEvaluatorConfig, request: Request, - project_id: Optional[str] = None, ): """Endpoint to fetch evaluator configurations for a specific app. @@ -145,11 +141,8 @@ async def create_new_evaluator_config( EvaluatorConfigDB: Evaluator configuration api model. """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) app_db = await db_manager.get_app_instance_by_id( - app_id=payload.app_id, project_id=project_id + app_id=payload.app_id, project_id=request.state.project_id ) if isCloudEE(): has_permission = await check_action_access( @@ -167,7 +160,7 @@ async def create_new_evaluator_config( ) evaluator_config = await evaluator_manager.create_evaluator_config( - project_id=project_id, + project_id=request.state.project_id, app_name=app_db.app_name, name=payload.name, evaluator_key=payload.evaluator_key, @@ -188,7 +181,6 @@ async def update_evaluator_config( evaluator_config_id: str, payload: UpdateEvaluatorConfig, request: Request, - project_id: Optional[str] = None, ): """Endpoint to update evaluator configurations for a specific app. @@ -197,9 +189,6 @@ async def update_evaluator_config( """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -232,7 +221,6 @@ async def update_evaluator_config( async def delete_evaluator_config( evaluator_config_id: str, request: Request, - project_id: Optional[str] = None, ): """Endpoint to delete a specific evaluator configuration. diff --git a/agenta-backend/agenta_backend/routers/human_evaluation_router.py b/agenta-backend/agenta_backend/routers/human_evaluation_router.py index c785ec9a7f..89c21463df 100644 --- a/agenta-backend/agenta_backend/routers/human_evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/human_evaluation_router.py @@ -4,7 +4,7 @@ from fastapi import HTTPException, Body, Request, status, Response from agenta_backend.models import converters -from agenta_backend.utils import project_utils + from agenta_backend.services import results_service from agenta_backend.services import evaluation_service from agenta_backend.services import db_manager, app_manager @@ -47,7 +47,6 @@ async def create_evaluation( payload: NewHumanEvaluation, request: Request, - project_id: Optional[str] = None, ): """Creates a new comparison table document Raises: @@ -56,11 +55,8 @@ async def create_evaluation( _description_ """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) app = await db_manager.fetch_app_by_id( - app_id=payload.app_id, project_id=project_id + app_id=payload.app_id, project_id=request.state.project_id ) if app is None: raise HTTPException(status_code=404, detail="App not found") @@ -80,7 +76,7 @@ async def create_evaluation( ) new_human_evaluation_db = await evaluation_service.create_new_human_evaluation( - payload, request.state.user_id, project_id + payload, request.state.user_id, request.state.project_id ) return await converters.human_evaluation_db_to_simple_evaluation_output( new_human_evaluation_db @@ -102,7 +98,6 @@ async def create_evaluation( async def fetch_list_human_evaluations( app_id: str, request: Request, - project_id: Optional[str] = None, ): """Fetches a list of evaluations, optionally filtered by an app ID. @@ -114,9 +109,6 @@ async def fetch_list_human_evaluations( """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -131,7 +123,9 @@ async def fetch_list_human_evaluations( status_code=403, ) - return await evaluation_service.fetch_list_human_evaluations(app_id, project_id) + return await evaluation_service.fetch_list_human_evaluations( + app_id, request.state.project_id + ) except Exception as e: status_code = e.status_code if hasattr(e, "status_code") else 500 # type: ignore raise HTTPException(status_code=status_code, detail=str(e)) from e @@ -141,7 +135,6 @@ async def fetch_list_human_evaluations( async def fetch_human_evaluation( evaluation_id: str, request: Request, - project_id: Optional[str] = None, ): """Fetches a single evaluation based on its ID. @@ -152,11 +145,8 @@ async def fetch_human_evaluation( HumanEvaluation: The fetched evaluation. """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) human_evaluation = await db_manager.fetch_human_evaluation_by_id( - evaluation_id, project_id + evaluation_id, request.state.project_id ) if not human_evaluation: raise HTTPException(status_code=404, detail="Evaluation not found") @@ -189,7 +179,6 @@ async def fetch_human_evaluation( async def fetch_evaluation_scenarios( evaluation_id: str, request: Request, - project_id: Optional[str] = None, ): """Fetches evaluation scenarios for a given evaluation ID. @@ -204,11 +193,8 @@ async def fetch_evaluation_scenarios( """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) human_evaluation = await db_manager.fetch_human_evaluation_by_id( - evaluation_id, project_id + evaluation_id, request.state.project_id ) if human_evaluation is None: raise HTTPException( @@ -250,7 +236,6 @@ async def update_human_evaluation( request: Request, evaluation_id: str, update_data: HumanEvaluationUpdate = Body(...), - project_id: Optional[str] = None, ): """Updates an evaluation's status. @@ -262,11 +247,8 @@ async def update_human_evaluation( """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) human_evaluation = await db_manager.fetch_human_evaluation_by_id( - evaluation_id, project_id + evaluation_id, request.state.project_id ) if not human_evaluation: raise HTTPException(status_code=404, detail="Evaluation not found") @@ -303,7 +285,6 @@ async def update_evaluation_scenario_router( evaluation_type: EvaluationType, payload: HumanEvaluationScenarioUpdate, request: Request, - project_id: Optional[str] = None, ): """Updates an evaluation scenario's vote or score based on its type. @@ -314,11 +295,8 @@ async def update_evaluation_scenario_router( None: 204 No Content status code upon successful update. """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) evaluation_scenario_db = await db_manager.fetch_human_evaluation_scenario_by_id( - evaluation_scenario_id, project_id + evaluation_scenario_id, request.state.project_id ) if evaluation_scenario_db is None: raise HTTPException( @@ -357,7 +335,6 @@ async def update_evaluation_scenario_router( async def get_evaluation_scenario_score_router( evaluation_scenario_id: str, request: Request, - project_id: Optional[str] = None, ) -> Dict[str, str]: """ Fetch the score of a specific evaluation scenario. @@ -370,11 +347,8 @@ async def get_evaluation_scenario_score_router( Dictionary containing the scenario ID and its score. """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) evaluation_scenario = db_manager.fetch_evaluation_scenario_by_id( - evaluation_scenario_id, project_id + evaluation_scenario_id, request.state.project_id ) if evaluation_scenario is None: raise HTTPException( @@ -409,7 +383,6 @@ async def update_evaluation_scenario_score_router( evaluation_scenario_id: str, payload: EvaluationScenarioScoreUpdate, request: Request, - project_id: Optional[str] = None, ): """Updates the score of an evaluation scenario. @@ -420,11 +393,8 @@ async def update_evaluation_scenario_score_router( None: 204 No Content status code upon successful update. """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) evaluation_scenario = await db_manager.fetch_evaluation_scenario_by_id( - evaluation_scenario_id, project_id + evaluation_scenario_id, request.state.project_id ) if evaluation_scenario is None: raise HTTPException( @@ -459,7 +429,6 @@ async def update_evaluation_scenario_score_router( async def fetch_results( evaluation_id: str, request: Request, - project_id: Optional[str] = None, ): """Fetch all the results for one the comparison table @@ -471,11 +440,8 @@ async def fetch_results( """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) evaluation = await db_manager.fetch_human_evaluation_by_id( - evaluation_id, project_id + evaluation_id, request.state.project_id ) if evaluation is None: raise HTTPException( @@ -516,7 +482,6 @@ async def fetch_results( async def delete_evaluations( delete_evaluations: DeleteEvaluation, request: Request, - project_id: Optional[str] = None, ): """ Delete specific comparison tables based on their unique IDs. @@ -529,9 +494,6 @@ async def delete_evaluations( """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) if isCloudEE(): evaluation_id = random.choice(delete_evaluations.evaluations_ids) has_permission = await check_action_access( @@ -548,7 +510,7 @@ async def delete_evaluations( ) await evaluation_service.delete_human_evaluations( - delete_evaluations.evaluations_ids, project_id + delete_evaluations.evaluations_ids, request.state.project_id ) return Response(status_code=status.HTTP_204_NO_CONTENT) except Exception as e: diff --git a/agenta-backend/agenta_backend/routers/testset_router.py b/agenta-backend/agenta_backend/routers/testset_router.py index ea50d3c9dc..31f6cdd7bb 100644 --- a/agenta-backend/agenta_backend/routers/testset_router.py +++ b/agenta-backend/agenta_backend/routers/testset_router.py @@ -11,7 +11,7 @@ from fastapi.responses import JSONResponse from fastapi import HTTPException, UploadFile, File, Form, Request -from agenta_backend.utils import project_utils + from agenta_backend.services import db_manager from agenta_backend.utils.common import APIRouter, isCloudEE from agenta_backend.models.converters import testset_db_to_pydantic @@ -54,7 +54,6 @@ async def upload_file( file: UploadFile = File(...), testset_name: Optional[str] = File(None), app_id: str = Form(None), - project_id: Optional[str] = None, ): """ Uploads a CSV or JSON file and saves its data to MongoDB. @@ -68,8 +67,9 @@ async def upload_file( dict: The result of the upload process. """ - project_id = project_utils.get_project_id(request=request, project_id=project_id) - app = await db_manager.fetch_app_by_id(app_id=app_id, project_id=project_id) + app = await db_manager.fetch_app_by_id( + app_id=app_id, project_id=request.state.project_id + ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -116,7 +116,7 @@ async def upload_file( try: testset = await db_manager.create_testset( - app=app, project_id=project_id, testset_data=document + app=app, project_id=request.state.project_id, testset_data=document ) return TestSetSimpleResponse( id=str(testset.id), @@ -135,7 +135,6 @@ async def import_testset( endpoint: str = Form(None), testset_name: str = Form(None), app_id: str = Form(None), - project_id: Optional[str] = None, ): """ Import JSON testset data from an endpoint and save it to MongoDB. @@ -148,8 +147,9 @@ async def import_testset( dict: The result of the import process. """ - project_id = project_utils.get_project_id(request=request, project_id=project_id) - app = await db_manager.fetch_app_by_id(app_id=app_id, project_id=project_id) + app = await db_manager.fetch_app_by_id( + app_id=app_id, project_id=request.state.project_id + ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -184,7 +184,7 @@ async def import_testset( document["csvdata"].append(row) testset = await db_manager.create_testset( - app=app, project_id=project_id, testset_data=document + app=app, project_id=request.state.project_id, testset_data=document ) return TestSetSimpleResponse( id=str(testset.id), @@ -214,7 +214,6 @@ async def create_testset( app_id: str, csvdata: NewTestset, request: Request, - project_id: Optional[str] = None, ): """ Create a testset with given name and app_name, save the testset to MongoDB. @@ -228,8 +227,9 @@ async def create_testset( str: The id of the test set created. """ - project_id = project_utils.get_project_id(request=request, project_id=project_id) - app = await db_manager.fetch_app_by_id(app_id=app_id, project_id=project_id) + app = await db_manager.fetch_app_by_id( + app_id=app_id, project_id=request.state.project_id + ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -251,7 +251,7 @@ async def create_testset( "csvdata": csvdata.csvdata, } testset_instance = await db_manager.create_testset( - app=app, project_id=project_id, testset_data=testset_data + app=app, project_id=request.state.project_id, testset_data=testset_data ) if testset_instance is not None: return TestSetSimpleResponse( @@ -269,7 +269,6 @@ async def update_testset( testset_id: str, csvdata: NewTestset, request: Request, - project_id: Optional[str] = None, ): """ Update a testset with given id, update the testset in MongoDB. @@ -282,9 +281,8 @@ async def update_testset( str: The id of the test set updated. """ - project_id = project_utils.get_project_id(request=request, project_id=project_id) testset = await db_manager.fetch_testset_by_id( - testset_id=testset_id, project_id=project_id + testset_id=testset_id, project_id=request.state.project_id ) if testset is None: raise HTTPException(status_code=404, detail="testset not found") @@ -326,7 +324,6 @@ async def update_testset( async def get_testsets( app_id: str, request: Request, - project_id: Optional[str] = None, ) -> List[TestSetOutputResponse]: """ Get all testsets. @@ -338,8 +335,9 @@ async def get_testsets( - `HTTPException` with status code 404 if no testsets are found. """ - project_id = project_utils.get_project_id(request=request, project_id=project_id) - app = await db_manager.fetch_app_by_id(app_id=app_id, project_id=project_id) + app = await db_manager.fetch_app_by_id( + app_id=app_id, project_id=request.state.project_id + ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -355,7 +353,9 @@ async def get_testsets( status_code=403, ) - testsets = await db_manager.fetch_testsets_by_app_id(project_id=project_id) + testsets = await db_manager.fetch_testsets_by_app_id( + project_id=request.state.project_id + ) return [ TestSetOutputResponse( _id=str(testset.id), # type: ignore @@ -370,7 +370,6 @@ async def get_testsets( async def get_single_testset( testset_id: str, request: Request, - project_id: Optional[str] = None, ): """ Fetch a specific testset in a MongoDB collection using its _id. @@ -383,11 +382,8 @@ async def get_single_testset( """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) test_set = await db_manager.fetch_testset_by_id( - testset_id=testset_id, project_id=project_id + testset_id=testset_id, project_id=request.state.project_id ) if isCloudEE(): has_permission = await check_action_access( @@ -416,7 +412,6 @@ async def get_single_testset( async def delete_testsets( payload: DeleteTestsets, request: Request, - project_id: Optional[str] = None, ): """ Delete specific testsets based on their unique IDs. diff --git a/agenta-backend/agenta_backend/routers/variants_router.py b/agenta-backend/agenta_backend/routers/variants_router.py index 9c7224b3bb..3952608506 100644 --- a/agenta-backend/agenta_backend/routers/variants_router.py +++ b/agenta-backend/agenta_backend/routers/variants_router.py @@ -6,7 +6,7 @@ from fastapi import HTTPException, Request, Body from agenta_backend.models import converters -from agenta_backend.utils import project_utils + from agenta_backend.utils.common import APIRouter, isCloudEE from agenta_backend.services import ( app_manager, @@ -51,7 +51,6 @@ async def add_variant_from_base_and_config( payload: AddVariantFromBasePayload, request: Request, - project_id: Optional[str] = None, ) -> Union[AppVariantResponse, Any]: """Add a new variant based on an existing one. Same as POST /config @@ -69,10 +68,10 @@ async def add_variant_from_base_and_config( try: logger.debug("Initiating process to add a variant based on a previous one.") logger.debug(f"Received payload: {payload}") - project_id = project_utils.get_project_id( - request=request, project_id=project_id + + base_db = await db_manager.fetch_base_by_id( + payload.base_id, request.state.project_id ) - base_db = await db_manager.fetch_base_by_id(payload.base_id) # Check user has permission to add variant if isCloudEE(): @@ -98,7 +97,7 @@ async def add_variant_from_base_and_config( new_config_name=payload.new_config_name, parameters=payload.parameters, user_uid=request.state.user_id, - project_id=project_id, + project_id=request.state.project_id, ) logger.debug(f"Successfully added new variant: {db_app_variant}") @@ -107,12 +106,12 @@ async def add_variant_from_base_and_config( user_uid=request.state.user_id, object_id=str(db_app_variant.app_id), object_type="app", - project_id=project_id, + project_id=request.state.project_id, ) logger.debug("Successfully updated last_modified_by app information") app_variant_db = await db_manager.get_app_variant_instance_by_id( - str(db_app_variant.id), project_id + str(db_app_variant.id), request.state.project_id ) return await converters.app_variant_db_to_output(app_variant_db) @@ -128,7 +127,6 @@ async def add_variant_from_base_and_config( async def remove_variant( variant_id: str, request: Request, - project_id: Optional[str] = None, ): """Remove a variant from the server. In the case it's the last variant using the image, stop the container and remove the image. @@ -141,9 +139,6 @@ async def remove_variant( """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -165,12 +160,12 @@ async def remove_variant( user_uid=request.state.user_id, object_id=variant_id, object_type="variant", - project_id=project_id, + project_id=request.state.project_id, ) logger.debug("Successfully updated last_modified_by app information") await app_manager.terminate_and_remove_app_variant( - project_id=project_id, app_variant_id=variant_id + project_id=request.state.project_id, app_variant_id=variant_id ) except DockerException as e: detail = f"Docker error while trying to remove the app variant: {str(e)}" @@ -188,7 +183,6 @@ async def update_variant_parameters( request: Request, variant_id: str, payload: UpdateVariantParameterPayload = Body(...), - project_id: Optional[str] = None, ): """ Updates the parameters for an app variant. @@ -205,9 +199,6 @@ async def update_variant_parameters( JSONResponse: A JSON response containing the updated app variant parameters. """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -230,7 +221,7 @@ async def update_variant_parameters( app_variant_id=variant_id, parameters=payload.parameters, user_uid=request.state.user_id, - project_id=project_id, + project_id=request.state.project_id, ) # Update last_modified_by app information @@ -238,7 +229,7 @@ async def update_variant_parameters( user_uid=request.state.user_id, object_id=variant_id, object_type="variant", - project_id=project_id, + project_id=request.state.project_id, ) logger.debug("Successfully updated last_modified_by app information") except ValueError as e: @@ -257,7 +248,6 @@ async def update_variant_image( variant_id: str, image: Image, request: Request, - project_id: Optional[str] = None, ): """ Updates the image used in an app variant. @@ -273,17 +263,14 @@ async def update_variant_image( JSONResponse: A JSON response indicating whether the update was successful or not. """ try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) db_app_variant = await db_manager.fetch_app_variant_by_id( - app_variant_id=variant_id, project_id=project_id + app_variant_id=variant_id, project_id=request.state.project_id ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=project_id, + project_id=request.state.project_id, permission=Permission.CREATE_APPLICATION, ) logger.debug( @@ -298,7 +285,7 @@ async def update_variant_image( ) await app_manager.update_variant_image( - db_app_variant, project_id, image, request.state.user_id + db_app_variant, request.state.project_id, image, request.state.user_id ) # Update last_modified_by app information @@ -306,7 +293,7 @@ async def update_variant_image( user_uid=request.state.user_id, object_id=str(db_app_variant.app_id), object_type="app", - project_id=project_id, + project_id=request.state.project_id, ) logger.debug("Successfully updated last_modified_by app information") except ValueError as e: @@ -335,7 +322,6 @@ async def start_variant( variant_id: str, action: VariantAction, env_vars: Optional[DockerEnvVars] = None, - project_id: Optional[str] = None, ) -> URI: """ Start a variant of an app. @@ -353,16 +339,15 @@ async def start_variant( HTTPException: If the app container cannot be started. """ - project_id = project_utils.get_project_id(request=request, project_id=project_id) app_variant_db = await db_manager.fetch_app_variant_by_id( - app_variant_id=variant_id, project_id=project_id + app_variant_id=variant_id, project_id=request.state.project_id ) # Check user has permission to start variant if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=project_id, + project_id=request.state.project_id, permission=Permission.CREATE_APPLICATION, ) logger.debug(f"User has Permission to start variant: {has_permission}") @@ -380,13 +365,15 @@ async def start_variant( envvars = {} if env_vars is None else env_vars.env_vars if action.action == VariantActionEnum.START: - url: URI = await app_manager.start_variant(app_variant_db, project_id, envvars) + url: URI = await app_manager.start_variant( + app_variant_db, request.state.project_id, envvars + ) # Deploy to production await db_manager.deploy_to_environment( environment_name="production", variant_id=str(app_variant_db.id), - project_id=project_id, + project_id=request.state.project_id, user_uid=request.state.user_id, ) return url @@ -396,7 +383,6 @@ async def start_variant( async def retrieve_variant_logs( variant_id: str, request: Request, - project_id: Optional[str] = None, ): try: app_variant = await db_manager.fetch_app_variant_by_id(variant_id) @@ -416,21 +402,17 @@ async def retrieve_variant_logs( async def get_variant( variant_id: str, request: Request, - project_id: Optional[str] = None, ): logger.debug("getting variant " + variant_id) try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) app_variant = await db_manager.fetch_app_variant_by_id( - app_variant_id=variant_id, project_id=project_id + app_variant_id=variant_id, project_id=request.state.project_id ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=project_id, + project_id=request.state.project_id, permission=Permission.VIEW_APPLICATION, ) logger.debug(f"User has Permission to get variant: {has_permission}") @@ -456,15 +438,11 @@ async def get_variant( async def get_variant_revisions( variant_id: str, request: Request, - project_id: Optional[str] = None, ): logger.debug("getting variant revisions: ", variant_id) try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) app_variant = await db_manager.fetch_app_variant_by_id( - app_variant_id=variant_id, project_id=project_id + app_variant_id=variant_id, project_id=request.state.project_id ) if isCloudEE(): @@ -483,7 +461,7 @@ async def get_variant_revisions( ) app_variant_revisions = await db_manager.list_app_variant_revisions_by_variant( - app_variant=app_variant, project_id=project_id + app_variant=app_variant, project_id=request.state.project_id ) return await converters.app_variant_db_revisions_to_output( app_variant_revisions @@ -502,24 +480,20 @@ async def get_variant_revision( variant_id: str, revision_number: int, request: Request, - project_id: Optional[str] = None, ): logger.debug("getting variant revision: ", variant_id, revision_number) try: - project_id = project_utils.get_project_id( - request=request, project_id=project_id - ) assert ( variant_id != "undefined" ), "Variant id is required to retrieve variant revision" app_variant = await db_manager.fetch_app_variant_by_id( - app_variant_id=variant_id, project_id=project_id + app_variant_id=variant_id, project_id=request.state.project_id ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=project_id, + project_id=request.state.project_id, permission=Permission.VIEW_APPLICATION, ) logger.debug(f"User has Permission to get variant: {has_permission}") diff --git a/agenta-backend/agenta_backend/services/auth_helper.py b/agenta-backend/agenta_backend/services/auth_helper.py index 27b3fbb811..ba9f0f9caa 100644 --- a/agenta-backend/agenta_backend/services/auth_helper.py +++ b/agenta-backend/agenta_backend/services/auth_helper.py @@ -37,10 +37,18 @@ async def authentication_middleware(request: Request, call_next): if project_id_from_request and not hasattr(request.state, "project_id"): setattr(request.state, "project_id", project_id_from_request) elif not project_id_from_request: + logger.info("Retrieving default project from database...") project = await fetch_default_project() # Fetch the default project if project is None: raise NoResultFound("Default project not found.") + setattr(request.state, "project_id", str(project.id)) + logger.info( + f"Default project fetched: {str(project.id)} and set in request.state" + ) + + if not hasattr(request.state, "user_id"): + setattr(request.state, "user_id", "0") # Call the next middleware or route handler response = await call_next(request) @@ -49,8 +57,3 @@ async def authentication_middleware(request: Request, call_next): # Handle exceptions, set status code status_code = e.status_code if hasattr(e, "status_code") else 500 raise HTTPException(status_code=status_code, detail=str(e)) - finally: - # Set default user_id if not already set - if not hasattr(request.state, "user_id"): - logger.warning("user_id not found in request.state, setting to default '0'") - setattr(request.state, "user_id", "0") From 17b4888c216cb967affbe3eb04488a73378f8e5f Mon Sep 17 00:00:00 2001 From: Abram Date: Tue, 17 Sep 2024 08:54:23 +0100 Subject: [PATCH 26/57] refactor (backend): remove body check for project_id in route request --- .../agenta_backend/utils/project_utils.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/agenta-backend/agenta_backend/utils/project_utils.py b/agenta-backend/agenta_backend/utils/project_utils.py index 798b304ac7..a1a833ead8 100644 --- a/agenta-backend/agenta_backend/utils/project_utils.py +++ b/agenta-backend/agenta_backend/utils/project_utils.py @@ -1,4 +1,3 @@ -import json import logging from typing import Optional @@ -16,7 +15,6 @@ async def retrieve_project_id_from_request(request: Request) -> Optional[str]: This function attempts to extract the `project_id` from various parts of the request: 1. Path parameters 2. Query parameters - 3. Request body (assuming it is JSON) Args: request (Request): The FastAPI `Request` object from which to extract the `project_id`. @@ -37,17 +35,5 @@ async def retrieve_project_id_from_request(request: Request) -> Optional[str]: logger.info("Project ID found in query params") return project_id_from_query_params - try: - request_body = await request.body() - if request_body: - project_id_response_str = request_body.decode("utf-8") - response_json = json.loads(project_id_response_str) - project_id_from_request_body = response_json.get("project_id") - if project_id_from_request_body: - logger.info("Project ID found in request body") - return project_id_from_request_body - except (json.JSONDecodeError, UnicodeDecodeError) as e: - logger.error(f"Error decoding request body: {e}") - logger.info("No project ID found in the request") return None From 7e2c5422511850624dfdd50c8b3af6a4a945c8c9 Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 19 Sep 2024 05:18:57 +0100 Subject: [PATCH 27/57] minor refactor (migrations): query models to fetch records that have no project_id in the case of downgrade migration to remove project_id --- .../migrations/postgres/data_migrations/projects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agenta-backend/agenta_backend/migrations/postgres/data_migrations/projects.py b/agenta-backend/agenta_backend/migrations/postgres/data_migrations/projects.py index 347be5fd6c..5ba0acc0f6 100644 --- a/agenta-backend/agenta_backend/migrations/postgres/data_migrations/projects.py +++ b/agenta-backend/agenta_backend/migrations/postgres/data_migrations/projects.py @@ -160,7 +160,7 @@ def remove_project_id_from_db_entities(): records = ( session.execute( select(model) - .where(model.project_id == None) + .where(model.project_id != None) .offset(offset) .limit(BATCH_SIZE) ) From d28a70b2266168359ee6f3d0fc0644e41f6f89e3 Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 19 Sep 2024 05:19:38 +0100 Subject: [PATCH 28/57] refactor (backend): remove object and object_type when checking for action access in rbac --- .../agenta_backend/routers/app_router.py | 20 ++++----- .../agenta_backend/routers/bases_router.py | 3 +- .../agenta_backend/routers/configs_router.py | 6 +-- .../routers/container_router.py | 2 +- .../routers/environment_router.py | 3 +- .../routers/evaluation_router.py | 45 +++++++++---------- .../routers/evaluators_router.py | 14 +++--- .../routers/human_evaluation_router.py | 26 +++++------ .../agenta_backend/routers/testset_router.py | 16 +++---- .../agenta_backend/routers/variants_router.py | 10 ++--- 10 files changed, 61 insertions(+), 84 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index b54deb6daa..e686e9ea9b 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -105,8 +105,7 @@ async def list_app_variants( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=app_id, - object_type="app", + project_id=request.state.project_id, permission=Permission.VIEW_APPLICATION, ) logger.debug(f"User has Permission to list app variants: {has_permission}") @@ -160,8 +159,7 @@ async def get_variant_by_env( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=app_id, - object_type="app", + project_id=request.state.project_id, permission=Permission.VIEW_APPLICATION, ) logger.debug( @@ -283,7 +281,7 @@ async def update_app( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=app, + project_id=request.state.project_id, permission=Permission.EDIT_APPLICATION, ) logger.debug(f"User has Permission to update app: {has_permission}") @@ -323,7 +321,9 @@ async def list_apps( """ try: apps = await db_manager.list_apps( - project_id=request.state.project_id, app_name=app_name + project_id=request.state.project_id, + user_uid=request.state.user_id, + app_name=app_name, ) return apps except Exception as e: @@ -427,7 +427,7 @@ async def remove_app( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=app, + project_id=request.state.project_id, permission=Permission.DELETE_APPLICATION, ) logger.debug(f"User has Permission to delete app: {has_permission}") @@ -649,8 +649,7 @@ async def list_environments( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=app_id, - object_type="app", + project_id=request.state.project_id, permission=Permission.VIEW_APPLICATION, ) logger.debug(f"User has Permission to list environments: {has_permission}") @@ -691,8 +690,7 @@ async def list_app_environment_revisions( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=app_id, - object_type="app", + project_id=request.state.project_id, permission=Permission.VIEW_APPLICATION, ) logger.debug(f"User has Permission to list environments: {has_permission}") diff --git a/agenta-backend/agenta_backend/routers/bases_router.py b/agenta-backend/agenta_backend/routers/bases_router.py index 6e98a06656..936b84d4e4 100644 --- a/agenta-backend/agenta_backend/routers/bases_router.py +++ b/agenta-backend/agenta_backend/routers/bases_router.py @@ -43,8 +43,7 @@ async def list_bases( if isCloudEE() and app_id is not None: has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=app_id, - object_type="app", + project_id=request.state.project_id, permission=Permission.VIEW_APPLICATION, ) if not has_permission: diff --git a/agenta-backend/agenta_backend/routers/configs_router.py b/agenta-backend/agenta_backend/routers/configs_router.py index 538b46c466..467fd72d59 100644 --- a/agenta-backend/agenta_backend/routers/configs_router.py +++ b/agenta-backend/agenta_backend/routers/configs_router.py @@ -38,7 +38,7 @@ async def save_config( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=base_db, + project_id=request.state.project_id, permission=Permission.MODIFY_VARIANT_CONFIGURATIONS, ) if not has_permission: @@ -118,7 +118,7 @@ async def get_config( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=base_db, + project_id=request.state.project_id, permission=Permission.MODIFY_VARIANT_CONFIGURATIONS, ) if not has_permission: @@ -261,7 +261,7 @@ async def revert_deployment_revision( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=environment_revision, + project_id=request.state.project_id, permission=Permission.EDIT_APP_ENVIRONMENT_DEPLOYMENT, ) if not has_permission: diff --git a/agenta-backend/agenta_backend/routers/container_router.py b/agenta-backend/agenta_backend/routers/container_router.py index ed8a737a32..973463297a 100644 --- a/agenta-backend/agenta_backend/routers/container_router.py +++ b/agenta-backend/agenta_backend/routers/container_router.py @@ -188,7 +188,7 @@ async def construct_app_container_url( if isCloudEE() and object_db is not None: has_permission = await check_action_access( user_uid=request.state.user_id, - object=object_db, + project_id=request.state.project_id, permission=Permission.VIEW_APPLICATION, ) if not has_permission: diff --git a/agenta-backend/agenta_backend/routers/environment_router.py b/agenta-backend/agenta_backend/routers/environment_router.py index d93a860742..573b268c3a 100644 --- a/agenta-backend/agenta_backend/routers/environment_router.py +++ b/agenta-backend/agenta_backend/routers/environment_router.py @@ -37,8 +37,7 @@ async def deploy_to_environment( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=payload.variant_id, - object_type="app_variant", + project_id=request.state.project_id, permission=Permission.DEPLOY_APPLICATION, ) logger.debug(f"User has permission deploy to environment: {has_permission}") diff --git a/agenta-backend/agenta_backend/routers/evaluation_router.py b/agenta-backend/agenta_backend/routers/evaluation_router.py index 51125ac4f8..84936d3220 100644 --- a/agenta-backend/agenta_backend/routers/evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/evaluation_router.py @@ -187,7 +187,7 @@ async def fetch_evaluation_status( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=evaluation, + project_id=request.state.project_id, permission=Permission.VIEW_EVALUATION, ) logger.debug( @@ -228,7 +228,7 @@ async def fetch_evaluation_results( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=evaluation, + project_id=request.state.project_id, permission=Permission.VIEW_EVALUATION, ) logger.debug( @@ -283,7 +283,7 @@ async def fetch_evaluation_scenarios( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=evaluation, + project_id=request.state.project_id, permission=Permission.VIEW_EVALUATION, ) logger.debug( @@ -330,7 +330,7 @@ async def fetch_list_evaluations( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=app, + project_id=request.state.project_id, permission=Permission.VIEW_EVALUATION, ) logger.debug( @@ -385,8 +385,7 @@ async def fetch_evaluation( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=evaluation_id, - object_type="evaluation", + project_id=request.state.project_id, permission=Permission.VIEW_EVALUATION, ) logger.debug( @@ -423,11 +422,9 @@ async def delete_evaluations( try: if isCloudEE(): - evaluation_id = random.choice(payload.evaluations_ids) has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=evaluation_id, - object_type="evaluation", + project_id=request.state.project_id, permission=Permission.DELETE_EVALUATION, ) logger.debug(f"User has permission to delete evaluation: {has_permission}") @@ -479,23 +476,21 @@ async def fetch_evaluation_scenarios( evaluations_ids_list = evaluations_ids.split(",") if isCloudEE(): - for evaluation_id in evaluations_ids_list: - has_permission = await check_action_access( - user_uid=request.state.user_id, - object_id=evaluation_id, - object_type="evaluation", - permission=Permission.VIEW_EVALUATION, - ) - logger.debug( - f"User has permission to get evaluation scenarios: {has_permission}" + has_permission = await check_action_access( + user_uid=request.state.user_id, + project_id=request.state.project_id, + permission=Permission.VIEW_EVALUATION, + ) + logger.debug( + f"User has permission to get evaluation scenarios: {has_permission}" + ) + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, ) - if not has_permission: - error_msg = f"You do not have permission to perform this action. Please contact your organization admin." - logger.error(error_msg) - return JSONResponse( - {"detail": error_msg}, - status_code=403, - ) eval_scenarios = await evaluation_service.compare_evaluations_scenarios( evaluations_ids_list, request.state.project_id diff --git a/agenta-backend/agenta_backend/routers/evaluators_router.py b/agenta-backend/agenta_backend/routers/evaluators_router.py index 2a4c63a642..80b610abfa 100644 --- a/agenta-backend/agenta_backend/routers/evaluators_router.py +++ b/agenta-backend/agenta_backend/routers/evaluators_router.py @@ -66,8 +66,7 @@ async def get_evaluator_configs( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=app_id, - object_type="app", + project_id=request.state.project_id, permission=Permission.VIEW_EVALUATION, ) if not has_permission: @@ -106,7 +105,7 @@ async def get_evaluator_config( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=evaluator_config_db.app, + project_id=request.state.project_id, permission=Permission.VIEW_EVALUATION, ) if not has_permission: @@ -147,8 +146,7 @@ async def create_new_evaluator_config( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=payload.app_id, - object_type="app", + project_id=request.state.project_id, permission=Permission.CREATE_EVALUATION, ) if not has_permission: @@ -192,8 +190,7 @@ async def update_evaluator_config( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=evaluator_config_id, - object_type="evaluator_config", + project_id=request.state.project_id, permission=Permission.EDIT_EVALUATION, ) if not has_permission: @@ -234,8 +231,7 @@ async def delete_evaluator_config( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=evaluator_config_id, - object_type="evaluator_config", + project_id=request.state.project_id, permission=Permission.DELETE_EVALUATION, ) if not has_permission: diff --git a/agenta-backend/agenta_backend/routers/human_evaluation_router.py b/agenta-backend/agenta_backend/routers/human_evaluation_router.py index 89c21463df..e0733b542e 100644 --- a/agenta-backend/agenta_backend/routers/human_evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/human_evaluation_router.py @@ -64,8 +64,7 @@ async def create_evaluation( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=payload.app_id, - object_type="app", + project_id=request.state.project_id, permission=Permission.CREATE_EVALUATION, ) if not has_permission: @@ -112,8 +111,7 @@ async def fetch_list_human_evaluations( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=app_id, - object_type="app", + project_id=request.state.project_id, permission=Permission.VIEW_EVALUATION, ) if not has_permission: @@ -154,8 +152,7 @@ async def fetch_human_evaluation( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=evaluation_id, - object_type="human_evaluation", + project_id=request.state.project_id, permission=Permission.VIEW_EVALUATION, ) if not has_permission: @@ -205,8 +202,7 @@ async def fetch_evaluation_scenarios( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=evaluation_id, - object_type="human_evaluation", + project_id=request.state.project_id, permission=Permission.VIEW_EVALUATION, ) if not has_permission: @@ -256,7 +252,7 @@ async def update_human_evaluation( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=human_evaluation, + project_id=request.state.project_id, permission=Permission.EDIT_EVALUATION, ) if not has_permission: @@ -307,7 +303,7 @@ async def update_evaluation_scenario_router( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=evaluation_scenario_db, + project_id=request.state.project_id, permission=Permission.EDIT_EVALUATION, ) if not has_permission: @@ -359,7 +355,7 @@ async def get_evaluation_scenario_score_router( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=evaluation_scenario, + project_id=request.state.project_id, permission=Permission.VIEW_EVALUATION, ) if not has_permission: @@ -405,7 +401,7 @@ async def update_evaluation_scenario_score_router( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=evaluation_scenario, + project_id=request.state.project_id, permission=Permission.VIEW_EVALUATION, ) if not has_permission: @@ -451,7 +447,7 @@ async def fetch_results( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=evaluation, + project_id=request.state.project_id, permission=Permission.VIEW_EVALUATION, ) if not has_permission: @@ -495,11 +491,9 @@ async def delete_evaluations( try: if isCloudEE(): - evaluation_id = random.choice(delete_evaluations.evaluations_ids) has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=evaluation_id, - object_type="human_evaluation", + project_id=request.state.project_id, permission=Permission.DELETE_EVALUATION, ) if not has_permission: diff --git a/agenta-backend/agenta_backend/routers/testset_router.py b/agenta-backend/agenta_backend/routers/testset_router.py index 31f6cdd7bb..afa55a93fd 100644 --- a/agenta-backend/agenta_backend/routers/testset_router.py +++ b/agenta-backend/agenta_backend/routers/testset_router.py @@ -73,7 +73,7 @@ async def upload_file( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=app, + project_id=request.state.project_id, permission=Permission.CREATE_TESTSET, ) logger.debug(f"User has Permission to upload Testset: {has_permission}") @@ -153,7 +153,7 @@ async def import_testset( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=app, + project_id=request.state.project_id, permission=Permission.CREATE_TESTSET, ) logger.debug(f"User has Permission to import Testset: {has_permission}") @@ -233,7 +233,7 @@ async def create_testset( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=app, + project_id=request.state.project_id, permission=Permission.CREATE_TESTSET, ) logger.debug(f"User has Permission to create Testset: {has_permission}") @@ -290,7 +290,7 @@ async def update_testset( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=testset, + project_id=request.state.project_id, permission=Permission.EDIT_TESTSET, ) logger.debug(f"User has Permission to update Testset: {has_permission}") @@ -341,7 +341,7 @@ async def get_testsets( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=app, + project_id=request.state.project_id, permission=Permission.VIEW_TESTSET, ) logger.debug(f"User has Permission to view Testsets: {has_permission}") @@ -388,7 +388,7 @@ async def get_single_testset( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=test_set, + project_id=request.state.project_id, permission=Permission.VIEW_TESTSET, ) logger.debug(f"User has Permission to view Testset: {has_permission}") @@ -424,11 +424,9 @@ async def delete_testsets( """ if isCloudEE(): - testset_id = random.choice(payload.testset_ids) has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=testset_id, - object_type="testset", + project_id=request.state.project_id, permission=Permission.DELETE_TESTSET, ) logger.debug(f"User has Permission to delete Testset: {has_permission}") diff --git a/agenta-backend/agenta_backend/routers/variants_router.py b/agenta-backend/agenta_backend/routers/variants_router.py index 3952608506..0d242910ab 100644 --- a/agenta-backend/agenta_backend/routers/variants_router.py +++ b/agenta-backend/agenta_backend/routers/variants_router.py @@ -77,7 +77,7 @@ async def add_variant_from_base_and_config( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=base_db, + project_id=request.state.project_id, permission=Permission.CREATE_APPLICATION, ) logger.debug( @@ -142,8 +142,7 @@ async def remove_variant( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=variant_id, - object_type="app_variant", + project_id=request.state.project_id, permission=Permission.DELETE_APPLICATION_VARIANT, ) logger.debug(f"User has Permission to delete app variant: {has_permission}") @@ -202,8 +201,7 @@ async def update_variant_parameters( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object_id=variant_id, - object_type="app_variant", + project_id=request.state.project_id, permission=Permission.MODIFY_VARIANT_CONFIGURATIONS, ) logger.debug( @@ -448,7 +446,7 @@ async def get_variant_revisions( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - object=app_variant, + project_id=request.state.project_id, permission=Permission.VIEW_APPLICATION, ) logger.debug(f"User has Permission to get variant: {has_permission}") From 5e93801747596ca17cce912dc162aa1dbaf6a641 Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 19 Sep 2024 05:20:11 +0100 Subject: [PATCH 29/57] refactor (backend): cleanup use of organization and workspace --- .../agenta_backend/models/api/api_models.py | 2 ++ .../agenta_backend/models/converters.py | 25 +++--------------- .../agenta_backend/services/db_manager.py | 26 +++---------------- 3 files changed, 8 insertions(+), 45 deletions(-) diff --git a/agenta-backend/agenta_backend/models/api/api_models.py b/agenta-backend/agenta_backend/models/api/api_models.py index dc2eea46f0..df9dd17972 100644 --- a/agenta-backend/agenta_backend/models/api/api_models.py +++ b/agenta-backend/agenta_backend/models/api/api_models.py @@ -84,6 +84,7 @@ class UpdateVariantParameterPayload(BaseModel): class AppVariant(BaseModel): app_id: str app_name: str + project_id: Optional[str] = None variant_name: str parameters: Optional[Dict[str, Any]] previous_variant_name: Optional[str] @@ -198,6 +199,7 @@ class AddVariantFromImagePayload(BaseModel): class ImageExtended(Image): # includes the mongodb image id id: str + project_id: Optional[str] = None class TemplateImageInfo(BaseModel): diff --git a/agenta-backend/agenta_backend/models/converters.py b/agenta-backend/agenta_backend/models/converters.py index b6b994deb8..180d3bbd24 100644 --- a/agenta-backend/agenta_backend/models/converters.py +++ b/agenta-backend/agenta_backend/models/converters.py @@ -274,17 +274,13 @@ def app_variant_db_to_pydantic( app_variant = AppVariant( app_id=str(app_variant_db.app.id), app_name=app_variant_db.app.app_name, + project_id=str(app_variant_db.project_id), variant_name=app_variant_db.variant_name, parameters=app_variant_db.config.parameters, previous_variant_name=app_variant_db.previous_variant_name, base_name=app_variant_db.base_name, config_name=app_variant_db.config_name, ) - - if isCloudEE(): - app_variant.organization_id = str(app_variant_db.organization_id) - app_variant.workspace_id = str(app_variant_db.workspace_id) - return app_variant @@ -313,11 +309,6 @@ async def app_variant_db_to_output(app_variant_db: AppVariantDB) -> AppVariantRe updated_at=str(app_variant_db.created_at), modified_by_id=str(app_variant_db.modified_by_id), ) - - if isCloudEE(): - variant_response.organization_id = str(app_variant_db.organization_id) - variant_response.workspace_id = str(app_variant_db.workspace_id) - return variant_response @@ -399,7 +390,7 @@ async def environment_db_and_revision_to_extended_output( ) if deployed_app_variant_id: deployed_app_variant = await db_manager.get_app_variant_instance_by_id( - deployed_app_variant_id + deployed_app_variant_id, str(environment_db.project_id) ) deployed_variant_name = deployed_app_variant.variant_name else: @@ -430,12 +421,6 @@ async def environment_db_and_revision_to_extended_output( revision=environment_db.revision, revisions=app_environment_revisions, ) - - if isCloudEE(): - environment_output_extended.organization_id = str( - environment_db.organization_id - ) - environment_output_extended.workspace_id = str(environment_db.workspace_id) return environment_output_extended @@ -454,14 +439,10 @@ def app_db_to_pydantic(app_db: AppDB) -> App: def image_db_to_pydantic(image_db: ImageDB) -> ImageExtended: image = ImageExtended( docker_id=image_db.docker_id, + project_id=str(image_db.project_id), tags=image_db.tags, id=str(image_db.id), ) - - if isCloudEE(): - image.organization_id = str(image_db.organization_id) - image.workspace_id = str(image_db.workspace_id) - return image diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 264bcf2e69..dbf598e2db 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -100,7 +100,6 @@ async def add_testset_to_app_variant( """Add testset to app variant. Args: - org_id (str): The id of the organization template_name (str): The name of the app template image app_name (str): The name of the app project_id (str): The ID of the project @@ -192,7 +191,6 @@ async def fetch_app_variant_by_id( ) if isCloudEE(): query = base_query.options( - joinedload(AppVariantDB.organization), joinedload(AppVariantDB.image.of_type(ImageDB)).load_only(ImageDB.docker_id, ImageDB.tags), # type: ignore ) else: @@ -855,14 +853,11 @@ async def get_orga_image_instance_by_docker_id( async def get_orga_image_instance_by_uri( - template_uri: str, - organization_id: Optional[str] = None, - workspace_id: Optional[str] = None, + template_uri: str ) -> ImageDB: """Get the image object from the database with the provided id. Arguments: - organization_id (str): The organization unique identifier template_uri (url): The image template url Returns: @@ -875,18 +870,6 @@ async def get_orga_image_instance_by_uri( async with db_engine.get_session() as session: query = select(ImageDB).filter_by(template_uri=template_uri) - - if isCloudEE(): - # assert that if organization is provided, workspace_id is also provided, and vice versa - assert ( - organization_id is not None and workspace_id is not None - ), "organization and workspace must be provided together" - - query = query.filter_by( - organization_id=uuid.UUID(organization_id), - workspace_id=workspace_id, - ) - result = await session.execute(query) image = result.scalars().first() return image @@ -965,10 +948,6 @@ async def add_variant_from_base_and_config( config_parameters=parameters, ) - if isCloudEE(): - db_app_variant.organization_id = previous_app_variant_db.organization_id - db_app_variant.workspace_id = previous_app_variant_db.workspace_id - session.add(db_app_variant) await session.commit() await session.refresh(db_app_variant) @@ -992,6 +971,7 @@ async def add_variant_from_base_and_config( async def list_apps( project_id: str, + user_uid: str, app_name: Optional[str] = None, ): """ @@ -2562,7 +2542,7 @@ async def update_app_variant( async def fetch_app_by_name_and_parameters(app_name: str, project_id: str): - """Fetch an app by its name, organization id, and workspace_id. + """Fetch an app by its name and project identifier. Args: app_name (str): The name of the app From 00765289c3c39a657cb20e7ec9328ec75083c436 Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 19 Sep 2024 05:20:55 +0100 Subject: [PATCH 30/57] chore (style): format code --- agenta-backend/agenta_backend/services/db_manager.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index dbf598e2db..e7412b0d1d 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -852,9 +852,7 @@ async def get_orga_image_instance_by_docker_id( return image -async def get_orga_image_instance_by_uri( - template_uri: str -) -> ImageDB: +async def get_orga_image_instance_by_uri(template_uri: str) -> ImageDB: """Get the image object from the database with the provided id. Arguments: From 8c6080973d3e913c2d08aec1343c974097335cf3 Mon Sep 17 00:00:00 2001 From: Abram Date: Tue, 24 Sep 2024 13:02:41 +0100 Subject: [PATCH 31/57] refactor (backend): remove deprecated organization and workspace IDs from app when getting image instance by uri, and pass project_id in place of workspace_id when creating an api key --- agenta-backend/agenta_backend/services/app_manager.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/agenta-backend/agenta_backend/services/app_manager.py b/agenta-backend/agenta_backend/services/app_manager.py index 5de31f2060..658670ac52 100644 --- a/agenta-backend/agenta_backend/services/app_manager.py +++ b/agenta-backend/agenta_backend/services/app_manager.py @@ -105,7 +105,7 @@ async def start_variant( api_key = await api_key_service.create_api_key( str(db_app_variant.user.uid), project_id=project_id, - workspace_id=str(db_app_variant.workspace_id), + project_id=project_id, expiration_date=None, hidden=True, ) @@ -490,8 +490,6 @@ async def add_variant_based_on_image( if parsed_url.scheme and parsed_url.netloc: db_image = await db_manager.get_orga_image_instance_by_uri( template_uri=docker_id_or_template_uri, - organization_id=str(app.organization_id) if isCloudEE() else None, # type: ignore - workspace_id=str(app.workspace_id) if isCloudEE() else None, # type: ignore ) else: db_image = await db_manager.get_orga_image_instance_by_docker_id( From a0540eb0775c4480c86b93e5beb39da0d09b9908 Mon Sep 17 00:00:00 2001 From: Abram Date: Tue, 24 Sep 2024 13:22:59 +0100 Subject: [PATCH 32/57] fix (bug): resolve SyntaxError: keyword argument repeated: project_id --- agenta-backend/agenta_backend/services/app_manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/agenta-backend/agenta_backend/services/app_manager.py b/agenta-backend/agenta_backend/services/app_manager.py index 658670ac52..d21781e6a3 100644 --- a/agenta-backend/agenta_backend/services/app_manager.py +++ b/agenta-backend/agenta_backend/services/app_manager.py @@ -105,7 +105,6 @@ async def start_variant( api_key = await api_key_service.create_api_key( str(db_app_variant.user.uid), project_id=project_id, - project_id=project_id, expiration_date=None, hidden=True, ) From a1e94bca3280d06ee9bbecd16ddf1ce924d03210 Mon Sep 17 00:00:00 2001 From: Abram Date: Tue, 24 Sep 2024 13:40:22 +0100 Subject: [PATCH 33/57] refactor (backend): remove organization and workspace attributes to refresh after app variant creation --- .../agenta_backend/services/db_manager.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index e7412b0d1d..bc1ff791dd 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -430,18 +430,14 @@ async def create_new_app_variant( session.add(variant) - attributes_to_refresh = [ - "app", - "image", - "base", - ] - if isCloudEE(): - attributes_to_refresh.extend(["organization", "workspace"]) - await session.commit() await session.refresh( variant, - attribute_names=attributes_to_refresh, + attribute_names=[ + "app", + "image", + "base", + ], ) # Ensures the app, image, user and base relationship are loaded variant_revision = AppVariantRevisionsDB( From b55ac52b44836699a067f3872718d16a94159820 Mon Sep 17 00:00:00 2001 From: Abram Date: Tue, 24 Sep 2024 13:56:51 +0100 Subject: [PATCH 34/57] refactor (backend): explictly fetch user when creating api_key for variant in cloud/ee --- agenta-backend/agenta_backend/routers/app_router.py | 5 ++++- agenta-backend/agenta_backend/services/app_manager.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index e686e9ea9b..421d1680c9 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -611,7 +611,10 @@ async def create_app_and_variant_from_template( else: envvars = {} if payload.env_vars is None else payload.env_vars await app_manager.start_variant( - app_variant_db, request.state.project_id, envvars + app_variant_db, + request.state.project_id, + envvars, + user_uid=request.state.user_id, ) logger.debug("End: Successfully created app and variant") diff --git a/agenta-backend/agenta_backend/services/app_manager.py b/agenta-backend/agenta_backend/services/app_manager.py index d21781e6a3..274bbdd49e 100644 --- a/agenta-backend/agenta_backend/services/app_manager.py +++ b/agenta-backend/agenta_backend/services/app_manager.py @@ -56,6 +56,7 @@ async def start_variant( db_app_variant: AppVariantDB, project_id: str, env_vars: Optional[DockerEnvVars] = None, + user_uid: Optional[str] = None, ) -> URI: """ Starts a Docker container for a given app variant. @@ -67,6 +68,7 @@ async def start_variant( app_variant (AppVariant): The app variant for which a container is to be started. project_id (str): The ID of the project the app variant belongs to. env_vars (DockerEnvVars): (optional) The environment variables to be passed to the container. + user_uid (str): (optional) The user ID. Returns: URI: The URI of the started Docker container. @@ -102,8 +104,9 @@ async def start_variant( } ) if isCloudEE(): + user = await db_manager.get_user(user_uid=user_uid) api_key = await api_key_service.create_api_key( - str(db_app_variant.user.uid), + str(user.id), project_id=project_id, expiration_date=None, hidden=True, From 63c9856d29fb660943ad97428076d0966ded8060 Mon Sep 17 00:00:00 2001 From: Abram Date: Tue, 24 Sep 2024 14:05:56 +0100 Subject: [PATCH 35/57] refactor (backend): fetch apps by their workspace/organization project id in cloud/ee --- agenta-backend/agenta_backend/routers/app_router.py | 4 ++++ agenta-backend/agenta_backend/services/db_manager.py | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index 421d1680c9..c0da62392f 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -304,6 +304,8 @@ async def update_app( async def list_apps( request: Request, app_name: Optional[str] = None, + org_id: Optional[str] = None, + workspace_id: Optional[str] = None, ) -> List[App]: """ Retrieve a list of apps filtered by app_name and org_id. @@ -324,6 +326,8 @@ async def list_apps( project_id=request.state.project_id, user_uid=request.state.user_id, app_name=app_name, + org_id=org_id, + workspace_id=workspace_id, ) return apps except Exception as e: diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index bc1ff791dd..5b726e2f07 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -967,6 +967,8 @@ async def list_apps( project_id: str, user_uid: str, app_name: Optional[str] = None, + org_id: Optional[str] = None, + workspace_id: Optional[str] = None, ): """ Lists all the unique app names and their IDs from the database @@ -999,9 +1001,12 @@ async def list_apps( detail="You do not have access to perform this action. Please contact your organization admin.", ) + project = await db_manager_ee.get_project_by_workspace_and_organization( + org_id=org_id, workspace_id=workspace_id + ) async with db_engine.get_session() as session: result = await session.execute( - select(AppDB).filter_by(project_id=uuid.UUID(project_id)) + select(AppDB).filter_by(project_id=project.id) ) apps = result.unique().scalars().all() return [converters.app_db_to_pydantic(app) for app in apps] From 2704959043f1c486670c7f8e6248f7043f0da901 Mon Sep 17 00:00:00 2001 From: Abram Date: Tue, 24 Sep 2024 14:23:29 +0100 Subject: [PATCH 36/57] refactor (backend): only make use of workspace_id from request query params when retrieving apps --- agenta-backend/agenta_backend/routers/app_router.py | 2 -- agenta-backend/agenta_backend/services/db_manager.py | 5 ++--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index c0da62392f..ee38dfce4b 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -304,7 +304,6 @@ async def update_app( async def list_apps( request: Request, app_name: Optional[str] = None, - org_id: Optional[str] = None, workspace_id: Optional[str] = None, ) -> List[App]: """ @@ -326,7 +325,6 @@ async def list_apps( project_id=request.state.project_id, user_uid=request.state.user_id, app_name=app_name, - org_id=org_id, workspace_id=workspace_id, ) return apps diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 5b726e2f07..7f62a68e35 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -967,7 +967,6 @@ async def list_apps( project_id: str, user_uid: str, app_name: Optional[str] = None, - org_id: Optional[str] = None, workspace_id: Optional[str] = None, ): """ @@ -1001,8 +1000,8 @@ async def list_apps( detail="You do not have access to perform this action. Please contact your organization admin.", ) - project = await db_manager_ee.get_project_by_workspace_and_organization( - org_id=org_id, workspace_id=workspace_id + project = await db_manager_ee.get_project_by_workspace( + workspace_id=workspace_id ) async with db_engine.get_session() as session: result = await session.execute( From 920ec3cc3228ec495b8ff6267e2637acc15d9231 Mon Sep 17 00:00:00 2001 From: Abram Date: Tue, 24 Sep 2024 16:06:48 +0100 Subject: [PATCH 37/57] refactor (backend): make use of project_id from app_variant when setting deployment uri path for docker container --- agenta-backend/agenta_backend/services/app_manager.py | 1 + .../agenta_backend/services/deployment_manager.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/agenta-backend/agenta_backend/services/app_manager.py b/agenta-backend/agenta_backend/services/app_manager.py index 274bbdd49e..e59a78e56a 100644 --- a/agenta-backend/agenta_backend/services/app_manager.py +++ b/agenta-backend/agenta_backend/services/app_manager.py @@ -95,6 +95,7 @@ async def start_variant( "http://host.docker.internal" # unclear why this stopped working ) # domain_name = "http://localhost" + env_vars = {} if env_vars is None else env_vars # type: ignore env_vars.update( { diff --git a/agenta-backend/agenta_backend/services/deployment_manager.py b/agenta-backend/agenta_backend/services/deployment_manager.py index 5bfbdc4c07..9ec391ba54 100644 --- a/agenta-backend/agenta_backend/services/deployment_manager.py +++ b/agenta-backend/agenta_backend/services/deployment_manager.py @@ -31,8 +31,8 @@ async def start_service( """ if isCloudEE(): - uri_path = f"{app_variant_db.organization.id}/{app_variant_db.app.app_name}/{app_variant_db.base_name}" - container_name = f"{app_variant_db.app.app_name}-{app_variant_db.base_name}-{app_variant_db.organization.id}" + uri_path = f"{app_variant_db.project_id}/{app_variant_db.app.app_name}/{app_variant_db.base_name}" + container_name = f"{app_variant_db.app.app_name}-{app_variant_db.base_name}-{app_variant_db.project_id}" else: uri_path = f"{app_variant_db.project_id}/{app_variant_db.app.app_name}/{app_variant_db.base_name}" container_name = f"{app_variant_db.app.app_name}-{app_variant_db.base_name}-{app_variant_db.project_id}" @@ -137,7 +137,7 @@ async def validate_image(image: Image) -> bool: raise ValueError(msg) if isCloudEE(): - image = Image(**image.model_dump(exclude={"workspace", "organization"})) + image = Image(**image.model_dump()) if not image.tags.startswith(agenta_registry_repo): raise ValueError( From 1027d1886c63ca202529111727cc72869a14102b Mon Sep 17 00:00:00 2001 From: Abram Date: Tue, 24 Sep 2024 21:23:31 +0100 Subject: [PATCH 38/57] refactor (backend): update use of project_id for rbac --- .../agenta_backend/models/api/api_models.py | 4 + .../agenta_backend/routers/app_router.py | 51 ++--- .../agenta_backend/routers/bases_router.py | 3 +- .../agenta_backend/routers/configs_router.py | 23 +-- .../routers/container_router.py | 24 +-- .../routers/environment_router.py | 8 +- .../routers/evaluation_router.py | 60 +++--- .../routers/evaluators_router.py | 25 ++- .../routers/human_evaluation_router.py | 60 +++--- .../agenta_backend/routers/testset_router.py | 72 +++---- .../agenta_backend/routers/variants_router.py | 58 +++--- .../agenta_backend/services/app_manager.py | 25 +-- .../agenta_backend/services/db_manager.py | 190 +++++++----------- .../services/evaluation_service.py | 18 +- .../agenta_backend/tasks/evaluations.py | 15 +- 15 files changed, 278 insertions(+), 358 deletions(-) diff --git a/agenta-backend/agenta_backend/models/api/api_models.py b/agenta-backend/agenta_backend/models/api/api_models.py index df9dd17972..472734c9c3 100644 --- a/agenta-backend/agenta_backend/models/api/api_models.py +++ b/agenta-backend/agenta_backend/models/api/api_models.py @@ -58,6 +58,8 @@ class VariantAction(BaseModel): class CreateApp(BaseModel): app_name: str + project_id: Optional[str] = None + workspace_id: Optional[str] = None class CreateAppOutput(BaseModel): @@ -239,6 +241,8 @@ class DockerEnvVars(BaseModel): class CreateAppVariant(BaseModel): app_name: str template_id: str + project_id: Optional[str] = None + workspace_id: Optional[str] = None env_vars: Dict[str, str] diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index ee38dfce4b..e2885ac5a0 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -102,10 +102,11 @@ async def list_app_variants( List[AppVariantResponse]: A list of app variants for the given app ID. """ try: + app = await db_manager.get_app_instance_by_id(app_id=app_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(app.project_id), permission=Permission.VIEW_APPLICATION, ) logger.debug(f"User has Permission to list app variants: {has_permission}") @@ -116,9 +117,7 @@ async def list_app_variants( status_code=403, ) - app_variants = await db_manager.list_app_variants( - app_id=app_id, project_id=request.state.project_id - ) + app_variants = await db_manager.list_app_variants(app_id=app_id) return [ await converters.app_variant_db_to_output(app_variant) for app_variant in app_variants @@ -156,10 +155,11 @@ async def get_variant_by_env( AppVariantResponse: The retrieved app variant. """ try: + app = await db_manager.get_app_instance_by_id(app_id=app_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(app.project_id), permission=Permission.VIEW_APPLICATION, ) logger.debug( @@ -231,7 +231,7 @@ async def create_app( has_permission = await check_rbac_permission( user_org_workspace_data=user_org_workspace_data, - project_id=request.state.project_id, + project_id=payload.project_id or request.state.project_id, permission=Permission.CREATE_APPLICATION, ) logger.debug( @@ -247,7 +247,9 @@ async def create_app( raise HTTPException(status_code=500, detail=str(e)) app_db = await db_manager.create_app_and_envs( - payload.app_name, request.state.project_id + payload.app_name, + project_id=payload.project_id or request.state.project_id, + workspace_id=payload.workspace_id, ) return CreateAppOutput(app_id=str(app_db.id), app_name=str(app_db.app_name)) except Exception as e: @@ -277,11 +279,11 @@ async def update_app( """ try: - app = await db_manager.fetch_app_by_id(app_id, request.state.project_id) + app = await db_manager.fetch_app_by_id(app_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(app.project_id), permission=Permission.EDIT_APPLICATION, ) logger.debug(f"User has Permission to update app: {has_permission}") @@ -369,12 +371,11 @@ async def add_variant_from_image( raise HTTPException(status_code=404, detail="Image not found") try: - app = await db_manager.fetch_app_by_id(app_id, request.state.project_id) - + app = await db_manager.fetch_app_by_id(app_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(app.project_id), permission=Permission.CREATE_APPLICATION, ) logger.debug( @@ -389,7 +390,7 @@ async def add_variant_from_image( variant_db = await app_manager.add_variant_based_on_image( app=app, - project_id=request.state.project_id, + project_id=str(app.project_id), variant_name=payload.variant_name, docker_id_or_template_uri=payload.docker_id, tags=payload.tags, @@ -398,13 +399,11 @@ async def add_variant_from_image( is_template_image=False, user_uid=request.state.user_id, ) - app_variant_db = await db_manager.fetch_app_variant_by_id( - str(variant_db.id), request.state.project_id - ) + app_variant_db = await db_manager.fetch_app_variant_by_id(str(variant_db.id)) logger.debug("Step 8: We create ready-to use evaluators") await evaluator_manager.create_ready_to_use_evaluators( - app_name=app.app_name, project_id=request.state.project_id + app_name=app.app_name, project_id=str(app.project_id) ) return await converters.app_variant_db_to_output(app_variant_db) @@ -424,12 +423,11 @@ async def remove_app( app -- App to remove """ try: - app = await db_manager.fetch_app_by_id(app_id, request.state.project_id) - + app = await db_manager.fetch_app_by_id(app_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(app.project_id), permission=Permission.DELETE_APPLICATION, ) logger.debug(f"User has Permission to delete app: {has_permission}") @@ -440,7 +438,7 @@ async def remove_app( status_code=403, ) - await app_manager.remove_app(app, request.state.project_id) + await app_manager.remove_app(app) except DockerException as e: detail = f"Docker error while trying to remove the app: {str(e)}" logger.exception(f"Docker error while trying to remove the app: {str(e)}") @@ -489,9 +487,12 @@ async def create_app_and_variant_from_template( ) logger.debug("Step 3: Checking user has permission to create app") + project = await db_manager_ee.get_project_by_workspace( + workspace_id=payload.workspace_id + ) has_permission = await check_rbac_permission( user_org_workspace_data=user_org_workspace_data, - project_id=request.state.project_id, + project_id=str(project.id), permission=Permission.CREATE_APPLICATION, ) logger.debug( @@ -512,7 +513,7 @@ async def create_app_and_variant_from_template( ) app_name = payload.app_name.lower() app = await db_manager.fetch_app_by_name_and_parameters( - app_name, request.state.project_id + app_name, payload.workspace_id ) if app is not None: raise Exception( @@ -526,7 +527,9 @@ async def create_app_and_variant_from_template( ) if app is None: app = await db_manager.create_app_and_envs( - app_name, request.state.project_id + app_name, + project_id=payload.project_id or request.state.project_id, + workspace_id=payload.workspace_id, ) logger.debug( diff --git a/agenta-backend/agenta_backend/routers/bases_router.py b/agenta-backend/agenta_backend/routers/bases_router.py index 936b84d4e4..e698f9e6f6 100644 --- a/agenta-backend/agenta_backend/routers/bases_router.py +++ b/agenta-backend/agenta_backend/routers/bases_router.py @@ -40,10 +40,11 @@ async def list_bases( HTTPException: If there was an error retrieving the bases. """ try: + app = await db_manager.fetch_app_by_id(app_id=app_id) if isCloudEE() and app_id is not None: has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(app.project_i), permission=Permission.VIEW_APPLICATION, ) if not has_permission: diff --git a/agenta-backend/agenta_backend/routers/configs_router.py b/agenta-backend/agenta_backend/routers/configs_router.py index 467fd72d59..e79cdb0c55 100644 --- a/agenta-backend/agenta_backend/routers/configs_router.py +++ b/agenta-backend/agenta_backend/routers/configs_router.py @@ -31,14 +31,12 @@ async def save_config( request: Request, ): try: - base_db = await db_manager.fetch_base_by_id( - payload.base_id, request.state.project_id - ) + base_db = await db_manager.fetch_base_by_id(payload.base_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(base_db.project_id), permission=Permission.MODIFY_VARIANT_CONFIGURATIONS, ) if not has_permission: @@ -50,7 +48,7 @@ async def save_config( ) variants_db = await db_manager.list_variants_for_base( - base_db, request.state.project_id + base_db, str(base_db.project_id) ) variant_to_overwrite = None for variant_db in variants_db: @@ -65,14 +63,14 @@ async def save_config( app_variant_id=str(variant_to_overwrite.id), parameters=payload.parameters, user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(base_db.project_id), ) logger.debug("Deploying to production environment") await db_manager.deploy_to_environment( environment_name="production", variant_id=str(variant_to_overwrite.id), - project_id=request.state.project_id, + project_id=str(base_db.project_id), user_uid=request.state.user_id, ) else: @@ -89,7 +87,7 @@ async def save_config( new_config_name=payload.config_name, parameters=payload.parameters, user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(base_db.project_id), ) except HTTPException as e: @@ -118,7 +116,7 @@ async def get_config( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(base_db.project_id), permission=Permission.MODIFY_VARIANT_CONFIGURATIONS, ) if not has_permission: @@ -133,7 +131,6 @@ async def get_config( if environment_name: app_environments = await db_manager.list_environments( app_id=str(base_db.app_id), # type: ignore - project_id=request.state.project_id, ) found_variant_revision = next( ( @@ -160,9 +157,7 @@ async def get_config( "parameters": found_variant_revision.config_parameters, } elif config_name: - variants_db = await db_manager.list_variants_for_base( - base_db, request.state.project_id - ) + variants_db = await db_manager.list_variants_for_base(base_db) found_variant = next( ( variant_db @@ -261,7 +256,7 @@ async def revert_deployment_revision( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(environment_revision.project_id), permission=Permission.EDIT_APP_ENVIRONMENT_DEPLOYMENT, ) if not has_permission: diff --git a/agenta-backend/agenta_backend/routers/container_router.py b/agenta-backend/agenta_backend/routers/container_router.py index 973463297a..77fed296bf 100644 --- a/agenta-backend/agenta_backend/routers/container_router.py +++ b/agenta-backend/agenta_backend/routers/container_router.py @@ -65,13 +65,11 @@ async def build_image( Image: The Docker image that was built. """ try: - app_db = await db_manager.fetch_app_by_id(app_id, request.state.project_id) - - # Check app access + app_db = await db_manager.fetch_app_by_id(app_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(app_db.project_id), permission=Permission.CREATE_APPLICATION, ) if not has_permission: @@ -105,12 +103,10 @@ async def restart_docker_container( """ logger.debug(f"Restarting container for variant {payload.variant_id}") - app_variant_db = await db_manager.fetch_app_variant_by_id( - payload.variant_id, request.state.project_id - ) + app_variant_db = await db_manager.fetch_app_variant_by_id(payload.variant_id) try: deployment = await db_manager.get_deployment_by_id( - app_variant_db.base.deployment, request.state.project_id + app_variant_db.base.deployment ) container_id = deployment.container_id @@ -166,7 +162,7 @@ async def construct_app_container_url( assert base_id or variant_id, "Please provide either base_id or variant_id" if base_id: - object_db = await db_manager.fetch_base_by_id(base_id, request.state.project_id) + object_db = await db_manager.fetch_base_by_id(base_id) elif variant_id and variant_id != "None": # NOTE: Backward Compatibility # --------------------------- @@ -177,9 +173,7 @@ async def construct_app_container_url( # This change ensures that users can still view their evaluations; however, # they will no longer be able to access a deployment URL for the deleted variant. # Therefore, we ensure that variant_id is not "None". - object_db = await db_manager.fetch_app_variant_by_id( - variant_id, request.state.project_id - ) + object_db = await db_manager.fetch_app_variant_by_id(variant_id) else: # NOTE: required for backward compatibility object_db = None @@ -188,7 +182,7 @@ async def construct_app_container_url( if isCloudEE() and object_db is not None: has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(object_db.project_id), permission=Permission.VIEW_APPLICATION, ) if not has_permission: @@ -199,11 +193,11 @@ async def construct_app_container_url( try: if getattr(object_db, "deployment_id", None): # this is a base deployment = await db_manager.get_deployment_by_id( - str(object_db.deployment_id), request.state.project_id # type: ignore + str(object_db.deployment_id) # type: ignore ) elif getattr(object_db, "base_id", None): # this is a variant deployment = await db_manager.get_deployment_by_id( - str(object_db.base.deployment_id), request.state.project_id # type: ignore + str(object_db.base.deployment_id) # type: ignore ) else: raise HTTPException( diff --git a/agenta-backend/agenta_backend/routers/environment_router.py b/agenta-backend/agenta_backend/routers/environment_router.py index 573b268c3a..554ffec229 100644 --- a/agenta-backend/agenta_backend/routers/environment_router.py +++ b/agenta-backend/agenta_backend/routers/environment_router.py @@ -34,10 +34,13 @@ async def deploy_to_environment( HTTPException: If the deployment fails. """ try: + variant = await db_manager.fetch_app_variant_by_id( + app_variant_id=payload.variant_id + ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(variant.project_id), permission=Permission.DEPLOY_APPLICATION, ) logger.debug(f"User has permission deploy to environment: {has_permission}") @@ -52,7 +55,6 @@ async def deploy_to_environment( await db_manager.deploy_to_environment( environment_name=payload.environment_name, variant_id=payload.variant_id, - project_id=request.state.project_id, user_uid=request.state.user_id, ) @@ -61,7 +63,7 @@ async def deploy_to_environment( user_uid=request.state.user_id, object_id=payload.variant_id, object_type="variant", - project_id=request.state.project_id, + project_id=str(variant.project_id), ) logger.debug("Successfully updated last_modified_by app information") except Exception as e: diff --git a/agenta-backend/agenta_backend/routers/evaluation_router.py b/agenta-backend/agenta_backend/routers/evaluation_router.py index 84936d3220..2cb33bc55f 100644 --- a/agenta-backend/agenta_backend/routers/evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/evaluation_router.py @@ -54,10 +54,11 @@ async def fetch_evaluation_ids( List[str]: A list of evaluation ids. """ try: + app = await db_manager.fetch_app_by_id(app_id=app_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(app.project_id), permission=Permission.VIEW_EVALUATION, ) logger.debug( @@ -71,7 +72,7 @@ async def fetch_evaluation_ids( status_code=403, ) evaluations = await db_manager.fetch_evaluations_by_resource( - resource_type, request.state.project_id, resource_ids + resource_type, str(app.project_id), resource_ids ) return list(map(lambda x: str(x.id), evaluations)) except Exception as exc: @@ -93,16 +94,14 @@ async def create_evaluation( _description_ """ try: - app = await db_manager.fetch_app_by_id( - app_id=payload.app_id, project_id=request.state.project_id - ) + app = await db_manager.fetch_app_by_id(app_id=payload.app_id) if app is None: raise HTTPException(status_code=404, detail="App not found") if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(app.project_id), permission=Permission.CREATE_EVALUATION, ) logger.debug(f"User has permission to create evaluation: {has_permission}") @@ -125,14 +124,14 @@ async def create_evaluation( for variant_id in payload.variant_ids: evaluation = await evaluation_service.create_new_evaluation( app_id=payload.app_id, - project_id=request.state.project_id, + project_id=str(app.project_id), variant_id=variant_id, testset_id=payload.testset_id, ) evaluate.delay( app_id=payload.app_id, - project_id=request.state.project_id, + project_id=str(app.project_id), variant_id=variant_id, evaluators_config_ids=payload.evaluators_configs, testset_id=payload.testset_id, @@ -147,7 +146,7 @@ async def create_evaluation( user_uid=request.state.user_id, object_id=payload.app_id, object_type="app", - project_id=request.state.project_id, + project_id=str(app.project_id), ) logger.debug("Successfully updated last_modified_by app information") @@ -181,13 +180,11 @@ async def fetch_evaluation_status( """ try: - evaluation = await db_manager.fetch_evaluation_by_id( - evaluation_id, request.state.project_id - ) + evaluation = await db_manager.fetch_evaluation_by_id(evaluation_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(evaluation.project_id), permission=Permission.VIEW_EVALUATION, ) logger.debug( @@ -228,7 +225,7 @@ async def fetch_evaluation_results( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(evaluation.project_id), permission=Permission.VIEW_EVALUATION, ) logger.debug( @@ -272,9 +269,7 @@ async def fetch_evaluation_scenarios( """ try: - evaluation = await db_manager.fetch_evaluation_by_id( - evaluation_id, request.state.project_id - ) + evaluation = await db_manager.fetch_evaluation_by_id(evaluation_id) if not evaluation: raise HTTPException( status_code=404, detail=f"Evaluation with id {evaluation_id} not found" @@ -283,7 +278,7 @@ async def fetch_evaluation_scenarios( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(evaluation.project_id), permission=Permission.VIEW_EVALUATION, ) logger.debug( @@ -299,7 +294,7 @@ async def fetch_evaluation_scenarios( eval_scenarios = ( await evaluation_service.fetch_evaluation_scenarios_for_evaluation( - evaluation_id=str(evaluation.id), project_id=request.state.project_id + evaluation_id=str(evaluation.id), project_id=str(evaluation.project_id) ) ) return eval_scenarios @@ -326,11 +321,11 @@ async def fetch_list_evaluations( List[Evaluation]: A list of evaluations. """ try: - app = await db_manager.fetch_app_by_id(app_id, request.state.project_id) + app = await db_manager.fetch_app_by_id(app_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(app.project_id), permission=Permission.VIEW_EVALUATION, ) logger.debug( @@ -344,9 +339,7 @@ async def fetch_list_evaluations( status_code=403, ) - return await evaluation_service.fetch_list_evaluations( - app, request.state.project_id - ) + return await evaluation_service.fetch_list_evaluations(app, str(app.project_id)) except Exception as exc: import traceback @@ -374,9 +367,7 @@ async def fetch_evaluation( Evaluation: The fetched evaluation. """ try: - evaluation = await db_manager.fetch_evaluation_by_id( - evaluation_id, request.state.project_id - ) + evaluation = await db_manager.fetch_evaluation_by_id(evaluation_id) if not evaluation: raise HTTPException( status_code=404, detail=f"Evaluation with id {evaluation_id} not found" @@ -385,7 +376,7 @@ async def fetch_evaluation( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(evaluation.project_id), permission=Permission.VIEW_EVALUATION, ) logger.debug( @@ -421,10 +412,11 @@ async def delete_evaluations( """ try: + evaluation = await db_manager.fetch_evaluation_by_id(payload.evaluations_ids[0]) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(evaluation.project), permission=Permission.DELETE_EVALUATION, ) logger.debug(f"User has permission to delete evaluation: {has_permission}") @@ -441,12 +433,12 @@ async def delete_evaluations( user_uid=request.state.user_id, object_id=random.choice(payload.evaluations_ids), object_type="evaluation", - project_id=request.state.project_id, + project_id=str(evaluation.project_id), ) logger.debug("Successfully updated last_modified_by app information") await evaluation_service.delete_evaluations( - payload.evaluations_ids, request.state.project_id + payload.evaluations_ids, str(evaluation.project_id) ) return Response(status_code=status.HTTP_204_NO_CONTENT) except Exception as exc: @@ -474,11 +466,11 @@ async def fetch_evaluation_scenarios( """ try: evaluations_ids_list = evaluations_ids.split(",") - + evaluation = await db_manager.fetch_evaluation_by_id(evaluations_ids_list[0]) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(evaluation.project_id), permission=Permission.VIEW_EVALUATION, ) logger.debug( @@ -493,7 +485,7 @@ async def fetch_evaluation_scenarios( ) eval_scenarios = await evaluation_service.compare_evaluations_scenarios( - evaluations_ids_list, request.state.project_id + evaluations_ids_list, str(evaluation.project_id) ) return eval_scenarios diff --git a/agenta-backend/agenta_backend/routers/evaluators_router.py b/agenta-backend/agenta_backend/routers/evaluators_router.py index 80b610abfa..7729c97abc 100644 --- a/agenta-backend/agenta_backend/routers/evaluators_router.py +++ b/agenta-backend/agenta_backend/routers/evaluators_router.py @@ -63,10 +63,11 @@ async def get_evaluator_configs( """ try: + app_db = await db_manager.fetch_app_by_id(app_id=app_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(app_db.project_id), permission=Permission.VIEW_EVALUATION, ) if not has_permission: @@ -78,7 +79,7 @@ async def get_evaluator_configs( ) evaluators_configs = await evaluator_manager.get_evaluators_configs( - request.state.project_id + str(app_db.project_id) ) return evaluators_configs except Exception as e: @@ -105,7 +106,7 @@ async def get_evaluator_config( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(evaluator_config_db.project_id), permission=Permission.VIEW_EVALUATION, ) if not has_permission: @@ -140,13 +141,11 @@ async def create_new_evaluator_config( EvaluatorConfigDB: Evaluator configuration api model. """ try: - app_db = await db_manager.get_app_instance_by_id( - app_id=payload.app_id, project_id=request.state.project_id - ) + app_db = await db_manager.get_app_instance_by_id(app_id=payload.app_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(app_db.project_id), permission=Permission.CREATE_EVALUATION, ) if not has_permission: @@ -158,7 +157,7 @@ async def create_new_evaluator_config( ) evaluator_config = await evaluator_manager.create_evaluator_config( - project_id=request.state.project_id, + project_id=str(app_db.project_id), app_name=app_db.app_name, name=payload.name, evaluator_key=payload.evaluator_key, @@ -187,10 +186,13 @@ async def update_evaluator_config( """ try: + evaluator_config = await db_manager.fetch_evaluator_config( + evaluator_config_id=evaluator_config_id + ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(evaluator_config.project_id), permission=Permission.EDIT_EVALUATION, ) if not has_permission: @@ -228,10 +230,13 @@ async def delete_evaluator_config( bool: True if deletion was successful, False otherwise. """ try: + evaluator_config = await db_manager.fetch_evaluator_config( + evaluator_config_id=evaluator_config_id + ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(evaluator_config.project_id), permission=Permission.DELETE_EVALUATION, ) if not has_permission: diff --git a/agenta-backend/agenta_backend/routers/human_evaluation_router.py b/agenta-backend/agenta_backend/routers/human_evaluation_router.py index e0733b542e..846c551ee3 100644 --- a/agenta-backend/agenta_backend/routers/human_evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/human_evaluation_router.py @@ -55,16 +55,14 @@ async def create_evaluation( _description_ """ try: - app = await db_manager.fetch_app_by_id( - app_id=payload.app_id, project_id=request.state.project_id - ) + app = await db_manager.fetch_app_by_id(app_id=payload.app_id) if app is None: raise HTTPException(status_code=404, detail="App not found") if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(app.project_id), permission=Permission.CREATE_EVALUATION, ) if not has_permission: @@ -75,7 +73,7 @@ async def create_evaluation( ) new_human_evaluation_db = await evaluation_service.create_new_human_evaluation( - payload, request.state.user_id, request.state.project_id + payload ) return await converters.human_evaluation_db_to_simple_evaluation_output( new_human_evaluation_db @@ -108,10 +106,11 @@ async def fetch_list_human_evaluations( """ try: + app = await db_manager.fetch_app_by_id(app_id=app_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(app.project_id), permission=Permission.VIEW_EVALUATION, ) if not has_permission: @@ -122,7 +121,7 @@ async def fetch_list_human_evaluations( ) return await evaluation_service.fetch_list_human_evaluations( - app_id, request.state.project_id + app_id, str(app.project_id) ) except Exception as e: status_code = e.status_code if hasattr(e, "status_code") else 500 # type: ignore @@ -143,16 +142,14 @@ async def fetch_human_evaluation( HumanEvaluation: The fetched evaluation. """ try: - human_evaluation = await db_manager.fetch_human_evaluation_by_id( - evaluation_id, request.state.project_id - ) + human_evaluation = await db_manager.fetch_human_evaluation_by_id(evaluation_id) if not human_evaluation: raise HTTPException(status_code=404, detail="Evaluation not found") if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(human_evaluation.project_id), permission=Permission.VIEW_EVALUATION, ) if not has_permission: @@ -190,9 +187,7 @@ async def fetch_evaluation_scenarios( """ try: - human_evaluation = await db_manager.fetch_human_evaluation_by_id( - evaluation_id, request.state.project_id - ) + human_evaluation = await db_manager.fetch_human_evaluation_by_id(evaluation_id) if human_evaluation is None: raise HTTPException( status_code=404, @@ -202,7 +197,7 @@ async def fetch_evaluation_scenarios( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(human_evaluation.project_id), permission=Permission.VIEW_EVALUATION, ) if not has_permission: @@ -243,16 +238,14 @@ async def update_human_evaluation( """ try: - human_evaluation = await db_manager.fetch_human_evaluation_by_id( - evaluation_id, request.state.project_id - ) + human_evaluation = await db_manager.fetch_human_evaluation_by_id(evaluation_id) if not human_evaluation: raise HTTPException(status_code=404, detail="Evaluation not found") if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(human_evaluation.project_id), permission=Permission.EDIT_EVALUATION, ) if not has_permission: @@ -292,7 +285,7 @@ async def update_evaluation_scenario_router( """ try: evaluation_scenario_db = await db_manager.fetch_human_evaluation_scenario_by_id( - evaluation_scenario_id, request.state.project_id + evaluation_scenario_id ) if evaluation_scenario_db is None: raise HTTPException( @@ -303,7 +296,7 @@ async def update_evaluation_scenario_router( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(evaluation_scenario_db.project_id), permission=Permission.EDIT_EVALUATION, ) if not has_permission: @@ -344,7 +337,7 @@ async def get_evaluation_scenario_score_router( """ try: evaluation_scenario = db_manager.fetch_evaluation_scenario_by_id( - evaluation_scenario_id, request.state.project_id + evaluation_scenario_id ) if evaluation_scenario is None: raise HTTPException( @@ -355,7 +348,7 @@ async def get_evaluation_scenario_score_router( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(evaluation_scenario.project_id), permission=Permission.VIEW_EVALUATION, ) if not has_permission: @@ -390,7 +383,7 @@ async def update_evaluation_scenario_score_router( """ try: evaluation_scenario = await db_manager.fetch_evaluation_scenario_by_id( - evaluation_scenario_id, request.state.project_id + evaluation_scenario_id ) if evaluation_scenario is None: raise HTTPException( @@ -401,7 +394,7 @@ async def update_evaluation_scenario_score_router( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(evaluation_scenario.project_id), permission=Permission.VIEW_EVALUATION, ) if not has_permission: @@ -436,9 +429,7 @@ async def fetch_results( """ try: - evaluation = await db_manager.fetch_human_evaluation_by_id( - evaluation_id, request.state.project_id - ) + evaluation = await db_manager.fetch_human_evaluation_by_id(evaluation_id) if evaluation is None: raise HTTPException( status_code=404, @@ -447,7 +438,7 @@ async def fetch_results( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(evaluation.project_id), permission=Permission.VIEW_EVALUATION, ) if not has_permission: @@ -476,24 +467,25 @@ async def fetch_results( @router.delete("/", response_model=List[str]) async def delete_evaluations( - delete_evaluations: DeleteEvaluation, + payload: DeleteEvaluation, request: Request, ): """ Delete specific comparison tables based on their unique IDs. Args: - delete_evaluations (List[str]): The unique identifiers of the comparison tables to delete. + payload (List[str]): The unique identifiers of the comparison tables to delete. Returns: A list of the deleted comparison tables' IDs. """ try: + evaluation = await db_manager.fetch_evaluation_by_id(payload.evaluations_ids[0]) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(evaluation.project_id), permission=Permission.DELETE_EVALUATION, ) if not has_permission: @@ -503,9 +495,7 @@ async def delete_evaluations( status_code=403, ) - await evaluation_service.delete_human_evaluations( - delete_evaluations.evaluations_ids, request.state.project_id - ) + await evaluation_service.delete_human_evaluations(payload.evaluations_ids) return Response(status_code=status.HTTP_204_NO_CONTENT) except Exception as e: import traceback diff --git a/agenta-backend/agenta_backend/routers/testset_router.py b/agenta-backend/agenta_backend/routers/testset_router.py index afa55a93fd..ddd39bcc26 100644 --- a/agenta-backend/agenta_backend/routers/testset_router.py +++ b/agenta-backend/agenta_backend/routers/testset_router.py @@ -67,13 +67,11 @@ async def upload_file( dict: The result of the upload process. """ - app = await db_manager.fetch_app_by_id( - app_id=app_id, project_id=request.state.project_id - ) + app = await db_manager.fetch_app_by_id(app_id=app_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(app.project_id), permission=Permission.CREATE_TESTSET, ) logger.debug(f"User has Permission to upload Testset: {has_permission}") @@ -116,7 +114,7 @@ async def upload_file( try: testset = await db_manager.create_testset( - app=app, project_id=request.state.project_id, testset_data=document + app=app, project_id=str(app.project_id), testset_data=document ) return TestSetSimpleResponse( id=str(testset.id), @@ -147,13 +145,11 @@ async def import_testset( dict: The result of the import process. """ - app = await db_manager.fetch_app_by_id( - app_id=app_id, project_id=request.state.project_id - ) + app = await db_manager.fetch_app_by_id(app_id=app_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(app.project_id), permission=Permission.CREATE_TESTSET, ) logger.debug(f"User has Permission to import Testset: {has_permission}") @@ -184,7 +180,7 @@ async def import_testset( document["csvdata"].append(row) testset = await db_manager.create_testset( - app=app, project_id=request.state.project_id, testset_data=document + app=app, project_id=str(app.project_id), testset_data=document ) return TestSetSimpleResponse( id=str(testset.id), @@ -227,13 +223,11 @@ async def create_testset( str: The id of the test set created. """ - app = await db_manager.fetch_app_by_id( - app_id=app_id, project_id=request.state.project_id - ) + app = await db_manager.fetch_app_by_id(app_id=app_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(app.project_id), permission=Permission.CREATE_TESTSET, ) logger.debug(f"User has Permission to create Testset: {has_permission}") @@ -251,7 +245,7 @@ async def create_testset( "csvdata": csvdata.csvdata, } testset_instance = await db_manager.create_testset( - app=app, project_id=request.state.project_id, testset_data=testset_data + app=app, project_id=str(app.project_id), testset_data=testset_data ) if testset_instance is not None: return TestSetSimpleResponse( @@ -281,16 +275,14 @@ async def update_testset( str: The id of the test set updated. """ - testset = await db_manager.fetch_testset_by_id( - testset_id=testset_id, project_id=request.state.project_id - ) + testset = await db_manager.fetch_testset_by_id(testset_id=testset_id) if testset is None: raise HTTPException(status_code=404, detail="testset not found") if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(testset.project_id), permission=Permission.EDIT_TESTSET, ) logger.debug(f"User has Permission to update Testset: {has_permission}") @@ -335,13 +327,11 @@ async def get_testsets( - `HTTPException` with status code 404 if no testsets are found. """ - app = await db_manager.fetch_app_by_id( - app_id=app_id, project_id=request.state.project_id - ) + app = await db_manager.fetch_app_by_id(app_id=app_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(app.project_id), permission=Permission.VIEW_TESTSET, ) logger.debug(f"User has Permission to view Testsets: {has_permission}") @@ -353,8 +343,8 @@ async def get_testsets( status_code=403, ) - testsets = await db_manager.fetch_testsets_by_app_id( - project_id=request.state.project_id + testsets = await db_manager.fetch_testsets_by_project_id( + project_id=str(app.project_id) ) return [ TestSetOutputResponse( @@ -382,13 +372,11 @@ async def get_single_testset( """ try: - test_set = await db_manager.fetch_testset_by_id( - testset_id=testset_id, project_id=request.state.project_id - ) + test_set = await db_manager.fetch_testset_by_id(testset_id=testset_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(test_set.project_id), permission=Permission.VIEW_TESTSET, ) logger.debug(f"User has Permission to view Testset: {has_permission}") @@ -424,19 +412,21 @@ async def delete_testsets( """ if isCloudEE(): - has_permission = await check_action_access( - user_uid=request.state.user_id, - project_id=request.state.project_id, - permission=Permission.DELETE_TESTSET, - ) - logger.debug(f"User has Permission to delete Testset: {has_permission}") - if not has_permission: - error_msg = f"You do not have permission to perform this action. Please contact your organization admin." - logger.error(error_msg) - return JSONResponse( - {"detail": error_msg}, - status_code=403, + for testset_id in payload.testset_ids: + testset = await db_manager.fetch_testset_by_id(testset_id=testset_id) + has_permission = await check_action_access( + user_uid=request.state.user_id, + project_id=str(testset.project_id), + permission=Permission.DELETE_TESTSET, ) + logger.debug(f"User has Permission to delete Testset: {has_permission}") + if not has_permission: + error_msg = f"You do not have permission to perform this action. Please contact your organization admin." + logger.error(error_msg) + return JSONResponse( + {"detail": error_msg}, + status_code=403, + ) await db_manager.remove_testsets(testset_ids=payload.testset_ids) return payload.testset_ids diff --git a/agenta-backend/agenta_backend/routers/variants_router.py b/agenta-backend/agenta_backend/routers/variants_router.py index 0d242910ab..3cbfc3fd16 100644 --- a/agenta-backend/agenta_backend/routers/variants_router.py +++ b/agenta-backend/agenta_backend/routers/variants_router.py @@ -69,15 +69,13 @@ async def add_variant_from_base_and_config( logger.debug("Initiating process to add a variant based on a previous one.") logger.debug(f"Received payload: {payload}") - base_db = await db_manager.fetch_base_by_id( - payload.base_id, request.state.project_id - ) + base_db = await db_manager.fetch_base_by_id(payload.base_id) # Check user has permission to add variant if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(base_db.project_id), permission=Permission.CREATE_APPLICATION, ) logger.debug( @@ -97,7 +95,7 @@ async def add_variant_from_base_and_config( new_config_name=payload.new_config_name, parameters=payload.parameters, user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(base_db.project_id), ) logger.debug(f"Successfully added new variant: {db_app_variant}") @@ -106,12 +104,12 @@ async def add_variant_from_base_and_config( user_uid=request.state.user_id, object_id=str(db_app_variant.app_id), object_type="app", - project_id=request.state.project_id, + project_id=str(base_db.project_id), ) logger.debug("Successfully updated last_modified_by app information") app_variant_db = await db_manager.get_app_variant_instance_by_id( - str(db_app_variant.id), request.state.project_id + str(db_app_variant.id) ) return await converters.app_variant_db_to_output(app_variant_db) @@ -139,10 +137,11 @@ async def remove_variant( """ try: + variant = await db_manager.fetch_app_variant_by_id(variant_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(variant.project_id), permission=Permission.DELETE_APPLICATION_VARIANT, ) logger.debug(f"User has Permission to delete app variant: {has_permission}") @@ -159,12 +158,12 @@ async def remove_variant( user_uid=request.state.user_id, object_id=variant_id, object_type="variant", - project_id=request.state.project_id, + project_id=str(variant.project_id), ) logger.debug("Successfully updated last_modified_by app information") await app_manager.terminate_and_remove_app_variant( - project_id=request.state.project_id, app_variant_id=variant_id + project_id=str(variant.project_id), app_variant_id=variant_id ) except DockerException as e: detail = f"Docker error while trying to remove the app variant: {str(e)}" @@ -198,10 +197,11 @@ async def update_variant_parameters( JSONResponse: A JSON response containing the updated app variant parameters. """ try: + variant_db = await db_manager.fetch_app_variant_by_id(variant_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(variant_db.project_id), permission=Permission.MODIFY_VARIANT_CONFIGURATIONS, ) logger.debug( @@ -219,7 +219,7 @@ async def update_variant_parameters( app_variant_id=variant_id, parameters=payload.parameters, user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(variant_db.project_id), ) # Update last_modified_by app information @@ -227,7 +227,7 @@ async def update_variant_parameters( user_uid=request.state.user_id, object_id=variant_id, object_type="variant", - project_id=request.state.project_id, + project_id=str(variant_db.project_id), ) logger.debug("Successfully updated last_modified_by app information") except ValueError as e: @@ -262,13 +262,13 @@ async def update_variant_image( """ try: db_app_variant = await db_manager.fetch_app_variant_by_id( - app_variant_id=variant_id, project_id=request.state.project_id + app_variant_id=variant_id ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(db_app_variant.project_id), permission=Permission.CREATE_APPLICATION, ) logger.debug( @@ -283,7 +283,7 @@ async def update_variant_image( ) await app_manager.update_variant_image( - db_app_variant, request.state.project_id, image, request.state.user_id + db_app_variant, str(db_app_variant.project_id), image, request.state.user_id ) # Update last_modified_by app information @@ -291,7 +291,7 @@ async def update_variant_image( user_uid=request.state.user_id, object_id=str(db_app_variant.app_id), object_type="app", - project_id=request.state.project_id, + project_id=str(db_app_variant.project_id), ) logger.debug("Successfully updated last_modified_by app information") except ValueError as e: @@ -337,15 +337,13 @@ async def start_variant( HTTPException: If the app container cannot be started. """ - app_variant_db = await db_manager.fetch_app_variant_by_id( - app_variant_id=variant_id, project_id=request.state.project_id - ) + app_variant_db = await db_manager.fetch_app_variant_by_id(app_variant_id=variant_id) # Check user has permission to start variant if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(app_variant_db.project_id), permission=Permission.CREATE_APPLICATION, ) logger.debug(f"User has Permission to start variant: {has_permission}") @@ -364,14 +362,14 @@ async def start_variant( if action.action == VariantActionEnum.START: url: URI = await app_manager.start_variant( - app_variant_db, request.state.project_id, envvars + app_variant_db, str(app_variant_db.project_id), envvars ) # Deploy to production await db_manager.deploy_to_environment( environment_name="production", variant_id=str(app_variant_db.id), - project_id=request.state.project_id, + project_id=str(app_variant_db.project_id), user_uid=request.state.user_id, ) return url @@ -404,13 +402,13 @@ async def get_variant( logger.debug("getting variant " + variant_id) try: app_variant = await db_manager.fetch_app_variant_by_id( - app_variant_id=variant_id, project_id=request.state.project_id + app_variant_id=variant_id ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(app_variant.project_id), permission=Permission.VIEW_APPLICATION, ) logger.debug(f"User has Permission to get variant: {has_permission}") @@ -440,13 +438,13 @@ async def get_variant_revisions( logger.debug("getting variant revisions: ", variant_id) try: app_variant = await db_manager.fetch_app_variant_by_id( - app_variant_id=variant_id, project_id=request.state.project_id + app_variant_id=variant_id ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(app_variant.project_id), permission=Permission.VIEW_APPLICATION, ) logger.debug(f"User has Permission to get variant: {has_permission}") @@ -459,7 +457,7 @@ async def get_variant_revisions( ) app_variant_revisions = await db_manager.list_app_variant_revisions_by_variant( - app_variant=app_variant, project_id=request.state.project_id + app_variant=app_variant, project_id=str(app_variant.project_id) ) return await converters.app_variant_db_revisions_to_output( app_variant_revisions @@ -485,13 +483,13 @@ async def get_variant_revision( variant_id != "undefined" ), "Variant id is required to retrieve variant revision" app_variant = await db_manager.fetch_app_variant_by_id( - app_variant_id=variant_id, project_id=request.state.project_id + app_variant_id=variant_id ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=request.state.project_id, + project_id=str(app_variant.project_id), permission=Permission.VIEW_APPLICATION, ) logger.debug(f"User has Permission to get variant: {has_permission}") diff --git a/agenta-backend/agenta_backend/services/app_manager.py b/agenta-backend/agenta_backend/services/app_manager.py index e59a78e56a..09ea01d020 100644 --- a/agenta-backend/agenta_backend/services/app_manager.py +++ b/agenta-backend/agenta_backend/services/app_manager.py @@ -152,7 +152,7 @@ async def update_variant_image( if not valid_image: raise ValueError("Image could not be found in registry.") - base = await db_manager.fetch_base_by_id(str(app_variant_db.base_id), project_id) + base = await db_manager.fetch_base_by_id(str(app_variant_db.base_id)) deployment = await db_manager.get_deployment_by_id( str(base.deployment_id), project_id ) @@ -204,9 +204,7 @@ async def get_appdb_str_by_id(object_id: str, object_type: str) -> str: if object_type == "app": return object_id elif object_type == "variant": - app_variant_db = await db_manager.fetch_app_variant_by_id( - object_id, project_id - ) + app_variant_db = await db_manager.fetch_app_variant_by_id(object_id) if app_variant_db is None: raise db_manager.NoResultFound(f"Variant with id {object_id} not found") return str(app_variant_db.app_id) @@ -257,9 +255,7 @@ async def terminate_and_remove_app_variant( ), "Only one of app_variant_id or app_variant_db must be provided" if app_variant_id: - app_variant_db = await db_manager.fetch_app_variant_by_id( - app_variant_id, project_id - ) + app_variant_db = await db_manager.fetch_app_variant_by_id(app_variant_id) logger.debug(f"Fetched app variant {app_variant_db}") app_id = str(app_variant_db.app_id) # type: ignore @@ -274,7 +270,7 @@ async def terminate_and_remove_app_variant( ) if is_last_variant_for_image: base_db = await db_manager.fetch_base_by_id( - base_id=str(app_variant_db.base_id), project_id=project_id + base_id=str(app_variant_db.base_id) ) if not base_db: raise db_manager.NoResultFound( @@ -323,7 +319,7 @@ async def terminate_and_remove_app_variant( logger.debug("remove_app_variant_from_db") await db_manager.remove_app_variant_from_db(app_variant_db, project_id) - app_variants = await db_manager.list_app_variants(app_id, project_id) + app_variants = await db_manager.list_app_variants(app_id) logger.debug(f"Count of app variants available: {len(app_variants)}") if ( len(app_variants) == 0 @@ -358,14 +354,13 @@ async def remove_app_related_resources(app_id: str, project_id: str): raise e from None -async def remove_app(app: AppDB, project_id: str): +async def remove_app(app: AppDB): """Removes all app variants from db, if it is the last one using an image, then deletes the image from the db, shutdowns the container, deletes it and remove the image from the registry Args: app (AppDB): The application instance to remove from database. - project_id (str): The ID of the project. """ if app is None: @@ -373,11 +368,11 @@ async def remove_app(app: AppDB, project_id: str): logger.error(error_msg) raise ValueError(error_msg) - app_variants = await db_manager.list_app_variants(str(app.id), project_id) + app_variants = await db_manager.list_app_variants(str(app.id)) try: for app_variant_db in app_variants: await terminate_and_remove_app_variant( - project_id=project_id, app_variant_db=app_variant_db + project_id=str(app_variant_db.project_id), app_variant_db=app_variant_db ) logger.info( f"Successfully deleted app variant {app_variant_db.app.app_name}/{app_variant_db.variant_name}." @@ -385,14 +380,14 @@ async def remove_app(app: AppDB, project_id: str): if len(app_variants) == 0: logger.debug("remove_app_related_resources") - await remove_app_related_resources(str(app.id), project_id) + await remove_app_related_resources(str(app.id), str(app.project_id)) except Exception as e: # Failsafe: in case something went wrong, # delete app and its related resources try: logger.debug("remove_app_related_resources") - await remove_app_related_resources(str(app.id), project_id) + await remove_app_related_resources(str(app.id), str(app.project_id)) except Exception as e: logger.error( f"An error occurred while deleting app {app.id} and its associated resources: {str(e)}" diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 7f62a68e35..c6554dd6cc 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -151,7 +151,7 @@ async def get_image_by_id(image_id: str) -> ImageDB: return image -async def fetch_app_by_id(app_id: str, project_id: str) -> AppDB: +async def fetch_app_by_id(app_id: str) -> AppDB: """Fetches an app by its ID. Args: @@ -161,23 +161,18 @@ async def fetch_app_by_id(app_id: str, project_id: str) -> AppDB: assert app_id is not None, "app_id cannot be None" app_uuid = await get_object_uuid(object_id=app_id, table_name="app_db") async with db_engine.get_session() as session: - base_query = select(AppDB).filter_by( - id=uuid.UUID(app_uuid), project_id=uuid.UUID(project_id) - ) + base_query = select(AppDB).filter_by(id=uuid.UUID(app_uuid)) result = await session.execute(base_query) app = result.unique().scalars().first() return app -async def fetch_app_variant_by_id( - app_variant_id: str, project_id: str -) -> Optional[AppVariantDB]: +async def fetch_app_variant_by_id(app_variant_id: str) -> Optional[AppVariantDB]: """ Fetches an app variant by its ID. Args: app_variant_id (str): The ID of the app variant to fetch. - project_id (str): The ID of the project the app variant belongs to. Returns: AppVariantDB: The fetched app variant, or None if no app variant was found. @@ -198,11 +193,7 @@ async def fetch_app_variant_by_id( joinedload(AppVariantDB.image).load_only(ImageDB.docker_id, ImageDB.tags), # type: ignore ) - result = await session.execute( - query.filter_by( - id=uuid.UUID(app_variant_id), project_id=uuid.UUID(project_id) - ) - ) + result = await session.execute(query.filter_by(id=uuid.UUID(app_variant_id))) app_variant = result.scalars().first() return app_variant @@ -285,13 +276,12 @@ async def fetch_app_variant_revision_by_variant( return app_variant_revision -async def fetch_base_by_id(base_id: str, project_id: str) -> Optional[VariantBaseDB]: +async def fetch_base_by_id(base_id: str) -> Optional[VariantBaseDB]: """ Fetches a base by its ID. Args: base_id (str): The ID of the base to fetch. - project_id (str): The ID of the project. Returns: VariantBaseDB: The fetched base, or None if no base was found. @@ -305,7 +295,7 @@ async def fetch_base_by_id(base_id: str, project_id: str) -> Optional[VariantBas .options( joinedload(VariantBaseDB.image), joinedload(VariantBaseDB.deployment) ) - .filter_by(id=uuid.UUID(base_uuid), project_id=uuid.UUID(project_id)) + .filter_by(id=uuid.UUID(base_uuid)) ) base = result.scalars().first() return base @@ -560,7 +550,9 @@ async def create_deployment( raise Exception(f"Error while creating deployment: {e}") -async def create_app_and_envs(app_name: str, project_id: str) -> AppDB: +async def create_app_and_envs( + app_name: str, project_id: Optional[str] = None, workspace_id: Optional[str] = None +) -> AppDB: """ Create a new app with the given name and organization ID. @@ -575,10 +567,14 @@ async def create_app_and_envs(app_name: str, project_id: str) -> AppDB: ValueError: If an app with the same name already exists. """ - app = await fetch_app_by_name_and_parameters(app_name, project_id) + app = await fetch_app_by_name_and_parameters(app_name) if app is not None: raise ValueError("App with the same name already exists") + if isCloudEE(): + project = await db_manager_ee.get_project_by_workspace(workspace_id) + project_id = str(project.id) + async with db_engine.get_session() as session: app = AppDB(app_name=app_name, project_id=uuid.UUID(project_id)) @@ -586,9 +582,7 @@ async def create_app_and_envs(app_name: str, project_id: str) -> AppDB: await session.commit() await session.refresh(app) - await initialize_environments( - session=session, app_db=app, project_id=project_id - ) + await initialize_environments(session=session, app_db=app) return app @@ -613,12 +607,11 @@ async def update_app(app_id: str, values_to_update: dict) -> None: await session.commit() -async def get_deployment_by_id(deployment_id: str, project_id: str) -> DeploymentDB: +async def get_deployment_by_id(deployment_id: str) -> DeploymentDB: """Get the deployment object from the database with the provided id. Arguments: deployment_id (str): The deployment id - project_id (str): The ID of the project Returns: DeploymentDB: instance of deployment object @@ -626,9 +619,7 @@ async def get_deployment_by_id(deployment_id: str, project_id: str) -> Deploymen async with db_engine.get_session() as session: result = await session.execute( - select(DeploymentDB).filter_by( - id=uuid.UUID(deployment_id), project_id=uuid.UUID(project_id) - ) + select(DeploymentDB).filter_by(id=uuid.UUID(deployment_id)) ) deployment = result.scalars().first() return deployment @@ -698,13 +689,12 @@ async def list_bases_for_app_id(app_id: str, base_name: Optional[str] = None): return bases -async def list_variants_for_base(base: VariantBaseDB, project_id: str): +async def list_variants_for_base(base: VariantBaseDB): """ Lists all the app variants from the db for a base Args: base (VariantBaseDB): if specified, only returns the variants for the base - project_id (str): The ID of the project Returns: List[AppVariant]: List of AppVariant objects @@ -714,7 +704,7 @@ async def list_variants_for_base(base: VariantBaseDB, project_id: str): async with db_engine.get_session() as session: result = await session.execute( select(AppVariantDB) - .filter_by(base_id=base.id, project_id=uuid.UUID(project_id)) + .filter_by(base_id=base.id) .order_by(AppVariantDB.variant_name.asc()) ) app_variants = result.scalars().all() @@ -869,7 +859,7 @@ async def get_orga_image_instance_by_uri(template_uri: str) -> ImageDB: return image -async def get_app_instance_by_id(app_id: str, project_id: str) -> AppDB: +async def get_app_instance_by_id(app_id: str) -> AppDB: """Get the app object from the database with the provided id. Arguments: @@ -880,11 +870,7 @@ async def get_app_instance_by_id(app_id: str, project_id: str) -> AppDB: """ async with db_engine.get_session() as session: - result = await session.execute( - select(AppDB).filter_by( - id=uuid.UUID(app_id), project_id=uuid.UUID(project_id) - ) - ) + result = await session.execute(select(AppDB).filter_by(id=uuid.UUID(app_id))) app = result.scalars().first() return app @@ -919,7 +905,7 @@ async def add_variant_from_base_and_config( raise HTTPException(status_code=404, detail="Previous app variant not found") logger.debug(f"Located previous variant: {previous_app_variant_db}") - app_variant_for_base = await list_variants_for_base(base_db, project_id) + app_variant_for_base = await list_variants_for_base(base_db) already_exists = any( av for av in app_variant_for_base if av.config_name == new_config_name # type: ignore @@ -980,17 +966,18 @@ async def list_apps( """ if app_name is not None: - app_db = await fetch_app_by_name_and_parameters( - app_name=app_name, project_id=project_id - ) + app_db = await fetch_app_by_name_and_parameters(app_name=app_name) return [converters.app_db_to_pydantic(app_db)] elif isCloudEE(): if isCloudEE(): + project = await db_manager_ee.get_project_by_workspace( + workspace_id=workspace_id + ) user_org_workspace_data = await get_user_org_and_workspace_id(user_uid) # type: ignore has_permission = await check_rbac_permission( # type: ignore user_org_workspace_data=user_org_workspace_data, - project_id=project_id, + project_id=str(project.id), permission=Permission.VIEW_APPLICATION, # type: ignore ) logger.debug(f"User has Permission to list apps: {has_permission}") @@ -1000,9 +987,6 @@ async def list_apps( detail="You do not have access to perform this action. Please contact your organization admin.", ) - project = await db_manager_ee.get_project_by_workspace( - workspace_id=workspace_id - ) async with db_engine.get_session() as session: result = await session.execute( select(AppDB).filter_by(project_id=project.id) @@ -1019,7 +1003,7 @@ async def list_apps( return [converters.app_db_to_pydantic(app) for app in apps] -async def list_app_variants(app_id: str, project_id: str): +async def list_app_variants(app_id: str): """ Lists all the app variants from the db @@ -1039,7 +1023,7 @@ async def list_app_variants(app_id: str, project_id: str): joinedload(AppVariantDB.app.of_type(AppDB)).load_only(AppDB.id, AppDB.app_name), # type: ignore joinedload(AppVariantDB.base.of_type(VariantBaseDB)).joinedload(VariantBaseDB.deployment.of_type(DeploymentDB)).load_only(DeploymentDB.uri), # type: ignore ) - .filter_by(app_id=uuid.UUID(app_uuid), project_id=uuid.UUID(project_id)) + .filter_by(app_id=uuid.UUID(app_uuid)) ) app_variants = result.scalars().all() return app_variants @@ -1138,7 +1122,7 @@ async def remove_app_variant_from_db(app_variant_db: AppVariantDB, project_id: s async def deploy_to_environment( - environment_name: str, variant_id: str, project_id: str, **user_org_data + environment_name: str, variant_id: str, **user_org_data ): """ Deploys an app variant to a specified environment. @@ -1146,7 +1130,6 @@ async def deploy_to_environment( Args: environment_name (str): The name of the environment to deploy the app variant to. variant_id (str): The ID of the app variant to deploy. - project_id (str): The ID of the project. Raises: ValueError: If the app variant is not found or if the environment is not found or if the app variant is already @@ -1155,9 +1138,9 @@ async def deploy_to_environment( None """ - app_variant_db = await fetch_app_variant_by_id(variant_id, project_id) + app_variant_db = await fetch_app_variant_by_id(variant_id) app_variant_revision_db = await fetch_app_variant_revision_by_variant( - app_variant_id=variant_id, project_id=project_id, revision=app_variant_db.revision # type: ignore + app_variant_id=variant_id, project_id=str(app_variant_db.project_id), revision=app_variant_db.revision # type: ignore ) if app_variant_db is None: raise ValueError("App variant not found") @@ -1174,7 +1157,7 @@ async def deploy_to_environment( result = await session.execute( select(AppEnvironmentDB).filter_by( app_id=app_variant_db.app_id, - project_id=uuid.UUID(project_id), + project_id=app_variant_db.project_id, name=environment_name, ) ) @@ -1193,7 +1176,7 @@ async def deploy_to_environment( session, environment_db, user, - project_id, + str(app_variant_db.project_id), deployed_app_variant_revision=app_variant_revision_db, deployment=deployment, ) @@ -1339,20 +1322,19 @@ async def update_app_environment_deployed_variant_revision( await session.refresh(app_environment) -async def list_environments(app_id: str, project_id: str, **kwargs: dict): +async def list_environments(app_id: str, **kwargs: dict): """ List all environments for a given app ID. Args: app_id (str): The ID of the app to list environments for. - project_id (str): The ID of the project to list environments for. Returns: List[AppEnvironmentDB]: A list of AppEnvironmentDB objects representing the environments for the given app ID. """ logging.debug("Listing environments for app %s", app_id) - app_instance = await fetch_app_by_id(app_id=app_id, project_id=project_id) + app_instance = await fetch_app_by_id(app_id=app_id) if app_instance is None: logging.error(f"App with id {app_id} not found") raise ValueError("App not found") @@ -1368,21 +1350,20 @@ async def list_environments(app_id: str, project_id: str, **kwargs: dict): AppVariantRevisionsDB.config_parameters, # type: ignore ) ) - .filter_by(app_id=uuid.UUID(app_id), project_id=uuid.UUID(project_id)) + .filter_by(app_id=uuid.UUID(app_id), project_id=app_instance.project_id) ) environments_db = result.scalars().all() return environments_db async def initialize_environments( - session: AsyncSession, app_db: AppDB, project_id: str + session: AsyncSession, app_db: AppDB ) -> List[AppEnvironmentDB]: """ Initializes the environments for the app with the given database. Args: app_db (AppDB): The database for the app. - project_id (str): The ID of the project. Returns: List[AppEnvironmentDB]: A list of the initialized environments. @@ -1390,15 +1371,13 @@ async def initialize_environments( environments = [] for env_name in ["development", "staging", "production"]: - env = await create_environment( - session=session, name=env_name, app_db=app_db, project_id=project_id - ) + env = await create_environment(session=session, name=env_name, app_db=app_db) environments.append(env) return environments async def create_environment( - session: AsyncSession, name: str, app_db: AppDB, project_id: str + session: AsyncSession, name: str, app_db: AppDB ) -> AppEnvironmentDB: """ Creates a new environment in the database. @@ -1406,14 +1385,13 @@ async def create_environment( Args: name (str): The name of the environment. app_db (AppDB): The AppDB object representing the app that the environment belongs to. - project_id (str): The ID of the project. Returns: AppEnvironmentDB: The newly created AppEnvironmentDB object. """ environment_db = AppEnvironmentDB( - app_id=app_db.id, name=name, project_id=uuid.UUID(project_id), revision=0 + app_id=app_db.id, name=name, project_id=app_db.project_id, revision=0 ) session.add(environment_db) @@ -1770,7 +1748,7 @@ async def get_app_variant_instance_by_id( return app_variant_db -async def fetch_testset_by_id(testset_id: str, project_id: str) -> Optional[TestSetDB]: +async def fetch_testset_by_id(testset_id: str) -> Optional[TestSetDB]: """Fetches a testset by its ID. Args: testset_id (str): The ID of the testset to fetch. @@ -1787,11 +1765,7 @@ async def fetch_testset_by_id(testset_id: str, project_id: str) -> Optional[Test raise ValueError(f"testset_id {testset_id} is not a valid UUID") from e async with db_engine.get_session() as session: - result = await session.execute( - select(TestSetDB).filter_by( - id=testset_uuid, project_id=uuid.UUID(project_id) - ) - ) + result = await session.execute(select(TestSetDB).filter_by(id=testset_uuid)) testset = result.scalars().first() return testset @@ -1842,8 +1816,8 @@ async def update_testset(testset_id: str, values_to_update: dict) -> None: await session.refresh(testset) -async def fetch_testsets_by_app_id(project_id: str): - """Fetches all testsets for a given app. +async def fetch_testsets_by_project_id(project_id: str): + """Fetches all testsets for a given project. Args: project_id (str): The ID of the project. @@ -1860,9 +1834,7 @@ async def fetch_testsets_by_app_id(project_id: str): return testsets -async def fetch_evaluation_by_id( - evaluation_id: str, project_id: str -) -> Optional[EvaluationDB]: +async def fetch_evaluation_by_id(evaluation_id: str) -> Optional[EvaluationDB]: """Fetches a evaluation by its ID. Args: @@ -1874,9 +1846,7 @@ async def fetch_evaluation_by_id( assert evaluation_id is not None, "evaluation_id cannot be None" async with db_engine.get_session() as session: - base_query = select(EvaluationDB).filter_by( - id=uuid.UUID(evaluation_id), project_id=uuid.UUID(project_id) - ) + base_query = select(EvaluationDB).filter_by(id=uuid.UUID(evaluation_id)) if isCloudEE(): query = base_query.options( joinedload(EvaluationDB.testset.of_type(TestSetDB)).load_only(TestSetDB.id, TestSetDB.name), # type: ignore @@ -1930,7 +1900,6 @@ async def list_human_evaluations(app_id: str, project_id: str): async def create_human_evaluation( app: AppDB, - project_id: str, status: str, evaluation_type: str, testset_id: str, @@ -1941,7 +1910,6 @@ async def create_human_evaluation( Args: app (AppDB: The app object - project_id (str): The ID of the project status (str): The status of the evaluation evaluation_type (str): The evaluation type testset_id (str): The ID of the evaluation testset @@ -1951,7 +1919,7 @@ async def create_human_evaluation( async with db_engine.get_session() as session: human_evaluation = HumanEvaluationDB( app_id=app.id, - project_id=uuid.UUID(project_id), + project_id=app.project_id, status=status, evaluation_type=evaluation_type, testset_id=testset_id, @@ -1965,7 +1933,6 @@ async def create_human_evaluation( await create_human_evaluation_variants( human_evaluation_id=str(human_evaluation.id), variants_ids=variants_ids, - project_id=project_id, ) return human_evaluation @@ -2005,7 +1972,7 @@ async def fetch_human_evaluation_variants(human_evaluation_id: str): async def create_human_evaluation_variants( - human_evaluation_id: str, variants_ids: List[str], project_id: str + human_evaluation_id: str, variants_ids: List[str] ): """ Creates human evaluation variants. @@ -2018,16 +1985,14 @@ async def create_human_evaluation_variants( variants_dict = {} for variant_id in variants_ids: - variant = await fetch_app_variant_by_id( - app_variant_id=variant_id, project_id=project_id - ) + variant = await fetch_app_variant_by_id(app_variant_id=variant_id) if variant: variants_dict[variant_id] = variant variants_revisions_dict = {} for variant_id, variant in variants_dict.items(): variant_revision = await fetch_app_variant_revision_by_variant( - app_variant_id=str(variant.id), project_id=project_id, revision=variant.revision # type: ignore + app_variant_id=str(variant.id), project_id=str(variant.project_id), revision=variant.revision # type: ignore ) if variant_revision: variants_revisions_dict[variant_id] = variant_revision @@ -2050,20 +2015,21 @@ async def create_human_evaluation_variants( async def fetch_human_evaluation_by_id( - evaluation_id: str, project_id: str + evaluation_id: str, ) -> Optional[HumanEvaluationDB]: - """Fetches a evaluation by its ID. + """ + Fetches a evaluation by its ID. + Args: evaluation_id (str): The ID of the evaluation to fetch. + Returns: EvaluationDB: The fetched evaluation, or None if no evaluation was found. """ assert evaluation_id is not None, "evaluation_id cannot be None" async with db_engine.get_session() as session: - base_query = select(HumanEvaluationDB).filter_by( - id=uuid.UUID(evaluation_id), project_id=uuid.UUID(project_id) - ) + base_query = select(HumanEvaluationDB).filter_by(id=uuid.UUID(evaluation_id)) if isCloudEE(): query = base_query.options( joinedload(HumanEvaluationDB.testset.of_type(TestSetDB)).load_only(TestSetDB.id, TestSetDB.name), # type: ignore @@ -2104,7 +2070,7 @@ async def update_human_evaluation(evaluation_id: str, values_to_update: dict): await session.refresh(human_evaluation) -async def delete_human_evaluation(evaluation_id: str, project_id: str): +async def delete_human_evaluation(evaluation_id: str): """Delete the evaluation by its ID. Args: @@ -2114,9 +2080,7 @@ async def delete_human_evaluation(evaluation_id: str, project_id: str): assert evaluation_id is not None, "evaluation_id cannot be None" async with db_engine.get_session() as session: result = await session.execute( - select(HumanEvaluationDB).filter_by( - id=uuid.UUID(evaluation_id), project_id=uuid.UUID(project_id) - ) + select(HumanEvaluationDB).filter_by(id=uuid.UUID(evaluation_id)) ) evaluation = result.scalars().first() if not evaluation: @@ -2233,13 +2197,12 @@ async def fetch_evaluation_scenarios(evaluation_id: str, project_id: str): async def fetch_evaluation_scenario_by_id( - evaluation_scenario_id: str, project_id: str + evaluation_scenario_id: str, ) -> Optional[EvaluationScenarioDB]: """Fetches and evaluation scenario by its ID. Args: evaluation_scenario_id (str): The ID of the evaluation scenario to fetch. - project_id (str): The project ID to use in fetching the evaluation scenario. Returns: EvaluationScenarioDB: The fetched evaluation scenario, or None if no evaluation scenario was found. @@ -2248,22 +2211,19 @@ async def fetch_evaluation_scenario_by_id( assert evaluation_scenario_id is not None, "evaluation_scenario_id cannot be None" async with db_engine.get_session() as session: result = await session.execute( - select(EvaluationScenarioDB).filter_by( - id=uuid.UUID(evaluation_scenario_id), project_id=uuid.UUID(project_id) - ) + select(EvaluationScenarioDB).filter_by(id=uuid.UUID(evaluation_scenario_id)) ) evaluation_scenario = result.scalars().first() return evaluation_scenario async def fetch_human_evaluation_scenario_by_id( - evaluation_scenario_id: str, project_id: str + evaluation_scenario_id: str, ) -> Optional[HumanEvaluationScenarioDB]: """Fetches and evaluation scenario by its ID. Args: evaluation_scenario_id (str): The ID of the evaluation scenario to fetch. - project_id (str): The project ID to use in fetching the human evaluation. Returns: EvaluationScenarioDB: The fetched evaluation scenario, or None if no evaluation scenario was found. @@ -2273,7 +2233,7 @@ async def fetch_human_evaluation_scenario_by_id( async with db_engine.get_session() as session: result = await session.execute( select(HumanEvaluationScenarioDB).filter_by( - id=uuid.UUID(evaluation_scenario_id), project_id=uuid.UUID(project_id) + id=uuid.UUID(evaluation_scenario_id) ) ) evaluation_scenario = result.scalars().first() @@ -2539,21 +2499,28 @@ async def update_app_variant( return app_variant -async def fetch_app_by_name_and_parameters(app_name: str, project_id: str): +async def fetch_app_by_name_and_parameters( + app_name: str, workspace_id: Optional[str] = None +): """Fetch an app by its name and project identifier. Args: app_name (str): The name of the app - project_id (str): The ID of the project + workspace_id (str, optional): The ID of the workspace. Defaults to None. Returns: AppDB: the instance of the app """ - async with db_engine.get_session() as session: - query = select(AppDB).filter_by( - app_name=app_name, project_id=uuid.UUID(project_id) + if isCloudEE() and workspace_id is not None: + project = await db_manager_ee.get_project_by_workspace( + workspace_id=workspace_id ) + query = select(AppDB).filter_by(app_name=app_name, project_id=project.id) + else: + query = select(AppDB).filter_by(app_name=app_name) + + async with db_engine.get_session() as session: result = await session.execute(query) app_db = result.unique().scalars().first() return app_db @@ -2866,12 +2833,11 @@ async def fetch_evaluators_configs(project_id: str): return evaluators_configs -async def fetch_evaluator_config(evaluator_config_id: str, project_id: str): +async def fetch_evaluator_config(evaluator_config_id: str): """Fetch evaluator configurations from the database. Args: evaluator_config_id (str): The ID of the evaluator configuration. - project_id (str): The ID of the project. Returns: EvaluatorConfigDB: the evaluator configuration object. @@ -2879,9 +2845,7 @@ async def fetch_evaluator_config(evaluator_config_id: str, project_id: str): async with db_engine.get_session() as session: result = await session.execute( - select(EvaluatorConfigDB).filter_by( - id=uuid.UUID(evaluator_config_id), project_id=uuid.UUID(project_id) - ) + select(EvaluatorConfigDB).filter_by(id=uuid.UUID(evaluator_config_id)) ) evaluator_config = result.scalars().first() return evaluator_config diff --git a/agenta-backend/agenta_backend/services/evaluation_service.py b/agenta-backend/agenta_backend/services/evaluation_service.py index 9b98e3be48..70d2ee3da9 100644 --- a/agenta-backend/agenta_backend/services/evaluation_service.py +++ b/agenta-backend/agenta_backend/services/evaluation_service.py @@ -331,7 +331,7 @@ async def fetch_human_evaluation(human_evaluation_db) -> HumanEvaluation: return await converters.human_evaluation_db_to_pydantic(human_evaluation_db) -async def delete_human_evaluations(evaluation_ids: List[str], project_id: str) -> None: +async def delete_human_evaluations(evaluation_ids: List[str]) -> None: """ Delete evaluations by their IDs. @@ -344,9 +344,7 @@ async def delete_human_evaluations(evaluation_ids: List[str], project_id: str) - """ for evaluation_id in evaluation_ids: - await db_manager.delete_human_evaluation( - evaluation_id=evaluation_id, project_id=project_id - ) + await db_manager.delete_human_evaluation(evaluation_id=evaluation_id) async def delete_evaluations(evaluation_ids: List[str]) -> None: @@ -363,23 +361,18 @@ async def delete_evaluations(evaluation_ids: List[str]) -> None: await db_manager.delete_evaluations(evaluation_ids=evaluation_ids) -async def create_new_human_evaluation( - payload: NewHumanEvaluation, user_uid: str, project_id: str -) -> HumanEvaluationDB: +async def create_new_human_evaluation(payload: NewHumanEvaluation) -> HumanEvaluationDB: """ Create a new evaluation based on the provided payload and additional arguments. Args: payload (NewEvaluation): The evaluation payload. - user_uid (str): The user_uid of the user - project_id (str): The ID of the project Returns: HumanEvaluationDB """ - user = await db_manager.get_user(user_uid) - app = await db_manager.fetch_app_by_id(app_id=payload.app_id, project_id=project_id) + app = await db_manager.fetch_app_by_id(app_id=payload.app_id) if app is None: raise HTTPException( status_code=404, @@ -388,7 +381,6 @@ async def create_new_human_evaluation( human_evaluation = await db_manager.create_human_evaluation( app=app, - project_id=project_id, status=payload.status, evaluation_type=payload.evaluation_type, testset_id=payload.testset_id, @@ -402,7 +394,7 @@ async def create_new_human_evaluation( await prepare_csvdata_and_create_evaluation_scenario( human_evaluation.testset.csvdata, payload.inputs, - project_id, + str(app.project_id), payload.evaluation_type, human_evaluation, ) diff --git a/agenta-backend/agenta_backend/tasks/evaluations.py b/agenta-backend/agenta_backend/tasks/evaluations.py index 10619ea8b2..9d0dd14e80 100644 --- a/agenta-backend/agenta_backend/tasks/evaluations.py +++ b/agenta-backend/agenta_backend/tasks/evaluations.py @@ -27,7 +27,6 @@ create_new_evaluation_scenario, fetch_app_by_id, fetch_app_variant_by_id, - fetch_evaluation_by_id, fetch_evaluator_config, fetch_testset_by_id, get_deployment_by_id, @@ -102,26 +101,22 @@ def evaluate( ) # 1. Fetch data from the database - app = loop.run_until_complete(fetch_app_by_id(app_id, project_id)) - app_variant_db = loop.run_until_complete( - fetch_app_variant_by_id(variant_id, project_id) - ) + app = loop.run_until_complete(fetch_app_by_id(app_id)) + app_variant_db = loop.run_until_complete(fetch_app_variant_by_id(variant_id)) assert ( app_variant_db is not None ), f"App variant with id {variant_id} not found!" app_variant_parameters = app_variant_db.config_parameters - testset_db = loop.run_until_complete( - fetch_testset_by_id(testset_id, project_id) - ) + testset_db = loop.run_until_complete(fetch_testset_by_id(testset_id)) evaluator_config_dbs = [] for evaluator_config_id in evaluators_config_ids: evaluator_config = loop.run_until_complete( - fetch_evaluator_config(evaluator_config_id, project_id) + fetch_evaluator_config(evaluator_config_id) ) evaluator_config_dbs.append(evaluator_config) deployment_db = loop.run_until_complete( - get_deployment_by_id(str(app_variant_db.base.deployment_id), project_id) + get_deployment_by_id(str(app_variant_db.base.deployment_id)) ) uri = deployment_manager.get_deployment_uri(uri=deployment_db.uri) # type: ignore From d855d4c0eb0aede3aa7cbc852562e828eb8ca766 Mon Sep 17 00:00:00 2001 From: Abram Date: Wed, 25 Sep 2024 16:18:21 +0100 Subject: [PATCH 39/57] refactor (backend): resolve failing evaluation run --- agenta-backend/agenta_backend/services/db_manager.py | 3 +-- .../agenta_backend/services/evaluation_service.py | 8 +++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index c6554dd6cc..b23e2d30e5 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -2683,7 +2683,7 @@ async def fetch_evaluations_by_resource( ) -async def delete_evaluations(evaluation_ids: List[str], project_id: str) -> None: +async def delete_evaluations(evaluation_ids: List[str]) -> None: """Delete evaluations based on the ids provided from the db. Args: @@ -2694,7 +2694,6 @@ async def delete_evaluations(evaluation_ids: List[str], project_id: str) -> None async with db_engine.get_session() as session: query = select(EvaluationDB).where( EvaluationDB.id.in_(evaluation_ids), - EvaluationDB.project_id == uuid.UUID(project_id), ) result = await session.execute(query) evaluations = result.scalars().all() diff --git a/agenta-backend/agenta_backend/services/evaluation_service.py b/agenta-backend/agenta_backend/services/evaluation_service.py index 70d2ee3da9..43908791be 100644 --- a/agenta-backend/agenta_backend/services/evaluation_service.py +++ b/agenta-backend/agenta_backend/services/evaluation_service.py @@ -420,10 +420,8 @@ async def create_new_evaluation( Evaluation: The newly created evaluation. """ - app = await db_manager.fetch_app_by_id(app_id=app_id, project_id=project_id) - testset = await db_manager.fetch_testset_by_id( - testset_id=testset_id, project_id=project_id - ) + app = await db_manager.fetch_app_by_id(app_id=app_id) + testset = await db_manager.fetch_testset_by_id(testset_id=testset_id) variant_db = await db_manager.get_app_variant_instance_by_id( variant_id=variant_id, project_id=project_id ) @@ -460,7 +458,7 @@ async def retrieve_evaluation_results(evaluation_id: str) -> List[dict]: async def compare_evaluations_scenarios(evaluations_ids: List[str], project_id: str): - evaluation = await db_manager.fetch_evaluation_by_id(evaluations_ids[0], project_id) + evaluation = await db_manager.fetch_evaluation_by_id(evaluations_ids[0]) testset = evaluation.testset unique_testset_datapoints = remove_duplicates(testset.csvdata) formatted_inputs = extract_inputs_values_from_testset(unique_testset_datapoints) From 21435d7c6f2a60c7fa6f6cc7ad437271c5433405 Mon Sep 17 00:00:00 2001 From: Abram Date: Wed, 25 Sep 2024 16:20:27 +0100 Subject: [PATCH 40/57] fix (bug): resolve TypeError: list_variants_for_base() takes 1 positional argument but 2 were given --- agenta-backend/agenta_backend/routers/configs_router.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/configs_router.py b/agenta-backend/agenta_backend/routers/configs_router.py index e79cdb0c55..b72c0f6a69 100644 --- a/agenta-backend/agenta_backend/routers/configs_router.py +++ b/agenta-backend/agenta_backend/routers/configs_router.py @@ -47,9 +47,7 @@ async def save_config( status_code=403, ) - variants_db = await db_manager.list_variants_for_base( - base_db, str(base_db.project_id) - ) + variants_db = await db_manager.list_variants_for_base(base_db) variant_to_overwrite = None for variant_db in variants_db: if variant_db.config_name == payload.config_name: @@ -70,7 +68,6 @@ async def save_config( await db_manager.deploy_to_environment( environment_name="production", variant_id=str(variant_to_overwrite.id), - project_id=str(base_db.project_id), user_uid=request.state.user_id, ) else: From 6a3bdaee7cf5bb664e9bd144c8c52f36e2c85e7e Mon Sep 17 00:00:00 2001 From: Abram Date: Wed, 25 Sep 2024 16:24:57 +0100 Subject: [PATCH 41/57] fix (bug): resolve TypeError: get_deployment_by_id() takes 1 positional argument but 2 were given --- agenta-backend/agenta_backend/services/app_manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/agenta-backend/agenta_backend/services/app_manager.py b/agenta-backend/agenta_backend/services/app_manager.py index 09ea01d020..db533f8091 100644 --- a/agenta-backend/agenta_backend/services/app_manager.py +++ b/agenta-backend/agenta_backend/services/app_manager.py @@ -154,7 +154,7 @@ async def update_variant_image( base = await db_manager.fetch_base_by_id(str(app_variant_db.base_id)) deployment = await db_manager.get_deployment_by_id( - str(base.deployment_id), project_id + str(base.deployment_id) ) await deployment_manager.stop_and_delete_service(deployment) @@ -209,7 +209,7 @@ async def get_appdb_str_by_id(object_id: str, object_type: str) -> str: raise db_manager.NoResultFound(f"Variant with id {object_id} not found") return str(app_variant_db.app_id) elif object_type == "deployment": - deployment_db = await db_manager.get_deployment_by_id(object_id, project_id) + deployment_db = await db_manager.get_deployment_by_id(object_id) if deployment_db is None: raise db_manager.NoResultFound( f"Deployment with id {object_id} not found" @@ -288,7 +288,7 @@ async def terminate_and_remove_app_variant( logger.debug("_stop_and_delete_app_container") try: deployment = await db_manager.get_deployment_by_id( - str(base_db.deployment_id), project_id + str(base_db.deployment_id) ) except Exception as e: logger.error(f"Failed to get deployment {e}") From ad738db89a3a92334359ddc3e655426a6c4e4a2e Mon Sep 17 00:00:00 2001 From: Abram Date: Wed, 25 Sep 2024 16:35:27 +0100 Subject: [PATCH 42/57] refactor (backend): apply app name uniqueness only within project scope on app creation --- .../agenta_backend/routers/app_router.py | 4 ++- .../agenta_backend/services/app_manager.py | 4 +-- .../agenta_backend/services/db_manager.py | 26 +++++++++++++------ 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index e2885ac5a0..d2f2e2fc42 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -513,7 +513,9 @@ async def create_app_and_variant_from_template( ) app_name = payload.app_name.lower() app = await db_manager.fetch_app_by_name_and_parameters( - app_name, payload.workspace_id + app_name, + workspace_id=payload.workspace_id, + project_id=payload.project_id or request.state.project_id, ) if app is not None: raise Exception( diff --git a/agenta-backend/agenta_backend/services/app_manager.py b/agenta-backend/agenta_backend/services/app_manager.py index db533f8091..1ac1f2b286 100644 --- a/agenta-backend/agenta_backend/services/app_manager.py +++ b/agenta-backend/agenta_backend/services/app_manager.py @@ -153,9 +153,7 @@ async def update_variant_image( raise ValueError("Image could not be found in registry.") base = await db_manager.fetch_base_by_id(str(app_variant_db.base_id)) - deployment = await db_manager.get_deployment_by_id( - str(base.deployment_id) - ) + deployment = await db_manager.get_deployment_by_id(str(base.deployment_id)) await deployment_manager.stop_and_delete_service(deployment) await db_manager.remove_deployment(str(deployment.id)) diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index b23e2d30e5..26526d1554 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -567,14 +567,16 @@ async def create_app_and_envs( ValueError: If an app with the same name already exists. """ - app = await fetch_app_by_name_and_parameters(app_name) - if app is not None: - raise ValueError("App with the same name already exists") - if isCloudEE(): project = await db_manager_ee.get_project_by_workspace(workspace_id) project_id = str(project.id) + app = await fetch_app_by_name_and_parameters( + app_name=app_name, project_id=project_id + ) + if app is not None: + raise ValueError("App with the same name already exists") + async with db_engine.get_session() as session: app = AppDB(app_name=app_name, project_id=uuid.UUID(project_id)) @@ -966,7 +968,9 @@ async def list_apps( """ if app_name is not None: - app_db = await fetch_app_by_name_and_parameters(app_name=app_name) + app_db = await fetch_app_by_name_and_parameters( + app_name=app_name, project_id=project_id + ) return [converters.app_db_to_pydantic(app_db)] elif isCloudEE(): @@ -2500,13 +2504,16 @@ async def update_app_variant( async def fetch_app_by_name_and_parameters( - app_name: str, workspace_id: Optional[str] = None + app_name: str, + workspace_id: Optional[str] = None, + project_id: Optional[str] = None, ): """Fetch an app by its name and project identifier. Args: app_name (str): The name of the app workspace_id (str, optional): The ID of the workspace. Defaults to None. + project_id (str, optional): The ID of the project. Defaults to None. Returns: AppDB: the instance of the app @@ -2518,7 +2525,9 @@ async def fetch_app_by_name_and_parameters( ) query = select(AppDB).filter_by(app_name=app_name, project_id=project.id) else: - query = select(AppDB).filter_by(app_name=app_name) + query = select(AppDB).filter_by( + app_name=app_name, project_id=uuid.UUID(project_id) + ) async with db_engine.get_session() as session: result = await session.execute(query) @@ -2683,7 +2692,7 @@ async def fetch_evaluations_by_resource( ) -async def delete_evaluations(evaluation_ids: List[str]) -> None: +async def delete_evaluations(evaluation_ids: List[str], project_id: str) -> None: """Delete evaluations based on the ids provided from the db. Args: @@ -2694,6 +2703,7 @@ async def delete_evaluations(evaluation_ids: List[str]) -> None: async with db_engine.get_session() as session: query = select(EvaluationDB).where( EvaluationDB.id.in_(evaluation_ids), + EvaluationDB.project_id == uuid.UUID(project_id), ) result = await session.execute(query) evaluations = result.scalars().all() From 83ae0cccfb548253fbada0b8364d1a5528915219 Mon Sep 17 00:00:00 2001 From: Abram Date: Wed, 25 Sep 2024 17:52:01 +0100 Subject: [PATCH 43/57] refactor (tests0: resolve failing backend tests due to project structuring --- .../routers/evaluation_router.py | 6 ++---- .../tests/variants_main_router/conftest.py | 16 ++++++++++----- .../test_app_variant_router.py | 20 ++++++++++--------- .../test_variant_evaluators_router.py | 2 +- .../test_variant_testset_router.py | 8 ++++---- .../test_variant_versioning_deployment.py | 16 ++++++--------- .../test_user_profile.py | 2 ++ 7 files changed, 37 insertions(+), 33 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/evaluation_router.py b/agenta-backend/agenta_backend/routers/evaluation_router.py index 2cb33bc55f..479bf68e1f 100644 --- a/agenta-backend/agenta_backend/routers/evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/evaluation_router.py @@ -219,9 +219,7 @@ async def fetch_evaluation_results( """ try: - evaluation = await db_manager.fetch_evaluation_by_id( - evaluation_id, project_id=request.state.project_id - ) + evaluation = await db_manager.fetch_evaluation_by_id(evaluation_id) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, @@ -416,7 +414,7 @@ async def delete_evaluations( if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, - project_id=str(evaluation.project), + project_id=str(evaluation.project_id), permission=Permission.DELETE_EVALUATION, ) logger.debug(f"User has permission to delete evaluation: {has_permission}") diff --git a/agenta-backend/agenta_backend/tests/variants_main_router/conftest.py b/agenta-backend/agenta_backend/tests/variants_main_router/conftest.py index 0d86e074c9..f4361e9c95 100644 --- a/agenta-backend/agenta_backend/tests/variants_main_router/conftest.py +++ b/agenta-backend/agenta_backend/tests/variants_main_router/conftest.py @@ -6,6 +6,7 @@ from agenta_backend.models.db.postgres_engine import db_engine from agenta_backend.models.shared_models import ConfigDB from agenta_backend.models.db_models import ( + ProjectDB, AppDB, UserDB, DeploymentDB, @@ -70,7 +71,12 @@ async def get_first_user_app(get_first_user_object): user = await get_first_user_object async with db_engine.get_session() as session: - app = AppDB(app_name="myapp", user_id=user.id) + project = ProjectDB(project_name="default", is_default=True) + session.add(project) + await session.commit() + await session.refresh(project) + + app = AppDB(app_name="myapp", project_id=project.id) session.add(app) await session.commit() await session.refresh(app) @@ -78,7 +84,7 @@ async def get_first_user_app(get_first_user_object): db_image = ImageDB( docker_id="sha256:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", tags="agentaai/templates_v2:local_test_prompt", - user_id=user.id, + project_id=project.id, ) session.add(db_image) await session.commit() @@ -91,7 +97,7 @@ async def get_first_user_app(get_first_user_object): db_deployment = DeploymentDB( app_id=app.id, - user_id=user.id, + project_id=project.id, container_name="container_a_test", container_id="w243e34red", uri="http://localhost/app/w243e34red", @@ -102,7 +108,7 @@ async def get_first_user_app(get_first_user_object): db_base = VariantBaseDB( base_name="app", image_id=db_image.id, - user_id=user.id, + project_id=project.id, app_id=app.id, deployment_id=db_deployment.id, ) @@ -114,7 +120,7 @@ async def get_first_user_app(get_first_user_object): app_id=app.id, variant_name="app", image_id=db_image.id, - user_id=user.id, + project_id=project.id, config_parameters={}, base_name="app", config_name="default", diff --git a/agenta-backend/agenta_backend/tests/variants_main_router/test_app_variant_router.py b/agenta-backend/agenta_backend/tests/variants_main_router/test_app_variant_router.py index 150ff3f659..29a144f037 100644 --- a/agenta-backend/agenta_backend/tests/variants_main_router/test_app_variant_router.py +++ b/agenta-backend/agenta_backend/tests/variants_main_router/test_app_variant_router.py @@ -1,5 +1,6 @@ import os import httpx +import random import pytest import logging from bson import ObjectId @@ -9,6 +10,7 @@ from agenta_backend.models.db.postgres_engine import db_engine from agenta_backend.models.shared_models import ConfigDB from agenta_backend.models.db_models import ( + ProjectDB, AppDB, DeploymentDB, VariantBaseDB, @@ -113,22 +115,22 @@ async def test_create_app_variant(get_first_user_object): ) app = result.scalars().first() + project_result = await session.execute( + select(ProjectDB).filter_by(is_default=True) + ) + project = project_result.scalars().first() + db_image = ImageDB( docker_id="sha256:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", tags="agentaai/templates_v2:local_test_prompt", - user_id=user.id, + project_id=project.id, ) session.add(db_image) await session.commit() - db_config = ConfigDB( - config_name="default", - parameters={}, - ) - db_deployment = DeploymentDB( app_id=app.id, - user_id=user.id, + project_id=project.id, container_name="container_a_test", container_id="w243e34red", uri="http://localhost/app/w243e34red", @@ -140,7 +142,7 @@ async def test_create_app_variant(get_first_user_object): db_base = VariantBaseDB( base_name="app", app_id=app.id, - user_id=user.id, + project_id=project.id, image_id=db_image.id, deployment_id=db_deployment.id, ) @@ -151,7 +153,7 @@ async def test_create_app_variant(get_first_user_object): app_id=app.id, variant_name="app", image_id=db_image.id, - user_id=user.id, + project_id=project.id, config_parameters={}, base_name="app", config_name="default", diff --git a/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_evaluators_router.py b/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_evaluators_router.py index ecd5e0a02d..ecfd8e3334 100644 --- a/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_evaluators_router.py +++ b/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_evaluators_router.py @@ -214,7 +214,7 @@ async def create_evaluation_with_evaluator(evaluator_config_name): app_variant = app_variant_result.scalars().first() testset_result = await session.execute( - select(TestSetDB).filter_by(app_id=app.id) + select(TestSetDB).filter_by(project_id=app.project_id) ) testset = testset_result.scalars().first() diff --git a/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_testset_router.py b/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_testset_router.py index 7862ebdd93..84f29650c7 100644 --- a/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_testset_router.py +++ b/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_testset_router.py @@ -68,7 +68,7 @@ async def test_update_testset(): app = result.scalars().first() testset_result = await session.execute( - select(TestSetDB).filter_by(app_id=app.id) + select(TestSetDB).filter_by(project_id=app.project_id) ) testset = testset_result.scalars().first() @@ -112,7 +112,7 @@ async def test_get_testsets(): ) assert response.status_code == 200 - assert len(response.json()) == 1 + assert len(response.json()) == 2 @pytest.mark.asyncio() @@ -124,7 +124,7 @@ async def test_get_testset(): app = result.scalars().first() testset_result = await session.execute( - select(TestSetDB).filter_by(app_id=app.id) + select(TestSetDB).filter_by(project_id=app.project_id) ) testset = testset_result.scalars().first() @@ -146,7 +146,7 @@ async def test_delete_testsets(): app = result.scalars().first() testset_result = await session.execute( - select(TestSetDB).filter_by(app_id=app.id) + select(TestSetDB).filter_by(project_id=app.project_id) ) testsets = testset_result.scalars().all() diff --git a/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_versioning_deployment.py b/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_versioning_deployment.py index 3b1f733a0b..43d9c5ac0e 100644 --- a/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_versioning_deployment.py +++ b/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_versioning_deployment.py @@ -8,7 +8,6 @@ from agenta_backend.models.db.postgres_engine import db_engine from agenta_backend.models.db_models import ( AppDB, - TestSetDB, AppVariantDB, ) @@ -18,7 +17,6 @@ timeout = httpx.Timeout(timeout=5, read=None, write=5) # Set global variables -APP_NAME = "evaluation_in_backend" ENVIRONMENT = os.environ.get("ENVIRONMENT") VARIANT_DEPLOY_ENVIRONMENTS = ["development", "staging", "production"] OPEN_AI_KEY = os.environ.get("OPENAI_API_KEY") @@ -31,13 +29,10 @@ @pytest.mark.asyncio async def test_update_app_variant_parameters(app_variant_parameters_updated): async with db_engine.get_session() as session: - result = await session.execute(select(AppDB).filter_by(app_name=APP_NAME)) - app = result.scalars().first() - - testset_result = await session.execute( - select(TestSetDB).filter_by(app_id=app.id) + result = await session.execute( + select(AppDB).filter_by(app_name="evaluation_in_backend") ) - testset = testset_result.scalars().first() + app = result.scalars().first() app_variant_result = await session.execute( select(AppVariantDB).filter_by(app_id=app.id, variant_name="app.default") @@ -49,7 +44,6 @@ async def test_update_app_variant_parameters(app_variant_parameters_updated): parameters["temperature"] = random.uniform(0.9, 1.5) parameters["frequence_penalty"] = random.uniform(0.9, 1.5) parameters["frequence_penalty"] = random.uniform(0.9, 1.5) - parameters["inputs"] = [{"name": list(testset.csvdata[0].keys())[0]}] payload = {"parameters": parameters} response = await test_client.put( @@ -62,7 +56,9 @@ async def test_update_app_variant_parameters(app_variant_parameters_updated): @pytest.mark.asyncio async def test_deploy_to_environment(deploy_to_environment_payload): async with db_engine.get_session() as session: - result = await session.execute(select(AppDB).filter_by(app_name=APP_NAME)) + result = await session.execute( + select(AppDB).filter_by(app_name="evaluation_in_backend") + ) app = result.scalars().first() app_variant_result = await session.execute( diff --git a/agenta-backend/agenta_backend/tests/variants_user_profile_router/test_user_profile.py b/agenta-backend/agenta_backend/tests/variants_user_profile_router/test_user_profile.py index 1fd8a4aec6..d7fd237994 100644 --- a/agenta-backend/agenta_backend/tests/variants_user_profile_router/test_user_profile.py +++ b/agenta-backend/agenta_backend/tests/variants_user_profile_router/test_user_profile.py @@ -48,6 +48,7 @@ async def test_fetch_user_profile_without_user_id(): assert response.json()["username"] == user_db_dict["username"] +@pytest.mark.asyncio async def test_fetch_user_profile_with_valid_user_id(): async with db_engine.get_session() as session: result = await session.execute(select(UserDB).filter_by(uid="0")) @@ -75,6 +76,7 @@ async def test_fetch_user_profile_with_valid_user_id(): assert response.json()["username"] == user_db_dict["username"] +@pytest.mark.asyncio async def test_fetch_user_profile_with_non_existent_user_id_error(): user_non_existent_id = str(uuid4()) response = await test_client.get( From f1856b4d5ccae77409284b2541f5d78e5f810365 Mon Sep 17 00:00:00 2001 From: Abram Date: Wed, 25 Sep 2024 18:11:15 +0100 Subject: [PATCH 44/57] fix (bug): resolve TypeError when creating variant from base --- agenta-backend/agenta_backend/routers/variants_router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agenta-backend/agenta_backend/routers/variants_router.py b/agenta-backend/agenta_backend/routers/variants_router.py index 3cbfc3fd16..486d9fb83b 100644 --- a/agenta-backend/agenta_backend/routers/variants_router.py +++ b/agenta-backend/agenta_backend/routers/variants_router.py @@ -109,7 +109,7 @@ async def add_variant_from_base_and_config( logger.debug("Successfully updated last_modified_by app information") app_variant_db = await db_manager.get_app_variant_instance_by_id( - str(db_app_variant.id) + str(db_app_variant.id), str(db_app_variant.project_id) ) return await converters.app_variant_db_to_output(app_variant_db) From 04595f6acab4d0c490a47199b9b2d9a55c5cfc8d Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 26 Sep 2024 09:09:58 +0100 Subject: [PATCH 45/57] refactor (web): remove User column from AbTestingEvalOverview --- .../AbTestingEvalOverview.tsx | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/agenta-web/src/components/pages/overview/abTestingEvaluation/AbTestingEvalOverview.tsx b/agenta-web/src/components/pages/overview/abTestingEvaluation/AbTestingEvalOverview.tsx index 4ad02bb7ce..958997b032 100644 --- a/agenta-web/src/components/pages/overview/abTestingEvaluation/AbTestingEvalOverview.tsx +++ b/agenta-web/src/components/pages/overview/abTestingEvaluation/AbTestingEvalOverview.tsx @@ -258,33 +258,6 @@ const AbTestingEvalOverview = () => { }, ] - if (isDemo()) { - columns.push({ - title: "User", - dataIndex: ["user", "username"], - key: "username", - onHeaderCell: () => ({ - style: {minWidth: 160}, - }), - render: (_, record: any) => { - return ( - - - {getInitials(record.user.username)} - - {record.user.username} - - ) - }, - }) - } - columns.push( ...([ { From f70f38558ed965429f280bfeca4727060756bfb3 Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 26 Sep 2024 09:29:44 +0100 Subject: [PATCH 46/57] refactor (backend): added assertion to ensure that app_variant is not None when fetching app variant revision --- agenta-backend/agenta_backend/services/evaluation_service.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/agenta-backend/agenta_backend/services/evaluation_service.py b/agenta-backend/agenta_backend/services/evaluation_service.py index 43908791be..7f8f2290f7 100644 --- a/agenta-backend/agenta_backend/services/evaluation_service.py +++ b/agenta-backend/agenta_backend/services/evaluation_service.py @@ -425,6 +425,11 @@ async def create_new_evaluation( variant_db = await db_manager.get_app_variant_instance_by_id( variant_id=variant_id, project_id=project_id ) + + assert variant_db is not None, f"App variant with ID {variant_id} cannot be None." + assert ( + variant_db.revision is not None + ), f"Revision of App variant with ID {variant_id} cannot be None" variant_revision = await db_manager.fetch_app_variant_revision_by_variant( app_variant_id=variant_id, project_id=project_id, revision=variant_db.revision # type: ignore ) From b78633537467e86c241c16f40dc16496874ae68a Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 26 Sep 2024 09:50:44 +0100 Subject: [PATCH 47/57] chore (backend): resolve the warning: SAWarning: SELECT statement has a cartesian product between FROM element(s) 'evaluations' and FROM element 'human_evaluations'. Apply join condition(s) between each element to resolve. --- agenta-backend/agenta_backend/services/db_manager.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 26526d1554..0d9bd26622 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -2644,7 +2644,7 @@ async def fetch_evaluations_by_resource( .join(HumanEvaluationVariantDB) .filter( HumanEvaluationVariantDB.variant_id.in_(ids), - EvaluationDB.project_id == uuid.UUID(project_id), + HumanEvaluationDB.project_id == uuid.UUID(project_id), ) .options(load_only(HumanEvaluationDB.id)) # type: ignore ) @@ -2652,7 +2652,7 @@ async def fetch_evaluations_by_resource( res_human_evaluations = result_human_evaluations.scalars().all() return res_evaluations + res_human_evaluations - if resource_type == "testset": + elif resource_type == "testset": result_evaluations = await session.execute( select(EvaluationDB) .filter( @@ -2665,7 +2665,8 @@ async def fetch_evaluations_by_resource( select(HumanEvaluationDB) .filter( HumanEvaluationDB.testset_id.in_(ids), - EvaluationDB.project_id == uuid.UUID(project_id), + HumanEvaluationDB.project_id + == uuid.UUID(project_id), # Fixed to match HumanEvaluationDB ) .options(load_only(HumanEvaluationDB.id)) # type: ignore ) @@ -2673,7 +2674,7 @@ async def fetch_evaluations_by_resource( res_human_evaluations = result_human_evaluations.scalars().all() return res_evaluations + res_human_evaluations - if resource_type == "evaluator_config": + elif resource_type == "evaluator_config": query = ( select(EvaluationDB) .join(EvaluationDB.evaluator_configs) From a253eb548d1a9bcb957ee2340cf57a2329352612 Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 26 Sep 2024 10:33:17 +0100 Subject: [PATCH 48/57] chore (web): remove redundant User column from HumanEvaluationResult component --- .../src/components/Evaluations/HumanEvaluationResult.tsx | 8 -------- 1 file changed, 8 deletions(-) diff --git a/agenta-web/src/components/Evaluations/HumanEvaluationResult.tsx b/agenta-web/src/components/Evaluations/HumanEvaluationResult.tsx index 87720a4387..96bef32120 100644 --- a/agenta-web/src/components/Evaluations/HumanEvaluationResult.tsx +++ b/agenta-web/src/components/Evaluations/HumanEvaluationResult.tsx @@ -280,14 +280,6 @@ export default function HumanEvaluationResult({setIsEvalModalOpen}: HumanEvaluat }, ] - if (isDemo()) { - columns.push({ - title: "User", - dataIndex: ["user", "username"], - key: "username", - }) - } - columns.push( ...[ { From 4a77d084900c23599b097bcfe21b386b48b90441 Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 26 Sep 2024 11:07:44 +0100 Subject: [PATCH 49/57] chore (backend): make use of app project_id to create app testsets and evaluators --- agenta-backend/agenta_backend/routers/app_router.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index d2f2e2fc42..ae89ecae07 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -550,7 +550,7 @@ async def create_app_and_variant_from_template( ) app_variant_db = await app_manager.add_variant_based_on_image( app=app, - project_id=request.state.project_id, + project_id=str(app.project_id), variant_name="app.default", docker_id_or_template_uri=( # type: ignore template_db.template_uri if isCloudProd() else template_db.digest @@ -570,7 +570,7 @@ async def create_app_and_variant_from_template( await db_manager.add_testset_to_app_variant( template_name=template_db.name, # type: ignore app_name=app.app_name, # type: ignore - project_id=request.state.project_id, + project_id=str(app.project_id), ) logger.debug( @@ -579,7 +579,7 @@ async def create_app_and_variant_from_template( else "Step 6: We create ready-to use evaluators" ) await evaluator_manager.create_ready_to_use_evaluators( - app_name=app.app_name, project_id=request.state.project_id + app_name=app.app_name, project_id=str(app.project_id) ) logger.debug( @@ -619,7 +619,7 @@ async def create_app_and_variant_from_template( envvars = {} if payload.env_vars is None else payload.env_vars await app_manager.start_variant( app_variant_db, - request.state.project_id, + str(app.project_id), envvars, user_uid=request.state.user_id, ) From 3266bb7c1c8f6ce600ac5fcf27fb0a3f5aca590b Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 26 Sep 2024 12:03:24 +0100 Subject: [PATCH 50/57] refactor (backend): resolve 'NoneType' object has no attribute 'project_id' when deleting evaluations --- .../agenta_backend/routers/evaluation_router.py | 4 +--- .../agenta_backend/routers/human_evaluation_router.py | 4 +++- agenta-backend/agenta_backend/services/db_manager.py | 8 ++------ 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/agenta-backend/agenta_backend/routers/evaluation_router.py b/agenta-backend/agenta_backend/routers/evaluation_router.py index 479bf68e1f..90214d4a6a 100644 --- a/agenta-backend/agenta_backend/routers/evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/evaluation_router.py @@ -435,9 +435,7 @@ async def delete_evaluations( ) logger.debug("Successfully updated last_modified_by app information") - await evaluation_service.delete_evaluations( - payload.evaluations_ids, str(evaluation.project_id) - ) + await evaluation_service.delete_evaluations(payload.evaluations_ids) return Response(status_code=status.HTTP_204_NO_CONTENT) except Exception as exc: raise HTTPException(status_code=500, detail=str(exc)) diff --git a/agenta-backend/agenta_backend/routers/human_evaluation_router.py b/agenta-backend/agenta_backend/routers/human_evaluation_router.py index 846c551ee3..a5573923e3 100644 --- a/agenta-backend/agenta_backend/routers/human_evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/human_evaluation_router.py @@ -481,7 +481,9 @@ async def delete_evaluations( """ try: - evaluation = await db_manager.fetch_evaluation_by_id(payload.evaluations_ids[0]) + evaluation = await db_manager.fetch_human_evaluation_by_id( + payload.evaluations_ids[0] + ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 0d9bd26622..d7949377aa 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -2693,19 +2693,15 @@ async def fetch_evaluations_by_resource( ) -async def delete_evaluations(evaluation_ids: List[str], project_id: str) -> None: +async def delete_evaluations(evaluation_ids: List[str]) -> None: """Delete evaluations based on the ids provided from the db. Args: evaluations_ids (list[str]): The IDs of the evaluation - project_id (str): The ID of the project """ async with db_engine.get_session() as session: - query = select(EvaluationDB).where( - EvaluationDB.id.in_(evaluation_ids), - EvaluationDB.project_id == uuid.UUID(project_id), - ) + query = select(EvaluationDB).where(EvaluationDB.id.in_(evaluation_ids)) result = await session.execute(query) evaluations = result.scalars().all() for evaluation in evaluations: From 2f0a8c3d728d27475f596b5c8b619bf591136fdd Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 26 Sep 2024 14:29:17 +0100 Subject: [PATCH 51/57] chore (backend): added log when deleting evaluations --- agenta-backend/agenta_backend/routers/evaluation_router.py | 1 + 1 file changed, 1 insertion(+) diff --git a/agenta-backend/agenta_backend/routers/evaluation_router.py b/agenta-backend/agenta_backend/routers/evaluation_router.py index 90214d4a6a..f6b6981c56 100644 --- a/agenta-backend/agenta_backend/routers/evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/evaluation_router.py @@ -435,6 +435,7 @@ async def delete_evaluations( ) logger.debug("Successfully updated last_modified_by app information") + logger.debug(f"Deleting evaluations {payload.evaluations_ids}...") await evaluation_service.delete_evaluations(payload.evaluations_ids) return Response(status_code=status.HTTP_204_NO_CONTENT) except Exception as exc: From 648fe6dc1488e71347d9ba5ccaeb96d901b5e5cc Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 26 Sep 2024 15:35:12 +0100 Subject: [PATCH 52/57] refactor (backend): added evaluation object_type when updating app last_modified_by --- agenta-backend/agenta_backend/services/app_manager.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/agenta-backend/agenta_backend/services/app_manager.py b/agenta-backend/agenta_backend/services/app_manager.py index 1ac1f2b286..47cc3ea404 100644 --- a/agenta-backend/agenta_backend/services/app_manager.py +++ b/agenta-backend/agenta_backend/services/app_manager.py @@ -213,6 +213,17 @@ async def get_appdb_str_by_id(object_id: str, object_type: str) -> str: f"Deployment with id {object_id} not found" ) return str(deployment_db.app_id) + elif object_type == "evaluation": + evaluation_db = await db_manager.fetch_evaluation_by_id(object_id) + if evaluation_db is None: + raise db_manager.NoResultFound( + f"Evaluation with id {object_id} not found" + ) + return str(evaluation_db.app_id) + else: + raise ValueError( + f"Could not update application. Unsupported type: {object_type}" + ) user = await db_manager.get_user(user_uid=user_uid) app_id = await get_appdb_str_by_id(object_id=object_id, object_type=object_type) From 62dc92b84f37092335a51421f7809854a6afbd7b Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 26 Sep 2024 15:37:38 +0100 Subject: [PATCH 53/57] chore (backend): added assert to ensure app_id in db object cannot be None --- agenta-backend/agenta_backend/services/app_manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/agenta-backend/agenta_backend/services/app_manager.py b/agenta-backend/agenta_backend/services/app_manager.py index 47cc3ea404..3c76b9eb7e 100644 --- a/agenta-backend/agenta_backend/services/app_manager.py +++ b/agenta-backend/agenta_backend/services/app_manager.py @@ -227,6 +227,7 @@ async def get_appdb_str_by_id(object_id: str, object_type: str) -> str: user = await db_manager.get_user(user_uid=user_uid) app_id = await get_appdb_str_by_id(object_id=object_id, object_type=object_type) + assert app_id is not None, f"app_id in {object_type} cannot be None" await db_manager.update_app( app_id=app_id, values_to_update={ From 31f66fce6e21b1a13838cd812cbd8906f1977a56 Mon Sep 17 00:00:00 2001 From: Abram Date: Thu, 26 Sep 2024 16:10:19 +0100 Subject: [PATCH 54/57] chore (backend): improve error message in update_last_modified_by function --- agenta-backend/agenta_backend/services/app_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agenta-backend/agenta_backend/services/app_manager.py b/agenta-backend/agenta_backend/services/app_manager.py index 3c76b9eb7e..0a39de23e7 100644 --- a/agenta-backend/agenta_backend/services/app_manager.py +++ b/agenta-backend/agenta_backend/services/app_manager.py @@ -222,7 +222,7 @@ async def get_appdb_str_by_id(object_id: str, object_type: str) -> str: return str(evaluation_db.app_id) else: raise ValueError( - f"Could not update application. Unsupported type: {object_type}" + f"Could not update last_modified_by application information. Unsupported type: {object_type}" ) user = await db_manager.get_user(user_uid=user_uid) From 9270508dfd51bf1c43c75e8329764fa8d163b249 Mon Sep 17 00:00:00 2001 From: Abram Date: Fri, 27 Sep 2024 12:19:57 +0100 Subject: [PATCH 55/57] fix (tests): resolve sqlalchemy.exc.InvalidRequestError: Entity namespace for testsets has no property app_id in test_create_evaluation_with_no_llm_keys testcase --- .../variants_main_router/test_variant_evaluators_router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_evaluators_router.py b/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_evaluators_router.py index 1f306ee861..e8fd22c5e2 100644 --- a/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_evaluators_router.py +++ b/agenta-backend/agenta_backend/tests/variants_main_router/test_variant_evaluators_router.py @@ -280,7 +280,7 @@ async def test_create_evaluation_with_no_llm_keys(evaluators_requiring_llm_keys) app_variant = app_variant_result.scalars().first() testset_result = await session.execute( - select(TestSetDB).filter_by(app_id=app.id) + select(TestSetDB).filter_by(project_id=app.project_id) ) testset = testset_result.scalars().first() From e4efa5291a237be68d3d1e87af83608f04fa85ce Mon Sep 17 00:00:00 2001 From: Abram Date: Fri, 27 Sep 2024 14:57:07 +0100 Subject: [PATCH 56/57] refactor (migrations): modify update_evaluators_with_app_name logic to: - fetch evaluators with an app_id - delete evaluators with no app_id --- .../postgres/data_migrations/applications.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/agenta-backend/agenta_backend/migrations/postgres/data_migrations/applications.py b/agenta-backend/agenta_backend/migrations/postgres/data_migrations/applications.py index d0e76a8f85..56a842c47b 100644 --- a/agenta-backend/agenta_backend/migrations/postgres/data_migrations/applications.py +++ b/agenta-backend/agenta_backend/migrations/postgres/data_migrations/applications.py @@ -6,7 +6,7 @@ import click from sqlalchemy.future import select -from sqlalchemy import create_engine +from sqlalchemy import create_engine, delete from sqlalchemy.orm import sessionmaker, Session from agenta_backend.models.deprecated_models import ( @@ -34,6 +34,7 @@ def update_evaluators_with_app_name(): records = ( session.execute( select(DeprecatedEvaluatorConfigDB) + .filter(DeprecatedEvaluatorConfigDB.app_id.isnot(None)) .offset(offset) .limit(BATCH_SIZE) ) @@ -54,6 +55,13 @@ def update_evaluators_with_app_name(): session.commit() offset += BATCH_SIZE + # Delete deprecated evaluator configs with app_id as None + session.execute( + delete(DeprecatedEvaluatorConfigDB).where( + DeprecatedEvaluatorConfigDB.app_id.is_(None) + ) + ) + session.commit() except Exception as e: session.rollback() click.echo( From 805d43069aa17c460220169d42b917b01c26eba2 Mon Sep 17 00:00:00 2001 From: Kaosiso Ezealigo Date: Sat, 28 Sep 2024 11:48:51 +0200 Subject: [PATCH 57/57] test(frontend) --- agenta-web/cypress/e2e/eval.scenarios.cy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agenta-web/cypress/e2e/eval.scenarios.cy.ts b/agenta-web/cypress/e2e/eval.scenarios.cy.ts index 5c545b13bd..9478c51f3f 100644 --- a/agenta-web/cypress/e2e/eval.scenarios.cy.ts +++ b/agenta-web/cypress/e2e/eval.scenarios.cy.ts @@ -24,7 +24,7 @@ describe("Evaluation Scenarios Test", function () { it("Should double click on the Evaluation and successfully navigate to the evalaution results page", () => { cy.get(".ant-table-row").eq(0).should("exist") - cy.get(".ant-table-row").click() + cy.get(".ant-table-row").click({force: true}) cy.wait(1000) cy.contains(/Evaluation Results/i) cy.get('[data-cy="evalaution-scenarios-table"]').should("exist")