diff --git a/agenta-web/cypress/e2e/eval.evaluations.cy.ts b/agenta-web/cypress/e2e/eval.evaluations.cy.ts
index 248a4e4778..79fb6662e9 100644
--- a/agenta-web/cypress/e2e/eval.evaluations.cy.ts
+++ b/agenta-web/cypress/e2e/eval.evaluations.cy.ts
@@ -1,4 +1,7 @@
+import {randString} from "../../src/lib/helpers/utils"
+
 describe("Evaluations CRUD Operations Test", function () {
+    let newEvalName = randString(5)
     let app_id
     before(() => {
         cy.createVariant()
@@ -38,6 +41,53 @@ describe("Evaluations CRUD Operations Test", function () {
         })
     })
 
+    context("Executing Evaluation with different answer column", () => {
+        it("Should successfully rename the testset columns", () => {
+            cy.visit(`/apps/${app_id}/testsets`)
+            cy.location("pathname").should("include", "/testsets")
+            cy.get(".ant-table-row").eq(0).click()
+            cy.wait(1000)
+            cy.contains(/create a new test set/i).should("be.visible")
+            cy.get('[data-cy="testset-header-column-edit-button"]').eq(1).click()
+            cy.get('[data-cy="testset-header-column-edit-input"]').clear()
+            cy.get('[data-cy="testset-header-column-edit-input"]').type("answer")
+            cy.get('[data-cy="testset-header-column-save-button"]').click()
+            cy.get('[data-cy="testset-save-button"]').click()
+        })
+
+        it("Should successfully create an Evaluator", () => {
+            cy.visit(`/apps/${app_id}/evaluations/new-evaluator`)
+            cy.location("pathname").should("include", "/evaluations/new-evaluator")
+            cy.get('[data-cy="evaluator-card"]').should("exist")
+            cy.get(".ant-space > :nth-child(2) > .ant-btn").click()
+            cy.get('[data-cy="new-evaluator-modal"]').should("exist")
+            cy.get('[data-cy^="select-new-evaluator"]').eq(0).click()
+            cy.get('[data-cy="configure-new-evaluator-modal"]').should("exist")
+            cy.get('[data-cy="configure-new-evaluator-modal-input"]').type(newEvalName, {
+                force: true,
+            })
+            cy.get('[data-cy="new-evaluator-advance-settings"]').click()
+            cy.get('[data-cy="new-evaluator-column-name"]').clear()
+            cy.get('[data-cy="new-evaluator-column-name"]').type("answer")
+            cy.get('[data-cy="configure-new-evaluator-modal-save-btn"]').click()
+            cy.get('[data-cy="evaluator-card"]').should("have.length", 2)
+            cy.wait(1000)
+        })
+
+        it("Should successfully create an Evaluation", () => {
+            cy.visit(`/apps/${app_id}/evaluations/results`)
+            cy.location("pathname").should("include", "/evaluations/results")
+            cy.createNewEvaluation(newEvalName)
+        })
+
+        it("Should verify the successful creation and completion of the evaluation", () => {
+            cy.visit(`/apps/${app_id}/evaluations/results`)
+            cy.location("pathname").should("include", "/evaluations/results")
+            cy.get('.ag-row[row-index="0"]').should("exist")
+            cy.get('.ag-cell[col-id="status"]').should("contain.text", "Completed")
+        })
+    })
+
     after(() => {
         cy.cleanupVariantAndTestset()
     })
diff --git a/agenta-web/cypress/support/commands/evaluations.ts b/agenta-web/cypress/support/commands/evaluations.ts
index 78215ed34f..a690763e42 100644
--- a/agenta-web/cypress/support/commands/evaluations.ts
+++ b/agenta-web/cypress/support/commands/evaluations.ts
@@ -100,7 +100,7 @@ Cypress.Commands.add("removeLlmProviderKey", () => {
     removeLlmProviderKey()
 })
 
-Cypress.Commands.add("createNewEvaluation", () => {
+Cypress.Commands.add("createNewEvaluation", (evaluatorName = "Exact Match") => {
     cy.request({
         url: `${Cypress.env().baseApiURL}/evaluations/?app_id=${app_id}`,
         method: "GET",
@@ -121,8 +121,8 @@ Cypress.Commands.add("createNewEvaluation", () => {
     cy.get('[data-cy="select-variant-group"]').click()
 
     cy.get('[data-cy="select-evaluators-group"]').click()
-    cy.get('[data-cy="select-evaluators-option"]').eq(0).click()
-    cy.get('[data-cy="select-evaluators-group"]').click()
+    cy.get('[data-cy="select-evaluators-option"]').contains(evaluatorName).eq(0).click()
+    cy.get('[data-cy="select-evaluators-group"]').click({force: true})
 
     cy.get(".ant-modal-footer > .ant-btn-primary > .ant-btn-icon > .anticon > svg").click()
     cy.wait(1000)
diff --git a/agenta-web/cypress/support/commands/index.ts b/agenta-web/cypress/support/commands/index.ts
index 3944f7d7ed..d9a45d1a23 100644
--- a/agenta-web/cypress/support/commands/index.ts
+++ b/agenta-web/cypress/support/commands/index.ts
@@ -13,7 +13,7 @@ declare global {
             saveOpenAiKey(): Chainable<void>
             removeLlmProviderKey(): Chainable<void>
             addingOpenaiKey(): Chainable<void>
-            createNewEvaluation(): Chainable<void>
+            createNewEvaluation(evaluatorName?: string): Chainable<void>
         }
     }
 }
diff --git a/agenta-web/src/components/TestSetTable/TableHeaderComponent.tsx b/agenta-web/src/components/TestSetTable/TableHeaderComponent.tsx
index 943737cb0b..857f6f2bc9 100644
--- a/agenta-web/src/components/TestSetTable/TableHeaderComponent.tsx
+++ b/agenta-web/src/components/TestSetTable/TableHeaderComponent.tsx
@@ -153,6 +153,7 @@ const TableHeaderComponent = ({
                         value={scopedInputValues[index]}
                         onChange={(event) => handleInputChange(index, event)}
                         size="small"
+                        data-cy="testset-header-column-edit-input"
                     />
                 ) : (
                     displayName
@@ -165,12 +166,14 @@ const TableHeaderComponent = ({
                             onClick={handleSave}
                             type="default"
                             className={classes.saveBtn}
+                            data-cy="testset-header-column-save-button"
                         />
                     ) : (
                         <Button
                             icon={<EditOutlined />}
                             onClick={() => setIsEditInputOpen(true)}
                             type="text"
+                            data-cy="testset-header-column-edit-button"
                         />
                     )}
 
diff --git a/agenta-web/src/components/pages/evaluations/evaluators/AdvancedSettings.tsx b/agenta-web/src/components/pages/evaluations/evaluators/AdvancedSettings.tsx
index 3db209fdaa..f09e1e73c5 100644
--- a/agenta-web/src/components/pages/evaluations/evaluators/AdvancedSettings.tsx
+++ b/agenta-web/src/components/pages/evaluations/evaluators/AdvancedSettings.tsx
@@ -33,7 +33,12 @@ const AdvancedSettings: React.FC<AdvancedSettingsProps> = ({settings}) => {
             expandIcon={({isActive}) => <CaretRightOutlined rotate={isActive ? 90 : 0} />}
             className={"my-[10px]"}
         >
-            <Collapse.Panel key="1" header="Advanced Settings" forceRender>
+            <Collapse.Panel
+                key="1"
+                header="Advanced Settings"
+                data-cy="new-evaluator-advance-settings"
+                forceRender
+            >
                 {settings.map((field) => {
                     const rules = [
                         {required: field.required ?? true, message: "This field is required"},
@@ -59,7 +64,7 @@ const AdvancedSettings: React.FC<AdvancedSettingsProps> = ({settings}) => {
                             rules={rules}
                         >
                             {field.type === "string" || field.type === "regex" ? (
-                                <Input />
+                                <Input data-cy="new-evaluator-column-name" />
                             ) : field.type === "number" ? (
                                 <InputNumber min={field.min} max={field.max} step={0.1} />
                             ) : field.type === "boolean" || field.type === "bool" ? (