[DB-12682] Unique Indexes are not being considered for conflict detec…

…tion cache (#1701)

- updated the queries(oracle, pg) to fetch unique index columns also for conflict detection cache

* Bug fix: Conflicting evens were erroring out in case of Multiple columns with unique constraint/index in a table
- fix: check for all columns instead of returning false on first mismatch

* Added a automation test pg-unique-key-conflicts-test for unique key conflicts handling
* Added oracle-unique-key-conflicts-test for handling unique key conflicts in oracle live migration
* Renamed file: assessMigrationCommandBulk.go --> assessMigrationBulkCommand.go

.github/workflows/pg-migtests.yml

@@ -163,6 +163,9 @@ jobs:
 
       - name: "TEST: pg-basic-non-public-live-migration-test"
         run: migtests/scripts/live-migration-run-test.sh pg/basic-non-public-live-test
+
+      - name: "TEST: pg-unique-key-conflicts-test"
+        run: migtests/scripts/live-migration-run-test.sh pg/unique-key-conflicts-test
 
       # case sensitive table names are not yet supported in live migration, to restricting test only to a few tables.
       - name: "TEST: pg-live-migration-multiple-schemas"

migtests/tests/oracle/unique-key-conflicts-test/cleanup-db

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -e
+set -x
+
+source ${SCRIPTS}/functions.sh
+
+echo "Deleting existing data in source database"
+run_sqlplus_as_schema_owner ${SOURCE_DB_NAME} ${TESTS_DIR}/oracle/utils/delete_full_schema
+
+if [ "${SOURCE_REPLICA_DB_SCHEMA}" != "" ]
+then
+	echo "Deleting existing data in FF database"
+	run_sqlplus ${SOURCE_DB_NAME} ${SOURCE_REPLICA_DB_SCHEMA} ${SOURCE_REPLICA_DB_PASSWORD} ${TESTS_DIR}/oracle/utils/delete_full_schema
+fi

migtests/tests/oracle/unique-key-conflicts-test/env.sh

@@ -0,0 +1,4 @@
+export SOURCE_DB_TYPE="oracle"
+export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"TEST_SCHEMA"}
+export TARGET_DB_NAME=${TARGET_DB_NAME:-"live_test"}
+export TARGET_DB_SCHEMA="test_schema"

migtests/tests/oracle/unique-key-conflicts-test/init-db

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -e
+set -x
+
+source ${SCRIPTS}/functions.sh
+
+echo "Assigning permissions to the validate files"
+chmod +x ${TEST_DIR}/validate
+chmod +x ${TEST_DIR}/validateAfterChanges
+
+echo "Deleting existing data in source database"
+run_sqlplus_as_schema_owner ${SOURCE_DB_NAME} ${TESTS_DIR}/oracle/utils/delete_full_schema
+
+echo "Initialising source database & inserting data"
+run_sqlplus_as_schema_owner ${SOURCE_DB_NAME} snapshot_schema
+run_sqlplus_as_schema_owner ${SOURCE_DB_NAME} snapshot_data
+
+if [ -n "${SOURCE_REPLICA_DB_SCHEMA}" ] && [ "${SOURCE_REPLICA_DB_SCHEMA}" != "${SOURCE_DB_SCHEMA}" ]; then
+    echo "Initialising FF database"
+    run_sqlplus ${SOURCE_DB_NAME} ${SOURCE_REPLICA_DB_SCHEMA} ${SOURCE_REPLICA_DB_PASSWORD} ${TESTS_DIR}/oracle/utils/delete_full_schema
+    run_sqlplus ${SOURCE_DB_NAME} ${SOURCE_REPLICA_DB_SCHEMA} ${SOURCE_REPLICA_DB_PASSWORD} snapshot_schema
+fi

migtests/tests/oracle/unique-key-conflicts-test/snapshot_data.sql

@@ -0,0 +1,43 @@
+-- Insert data for single_unique_constr
+INSERT INTO single_unique_constr (email) VALUES ('user1@example.com');
+INSERT INTO single_unique_constr (email) VALUES ('user2@example.com');
+INSERT INTO single_unique_constr (email) VALUES ('user3@example.com');
+INSERT INTO single_unique_constr (email) VALUES ('user4@example.com');
+INSERT INTO single_unique_constr (email) VALUES ('user5@example.com');
+
+-- Insert data for multi_unique_constr
+INSERT INTO multi_unique_constr (first_name, last_name) VALUES ('John', 'Doe');
+INSERT INTO multi_unique_constr (first_name, last_name) VALUES ('Jane', 'Smith');
+INSERT INTO multi_unique_constr (first_name, last_name) VALUES ('Bob', 'Johnson');
+INSERT INTO multi_unique_constr (first_name, last_name) VALUES ('Alice', 'Williams');
+INSERT INTO multi_unique_constr (first_name, last_name) VALUES ('Tom', 'Clark');
+
+-- Insert data for single_unique_idx
+INSERT INTO single_unique_idx (ssn) VALUES ('SSN1');
+INSERT INTO single_unique_idx (ssn) VALUES ('SSN2');
+INSERT INTO single_unique_idx (ssn) VALUES ('SSN3');
+INSERT INTO single_unique_idx (ssn) VALUES ('SSN4');
+INSERT INTO single_unique_idx (ssn) VALUES ('SSN5');
+
+-- Insert data for multi_unique_idx
+INSERT INTO multi_unique_idx (first_name, last_name) VALUES ('John', 'Doe');
+INSERT INTO multi_unique_idx (first_name, last_name) VALUES ('Jane', 'Smith');
+INSERT INTO multi_unique_idx (first_name, last_name) VALUES ('Bob', 'Johnson');
+INSERT INTO multi_unique_idx (first_name, last_name) VALUES ('Alice', 'Williams');
+INSERT INTO multi_unique_idx (first_name, last_name) VALUES ('Tom', 'Clark');
+
+-- Insert data for diff_columns_constr_idx
+INSERT INTO diff_columns_constr_idx (email, phone_number) VALUES ('user1@example.com', '555-555-5551');
+INSERT INTO diff_columns_constr_idx (email, phone_number) VALUES ('user2@example.com', '555-555-5552');
+INSERT INTO diff_columns_constr_idx (email, phone_number) VALUES ('user3@example.com', '555-555-5553');
+INSERT INTO diff_columns_constr_idx (email, phone_number) VALUES ('user4@example.com', '555-555-5554');
+INSERT INTO diff_columns_constr_idx (email, phone_number) VALUES ('user5@example.com', '555-555-5555');
+
+-- Insert data for subset_columns_constr_idx
+INSERT INTO subset_columns_constr_idx (first_name, last_name, phone_number) VALUES ('John', 'Doe', '123-456-7890');
+INSERT INTO subset_columns_constr_idx (first_name, last_name, phone_number) VALUES ('Jane', 'Smith', '123-456-7891');
+INSERT INTO subset_columns_constr_idx (first_name, last_name, phone_number) VALUES ('Bob', 'Johnson', '123-456-7892');
+INSERT INTO subset_columns_constr_idx (first_name, last_name, phone_number) VALUES ('Alice', 'Williams', '123-456-7893');
+INSERT INTO subset_columns_constr_idx (first_name, last_name, phone_number) VALUES ('Tom', 'Clark', '123-456-7894');
+
+COMMIT;

migtests/tests/oracle/unique-key-conflicts-test/snapshot_schema.sql

@@ -0,0 +1,50 @@
+-- Table with Single Column Unique Constraint
+CREATE TABLE single_unique_constr (
+    id NUMBER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+    email VARCHAR2(255) UNIQUE
+);
+
+-- Table with Multiple Column Unique Constraint
+CREATE TABLE multi_unique_constr (
+    id NUMBER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+    first_name VARCHAR2(100),
+    last_name VARCHAR2(100),
+    CONSTRAINT unique_name UNIQUE (first_name, last_name)
+);
+
+-- Table with Single Column Unique Index
+CREATE TABLE single_unique_idx (
+    id NUMBER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+    ssn VARCHAR2(20)
+);
+CREATE UNIQUE INDEX idx_ssn_unique ON single_unique_idx (ssn);
+
+-- Table with Multiple Column Unique Index
+CREATE TABLE multi_unique_idx (
+    id NUMBER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+    first_name VARCHAR2(100),
+    last_name VARCHAR2(100)
+);
+CREATE UNIQUE INDEX idx_name_unique ON multi_unique_idx (first_name, last_name);
+
+-- Table with Unique Constraint and Unique Index on Different Columns
+CREATE TABLE diff_columns_constr_idx (
+    id NUMBER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+    email VARCHAR2(255) UNIQUE,
+    phone_number VARCHAR2(20)
+);
+CREATE UNIQUE INDEX idx_phone_unique ON diff_columns_constr_idx (phone_number);
+
+-- Table with Unique Constraint and Unique Index, with Overlapping Columns
+CREATE TABLE subset_columns_constr_idx (
+    id NUMBER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+    first_name VARCHAR2(100),
+    last_name VARCHAR2(100),
+    phone_number VARCHAR2(20)
+);
+
+-- Unique constraint on first_name and last_name
+ALTER TABLE subset_columns_constr_idx ADD CONSTRAINT unique_name_constraint UNIQUE (first_name, last_name);
+
+-- Unique index on first_name, last_name, and phone_number
+CREATE UNIQUE INDEX idx_name_phone_unique ON subset_columns_constr_idx (first_name, last_name, phone_number);

migtests/tests/oracle/unique-key-conflicts-test/source_delta.sql

@@ -0,0 +1,125 @@
+-- Conflicts for single_unique_constr
+-- DELETE-INSERT conflict
+DELETE FROM single_unique_constr WHERE id = 1;
+INSERT INTO single_unique_constr (id, email) VALUES (6, 'user1@example.com');
+COMMIT;
+
+-- DELETE-UPDATE conflict
+DELETE FROM single_unique_constr WHERE id = 2;
+UPDATE single_unique_constr SET email = 'user2@example.com' WHERE id = 3;
+COMMIT;
+
+-- UPDATE-INSERT conflict
+UPDATE single_unique_constr SET email = 'updated_user4@example.com' WHERE id = 4;
+INSERT INTO single_unique_constr (id, email) VALUES (7, 'user4@example.com');
+COMMIT;
+
+-- UPDATE-UPDATE conflict
+UPDATE single_unique_constr SET email = 'updated_user5@example.com' WHERE id = 5;
+UPDATE single_unique_constr SET email = 'user5@example.com' WHERE id = 6;
+COMMIT;
+
+-- Conflicts for multi_unique_constr
+-- DELETE-INSERT conflict
+DELETE FROM multi_unique_constr WHERE id = 1;
+INSERT INTO multi_unique_constr (id, first_name, last_name) VALUES (6, 'John', 'Doe');
+COMMIT;
+
+-- DELETE-UPDATE conflict
+DELETE FROM multi_unique_constr WHERE id = 2;
+UPDATE multi_unique_constr SET first_name = 'Jane', last_name = 'Smith' WHERE id = 4;
+COMMIT;
+
+-- UPDATE-INSERT conflict
+UPDATE multi_unique_constr SET first_name = 'Updated_Tom' WHERE id = 5;
+INSERT INTO multi_unique_constr (id, first_name, last_name) VALUES (7, 'Tom', 'Clark');
+COMMIT;
+
+-- UPDATE-UPDATE conflict
+UPDATE multi_unique_constr SET first_name = 'Updated_Alice' WHERE id = 4;
+UPDATE multi_unique_constr SET first_name = 'Alice', last_name = 'Williams' WHERE id = 5;
+COMMIT;
+
+-- Conflicts for single_unique_idx
+-- DELETE-INSERT conflict
+DELETE FROM single_unique_idx WHERE id = 1;
+INSERT INTO single_unique_idx (id, ssn) VALUES (6, 'SSN1');
+COMMIT;
+
+-- DELETE-UPDATE conflict
+DELETE FROM single_unique_idx WHERE id = 2;
+UPDATE single_unique_idx SET ssn = 'SSN2' WHERE id = 3;
+COMMIT;
+
+-- UPDATE-INSERT conflict
+UPDATE single_unique_idx SET ssn = 'updated_SSN4' WHERE id = 4;
+INSERT INTO single_unique_idx (id, ssn) VALUES (7, 'SSN4');
+COMMIT;
+
+-- UPDATE-UPDATE conflict
+UPDATE single_unique_idx SET ssn = 'updated_SSN5' WHERE id = 5;
+UPDATE single_unique_idx SET ssn = 'SSN5' WHERE id = 6;
+COMMIT;
+
+-- Conflicts for multi_unique_idx
+-- DELETE-INSERT conflict
+DELETE FROM multi_unique_idx WHERE id = 1;
+INSERT INTO multi_unique_idx (id, first_name, last_name) VALUES (6, 'John', 'Doe');
+COMMIT;
+
+-- DELETE-UPDATE conflict
+DELETE FROM multi_unique_idx WHERE id = 2;
+UPDATE multi_unique_idx SET first_name = 'Jane', last_name = 'Smith' WHERE id = 3;
+COMMIT;
+
+-- UPDATE-INSERT conflict
+UPDATE multi_unique_idx SET first_name = 'Updated_Tom' WHERE id = 5;
+INSERT INTO multi_unique_idx (id, first_name, last_name) VALUES (7, 'Tom', 'Clark');
+COMMIT;
+
+-- UPDATE-UPDATE conflict
+UPDATE multi_unique_idx SET first_name = 'Updated_Alice' WHERE id = 4;
+UPDATE multi_unique_idx SET first_name = 'Alice', last_name = 'Williams' WHERE id = 6;
+COMMIT;
+
+-- Conflicts for diff_columns_constr_idx
+-- DELETE-INSERT conflict
+DELETE FROM diff_columns_constr_idx WHERE id = 1;
+INSERT INTO diff_columns_constr_idx (id, email, phone_number) VALUES (6, 'user1@example.com', '555-555-5560');
+COMMIT; on email
+
+-- DELETE-UPDATE conflict
+DELETE FROM diff_columns_constr_idx WHERE id = 2;
+UPDATE diff_columns_constr_idx SET phone_number = '555-555-5552' WHERE id = 3;
+COMMIT; on phone_number
+
+-- UPDATE-INSERT conflict
+UPDATE diff_columns_constr_idx SET email = 'updated_user4@example.com' WHERE id = 4;
+INSERT INTO diff_columns_constr_idx (id, email, phone_number) VALUES (7, 'user4@example.com', '555-555-5561');
+COMMIT; on email
+
+-- UPDATE-UPDATE conflict
+UPDATE diff_columns_constr_idx SET phone_number = '555-555-5558' WHERE id = 5;
+UPDATE diff_columns_constr_idx SET phone_number = '555-555-5555' WHERE id = 6;
+COMMIT; on phone_number
+
+-- Conflicts for subset_columns_constr_idx
+-- DELETE-INSERT conflict
+DELETE FROM subset_columns_constr_idx WHERE id = 1;
+INSERT INTO subset_columns_constr_idx (id, first_name, last_name, phone_number) VALUES (6, 'John', 'Doe', '123-456-7890');
+COMMIT;
+
+-- DELETE-UPDATE conflict
+DELETE FROM subset_columns_constr_idx WHERE id = 2;
+UPDATE subset_columns_constr_idx SET first_name = 'Jane', last_name = 'Smith' WHERE id = 3;
+COMMIT;
+
+-- UPDATE-INSERT conflict
+UPDATE subset_columns_constr_idx SET first_name = 'Updated_Bob' WHERE id = 3;
+INSERT INTO subset_columns_constr_idx (id, first_name, last_name, phone_number) VALUES (7, 'Bob', 'Johnson', '123-456-7892');
+COMMIT;
+
+-- UPDATE-UPDATE conflict
+UPDATE subset_columns_constr_idx SET first_name = 'Updated_Alice' WHERE id = 4;
+UPDATE subset_columns_constr_idx SET first_name = 'Alice', last_name = 'Williams', phone_number = '123-456-7893' WHERE id = 5;
+COMMIT;