MODDICONV-365: Automatically migration preparation for already migrated envs.

mod-di-converter-storage-server/src/main/java/org/folio/dao/association/ProfileWrapperDao.java

@@ -51,4 +51,12 @@ public interface ProfileWrapperDao {
    * @return future with founded ProfileWrapper
    */
   Future<List<ProfileWrapper>> getWrapperByProfileId(String profileId, ProfileType profileType, String tenantId);
+
+  /**
+   * Get count of the strings in the table
+   * @param tenantId - tenant id
+   * @param tableName - table name
+   * @return future with count of string in the table
+   */
+  Future<Integer> getLinesCount(String tenantId, String tableName);
 }

mod-di-converter-storage-server/src/main/java/org/folio/dao/association/ProfileWrapperDaoImpl.java

@@ -27,16 +27,12 @@
 @Repository
 public class ProfileWrapperDaoImpl implements ProfileWrapperDao {
   private static final Logger LOGGER = LogManager.getLogger();
-  private static final String ID_FIELD = "'id'";
   private static final String TABLE_NAME = "profile_wrappers";
-  private static final String INSERT_QUERY = "INSERT INTO %s.%s (id, profile_type, %s) VALUES ($1, $2, $3)";
-
+   private static final String INSERT_QUERY = "INSERT INTO %s.%s (id, profile_type, %s) VALUES ($1, $2, $3)";
   private static final String SELECT_ON_EMPTY_TABLE_QUERY = "SELECT EXISTS (SELECT * FROM %s.%s LIMIT 1)";
-
+  private static final String SQL_LINES_COUNT = "select count(id) from %s.%s";
   private static final String SELECT_QUERY = "SELECT * FROM %s.%s WHERE id = $1";
-
   private static final String SELECT_QUERY_ON_GETTING_PROFILE_WRAPPER = "SELECT * FROM %s.%s WHERE %s = $1";
-
   private static final Map<String, String> profileTypeToColumn;
 
   static {
@@ -92,6 +88,16 @@ public Future<Boolean> checkIfDataInTableExists(String tenantId) {
 
   }
 
+  @Override
+  public Future<Integer> getLinesCount(String tenantId, String tableName) {
+    Promise<RowSet<Row>> promise = Promise.promise();
+    String query = format(SQL_LINES_COUNT, convertToPsqlStandard(tenantId), tableName);
+
+    pgClientFactory.createInstance(tenantId).execute(query, promise);
+    return promise.future().map(resultSet -> resultSet.iterator().next().getInteger(0));
+
+  }
+
   @Override
   public Future<List<ProfileWrapper>> getWrapperByProfileId(String profileId, ProfileType profileType, String tenantId) {
     Promise<RowSet<Row>> promise = Promise.promise();

mod-di-converter-storage-server/src/main/java/org/folio/services/migration/ProfileMigrationServiceImpl.java

@@ -18,6 +18,7 @@
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.util.Map;
+import java.util.concurrent.CompletionException;
 
 import static java.lang.String.format;
 
@@ -26,8 +27,11 @@ public class ProfileMigrationServiceImpl implements ProfileMigrationService {
   private static final Logger LOGGER = LogManager.getLogger();
   private static final String UPDATE_SCHEMA_FOR_MIGRATION = "templates/db_scripts/associations-migration/actualize_schema_for_migrations.sql";
   private static final String INIT_WRAPPERS = "templates/db_scripts/associations-migration/init_wrappers.sql";
+  private static final String REMOVE_WRAPPERS = "templates/db_scripts/associations-migration/clean_profile_wrappers.sql";
   private static final String TENANT_PLACEHOLDER = "${myuniversity}";
   private static final String MODULE_PLACEHOLDER = "${mymodule}";
+  private static final String SYSTEM_TABLE_NAME = "metadata_internal";
+
   @Autowired
   protected PostgresClientFactory pgClientFactory;
   @Autowired
@@ -37,17 +41,37 @@ public class ProfileMigrationServiceImpl implements ProfileMigrationService {
   public Future<Boolean> migrateDataImportProfiles(Map<String, String> headers, Context context) {
     String tenantId = new OkapiConnectionParams(headers).getTenantId();
     LOGGER.info("Profile migration started...");
-    return profileWrapperDao.checkIfDataInTableExists(tenantId)
-      .compose(isDataPresent -> {
-        if (!isDataPresent) {
-          return runScript(tenantId, INIT_WRAPPERS)
-            .compose(ar -> runScript(tenantId, UPDATE_SCHEMA_FOR_MIGRATION));
+
+    return profileWrapperDao.getLinesCount(tenantId, SYSTEM_TABLE_NAME)
+      .compose(isRowCount -> {
+        if (isRowCount == 0) {
+          return profileWrapperDao.checkIfDataInTableExists(tenantId)
+            .compose(isDataPresent -> processMigration(isDataPresent, tenantId));
         } else {
-          LOGGER.info("migrateDataImportProfiles:: Migration will not execute. profile_wrappers table is NOT empty already.");
+          LOGGER.info("migrateDataImportProfiles:: Migration already executed.");
           return Future.succeededFuture(true);
         }
       })
-      .onFailure(th -> LOGGER.error("migrateDataImportProfiles:: Something happened during the profile migration", th));
+      .onFailure(th -> {
+        LOGGER.error("migrateDataImportProfiles:: Something happened during the profile migration", th);
+        throw new CompletionException(th);
+      });
+  }
+
+  private Future<Boolean> processMigration(Boolean isDataPresent, String tenantId) {
+    if (!isDataPresent) {
+      return runScriptChain(tenantId, INIT_WRAPPERS, UPDATE_SCHEMA_FOR_MIGRATION);
+    } else {
+      return runScriptChain(tenantId, REMOVE_WRAPPERS, INIT_WRAPPERS, UPDATE_SCHEMA_FOR_MIGRATION);
+    }
+  }
+
+  private Future<Boolean> runScriptChain(String tenantId, String... scripts) {
+    Future<Boolean> future = Future.succeededFuture(true);
+    for (String script : scripts) {
+      future = future.compose(ar -> runScript(tenantId, script));
+    }
+    return future;
   }
 
   private Future<Boolean> runScript(String tenantId, String sqlPath) {

mod-di-converter-storage-server/src/main/resources/templates/db_scripts/associations-migration/clean_profile_wrappers.sql

@@ -0,0 +1,61 @@
+/*
+ Migration will start automatically when profile_wrappers table is empty.
+ This function cleans this table: removes FKeys, disables triggers,
+ truncates data, enables triggers and restores FKeys.
+ */
+DO $$
+DECLARE
+r record;
+BEGIN
+  RAISE NOTICE '===';
+  RAISE NOTICE 'Preparing mod_di_converter_storage for migration.';
+  DROP TABLE IF EXISTS foreign_keys;
+
+  RAISE NOTICE 'Find FKeys for deletion.';
+  CREATE TEMP TABLE foreign_keys AS
+    SELECT conrelid :: regclass AS table_name, conname AS foreign_key
+    FROM pg_constraint
+    WHERE connamespace = (SELECT current_setting('SEARCH_PATH')) :: regnamespace
+        AND contype = 'f' AND conrelid :: regclass :: text like '%_to_%'
+    ORDER BY conrelid :: regclass :: text, contype desc;
+
+  RAISE NOTICE 'Removing FKeys:';
+  FOR r in
+  SELECT table_name, foreign_key
+  FROM foreign_keys
+  LOOP
+    RAISE NOTICE ' Delete FKey:: % : %', r.table_name, r.foreign_key;
+    EXECUTE 'ALTER TABLE ' || r.table_name || ' DROP CONSTRAINT ' || r.foreign_key || ';';
+  END LOOP;
+  RAISE NOTICE 'FKeys were removed.';
+
+  RAISE NOTICE 'Disable triggers.';
+  SET session_replication_role = replica;
+
+  RAISE NOTICE 'Removing old associations.';
+  FOR r in
+  SELECT distinct table_name
+  FROM foreign_keys
+  LOOP
+    EXECUTE 'UPDATE ' || r.table_name || ' SET masterwrapperid=null, detailwrapperid=null;';
+  END LOOP;
+
+  RAISE NOTICE 'Removing old wrappers.';
+  TRUNCATE profile_wrappers;
+
+  RAISE NOTICE 'Enabling triggers back.';
+  SET session_replication_role = DEFAULT;
+
+  RAISE NOTICE 'Creating FKeys back:';
+  FOR r in
+  SELECT table_name, foreign_key
+  FROM foreign_keys
+  LOOP
+    RAISE NOTICE ' Create FKey:: % : %', r.table_name, r.foreign_key;
+    EXECUTE 'ALTER TABLE ' || r.table_name || ' ADD CONSTRAINT ' || r.foreign_key || ' FOREIGN KEY (' ||
+      left(r.foreign_key, strpos(r.foreign_key, '_') -1)|| ') REFERENCES profile_wrappers(id) ON DELETE CASCADE;';
+  END LOOP;
+
+  DROP TABLE IF EXISTS foreign_keys;
+  RAISE NOTICE 'DB ready for migration.';
+END $$;

mod-di-converter-storage-server/src/main/resources/templates/db_scripts/associations-migration/init_wrappers.sql

@@ -1,9 +1,16 @@
 CREATE EXTENSION IF NOT EXISTS "uuid-ossp" WITH SCHEMA public;
+/**
+To be able to check the migration results in the future. Could be remove after migration.
+ */
+drop table if exists snapshots_old;
+create table snapshots_old as
+  select job_profile_id, s.get_profile_snapshot ->> 'association_id' as association_id, s.get_profile_snapshot as snapshot
+    from (select jp.id job_profile_id, get_profile_snapshot(jp.id, 'JOB_PROFILE', 'job_profiles', jp.id::TEXT) from job_profiles jp) s;
 
 /*
 This script will migrate job profiles to utilize profile wrappers. The order of DML is important to ensure consistent
 state before and after migration.
- */
+*/
 
 -- create unique wrappers for each job profile
 insert into profile_wrappers (id, profile_type, job_profile_id)
@@ -224,7 +231,7 @@ $$
         into match_wrapper_id
         from profile_wrappers
         where match_profile_id = (r.jsonb ->> 'masterProfileId')::uuid
-          and associated_job_profile_id = (r.jsonb ->> 'jobProfileId')::uuid;
+            and associated_job_profile_id = (r.jsonb ->> 'jobProfileId')::uuid;
 
         if match_wrapper_id is null or action_wrapper_id is null then
           raise debug 'Incorrect data: match_to_action_profiles id: %, jobProfileId: %, action_wrapper_id: %, match_wrapper_id: %',
@@ -280,3 +287,17 @@ $$
     RAISE NOTICE 'PROFILES_MIGRATION:: updated action_to_action_profiles';
   END
 $$;
+
+/*
+ System table for saving migration history.
+ */
+insert into metadata_internal(id, jsonb, creation_date)
+  values (public.uuid_generate_v4(), '{"name": "Migration of profiles to the use of wrappers"}', now()::timestamptz);
+
+/*
+  To check the results of the migration. Could be remove after migration.
+ */
+drop table if exists snapshots_new;
+create table snapshots_new as
+  select job_profile_id, s.get_profile_snapshot ->> 'association_id' as association_id, s.get_profile_snapshot as snapshot
+    from (select jp.id job_profile_id, get_profile_snapshot(jp.id, 'JOB_PROFILE', 'job_profiles', jp.id::TEXT)  from job_profiles jp) s;

mod-di-converter-storage-server/src/main/resources/templates/db_scripts/schema.json

@@ -302,6 +302,11 @@
           "removeAccents": false
         }
       ]
+    },
+    {
+      "tableName": "metadata_internal",
+      "fromModuleVersion": "mod-di-converter-storage-2.2.0",
+      "withMetadata": true
     }
   ],
   "scripts": [