MODDICONV-365: Automatically migration preparation for already migrat…

…ed envs. (#49)

* MODDICONV-365: Automatically migration preparation for already migrated envs.

mod-di-converter-storage-server/pom.xml

@@ -143,6 +143,7 @@
       <plugin>
         <groupId>org.folio</groupId>
         <artifactId>domain-models-maven-plugin</artifactId>
+        <version>${raml-module-builder.version}</version>
         <executions>
           <execution>
             <id>generate_interfaces</id>

mod-di-converter-storage-server/src/main/java/org/folio/dao/association/ProfileWrapperDao.java

@@ -51,4 +51,12 @@ public interface ProfileWrapperDao {
    * @return future with founded ProfileWrapper
    */
   Future<List<ProfileWrapper>> getWrapperByProfileId(String profileId, ProfileType profileType, String tenantId);
+
+  /**
+   * Get count of the strings in the table
+   * @param tenantId - tenant id
+   * @param tableName - table name
+   * @return future with count of string in the table
+   */
+  Future<Integer> getLinesCount(String tenantId, String tableName);
 }

mod-di-converter-storage-server/src/main/java/org/folio/dao/association/ProfileWrapperDaoImpl.java

@@ -27,16 +27,12 @@
 @Repository
 public class ProfileWrapperDaoImpl implements ProfileWrapperDao {
   private static final Logger LOGGER = LogManager.getLogger();
-  private static final String ID_FIELD = "'id'";
   private static final String TABLE_NAME = "profile_wrappers";
-  private static final String INSERT_QUERY = "INSERT INTO %s.%s (id, profile_type, %s) VALUES ($1, $2, $3)";
-
+   private static final String INSERT_QUERY = "INSERT INTO %s.%s (id, profile_type, %s) VALUES ($1, $2, $3)";
   private static final String SELECT_ON_EMPTY_TABLE_QUERY = "SELECT EXISTS (SELECT * FROM %s.%s LIMIT 1)";
-
+  private static final String SQL_LINES_COUNT = "select count(id) from %s.%s";
   private static final String SELECT_QUERY = "SELECT * FROM %s.%s WHERE id = $1";
-
   private static final String SELECT_QUERY_ON_GETTING_PROFILE_WRAPPER = "SELECT * FROM %s.%s WHERE %s = $1";
-
   private static final Map<String, String> profileTypeToColumn;
 
   static {
@@ -92,6 +88,16 @@ public Future<Boolean> checkIfDataInTableExists(String tenantId) {
 
   }
 
+  @Override
+  public Future<Integer> getLinesCount(String tenantId, String tableName) {
+    Promise<RowSet<Row>> promise = Promise.promise();
+    String query = format(SQL_LINES_COUNT, convertToPsqlStandard(tenantId), tableName);
+
+    pgClientFactory.createInstance(tenantId).execute(query, promise);
+    return promise.future().map(resultSet -> resultSet.iterator().next().getInteger(0));
+
+  }
+
   @Override
   public Future<List<ProfileWrapper>> getWrapperByProfileId(String profileId, ProfileType profileType, String tenantId) {
     Promise<RowSet<Row>> promise = Promise.promise();

mod-di-converter-storage-server/src/main/java/org/folio/services/migration/ProfileMigrationServiceImpl.java

@@ -18,6 +18,7 @@
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.util.Map;
+import java.util.concurrent.CompletionException;
 
 import static java.lang.String.format;
 
@@ -26,8 +27,11 @@ public class ProfileMigrationServiceImpl implements ProfileMigrationService {
   private static final Logger LOGGER = LogManager.getLogger();
   private static final String UPDATE_SCHEMA_FOR_MIGRATION = "templates/db_scripts/associations-migration/actualize_schema_for_migrations.sql";
   private static final String INIT_WRAPPERS = "templates/db_scripts/associations-migration/init_wrappers.sql";
+  private static final String REMOVE_WRAPPERS = "templates/db_scripts/associations-migration/clean_profile_wrappers.sql";
   private static final String TENANT_PLACEHOLDER = "${myuniversity}";
   private static final String MODULE_PLACEHOLDER = "${mymodule}";
+  private static final String SYSTEM_TABLE_NAME = "metadata_internal";
+
   @Autowired
   protected PostgresClientFactory pgClientFactory;
   @Autowired
@@ -37,17 +41,36 @@ public class ProfileMigrationServiceImpl implements ProfileMigrationService {
   public Future<Boolean> migrateDataImportProfiles(Map<String, String> headers, Context context) {
     String tenantId = new OkapiConnectionParams(headers).getTenantId();
     LOGGER.info("Profile migration started...");
-    return profileWrapperDao.checkIfDataInTableExists(tenantId)
-      .compose(isDataPresent -> {
-        if (!isDataPresent) {
-          return runScript(tenantId, INIT_WRAPPERS)
-            .compose(ar -> runScript(tenantId, UPDATE_SCHEMA_FOR_MIGRATION));
+
+    return profileWrapperDao.getLinesCount(tenantId, SYSTEM_TABLE_NAME)
+      .compose(isRowCount -> {
+        if (isRowCount == 0) {
+          return profileWrapperDao.checkIfDataInTableExists(tenantId)
+            .compose(isDataPresent -> processMigration(isDataPresent, tenantId));
         } else {
-          LOGGER.info("migrateDataImportProfiles:: Migration will not execute. profile_wrappers table is NOT empty already.");
+          LOGGER.info("migrateDataImportProfiles:: Migration already executed.");
           return Future.succeededFuture(true);
         }
       })
-      .onFailure(th -> LOGGER.error("migrateDataImportProfiles:: Something happened during the profile migration", th));
+      .onFailure(th -> {
+        LOGGER.error("migrateDataImportProfiles:: Something happened during the profile migration", th);
+      });
+  }
+
+  private Future<Boolean> processMigration(Boolean isDataPresent, String tenantId) {
+    if (!isDataPresent) {
+      return runScriptChain(tenantId, INIT_WRAPPERS, UPDATE_SCHEMA_FOR_MIGRATION);
+    } else {
+      return runScriptChain(tenantId, REMOVE_WRAPPERS, INIT_WRAPPERS, UPDATE_SCHEMA_FOR_MIGRATION);
+    }
+  }
+
+  private Future<Boolean> runScriptChain(String tenantId, String... scripts) {
+    Future<Boolean> future = Future.succeededFuture(true);
+    for (String script : scripts) {
+      future = future.compose(ar -> runScript(tenantId, script));
+    }
+    return future;
   }
 
   private Future<Boolean> runScript(String tenantId, String sqlPath) {

mod-di-converter-storage-server/src/main/resources/templates/db_scripts/associations-migration/actualize_schema_for_migrations.sql

@@ -9,3 +9,11 @@ CREATE OR REPLACE RULE delete_associations_with_details AS ON DELETE TO associat
 CREATE OR REPLACE FUNCTION get_profile_snapshot(profileId uuid, profile_type text, profile_table text, jobProfileId text) RETURNS TABLE(snapshot json) AS $$ BEGIN  RETURN query EXECUTE format('WITH RECURSIVE recursive_snapshot AS (SELECT job_profile.id AS association_id, CAST(NULL AS uuid) AS master_id, job_profile.id AS detail_id, CAST(NULL AS uuid) AS masterwrapperid, pw.id AS detailwrapperid, ''%s'' detail_type, 0 AS detail_order, json_agg(job_profile.jsonb) detail, null AS react_to FROM %s AS job_profile LEFT JOIN profile_wrappers pw ON pw.job_profile_id = job_profile.id WHERE job_profile.id = ''%s'' GROUP BY job_profile.id, pw.id UNION ALL SELECT associations_view.association_id, associations_view.master_id AS master_id, associations_view.detail_id AS detail_id, associations_view.masterwrapperid AS masterwrapperid, associations_view.detailwrapperid AS detailwrapperid, associations_view.detail_type AS detail_type, associations_view.detail_order AS detail_order, associations_view.detail AS detail, associations_view.react_to AS react_to FROM associations_view INNER JOIN recursive_snapshot ON associations_view.masterwrapperid = recursive_snapshot.detailwrapperid AND CASE WHEN associations_view.master_type = ''MATCH_PROFILE'' AND ''%s'' != ''null'' THEN associations_view.job_profile_id = NULLIF(''%s'',''null'')::uuid ELSE associations_view.job_profile_id IS NULL END) SELECT row_to_json(row) FROM recursive_snapshot row ORDER BY row.detail_order ASC', profile_type, profile_table, profileId, jobProfileId, jobProfileId); END $$language plpgsql;
 
 GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA ${myuniversity}_${mymodule} TO ${myuniversity}_${mymodule};
+
+/*
+  To check the results of the migration. Could be remove after migration.
+ */
+drop table if exists snapshots_new;
+create table snapshots_new as
+select job_profile_id, s.get_profile_snapshot ->> 'association_id' as association_id, s.get_profile_snapshot as snapshot
+  from (select jp.id job_profile_id, get_profile_snapshot(jp.id, 'JOB_PROFILE', 'job_profiles', jp.id::TEXT)  from job_profiles jp) s;

mod-di-converter-storage-server/src/main/resources/templates/db_scripts/associations-migration/actualize_wrappers_ids_at_relations_jsonb.sql

@@ -32,3 +32,4 @@ WHERE masterwrapperid IS NOT NULL AND detailwrapperid IS NOT NULL AND jsonb->>'m
 UPDATE ${myuniversity}_${mymodule}.job_to_match_profiles
 SET jsonb = jsonb_set(jsonb_set(jsonb, '{masterWrapperId}', to_jsonb(masterwrapperid), true),'{detailWrapperId}', to_jsonb(detailWrapperId), true)
 WHERE masterwrapperid IS NOT NULL AND detailwrapperid IS NOT NULL AND jsonb->>'masterWrapperId' IS NULL AND jsonb->>'detailWrapperId' IS NULL;
+

mod-di-converter-storage-server/src/main/resources/templates/db_scripts/associations-migration/clean_profile_wrappers.sql

@@ -0,0 +1,69 @@
+/**
+To be able to check the migration results in the future. Could be remove after migration.
+ */
+drop table if exists snapshots_old;
+create table snapshots_old as
+  select job_profile_id, s.get_profile_snapshot ->> 'association_id' as association_id, s.get_profile_snapshot as snapshot
+    from (select jp.id job_profile_id, get_profile_snapshot(jp.id, 'JOB_PROFILE', 'job_profiles', jp.id::TEXT) from job_profiles jp) s;
+
+/*
+ Migration will start automatically when profile_wrappers table is empty.
+ This function cleans this table: removes FKeys, disables triggers,
+ truncates data, enables triggers and restores FKeys.
+ */
+DO $$
+DECLARE
+r record;
+BEGIN
+  RAISE NOTICE '===';
+  RAISE NOTICE 'Preparing mod_di_converter_storage for migration.';
+  DROP TABLE IF EXISTS foreign_keys;
+
+  RAISE NOTICE 'Find FKeys for deletion.';
+  CREATE TEMP TABLE foreign_keys AS
+    SELECT conrelid :: regclass AS table_name, conname AS foreign_key
+    FROM pg_constraint
+    WHERE connamespace = (SELECT current_setting('SEARCH_PATH')) :: regnamespace
+        AND contype = 'f' AND conrelid :: regclass :: text like '%_to_%'
+    ORDER BY conrelid :: regclass :: text, contype desc;
+
+  RAISE NOTICE 'Removing FKeys:';
+  FOR r in
+  SELECT table_name, foreign_key
+  FROM foreign_keys
+  LOOP
+    RAISE NOTICE ' Delete FKey:: % : %', r.table_name, r.foreign_key;
+    EXECUTE 'ALTER TABLE ' || r.table_name || ' DROP CONSTRAINT ' || r.foreign_key || ';';
+  END LOOP;
+  RAISE NOTICE 'FKeys were removed.';
+
+  RAISE NOTICE 'Disable triggers.';
+  SET session_replication_role = replica;
+
+  RAISE NOTICE 'Removing old associations.';
+  FOR r in
+  SELECT distinct table_name
+  FROM foreign_keys
+  LOOP
+    EXECUTE 'UPDATE ' || r.table_name || ' SET masterwrapperid=null, detailwrapperid=null;';
+  END LOOP;
+
+  RAISE NOTICE 'Removing old wrappers.';
+  TRUNCATE profile_wrappers;
+
+  RAISE NOTICE 'Enabling triggers back.';
+  SET session_replication_role = DEFAULT;
+
+  RAISE NOTICE 'Creating FKeys back:';
+  FOR r in
+  SELECT table_name, foreign_key
+  FROM foreign_keys
+  LOOP
+    RAISE NOTICE ' Create FKey:: % : %', r.table_name, r.foreign_key;
+    EXECUTE 'ALTER TABLE ' || r.table_name || ' ADD CONSTRAINT ' || r.foreign_key || ' FOREIGN KEY (' ||
+      left(r.foreign_key, strpos(r.foreign_key, '_') -1)|| ') REFERENCES profile_wrappers(id) ON DELETE CASCADE;';
+  END LOOP;
+
+  DROP TABLE IF EXISTS foreign_keys;
+  RAISE NOTICE 'DB ready for migration.';
+END $$;

mod-di-converter-storage-server/src/main/resources/templates/db_scripts/associations-migration/init_wrappers.sql

@@ -3,7 +3,7 @@ CREATE EXTENSION IF NOT EXISTS "uuid-ossp" WITH SCHEMA public;
 /*
 This script will migrate job profiles to utilize profile wrappers. The order of DML is important to ensure consistent
 state before and after migration.
- */
+*/
 
 -- create unique wrappers for each job profile
 insert into profile_wrappers (id, profile_type, job_profile_id)
@@ -224,7 +224,7 @@ $$
         into match_wrapper_id
         from profile_wrappers
         where match_profile_id = (r.jsonb ->> 'masterProfileId')::uuid
-          and associated_job_profile_id = (r.jsonb ->> 'jobProfileId')::uuid;
+            and associated_job_profile_id = (r.jsonb ->> 'jobProfileId')::uuid;
 
         if match_wrapper_id is null or action_wrapper_id is null then
           raise debug 'Incorrect data: match_to_action_profiles id: %, jobProfileId: %, action_wrapper_id: %, match_wrapper_id: %',
@@ -280,3 +280,10 @@ $$
     RAISE NOTICE 'PROFILES_MIGRATION:: updated action_to_action_profiles';
   END
 $$;
+
+/*
+ System table for saving migration history.
+ */
+insert into metadata_internal(id, jsonb, creation_date)
+  values (public.uuid_generate_v4(), '{"name": "Migration of profiles to the use of wrappers"}', now()::timestamptz);
+

mod-di-converter-storage-server/src/main/resources/templates/db_scripts/schema.json

@@ -302,6 +302,11 @@
           "removeAccents": false
         }
       ]
+    },
+    {
+      "tableName": "metadata_internal",
+      "fromModuleVersion": "mod-di-converter-storage-2.1.7",
+      "withMetadata": true
     }
   ],
   "scripts": [
@@ -407,12 +412,12 @@
     },
     {
       "run": "after",
-      "snippetPath": "data-migration/set_account_no.sql",
-      "fromModuleVersion": "mod-di-converter-storage-2.2.0"
+      "snippet": "ALTER TABLE profile_wrappers ADD IF NOT EXISTS associated_job_profile_id uuid NULL;",
+      "fromModuleVersion": "mod-di-converter-storage-2.1.7"
     },
     {
       "run": "after",
-      "snippet": "ALTER TABLE profile_wrappers ADD IF NOT EXISTS associated_job_profile_id uuid NULL;",
+      "snippetPath": "data-migration/set_account_no.sql",
       "fromModuleVersion": "mod-di-converter-storage-2.2.0"
     }
   ]