diff --git a/mod-di-converter-storage-server/pom.xml b/mod-di-converter-storage-server/pom.xml index 0b848881..6b855543 100644 --- a/mod-di-converter-storage-server/pom.xml +++ b/mod-di-converter-storage-server/pom.xml @@ -143,6 +143,7 @@ org.folio domain-models-maven-plugin + ${raml-module-builder.version} generate_interfaces diff --git a/mod-di-converter-storage-server/src/main/java/org/folio/dao/association/ProfileWrapperDao.java b/mod-di-converter-storage-server/src/main/java/org/folio/dao/association/ProfileWrapperDao.java index d906aea8..1229d1b7 100644 --- a/mod-di-converter-storage-server/src/main/java/org/folio/dao/association/ProfileWrapperDao.java +++ b/mod-di-converter-storage-server/src/main/java/org/folio/dao/association/ProfileWrapperDao.java @@ -51,4 +51,12 @@ public interface ProfileWrapperDao { * @return future with founded ProfileWrapper */ Future> getWrapperByProfileId(String profileId, ProfileType profileType, String tenantId); + + /** + * Get count of the strings in the table + * @param tenantId - tenant id + * @param tableName - table name + * @return future with count of string in the table + */ + Future getLinesCount(String tenantId, String tableName); } diff --git a/mod-di-converter-storage-server/src/main/java/org/folio/dao/association/ProfileWrapperDaoImpl.java b/mod-di-converter-storage-server/src/main/java/org/folio/dao/association/ProfileWrapperDaoImpl.java index 03e1ba39..d9191ef2 100644 --- a/mod-di-converter-storage-server/src/main/java/org/folio/dao/association/ProfileWrapperDaoImpl.java +++ b/mod-di-converter-storage-server/src/main/java/org/folio/dao/association/ProfileWrapperDaoImpl.java @@ -27,16 +27,12 @@ @Repository public class ProfileWrapperDaoImpl implements ProfileWrapperDao { private static final Logger LOGGER = LogManager.getLogger(); - private static final String ID_FIELD = "'id'"; private static final String TABLE_NAME = "profile_wrappers"; - private static final String INSERT_QUERY = "INSERT INTO %s.%s (id, profile_type, %s) VALUES ($1, $2, $3)"; - + private static final String INSERT_QUERY = "INSERT INTO %s.%s (id, profile_type, %s) VALUES ($1, $2, $3)"; private static final String SELECT_ON_EMPTY_TABLE_QUERY = "SELECT EXISTS (SELECT * FROM %s.%s LIMIT 1)"; - + private static final String SQL_LINES_COUNT = "select count(id) from %s.%s"; private static final String SELECT_QUERY = "SELECT * FROM %s.%s WHERE id = $1"; - private static final String SELECT_QUERY_ON_GETTING_PROFILE_WRAPPER = "SELECT * FROM %s.%s WHERE %s = $1"; - private static final Map profileTypeToColumn; static { @@ -92,6 +88,16 @@ public Future checkIfDataInTableExists(String tenantId) { } + @Override + public Future getLinesCount(String tenantId, String tableName) { + Promise> promise = Promise.promise(); + String query = format(SQL_LINES_COUNT, convertToPsqlStandard(tenantId), tableName); + + pgClientFactory.createInstance(tenantId).execute(query, promise); + return promise.future().map(resultSet -> resultSet.iterator().next().getInteger(0)); + + } + @Override public Future> getWrapperByProfileId(String profileId, ProfileType profileType, String tenantId) { Promise> promise = Promise.promise(); diff --git a/mod-di-converter-storage-server/src/main/java/org/folio/services/migration/ProfileMigrationServiceImpl.java b/mod-di-converter-storage-server/src/main/java/org/folio/services/migration/ProfileMigrationServiceImpl.java index a1120659..7916a38b 100644 --- a/mod-di-converter-storage-server/src/main/java/org/folio/services/migration/ProfileMigrationServiceImpl.java +++ b/mod-di-converter-storage-server/src/main/java/org/folio/services/migration/ProfileMigrationServiceImpl.java @@ -18,6 +18,7 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.Map; +import java.util.concurrent.CompletionException; import static java.lang.String.format; @@ -26,8 +27,11 @@ public class ProfileMigrationServiceImpl implements ProfileMigrationService { private static final Logger LOGGER = LogManager.getLogger(); private static final String UPDATE_SCHEMA_FOR_MIGRATION = "templates/db_scripts/associations-migration/actualize_schema_for_migrations.sql"; private static final String INIT_WRAPPERS = "templates/db_scripts/associations-migration/init_wrappers.sql"; + private static final String REMOVE_WRAPPERS = "templates/db_scripts/associations-migration/clean_profile_wrappers.sql"; private static final String TENANT_PLACEHOLDER = "${myuniversity}"; private static final String MODULE_PLACEHOLDER = "${mymodule}"; + private static final String SYSTEM_TABLE_NAME = "metadata_internal"; + @Autowired protected PostgresClientFactory pgClientFactory; @Autowired @@ -37,17 +41,36 @@ public class ProfileMigrationServiceImpl implements ProfileMigrationService { public Future migrateDataImportProfiles(Map headers, Context context) { String tenantId = new OkapiConnectionParams(headers).getTenantId(); LOGGER.info("Profile migration started..."); - return profileWrapperDao.checkIfDataInTableExists(tenantId) - .compose(isDataPresent -> { - if (!isDataPresent) { - return runScript(tenantId, INIT_WRAPPERS) - .compose(ar -> runScript(tenantId, UPDATE_SCHEMA_FOR_MIGRATION)); + + return profileWrapperDao.getLinesCount(tenantId, SYSTEM_TABLE_NAME) + .compose(isRowCount -> { + if (isRowCount == 0) { + return profileWrapperDao.checkIfDataInTableExists(tenantId) + .compose(isDataPresent -> processMigration(isDataPresent, tenantId)); } else { - LOGGER.info("migrateDataImportProfiles:: Migration will not execute. profile_wrappers table is NOT empty already."); + LOGGER.info("migrateDataImportProfiles:: Migration already executed."); return Future.succeededFuture(true); } }) - .onFailure(th -> LOGGER.error("migrateDataImportProfiles:: Something happened during the profile migration", th)); + .onFailure(th -> { + LOGGER.error("migrateDataImportProfiles:: Something happened during the profile migration", th); + }); + } + + private Future processMigration(Boolean isDataPresent, String tenantId) { + if (!isDataPresent) { + return runScriptChain(tenantId, INIT_WRAPPERS, UPDATE_SCHEMA_FOR_MIGRATION); + } else { + return runScriptChain(tenantId, REMOVE_WRAPPERS, INIT_WRAPPERS, UPDATE_SCHEMA_FOR_MIGRATION); + } + } + + private Future runScriptChain(String tenantId, String... scripts) { + Future future = Future.succeededFuture(true); + for (String script : scripts) { + future = future.compose(ar -> runScript(tenantId, script)); + } + return future; } private Future runScript(String tenantId, String sqlPath) { diff --git a/mod-di-converter-storage-server/src/main/resources/templates/db_scripts/associations-migration/actualize_schema_for_migrations.sql b/mod-di-converter-storage-server/src/main/resources/templates/db_scripts/associations-migration/actualize_schema_for_migrations.sql index 2195e603..95ce53d2 100644 --- a/mod-di-converter-storage-server/src/main/resources/templates/db_scripts/associations-migration/actualize_schema_for_migrations.sql +++ b/mod-di-converter-storage-server/src/main/resources/templates/db_scripts/associations-migration/actualize_schema_for_migrations.sql @@ -9,3 +9,11 @@ CREATE OR REPLACE RULE delete_associations_with_details AS ON DELETE TO associat CREATE OR REPLACE FUNCTION get_profile_snapshot(profileId uuid, profile_type text, profile_table text, jobProfileId text) RETURNS TABLE(snapshot json) AS $$ BEGIN RETURN query EXECUTE format('WITH RECURSIVE recursive_snapshot AS (SELECT job_profile.id AS association_id, CAST(NULL AS uuid) AS master_id, job_profile.id AS detail_id, CAST(NULL AS uuid) AS masterwrapperid, pw.id AS detailwrapperid, ''%s'' detail_type, 0 AS detail_order, json_agg(job_profile.jsonb) detail, null AS react_to FROM %s AS job_profile LEFT JOIN profile_wrappers pw ON pw.job_profile_id = job_profile.id WHERE job_profile.id = ''%s'' GROUP BY job_profile.id, pw.id UNION ALL SELECT associations_view.association_id, associations_view.master_id AS master_id, associations_view.detail_id AS detail_id, associations_view.masterwrapperid AS masterwrapperid, associations_view.detailwrapperid AS detailwrapperid, associations_view.detail_type AS detail_type, associations_view.detail_order AS detail_order, associations_view.detail AS detail, associations_view.react_to AS react_to FROM associations_view INNER JOIN recursive_snapshot ON associations_view.masterwrapperid = recursive_snapshot.detailwrapperid AND CASE WHEN associations_view.master_type = ''MATCH_PROFILE'' AND ''%s'' != ''null'' THEN associations_view.job_profile_id = NULLIF(''%s'',''null'')::uuid ELSE associations_view.job_profile_id IS NULL END) SELECT row_to_json(row) FROM recursive_snapshot row ORDER BY row.detail_order ASC', profile_type, profile_table, profileId, jobProfileId, jobProfileId); END $$language plpgsql; GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA ${myuniversity}_${mymodule} TO ${myuniversity}_${mymodule}; + +/* + To check the results of the migration. Could be remove after migration. + */ +drop table if exists snapshots_new; +create table snapshots_new as +select job_profile_id, s.get_profile_snapshot ->> 'association_id' as association_id, s.get_profile_snapshot as snapshot + from (select jp.id job_profile_id, get_profile_snapshot(jp.id, 'JOB_PROFILE', 'job_profiles', jp.id::TEXT) from job_profiles jp) s; diff --git a/mod-di-converter-storage-server/src/main/resources/templates/db_scripts/associations-migration/actualize_wrappers_ids_at_relations_jsonb.sql b/mod-di-converter-storage-server/src/main/resources/templates/db_scripts/associations-migration/actualize_wrappers_ids_at_relations_jsonb.sql index bb8f97c2..3bcdf23a 100644 --- a/mod-di-converter-storage-server/src/main/resources/templates/db_scripts/associations-migration/actualize_wrappers_ids_at_relations_jsonb.sql +++ b/mod-di-converter-storage-server/src/main/resources/templates/db_scripts/associations-migration/actualize_wrappers_ids_at_relations_jsonb.sql @@ -32,3 +32,4 @@ WHERE masterwrapperid IS NOT NULL AND detailwrapperid IS NOT NULL AND jsonb->>'m UPDATE ${myuniversity}_${mymodule}.job_to_match_profiles SET jsonb = jsonb_set(jsonb_set(jsonb, '{masterWrapperId}', to_jsonb(masterwrapperid), true),'{detailWrapperId}', to_jsonb(detailWrapperId), true) WHERE masterwrapperid IS NOT NULL AND detailwrapperid IS NOT NULL AND jsonb->>'masterWrapperId' IS NULL AND jsonb->>'detailWrapperId' IS NULL; + diff --git a/mod-di-converter-storage-server/src/main/resources/templates/db_scripts/associations-migration/clean_profile_wrappers.sql b/mod-di-converter-storage-server/src/main/resources/templates/db_scripts/associations-migration/clean_profile_wrappers.sql new file mode 100644 index 00000000..79c9c14d --- /dev/null +++ b/mod-di-converter-storage-server/src/main/resources/templates/db_scripts/associations-migration/clean_profile_wrappers.sql @@ -0,0 +1,69 @@ +/** +To be able to check the migration results in the future. Could be remove after migration. + */ +drop table if exists snapshots_old; +create table snapshots_old as + select job_profile_id, s.get_profile_snapshot ->> 'association_id' as association_id, s.get_profile_snapshot as snapshot + from (select jp.id job_profile_id, get_profile_snapshot(jp.id, 'JOB_PROFILE', 'job_profiles', jp.id::TEXT) from job_profiles jp) s; + +/* + Migration will start automatically when profile_wrappers table is empty. + This function cleans this table: removes FKeys, disables triggers, + truncates data, enables triggers and restores FKeys. + */ +DO $$ +DECLARE +r record; +BEGIN + RAISE NOTICE '==='; + RAISE NOTICE 'Preparing mod_di_converter_storage for migration.'; + DROP TABLE IF EXISTS foreign_keys; + + RAISE NOTICE 'Find FKeys for deletion.'; + CREATE TEMP TABLE foreign_keys AS + SELECT conrelid :: regclass AS table_name, conname AS foreign_key + FROM pg_constraint + WHERE connamespace = (SELECT current_setting('SEARCH_PATH')) :: regnamespace + AND contype = 'f' AND conrelid :: regclass :: text like '%_to_%' + ORDER BY conrelid :: regclass :: text, contype desc; + + RAISE NOTICE 'Removing FKeys:'; + FOR r in + SELECT table_name, foreign_key + FROM foreign_keys + LOOP + RAISE NOTICE ' Delete FKey:: % : %', r.table_name, r.foreign_key; + EXECUTE 'ALTER TABLE ' || r.table_name || ' DROP CONSTRAINT ' || r.foreign_key || ';'; + END LOOP; + RAISE NOTICE 'FKeys were removed.'; + + RAISE NOTICE 'Disable triggers.'; + SET session_replication_role = replica; + + RAISE NOTICE 'Removing old associations.'; + FOR r in + SELECT distinct table_name + FROM foreign_keys + LOOP + EXECUTE 'UPDATE ' || r.table_name || ' SET masterwrapperid=null, detailwrapperid=null;'; + END LOOP; + + RAISE NOTICE 'Removing old wrappers.'; + TRUNCATE profile_wrappers; + + RAISE NOTICE 'Enabling triggers back.'; + SET session_replication_role = DEFAULT; + + RAISE NOTICE 'Creating FKeys back:'; + FOR r in + SELECT table_name, foreign_key + FROM foreign_keys + LOOP + RAISE NOTICE ' Create FKey:: % : %', r.table_name, r.foreign_key; + EXECUTE 'ALTER TABLE ' || r.table_name || ' ADD CONSTRAINT ' || r.foreign_key || ' FOREIGN KEY (' || + left(r.foreign_key, strpos(r.foreign_key, '_') -1)|| ') REFERENCES profile_wrappers(id) ON DELETE CASCADE;'; + END LOOP; + + DROP TABLE IF EXISTS foreign_keys; + RAISE NOTICE 'DB ready for migration.'; +END $$; diff --git a/mod-di-converter-storage-server/src/main/resources/templates/db_scripts/associations-migration/init_wrappers.sql b/mod-di-converter-storage-server/src/main/resources/templates/db_scripts/associations-migration/init_wrappers.sql index eaa8ed45..30681698 100644 --- a/mod-di-converter-storage-server/src/main/resources/templates/db_scripts/associations-migration/init_wrappers.sql +++ b/mod-di-converter-storage-server/src/main/resources/templates/db_scripts/associations-migration/init_wrappers.sql @@ -3,7 +3,7 @@ CREATE EXTENSION IF NOT EXISTS "uuid-ossp" WITH SCHEMA public; /* This script will migrate job profiles to utilize profile wrappers. The order of DML is important to ensure consistent state before and after migration. - */ +*/ -- create unique wrappers for each job profile insert into profile_wrappers (id, profile_type, job_profile_id) @@ -224,7 +224,7 @@ $$ into match_wrapper_id from profile_wrappers where match_profile_id = (r.jsonb ->> 'masterProfileId')::uuid - and associated_job_profile_id = (r.jsonb ->> 'jobProfileId')::uuid; + and associated_job_profile_id = (r.jsonb ->> 'jobProfileId')::uuid; if match_wrapper_id is null or action_wrapper_id is null then raise debug 'Incorrect data: match_to_action_profiles id: %, jobProfileId: %, action_wrapper_id: %, match_wrapper_id: %', @@ -280,3 +280,10 @@ $$ RAISE NOTICE 'PROFILES_MIGRATION:: updated action_to_action_profiles'; END $$; + +/* + System table for saving migration history. + */ +insert into metadata_internal(id, jsonb, creation_date) + values (public.uuid_generate_v4(), '{"name": "Migration of profiles to the use of wrappers"}', now()::timestamptz); + diff --git a/mod-di-converter-storage-server/src/main/resources/templates/db_scripts/schema.json b/mod-di-converter-storage-server/src/main/resources/templates/db_scripts/schema.json index 2a069ba1..216be050 100644 --- a/mod-di-converter-storage-server/src/main/resources/templates/db_scripts/schema.json +++ b/mod-di-converter-storage-server/src/main/resources/templates/db_scripts/schema.json @@ -302,6 +302,11 @@ "removeAccents": false } ] + }, + { + "tableName": "metadata_internal", + "fromModuleVersion": "mod-di-converter-storage-2.1.7", + "withMetadata": true } ], "scripts": [ @@ -407,12 +412,12 @@ }, { "run": "after", - "snippetPath": "data-migration/set_account_no.sql", - "fromModuleVersion": "mod-di-converter-storage-2.2.0" + "snippet": "ALTER TABLE profile_wrappers ADD IF NOT EXISTS associated_job_profile_id uuid NULL;", + "fromModuleVersion": "mod-di-converter-storage-2.1.7" }, { "run": "after", - "snippet": "ALTER TABLE profile_wrappers ADD IF NOT EXISTS associated_job_profile_id uuid NULL;", + "snippetPath": "data-migration/set_account_no.sql", "fromModuleVersion": "mod-di-converter-storage-2.2.0" } ]