EW-1052 Add migration of legacy TSP data and align new sync with lega…

…cy sync. (#5336)

* Add legacy migration.
* Add consent to new tsp user
*  Add permissions to new tsp schools
* Set source options during migration
* Search for account by externalId of user.
* change restart policy

---------

Co-authored-by: Simone Radtke <simone.radtke@capgemini.com>
Co-authored-by: Simone Radtke <94017602+SimoneRadtke-Cap@users.noreply.github.com>

ansible/roles/schulcloud-server-tspsync/templates/api-tsp-sync-init.yml.j2

@@ -50,5 +50,5 @@ spec:
           requests:
             cpu: {{ API_CPU_REQUESTS|default("100m", true) }}
             memory: {{ API_MEMORY_REQUESTS|default("150Mi", true) }}
-      restartPolicy: Never
+      restartPolicy: OnFailure
   backoffLimit: 5

apps/server/src/infra/sync/sync.module.ts

@@ -12,6 +12,7 @@ import { LoggerModule } from '@src/core/logger';
 import { ProvisioningModule } from '@src/modules/provisioning';
 import { SyncConsole } from './console/sync.console';
 import { SyncService } from './service/sync.service';
+import { TspLegacyMigrationService } from './tsp/tsp-legacy-migration.service';
 import { TspOauthDataMapper } from './tsp/tsp-oauth-data.mapper';
 import { TspSyncService } from './tsp/tsp-sync.service';
 import { TspSyncStrategy } from './tsp/tsp-sync.strategy';
@@ -40,7 +41,7 @@ import { TspFetchService } from './tsp/tsp-fetch.service';
 		SyncUc,
 		SyncService,
 		...((Configuration.get('FEATURE_TSP_SYNC_ENABLED') as boolean)
-			? [TspSyncStrategy, TspSyncService, TspOauthDataMapper, TspFetchService]
+			? [TspSyncStrategy, TspSyncService, TspOauthDataMapper, TspFetchService, TspLegacyMigrationService]
 			: []),
 	],
 	exports: [SyncConsole],

apps/server/src/infra/sync/tsp/loggable/tsp-legacy-migration-start.loggable.spec.ts

@@ -0,0 +1,23 @@
+import { TspLegacyMigrationStartLoggable } from './tsp-legacy-migration-start.loggable';
+
+describe(TspLegacyMigrationStartLoggable.name, () => {
+	let loggable: TspLegacyMigrationStartLoggable;
+
+	beforeAll(() => {
+		loggable = new TspLegacyMigrationStartLoggable();
+	});
+
+	describe('when loggable is initialized', () => {
+		it('should be defined', () => {
+			expect(loggable).toBeDefined();
+		});
+	});
+
+	describe('getLogMessage', () => {
+		it('should return a log message', () => {
+			expect(loggable.getLogMessage()).toEqual({
+				message: 'Running migration of legacy tsp data.',
+			});
+		});
+	});
+});

apps/server/src/infra/sync/tsp/loggable/tsp-legacy-migration-start.loggable.ts

@@ -0,0 +1,11 @@
+import { Loggable, LogMessage } from '@src/core/logger';
+
+export class TspLegacyMigrationStartLoggable implements Loggable {
+	getLogMessage(): LogMessage {
+		const message: LogMessage = {
+			message: 'Running migration of legacy tsp data.',
+		};
+
+		return message;
+	}
+}

apps/server/src/infra/sync/tsp/loggable/tsp-legacy-migration-system-missing.loggable.spec.ts

@@ -0,0 +1,23 @@
+import { TspLegacyMigrationSystemMissingLoggable } from './tsp-legacy-migration-system-missing.loggable';
+
+describe(TspLegacyMigrationSystemMissingLoggable.name, () => {
+	let loggable: TspLegacyMigrationSystemMissingLoggable;
+
+	beforeAll(() => {
+		loggable = new TspLegacyMigrationSystemMissingLoggable();
+	});
+
+	describe('when loggable is initialized', () => {
+		it('should be defined', () => {
+			expect(loggable).toBeDefined();
+		});
+	});
+
+	describe('getLogMessage', () => {
+		it('should return a log message', () => {
+			expect(loggable.getLogMessage()).toEqual({
+				message: 'No legacy system found',
+			});
+		});
+	});
+});

apps/server/src/infra/sync/tsp/loggable/tsp-legacy-migration-system-missing.loggable.ts

@@ -0,0 +1,11 @@
+import { Loggable, LogMessage } from '@src/core/logger';
+
+export class TspLegacyMigrationSystemMissingLoggable implements Loggable {
+	getLogMessage(): LogMessage {
+		const message: LogMessage = {
+			message: 'No legacy system found',
+		};
+
+		return message;
+	}
+}

apps/server/src/infra/sync/tsp/loggable/tsp-legacy-school-migration-count.loggable.spec.ts

@@ -0,0 +1,26 @@
+import { TspLegacySchoolMigrationCountLoggable } from './tsp-legacy-school-migration-count.loggable';
+
+describe(TspLegacySchoolMigrationCountLoggable.name, () => {
+	let loggable: TspLegacySchoolMigrationCountLoggable;
+
+	beforeAll(() => {
+		loggable = new TspLegacySchoolMigrationCountLoggable(10);
+	});
+
+	describe('when loggable is initialized', () => {
+		it('should be defined', () => {
+			expect(loggable).toBeDefined();
+		});
+	});
+
+	describe('getLogMessage', () => {
+		it('should return a log message', () => {
+			expect(loggable.getLogMessage()).toEqual({
+				message: `Found 10 legacy tsp schools to migrate`,
+				data: {
+					total: 10,
+				},
+			});
+		});
+	});
+});

apps/server/src/infra/sync/tsp/loggable/tsp-legacy-school-migration-count.loggable.ts

@@ -0,0 +1,16 @@
+import { Loggable, LogMessage } from '@src/core/logger';
+
+export class TspLegacySchoolMigrationCountLoggable implements Loggable {
+	constructor(private readonly total: number) {}
+
+	getLogMessage(): LogMessage {
+		const message: LogMessage = {
+			message: `Found ${this.total} legacy tsp schools to migrate`,
+			data: {
+				total: this.total,
+			},
+		};
+
+		return message;
+	}
+}

apps/server/src/infra/sync/tsp/loggable/tsp-legacy-school-migration-success.loggable.spec.ts

@@ -0,0 +1,27 @@
+import { TspLegacySchoolMigrationSuccessLoggable } from './tsp-legacy-school-migration-success.loggable';
+
+describe(TspLegacySchoolMigrationSuccessLoggable.name, () => {
+	let loggable: TspLegacySchoolMigrationSuccessLoggable;
+
+	beforeAll(() => {
+		loggable = new TspLegacySchoolMigrationSuccessLoggable(10, 5);
+	});
+
+	describe('when loggable is initialized', () => {
+		it('should be defined', () => {
+			expect(loggable).toBeDefined();
+		});
+	});
+
+	describe('getLogMessage', () => {
+		it('should return a log message', () => {
+			expect(loggable.getLogMessage()).toEqual({
+				message: `Legacy tsp data migration finished. Total schools: 10, migrated schools: 5`,
+				data: {
+					total: 10,
+					migrated: 5,
+				},
+			});
+		});
+	});
+});

apps/server/src/infra/sync/tsp/loggable/tsp-legacy-school-migration-success.loggable.ts

@@ -0,0 +1,17 @@
+import { Loggable, LogMessage } from '@src/core/logger';
+
+export class TspLegacySchoolMigrationSuccessLoggable implements Loggable {
+	constructor(private readonly total: number, private readonly migrated: number) {}
+
+	getLogMessage(): LogMessage {
+		const message: LogMessage = {
+			message: `Legacy tsp data migration finished. Total schools: ${this.total}, migrated schools: ${this.migrated}`,
+			data: {
+				total: this.total,
+				migrated: this.migrated,
+			},
+		};
+
+		return message;
+	}
+}

apps/server/src/infra/sync/tsp/tsp-legacy-migration.service.integration.spec.ts

@@ -0,0 +1,106 @@
+import { createMock, DeepMocked } from '@golevelup/ts-jest';
+import { EntityManager } from '@mikro-orm/mongodb';
+import { Test, TestingModule } from '@nestjs/testing';
+import { SchoolEntity } from '@shared/domain/entity';
+import { SystemProvisioningStrategy } from '@shared/domain/interface/system-provisioning.strategy';
+import { SchoolFeature } from '@shared/domain/types';
+import { cleanupCollections, schoolEntityFactory, systemEntityFactory } from '@shared/testing';
+import { Logger } from '@src/core/logger';
+import { MongoMemoryDatabaseModule } from '@src/infra/database';
+import { SystemType } from '@src/modules/system';
+import { TspLegacyMigrationSystemMissingLoggable } from './loggable/tsp-legacy-migration-system-missing.loggable';
+import { TspLegacyMigrationService } from './tsp-legacy-migration.service';
+
+describe('account repo', () => {
+	let module: TestingModule;
+	let em: EntityManager;
+	let sut: TspLegacyMigrationService;
+	let logger: DeepMocked<Logger>;
+
+	beforeAll(async () => {
+		module = await Test.createTestingModule({
+			imports: [MongoMemoryDatabaseModule.forRoot()],
+			providers: [
+				TspLegacyMigrationService,
+				{
+					provide: Logger,
+					useValue: createMock<Logger>(),
+				},
+			],
+		}).compile();
+		sut = module.get(TspLegacyMigrationService);
+		em = module.get(EntityManager);
+		logger = module.get(Logger);
+	});
+
+	afterAll(async () => {
+		await module.close();
+	});
+
+	afterEach(async () => {
+		jest.resetAllMocks();
+		jest.clearAllMocks();
+		jest.restoreAllMocks();
+		await cleanupCollections(em);
+	});
+
+	describe('migrateLegacyData', () => {
+		describe('when legacy system is not found', () => {
+			it('should log TspLegacyMigrationSystemMissingLoggable', async () => {
+				await sut.migrateLegacyData('');
+
+				expect(logger.info).toHaveBeenCalledWith(new TspLegacyMigrationSystemMissingLoggable());
+			});
+		});
+
+		describe('when migrating legacy data', () => {
+			const setup = async () => {
+				const legacySystem = systemEntityFactory.buildWithId({
+					type: 'tsp-school',
+				});
+				const newSystem = systemEntityFactory.buildWithId({
+					type: SystemType.OAUTH,
+					provisioningStrategy: SystemProvisioningStrategy.TSP,
+				});
+
+				const schoolIdentifier = '123';
+				const legacySchool = schoolEntityFactory.buildWithId({
+					systems: [legacySystem],
+					features: [],
+				});
+
+				await em.persistAndFlush([legacySystem, newSystem, legacySchool]);
+				em.clear();
+
+				await em.getCollection('schools').findOneAndUpdate(
+					{
+						systems: [legacySystem._id],
+					},
+					{
+						$set: {
+							sourceOptions: {
+								schoolIdentifier,
+							},
+							source: 'tsp',
+						},
+					}
+				);
+
+				return { legacySystem, newSystem, legacySchool, schoolId: schoolIdentifier };
+			};
+
+			it('should update the school to the new format', async () => {
+				const { newSystem, legacySchool, schoolId: schoolIdentifier } = await setup();
+
+				await sut.migrateLegacyData(newSystem.id);
+
+				const migratedSchool = await em.findOne<SchoolEntity>(SchoolEntity.name, {
+					id: legacySchool.id,
+				});
+				expect(migratedSchool?.externalId).toBe(schoolIdentifier);
+				expect(migratedSchool?.systems[0].id).toBe(newSystem.id);
+				expect(migratedSchool?.features).toContain(SchoolFeature.OAUTH_PROVISIONING_ENABLED);
+			});
+		});
+	});
+});

apps/server/src/infra/sync/tsp/tsp-legacy-migration.service.ts

@@ -0,0 +1,93 @@
+import { EntityManager, ObjectId } from '@mikro-orm/mongodb';
+import { Injectable } from '@nestjs/common';
+import { EntityId, SchoolFeature } from '@shared/domain/types';
+import { Logger } from '@src/core/logger';
+import { TspLegacyMigrationStartLoggable } from './loggable/tsp-legacy-migration-start.loggable';
+import { TspLegacyMigrationSystemMissingLoggable } from './loggable/tsp-legacy-migration-system-missing.loggable';
+import { TspLegacySchoolMigrationCountLoggable } from './loggable/tsp-legacy-school-migration-count.loggable';
+import { TspLegacySchoolMigrationSuccessLoggable } from './loggable/tsp-legacy-school-migration-success.loggable';
+
+type LegacyTspSchoolProperties = {
+	sourceOptions: {
+		schoolIdentifier: number;
+	};
+};
+
+const TSP_LEGACY_SYSTEM_TYPE = 'tsp-school';
+const TSP_LEGACY_SOURCE_TYPE = 'tsp';
+const SCHOOLS_COLLECTION = 'schools';
+const SYSTEMS_COLLECTION = 'systems';
+
+@Injectable()
+export class TspLegacyMigrationService {
+	constructor(private readonly em: EntityManager, private readonly logger: Logger) {
+		logger.setContext(TspLegacyMigrationService.name);
+	}
+
+	public async migrateLegacyData(newSystemId: EntityId): Promise<void> {
+		this.logger.info(new TspLegacyMigrationStartLoggable());
+
+		const legacySystemId = await this.findLegacySystemId();
+
+		if (!legacySystemId) {
+			this.logger.info(new TspLegacyMigrationSystemMissingLoggable());
+			return;
+		}
+
+		const schoolIds = await this.findIdsOfLegacyTspSchools(legacySystemId);
+
+		this.logger.info(new TspLegacySchoolMigrationCountLoggable(schoolIds.length));
+
+		const promises = schoolIds.map(async (oldId): Promise<number> => {
+			const legacySchoolFilter = {
+				systems: [legacySystemId],
+				source: TSP_LEGACY_SOURCE_TYPE,
+				sourceOptions: {
+					schoolIdentifier: oldId,
+				},
+			};
+
+			const featureUpdateCount = await this.em.nativeUpdate(SCHOOLS_COLLECTION, legacySchoolFilter, {
+				$addToSet: {
+					features: SchoolFeature.OAUTH_PROVISIONING_ENABLED,
+				},
+			});
+			const idUpdateCount = await this.em.nativeUpdate(SCHOOLS_COLLECTION, legacySchoolFilter, {
+				ldapSchoolIdentifier: oldId,
+				systems: [new ObjectId(newSystemId)],
+			});
+
+			return featureUpdateCount === 1 && idUpdateCount === 1 ? 1 : 0;
+		});
+
+		const results = await Promise.allSettled(promises);
+		const successfulMigrations = results
+			.filter((r) => r.status === 'fulfilled')
+			.map((r) => r.value)
+			.reduce((previousValue, currentValue) => previousValue + currentValue, 0);
+
+		this.logger.info(new TspLegacySchoolMigrationSuccessLoggable(schoolIds.length, successfulMigrations));
+	}
+
+	private async findLegacySystemId() {
+		const tspLegacySystem = await this.em.getCollection(SYSTEMS_COLLECTION).findOne({
+			type: TSP_LEGACY_SYSTEM_TYPE,
+		});
+
+		return tspLegacySystem?._id;
+	}
+
+	private async findIdsOfLegacyTspSchools(legacySystemId: ObjectId) {
+		const schools = await this.em
+			.getCollection<LegacyTspSchoolProperties>(SCHOOLS_COLLECTION)
+			.find({
+				systems: [legacySystemId],
+				source: TSP_LEGACY_SOURCE_TYPE,
+			})
+			.toArray();
+
+		const schoolIds = schools.map((school) => school.sourceOptions.schoolIdentifier);
+
+		return schoolIds;
+	}
+}