refactor: adjust group migration to use flattened groups

services/api/src/migrations/lagoon/migrations/20231130000000_group_projects.ts

@@ -19,21 +19,30 @@ export const up = async (migrate) => {
 
   // load all groups from keycloak
   const allGroups = await GroupModel.loadAllGroups();
-  // transform them into lagoon expected format
-  const keycloakGroups = await GroupModel.transformKeycloakGroups(allGroups);
-  for (const kg of keycloakGroups) {
-    // extract the project ids from the group
-    const groupProjects = await GroupModel.getProjectsFromGroupAndSubgroups(kg);
-    for (const pid of groupProjects) {
-      // add them to the database
-      logger.info(`Migrating project ${pid} and groupId ${kg.id} to database`)
-      await Helpers(sqlClientPool).addProjectToGroup(pid, kg.id)
-    }
-    // if the group is in an organization
-    if (R.prop('lagoon-organization', kg.attributes)) {
-      // add it to the database
-      logger.info(`Migrating groupId ${kg.id} in organization ${R.prop('lagoon-organization', kg.attributes)} to database`)
-      await Helpers(sqlClientPool).addOrganizationToGroup(parseInt(R.prop('lagoon-organization', kg.attributes)[0], 10), kg.id)
+  // flatten them out
+  const flattenGroups = (groups, group) => {
+    groups.push(R.omit(['subGroups'], group));
+    const flatSubGroups = group.subGroups.reduce(flattenGroups, []);
+    return groups.concat(flatSubGroups);
+  };
+  const fgs = R.pipe(
+    R.reduce(flattenGroups, []),
+    )(allGroups)
+  // loop over the groups ignoring `role-subgroup` groups
+  for (const fg of fgs) {
+    if (fg.attributes['type'] != "role-subgroup") {
+      const groupProjects = await GroupModel.getProjectsFromGroup(fg);
+      for (const pid of groupProjects) {
+        logger.info(`Migrating project ${pid} and group ${fg.name}/${fg.id} to database`)
+        // add the project group association to the database
+        await Helpers(sqlClientPool).addProjectToGroup(pid, fg.id)
+      }
+      // if the group is in an organization
+      if (R.prop('lagoon-organization', fg.attributes)) {
+        // add the organization group association to the database
+        logger.info(`Migrating group ${fg.name}/${fg.id} in organization ${R.prop('lagoon-organization', fg.attributes)} to database`)
+        await Helpers(sqlClientPool).addOrganizationToGroup(parseInt(R.prop('lagoon-organization', fg.attributes)[0], 10), fg.id)
+      }
     }
   }
 

services/api/src/models/group.ts

@@ -263,9 +263,10 @@ export const Group = (clients: {
     // briefRepresentation pulls all the group information from keycloak including the attributes
     // this means we don't need to iterate over all the groups one by one anymore to get the full group information
     const fullGroups = await keycloakAdminClient.groups.find({briefRepresentation: false});
-    const keycloakGroups = await transformKeycloakGroups(fullGroups);
-
-    return keycloakGroups;
+    // no need to transform, just return the full response, only the `allGroups` and `deleteAllGroups` resolvers use this
+    // and the `sync-groups-opendistro-security` consumption of this helper sync script is going to
+    // go away in the future when we move to the `lagoon-opensearch-sync` supporting service
+    return fullGroups;
   };
 
   const loadParentGroup = async (groupInput: Group): Promise<Group> =>
@@ -468,6 +469,23 @@ export const Group = (clients: {
     }
   };
 
+  // return only project ids that still exist in lagoon in the response for which projects this group has assigned
+  // in the past some groups could have been deleted from lagoon and their `attribute` in keycloak remained
+  const getProjectsFromGroup = async (
+    group: Group
+  ): Promise<number[]> => {
+    try {
+      const groupProjectIds = getProjectIdsFromGroup(group);
+      // remove deleted projects from the result to prevent null errors in user queries
+      const existingProjects = await projectHelpers(sqlClientPool).getAllProjectsIn(groupProjectIds);
+      let existingProjectsIds = [];
+      existingProjectsIds.push(...existingProjects.map(epi => epi.id));
+      return existingProjectsIds
+    } catch (err) {
+      return [];
+    }
+  };
+
   const getGroupMembership = async (
     group: Group
   ): Promise<GroupMembership[]> => {
@@ -970,6 +988,7 @@ export const Group = (clients: {
     loadGroupsByProjectIdFromGroups,
     getProjectsFromGroupAndParents,
     getProjectsFromGroupAndSubgroups,
+    getProjectsFromGroup,
     addGroup,
     updateGroup,
     deleteGroup,