refactor: store project-group association in api instead of keycloak

Makefile

@@ -371,7 +371,7 @@ STERN_VERSION = v2.6.1
 CHART_TESTING_VERSION = v3.10.1
 K3D_IMAGE = docker.io/rancher/k3s:v1.28.6-k3s2
 TESTS = [nginx,api,features-kubernetes,bulk-deployment,features-kubernetes-2,features-variables,active-standby-kubernetes,tasks,drush,python,gitlab,github,bitbucket,services,workflows]
-CHARTS_TREEISH = prerelease/lagoon_v218
+CHARTS_TREEISH = keycloak-migrations
 TASK_IMAGES = task-activestandby
 
 # Symlink the installed kubectl client if the correct version is already

docker-compose.yaml

@@ -43,7 +43,7 @@ services:
       - ./node-packages:/app/node-packages:delegated
     environment:
       - CONSOLE_LOGGING_LEVEL=trace
-  api-init:
+  api-db-init:
     image: ${IMAGE_REPO:-lagoon}/api
     command: >
       sh -c "./node_modules/.bin/knex migrate:list --cwd /app/services/api/database
@@ -53,8 +53,24 @@ services:
       - ./node-packages:/app/node-packages:delegated
       - /app/node-packages/commons/dist
     depends_on:
-      - api-db
-      - keycloak
+      api-db:
+        condition: service_started
+      keycloak:
+        condition: service_started
+  api-lagoon-migrations:
+    image: ${IMAGE_REPO:-lagoon}/api
+    command: sh -c "./node_modules/.bin/tsc && node -r dotenv-extended/config dist/migrations/lagoon/migration.js"
+    volumes:
+      - ./services/api/src:/app/services/api/src
+    environment:
+      - KEYCLOAK_URL=http://172.17.0.1:8088
+      - NODE_ENV=development
+      - CONSOLE_LOGGING_LEVEL=trace
+    depends_on:
+      api-db-init:
+        condition: service_completed_successfully # don't start the lagoon migrations until the db migrations is completed
+      keycloak:
+        condition: service_started
   api:
     image: ${IMAGE_REPO:-lagoon}/api
     command: ./node_modules/.bin/tsc-watch --build --incremental --onSuccess "node -r dotenv-extended/config dist/index"
@@ -74,8 +90,10 @@ services:
       - S3_BAAS_SECRET_ACCESS_KEY=minio123
       - CONSOLE_LOGGING_LEVEL=debug
     depends_on:
-      - api-init
-      - keycloak
+      api-lagoon-migrations:
+        condition: service_started
+      keycloak:
+        condition: service_started
     ports:
       - '3000:3000'
     # Uncomment for local new relic tracking

services/api/database/migrations/20240312000000_group_projects.js

@@ -0,0 +1,40 @@
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.up = async function(knex) {
+    group_projects = await knex.schema.hasTable('kc_group_projects');
+    if (!group_projects) {
+        return knex.schema
+        // this table holds the main group to organization id association
+        .createTable('kc_group_organization', function (table) {
+            table.increments('id').notNullable().primary();
+            table.string('group_id', 50).notNullable();
+            table.integer('organization_id').notNullable();
+            table.unique(['group_id', 'organization_id'], {indexName: 'group_organization'});
+        })
+        // this table holds the main group to organization id association
+        .createTable('kc_group_projects', function (table) {
+            table.increments('id').notNullable().primary();
+            table.string('group_id', 50).notNullable();
+            table.integer('project_id').notNullable();
+            table.unique(['group_id', 'project_id'], {indexName: 'group_project'});
+        })
+    }
+    else {
+        return knex.schema
+    }
+};
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.down = async function(knex) {
+    // caveats around this are that the rollback can only work while data is still saved in keycloak attributes
+    // once we remove that duplication of attribute into keycloak, this rollback would result in data loss for group>project associations
+    // for any group project associations made after the attribute removal
+    return knex.schema
+    .dropTable('kc_group_organization')
+    .dropTable('kc_group_projects')
+};

services/api/package.json

@@ -15,7 +15,8 @@
     "sync:gitlab:all": "yarn run sync:gitlab:users && yarn run sync:gitlab:groups && yarn run sync:gitlab:projects",
     "sync:opendistro-security": "node --max-http-header-size=80000 dist/helpers/sync-groups-opendistro-security",
     "sync:bitbucket:repo-permissions": "node dist/bitbucket-sync/repo-permissions",
-    "sync:harbor:projects": "node dist/migrations/2-harbor/harborSync.js"
+    "sync:harbor:projects": "node dist/migrations/2-harbor/harborSync.js",
+    "migrations:lagoon": "node dist/migrations/lagoon/migrations.js"
   },
   "keywords": [],
   "author": "amazee.io <hello@amazee.io> (http://www.amazee.io)",
@@ -49,7 +50,7 @@
     "jsonwebtoken": "^8.0.1",
     "keycloak-admin": "https://github.com/amazeeio/keycloak-admin.git#bd015d2e34634f262c0827f00620657427e3c252",
     "keycloak-connect": "^5.0.0",
-    "knex": "^0.95.15",
+    "knex": "^3.0.1",
     "mariadb": "^2.5.2",
     "moment": "^2.24.0",
     "morgan": "^1.9.0",

services/api/src/apolloServer.js

@@ -120,28 +120,6 @@ const apolloServer = new ApolloServer({
         esClient,
       };
 
-      // get all keycloak groups, do this early to reduce the number of times this is called otherwise
-      // but doing this early and once is pretty cheap
-      let keycloakGroups = []
-      try {
-        // check redis for the allgroups cache value
-        const data = await getRedisKeycloakCache("allgroups");
-        let buff = new Buffer(data, 'base64');
-        keycloakGroups = JSON.parse(buff.toString('utf-8'));
-      } catch (err) {
-        logger.warn(`Couldn't check redis keycloak cache: ${err.message}`);
-        // if it can't be recalled from redis, get the data from keycloak
-        const allGroups = await Group.Group(modelClients).loadAllGroups();
-        keycloakGroups = await Group.Group(modelClients).transformKeycloakGroups(allGroups);
-        const data = Buffer.from(JSON.stringify(keycloakGroups)).toString('base64')
-        try {
-          // then attempt to save it to redis
-          await saveRedisKeycloakCache("allgroups", data);
-        } catch (err) {
-          logger.warn(`Couldn't save redis keycloak cache: ${err.message}`);
-        }
-      }
-
       let currentUser = {};
       let serviceAccount = {};
       // if this is a user request, get the users keycloak groups too, do this one to reduce the number of times it is called elsewhere
@@ -151,11 +129,12 @@ const apolloServer = new ApolloServer({
       const keycloakGrant = grant
       let legacyGrant = legacyCredentials ? legacyCredentials : null
       if (keycloakGrant) {
+        // get all the users keycloak groups, do this early to reduce the number of times this is called otherwise
         keycloakUsersGroups = await User.User(modelClients).getAllGroupsForUser(keycloakGrant.access_token.content.sub);
         serviceAccount = await keycloakGrantManager.obtainFromClientCredentials();
         currentUser = await User.User(modelClients).loadUserById(keycloakGrant.access_token.content.sub);
         // grab the users project ids and roles in the first request
-        groupRoleProjectIds = await User.User(modelClients).getAllProjectsIdsForUser(currentUser, keycloakUsersGroups);
+        groupRoleProjectIds = await User.User(modelClients).getAllProjectsIdsForUser(currentUser.id, keycloakUsersGroups);
       }
 
       return {
@@ -173,7 +152,6 @@ const apolloServer = new ApolloServer({
           ProjectModel: ProjectModel.ProjectModel(modelClients),
           EnvironmentModel: EnvironmentModel.EnvironmentModel(modelClients)
         },
-        keycloakGroups,
         keycloakUsersGroups,
       };
     },
@@ -206,28 +184,6 @@ const apolloServer = new ApolloServer({
         esClient,
       };
 
-      // get all keycloak groups, do this early to reduce the number of times this is called otherwise
-      // but doing this early and once is pretty cheap
-      let keycloakGroups = []
-      try {
-        // check redis for the allgroups cache value
-        const data = await getRedisKeycloakCache("allgroups");
-        let buff = new Buffer(data, 'base64');
-        keycloakGroups = JSON.parse(buff.toString('utf-8'));
-      } catch (err) {
-        logger.warn(`Couldn't check redis keycloak cache: ${err.message}`);
-        // if it can't be recalled from redis, get the data from keycloak
-        const allGroups = await Group.Group(modelClients).loadAllGroups();
-        keycloakGroups = await Group.Group(modelClients).transformKeycloakGroups(allGroups);
-        const data = Buffer.from(JSON.stringify(keycloakGroups)).toString('base64')
-        try {
-          // then attempt to save it to redis
-          await saveRedisKeycloakCache("allgroups", data);
-        } catch (err) {
-          logger.warn(`Couldn't save redis keycloak cache: ${err.message}`);
-        }
-      }
-
       let currentUser = {};
       let serviceAccount = {};
       // if this is a user request, get the users keycloak groups too, do this one to reduce the number of times it is called elsewhere
@@ -237,11 +193,12 @@ const apolloServer = new ApolloServer({
       const keycloakGrant = req.kauth ? req.kauth.grant : null
       let legacyGrant = req.legacyCredentials ? req.legacyCredentials : null
       if (keycloakGrant) {
+        // get all the users keycloak groups, do this early to reduce the number of times this is called otherwise
         keycloakUsersGroups = await User.User(modelClients).getAllGroupsForUser(keycloakGrant.access_token.content.sub);
         serviceAccount = await keycloakGrantManager.obtainFromClientCredentials();
         currentUser = await User.User(modelClients).loadUserById(keycloakGrant.access_token.content.sub);
         // grab the users project ids and roles in the first request
-        groupRoleProjectIds = await User.User(modelClients).getAllProjectsIdsForUser(currentUser, keycloakUsersGroups);
+        groupRoleProjectIds = await User.User(modelClients).getAllProjectsIdsForUser(currentUser.id, keycloakUsersGroups);
       }
 
       // do a permission check to see if the user is platform admin/owner, or has permission for `viewAll` on certain resources
@@ -308,7 +265,6 @@ const apolloServer = new ApolloServer({
           ProjectModel: ProjectModel.ProjectModel(modelClients),
           EnvironmentModel: EnvironmentModel.EnvironmentModel(modelClients)
         },
-        keycloakGroups,
         keycloakUsersGroups,
         adminScopes: {
           projectViewAll: projectViewAll,

services/api/src/clients/redisClient.ts

@@ -27,6 +27,8 @@ redisClient.on('error', function(error) {
   console.error(error);
 });
 
+export const groupCacheExpiry = toNumber(getConfigFromEnv('REDIS_GROUP_CACHE_EXPIRY', '172800'))
+
 export const get = promisify(redisClient.get).bind(redisClient);
 const hgetall = promisify(redisClient.hgetall).bind(redisClient);
 const smembers = promisify(redisClient.smembers).bind(redisClient);

services/api/src/migrations/lagoon/migration.ts

@@ -0,0 +1,30 @@
+import { waitForKeycloak } from '../../util/waitForKeycloak';
+import { envHasConfig } from '../../util/config';
+import { logger } from '../../loggers/logger';
+import { migrate } from '../../util/db'
+
+
+(async () => {
+    await waitForKeycloak();
+
+    // run any migrations that need keycloak before starting the api
+    try {
+        // run the migrations
+        logger.info('previous migrations:');
+        const before = await migrate.migrate.list();
+        for (const l of before) {
+            if (l.length) {
+                logger.info(`- ${l.name}`)
+            }
+        }
+        logger.info('performing migrations if required');
+        // actually run the migrations
+        await migrate.migrate.latest();
+        logger.info('migrations completed');
+    } catch (e) {
+        logger.fatal(`Couldn't run migrations: ${e.message}`);
+        process.exit(1)
+    }
+
+    process.exit()
+})();

services/api/src/migrations/lagoon/migrations/20240312000000_group_projects.ts

@@ -0,0 +1,54 @@
+import * as R from 'ramda';
+import { logger } from '@lagoon/commons/dist/logs/local-logger';
+import { getKeycloakAdminClient } from '../../../clients/keycloak-admin';
+import { sqlClientPool } from '../../../clients/sqlClient';
+import { esClient } from '../../../clients/esClient';
+import redisClient from '../../../clients/redisClient';
+import { Group } from '../../../models/group';
+import { Helpers } from '../../../resources/group/helpers';
+
+export const up = async (migrate) => {
+  const keycloakAdminClient = await getKeycloakAdminClient();
+
+  const GroupModel = Group({
+    sqlClientPool,
+    keycloakAdminClient,
+    esClient,
+    redisClient
+  });
+
+  // load all groups from keycloak
+  const allGroups = await GroupModel.loadAllGroups();
+  // flatten them out
+  const flattenGroups = (groups, group) => {
+    groups.push(R.omit(['subGroups'], group));
+    const flatSubGroups = group.subGroups.reduce(flattenGroups, []);
+    return groups.concat(flatSubGroups);
+  };
+  const fgs = R.pipe(
+    R.reduce(flattenGroups, []),
+    )(allGroups)
+  // loop over the groups ignoring `role-subgroup` groups
+  for (const fg of fgs) {
+    if (fg.attributes['type'] != "role-subgroup") {
+      const groupProjects = await GroupModel.getProjectsFromGroup(fg);
+      for (const pid of groupProjects) {
+        logger.info(`Migrating project ${pid} and group ${fg.name}/${fg.id} to database`)
+        // add the project group association to the database
+        await Helpers(sqlClientPool).addProjectToGroup(pid, fg.id)
+      }
+      // if the group is in an organization
+      if (R.prop('lagoon-organization', fg.attributes)) {
+        // add the organization group association to the database
+        logger.info(`Migrating group ${fg.name}/${fg.id} in organization ${R.prop('lagoon-organization', fg.attributes)} to database`)
+        await Helpers(sqlClientPool).addOrganizationToGroup(parseInt(R.prop('lagoon-organization', fg.attributes)[0], 10), fg.id)
+      }
+    }
+  }
+
+  return migrate.schema
+}
+
+export const down = async (migrate) => {
+  return migrate.schema
+}