-
Notifications
You must be signed in to change notification settings - Fork 150
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #3612 from uselagoon/project-groups-db-cache
refactor: store project-group association in api instead of keycloak
- Loading branch information
Showing
30 changed files
with
1,034 additions
and
489 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
40 changes: 40 additions & 0 deletions
40
services/api/database/migrations/20240312000000_group_projects.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| /** | ||
| * @param { import("knex").Knex } knex | ||
| * @returns { Promise<void> } | ||
| */ | ||
| exports.up = async function(knex) { | ||
| group_projects = await knex.schema.hasTable('kc_group_projects'); | ||
| if (!group_projects) { | ||
| return knex.schema | ||
| // this table holds the main group to organization id association | ||
| .createTable('kc_group_organization', function (table) { | ||
| table.increments('id').notNullable().primary(); | ||
| table.string('group_id', 50).notNullable(); | ||
| table.integer('organization_id').notNullable(); | ||
| table.unique(['group_id', 'organization_id'], {indexName: 'group_organization'}); | ||
| }) | ||
| // this table holds the main group to organization id association | ||
| .createTable('kc_group_projects', function (table) { | ||
| table.increments('id').notNullable().primary(); | ||
| table.string('group_id', 50).notNullable(); | ||
| table.integer('project_id').notNullable(); | ||
| table.unique(['group_id', 'project_id'], {indexName: 'group_project'}); | ||
| }) | ||
| } | ||
| else { | ||
| return knex.schema | ||
| } | ||
| }; | ||
|
|
||
| /** | ||
| * @param { import("knex").Knex } knex | ||
| * @returns { Promise<void> } | ||
| */ | ||
| exports.down = async function(knex) { | ||
| // caveats around this are that the rollback can only work while data is still saved in keycloak attributes | ||
| // once we remove that duplication of attribute into keycloak, this rollback would result in data loss for group>project associations | ||
| // for any group project associations made after the attribute removal | ||
| return knex.schema | ||
| .dropTable('kc_group_organization') | ||
| .dropTable('kc_group_projects') | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| import { waitForKeycloak } from '../../util/waitForKeycloak'; | ||
| import { envHasConfig } from '../../util/config'; | ||
| import { logger } from '../../loggers/logger'; | ||
| import { migrate } from '../../util/db' | ||
|
|
||
|
|
||
| (async () => { | ||
| await waitForKeycloak(); | ||
|
|
||
| // run any migrations that need keycloak before starting the api | ||
| try { | ||
| // run the migrations | ||
| logger.info('previous migrations:'); | ||
| const before = await migrate.migrate.list(); | ||
| for (const l of before) { | ||
| if (l.length) { | ||
| logger.info(`- ${l.name}`) | ||
| } | ||
| } | ||
| logger.info('performing migrations if required'); | ||
| // actually run the migrations | ||
| await migrate.migrate.latest(); | ||
| logger.info('migrations completed'); | ||
| } catch (e) { | ||
| logger.fatal(`Couldn't run migrations: ${e.message}`); | ||
| process.exit(1) | ||
| } | ||
|
|
||
| process.exit() | ||
| })(); |
54 changes: 54 additions & 0 deletions
54
services/api/src/migrations/lagoon/migrations/20240312000000_group_projects.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,54 @@ | ||
| import * as R from 'ramda'; | ||
| import { logger } from '@lagoon/commons/dist/logs/local-logger'; | ||
| import { getKeycloakAdminClient } from '../../../clients/keycloak-admin'; | ||
| import { sqlClientPool } from '../../../clients/sqlClient'; | ||
| import { esClient } from '../../../clients/esClient'; | ||
| import redisClient from '../../../clients/redisClient'; | ||
| import { Group } from '../../../models/group'; | ||
| import { Helpers } from '../../../resources/group/helpers'; | ||
|
|
||
| export const up = async (migrate) => { | ||
| const keycloakAdminClient = await getKeycloakAdminClient(); | ||
|
|
||
| const GroupModel = Group({ | ||
| sqlClientPool, | ||
| keycloakAdminClient, | ||
| esClient, | ||
| redisClient | ||
| }); | ||
|
|
||
| // load all groups from keycloak | ||
| const allGroups = await GroupModel.loadAllGroups(); | ||
| // flatten them out | ||
| const flattenGroups = (groups, group) => { | ||
| groups.push(R.omit(['subGroups'], group)); | ||
| const flatSubGroups = group.subGroups.reduce(flattenGroups, []); | ||
| return groups.concat(flatSubGroups); | ||
| }; | ||
| const fgs = R.pipe( | ||
| R.reduce(flattenGroups, []), | ||
| )(allGroups) | ||
| // loop over the groups ignoring `role-subgroup` groups | ||
| for (const fg of fgs) { | ||
| if (fg.attributes['type'] != "role-subgroup") { | ||
| const groupProjects = await GroupModel.getProjectsFromGroup(fg); | ||
| for (const pid of groupProjects) { | ||
| logger.info(`Migrating project ${pid} and group ${fg.name}/${fg.id} to database`) | ||
| // add the project group association to the database | ||
| await Helpers(sqlClientPool).addProjectToGroup(pid, fg.id) | ||
| } | ||
| // if the group is in an organization | ||
| if (R.prop('lagoon-organization', fg.attributes)) { | ||
| // add the organization group association to the database | ||
| logger.info(`Migrating group ${fg.name}/${fg.id} in organization ${R.prop('lagoon-organization', fg.attributes)} to database`) | ||
| await Helpers(sqlClientPool).addOrganizationToGroup(parseInt(R.prop('lagoon-organization', fg.attributes)[0], 10), fg.id) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return migrate.schema | ||
| } | ||
|
|
||
| export const down = async (migrate) => { | ||
| return migrate.schema | ||
| } |
Oops, something went wrong.